├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md ├── images │ ├── geofence.gif │ ├── logo-auto.svg │ ├── logo-dark.svg │ ├── logo-light.svg │ ├── logo-small.png │ ├── logo.png │ ├── roaming.gif │ ├── search-intersects.png │ ├── search-nearby.png │ ├── search-plain.png │ ├── search-within.png │ ├── sparse-1.png │ ├── sparse-2.png │ ├── sparse-3.png │ ├── sparse-4.png │ ├── sparse-5.png │ ├── sparse-6.png │ └── sparse-none.png └── workflows │ └── main.yml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── tile38-benchmark │ ├── az │ │ └── az.go │ └── main.go ├── tile38-cli │ └── main.go ├── tile38-luamemtest │ └── main.go └── tile38-server │ └── main.go ├── core ├── commands.go ├── commands.json ├── commands_gen.go ├── commands_test.go ├── gen.sh └── version.go ├── go.mod ├── go.sum ├── internal ├── bing │ ├── bing.go │ ├── bing_test.go │ ├── ext.go │ └── ext_test.go ├── buffer │ ├── buffer.go │ └── buffer_test.go ├── clip │ ├── clip.go │ ├── clip_test.go │ ├── collection.go │ ├── feature.go │ ├── linestring.go │ ├── point.go │ ├── polygon.go │ └── rect.go ├── collection │ ├── collection.go │ ├── collection_test.go │ ├── geodesic.go │ └── string.go ├── deadline │ └── deadline.go ├── endpoint │ ├── amqp.go │ ├── disque.go │ ├── endpoint.go │ ├── eventHub.go │ ├── grpc.go │ ├── http.go │ ├── kafka.go │ ├── local.go │ ├── mqtt.go │ ├── nats.go │ ├── pubsub.go │ ├── redis.go │ ├── scram_client.go │ └── sqs.go ├── field │ ├── field.go │ ├── field_test.go │ ├── list_binary.go │ ├── list_struct.go │ └── list_test.go ├── glob │ ├── glob.go │ ├── glob_test.go │ └── match.go ├── hservice │ ├── gen.sh │ ├── hservice.pb.go │ └── hservice.proto ├── log │ ├── log.go │ └── log_test.go ├── object │ ├── object_binary.go │ ├── object_struct.go │ └── object_test.go ├── server │ ├── aof.go │ ├── aofmigrate.go │ ├── aofshrink.go │ ├── bson.go │ ├── bson_test.go │ ├── checksum.go │ ├── client.go │ ├── config.go │ ├── crud.go │ ├── dev.go │ ├── expire.go │ ├── expr.go │ ├── expression.go │ ├── fence.go │ ├── follow.go │ ├── group.go │ ├── hooks.go │ ├── json.go │ ├── json_test.go │ ├── keys.go │ ├── live.go │ ├── metrics.go │ ├── monitor.go │ ├── must.go │ ├── must_test.go │ ├── output.go │ ├── pubqueue.go │ ├── pubsub.go │ ├── readonly.go │ ├── respconn.go │ ├── scan.go │ ├── scanner.go │ ├── scanner_test.go │ ├── scripts.go │ ├── search.go │ ├── server.go │ ├── stats.go │ ├── stats_cpu.go │ ├── stats_cpu_darlin.go │ ├── test.go │ ├── token.go │ └── token_test.go └── sstring │ ├── sstring.go │ └── sstring_test.go ├── scripts ├── RELEASE.md ├── build.sh ├── docker-push.sh ├── package.sh └── test.sh └── tests ├── 107 ├── .gitignore ├── LINK └── main.go ├── 616 └── main.go ├── README.md ├── aof_legacy ├── aof_test.go ├── client_test.go ├── fence_roaming_test.go ├── fence_test.go ├── follower_test.go ├── json_test.go ├── keys_search_test.go ├── keys_test.go ├── metrics_test.go ├── mock_io_test.go ├── mock_test.go ├── monitor_test.go ├── proto_test.go ├── scripts_test.go ├── stats_test.go ├── testcmd_test.go ├── tests_test.go └── timeout_test.go /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## How to contribute to Tile38 2 | 3 | Before getting starting with contributing, please know that we currently use [Tile38 Slack](https://tile38.com/slack) channel for casual questions and user chat. 4 | 5 | ### Did you find a bug? 6 | 7 | - **Do not open up a GitHub issue if the bug is a security vulnerability in Tile38**. Sensitive security-related issues should be reported to [security@tile38.com](mailto:security@tile38.com). 8 | 9 | - **Ensure the bug was not already reported** by searching on GitHub under [Issues](https://github.com/tidwall/tile38/issues). 10 | - If you're unable to find an open issue addressing the problem, [open a new one](https://github.com/tidwall/tile38/issues/new). Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring. 11 | 12 | ### Did you fix whitespace, format code, or make a purely cosmetic patch? 13 | 14 | Changes that are cosmetic in nature and do not add anything substantial to the stability, functionality, or testability of Tile38 will generally not be accepted. 15 | 16 | ### Do you intend to add a new feature or change an existing one? 17 | 18 | - New features will probably not be approved without prior discussion. If you need a specialized feature, make sure to express your willingness to fund the work and maintenance. 19 | - Please do not open a pull request without filing an issue and/or discussing it with a maintainer beforehand. 20 | 21 | Thanks! 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Logs** 20 | If applicable, provide logs, panics, system messages to help explain your problem. 21 | 22 | **Operating System (please complete the following information):** 23 | - OS: [e.g. Linux / Windows / Mac OS] 24 | - CPU: [e.g. amd64 / arm64 / Apple Silicon / Intel] 25 | - Version: [e.g. 1.19.0] 26 | - Container: [e.g. Docker / None] 27 | 28 | **Additional context** 29 | Add any other context about the problem here. 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Community Support 4 | url: https://tile38.com/slack/ 5 | about: Please ask and answer questions here. 6 | - name: Documenation Issues 7 | url: https://github.com/tile38/tile38.github.io/issues 8 | about: Please documenation related issues here. 9 | 10 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please do not open a pull request without first filing an issue and/or discussing the feature directly with the project maintainer. 2 | 3 | ### Please ensure you adhere to every item in this list 4 | 5 | - [ ] This PR was pre-approved by the project maintainer 6 | - [ ] I have self-reviewed the code 7 | - [ ] I have added all necessary tests 8 | 9 | ### Describe your changes 10 | 11 | Please provide detailed description of the changes. 12 | 13 | ### Issue number and link 14 | 15 | Pull request require a prior issue with discussion. 16 | Include the issue number of link here. 17 | -------------------------------------------------------------------------------- /.github/images/geofence.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/geofence.gif -------------------------------------------------------------------------------- /.github/images/logo-auto.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 21 | 22 | 23 | 24 | 26 | 28 | 30 | 32 | 34 | 36 | 38 | 40 | 42 | 43 | 44 | 45 | 46 | 47 | 49 | 51 | 53 | 55 | 57 | 59 | 61 | 63 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 74 | 75 | 76 | 77 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 87 | 88 | 89 | 90 | 91 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 101 | 102 | 103 | 104 | 105 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 115 | 117 | 119 | 121 | 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /.github/images/logo-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | 13 | 14 | 15 | 17 | 19 | 21 | 23 | 25 | 27 | 29 | 31 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 42 | 43 | 44 | 45 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 55 | 56 | 57 | 58 | 59 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 69 | 70 | 71 | 72 | 73 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 83 | 85 | 87 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /.github/images/logo-light.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | 13 | 14 | 15 | 17 | 19 | 21 | 23 | 25 | 27 | 29 | 31 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 42 | 43 | 44 | 45 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 55 | 56 | 57 | 58 | 59 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 69 | 70 | 71 | 72 | 73 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 83 | 85 | 87 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /.github/images/logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/logo-small.png -------------------------------------------------------------------------------- /.github/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/logo.png -------------------------------------------------------------------------------- /.github/images/roaming.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/roaming.gif -------------------------------------------------------------------------------- /.github/images/search-intersects.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/search-intersects.png -------------------------------------------------------------------------------- /.github/images/search-nearby.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/search-nearby.png -------------------------------------------------------------------------------- /.github/images/search-plain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/search-plain.png -------------------------------------------------------------------------------- /.github/images/search-within.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/search-within.png -------------------------------------------------------------------------------- /.github/images/sparse-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-1.png -------------------------------------------------------------------------------- /.github/images/sparse-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-2.png -------------------------------------------------------------------------------- /.github/images/sparse-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-3.png -------------------------------------------------------------------------------- /.github/images/sparse-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-4.png -------------------------------------------------------------------------------- /.github/images/sparse-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-5.png -------------------------------------------------------------------------------- /.github/images/sparse-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-6.png -------------------------------------------------------------------------------- /.github/images/sparse-none.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tidwall/tile38/3fe57a76be777436a186ed9d0f246d61c413eae7/.github/images/sparse-none.png -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | build: 11 | name: Build 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Set up Go 1.x 15 | uses: actions/setup-go@v2 16 | with: 17 | go-version: ^1.24 18 | 19 | - name: Check out code 20 | uses: actions/checkout@v2 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Test 25 | run: make test 26 | 27 | - name: Package 28 | run: make package 29 | 30 | - name: Docker push 31 | env: 32 | DOCKER_LOGIN: tidwall 33 | DOCKER_USER: tile38 34 | DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} 35 | run: ./scripts/docker-push.sh 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | tile38-* 3 | !cmd/tile38-* 4 | *.test 5 | data*/ 6 | coverage.out 7 | packages/ 8 | 9 | # Ignore IDE folders 10 | .idea/ 11 | .vscode/ 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.20 2 | 3 | ARG VERSION 4 | ARG TARGETOS 5 | ARG TARGETARCH 6 | 7 | RUN apk add --no-cache ca-certificates 8 | 9 | ADD packages/tile38-$VERSION-$TARGETOS-$TARGETARCH/tile38-server /usr/local/bin 10 | ADD packages/tile38-$VERSION-$TARGETOS-$TARGETARCH/tile38-cli /usr/local/bin 11 | ADD packages/tile38-$VERSION-$TARGETOS-$TARGETARCH/tile38-benchmark /usr/local/bin 12 | 13 | RUN addgroup -S tile38 && \ 14 | adduser -S -G tile38 tile38 && \ 15 | mkdir /data && chown tile38:tile38 /data 16 | 17 | VOLUME /data 18 | 19 | EXPOSE 9851 20 | CMD ["tile38-server", "-d", "/data"] 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Josh Baker 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: tile38-server tile38-cli tile38-benchmark tile38-luamemtest 2 | 3 | .PHONY: tile38-server 4 | tile38-server: 5 | @./scripts/build.sh tile38-server 6 | 7 | .PHONY: tile38-cli 8 | tile38-cli: 9 | @./scripts/build.sh tile38-cli 10 | 11 | .PHONY: tile38-benchmark 12 | tile38-benchmark: 13 | @./scripts/build.sh tile38-benchmark 14 | 15 | .PHONY: tile38-luamemtest 16 | tile38-luamemtest: 17 | @./scripts/build.sh tile38-luamemtest 18 | 19 | test: all 20 | @./scripts/test.sh 21 | 22 | package: 23 | @rm -rf packages/ 24 | @scripts/package.sh Windows windows amd64 25 | @scripts/package.sh Mac darwin amd64 26 | @scripts/package.sh Linux linux amd64 27 | @scripts/package.sh FreeBSD freebsd amd64 28 | @scripts/package.sh ARM linux arm 29 | @scripts/package.sh ARM64 linux arm64 30 | 31 | clean: 32 | rm -rf tile38-server tile38-cli tile38-benchmark tile38-luamemtest 33 | 34 | distclean: clean 35 | rm -rf packages/ 36 | 37 | install: all 38 | cp tile38-server /usr/local/bin 39 | cp tile38-cli /usr/local/bin 40 | cp tile38-benchmark /usr/local/bin 41 | 42 | uninstall: 43 | rm -f /usr/local/bin/tile38-server 44 | rm -f /usr/local/bin/tile38-cli 45 | rm -f /usr/local/bin/tile38-benchmark 46 | -------------------------------------------------------------------------------- /core/commands.go: -------------------------------------------------------------------------------- 1 | //go:build ignore 2 | 3 | package core 4 | 5 | import ( 6 | "encoding/json" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | clear = "\x1b[0m" 12 | bright = "\x1b[1m" 13 | gray = "\x1b[90m" 14 | yellow = "\x1b[33m" 15 | ) 16 | 17 | // Command represents a Tile38 command. 18 | type Command struct { 19 | Name string `json:"-"` 20 | Summary string `json:"summary"` 21 | Complexity string `json:"complexity"` 22 | Arguments []Argument `json:"arguments"` 23 | Since string `json:"since"` 24 | Group string `json:"group"` 25 | DevOnly bool `json:"dev"` 26 | } 27 | 28 | // String returns a string representation of the command. 29 | func (c Command) String() string { 30 | var s = c.Name 31 | for _, arg := range c.Arguments { 32 | s += " " + arg.String() 33 | } 34 | return s 35 | } 36 | 37 | // TermOutput returns a string representation of the command suitable for displaying in a terminal. 38 | func (c Command) TermOutput(indent string) string { 39 | line := c.String() 40 | var line1 string 41 | if strings.HasPrefix(line, c.Name) { 42 | line1 = bright + c.Name + clear + gray + line[len(c.Name):] + clear 43 | } else { 44 | line1 = bright + strings.Replace(c.String(), " ", " "+clear+gray, 1) + clear 45 | } 46 | line2 := yellow + "summary: " + clear + c.Summary 47 | //line3 := yellow + "since: " + clear + c.Since 48 | return indent + line1 + "\n" + indent + line2 + "\n" //+ indent + line3 + "\n" 49 | } 50 | 51 | // EnumArg represents a enum arguments. 52 | type EnumArg struct { 53 | Name string `json:"name"` 54 | Arguments []Argument `json:"arguments"` 55 | } 56 | 57 | // String returns a string representation of an EnumArg. 58 | func (a EnumArg) String() string { 59 | var s = a.Name 60 | for _, arg := range a.Arguments { 61 | s += " " + arg.String() 62 | } 63 | return s 64 | } 65 | 66 | // Argument represents a command argument. 67 | type Argument struct { 68 | Command string `json:"command"` 69 | NameAny interface{} `json:"name"` 70 | TypeAny interface{} `json:"type"` 71 | Optional bool `json:"optional"` 72 | Multiple bool `json:"multiple"` 73 | Variadic bool `json:"variadic"` 74 | Enum []string `json:"enum"` 75 | EnumArgs []EnumArg `json:"enumargs"` 76 | } 77 | 78 | // String returns a string representation of an Argument. 79 | func (a Argument) String() string { 80 | var s string 81 | if a.Command != "" { 82 | s += " " + a.Command 83 | } 84 | if len(a.EnumArgs) > 0 { 85 | eargs := "" 86 | for _, arg := range a.EnumArgs { 87 | v := arg.String() 88 | if strings.Contains(v, " ") { 89 | v = "(" + v + ")" 90 | } 91 | eargs += v + "|" 92 | } 93 | if len(eargs) > 0 { 94 | eargs = eargs[:len(eargs)-1] 95 | } 96 | s += " " + eargs 97 | } else if len(a.Enum) > 0 { 98 | s += " " + strings.Join(a.Enum, "|") 99 | } else { 100 | names, _ := a.NameTypes() 101 | subs := "" 102 | for _, name := range names { 103 | subs += " " + name 104 | } 105 | subs = strings.TrimSpace(subs) 106 | s += " " + subs 107 | if a.Variadic { 108 | if len(names) == 0 { 109 | s += " [" + subs + " ...]" 110 | } else { 111 | s += " [" + names[len(names)-1] + " ...]" 112 | } 113 | } 114 | if a.Multiple { 115 | s += " ..." 116 | } 117 | } 118 | s = strings.TrimSpace(s) 119 | if a.Optional { 120 | s = "[" + s + "]" 121 | } 122 | return s 123 | } 124 | 125 | func parseAnyStringArray(any interface{}) []string { 126 | if str, ok := any.(string); ok { 127 | return []string{str} 128 | } else if any, ok := any.([]interface{}); ok { 129 | arr := []string{} 130 | for _, any := range any { 131 | if str, ok := any.(string); ok { 132 | arr = append(arr, str) 133 | } 134 | } 135 | return arr 136 | } 137 | return []string{} 138 | } 139 | 140 | // NameTypes returns the types and names of an argument as separate arrays. 141 | func (a Argument) NameTypes() (names, types []string) { 142 | names = parseAnyStringArray(a.NameAny) 143 | types = parseAnyStringArray(a.TypeAny) 144 | if len(types) > len(names) { 145 | types = types[:len(names)] 146 | } else { 147 | for len(types) < len(names) { 148 | types = append(types, "") 149 | } 150 | } 151 | return 152 | } 153 | 154 | // Commands is a map of all of the commands. 155 | var Commands = func() map[string]Command { 156 | var commands map[string]Command 157 | if err := json.Unmarshal([]byte(commandsJSON), &commands); err != nil { 158 | panic(err.Error()) 159 | } 160 | for name, command := range commands { 161 | command.Name = strings.ToUpper(name) 162 | commands[name] = command 163 | } 164 | return commands 165 | }() 166 | 167 | var commandsJSON = `{{.CommandsJSON}}` 168 | -------------------------------------------------------------------------------- /core/commands_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "testing" 7 | ) 8 | 9 | func TestCommands(t *testing.T) { 10 | var names []string 11 | for name := range Commands { 12 | names = append(names, name) 13 | } 14 | sort.Strings(names) 15 | for _, name := range names { 16 | cmd := Commands[name] 17 | if cmd.Group == "server" { 18 | fmt.Printf("%v\n", cmd.String()) 19 | } 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /core/gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | export CommandsJSON="$(cat commands.json)" 7 | 8 | # replace out the json 9 | perl -pe ' 10 | while (($i = index($_, "{{.CommandsJSON}}")) != -1) { 11 | substr($_, $i, length("{{.CommandsJSON}}")) = $ENV{"CommandsJSON"}; 12 | } 13 | ' commands.go > commands_gen.go 14 | 15 | # remove the ignore 16 | sed -i -e 's/\/\/go:build ignore/\/\/ This file was autogenerated. DO NOT EDIT./g' commands_gen.go 17 | rm -rf commands_gen.go-e 18 | -------------------------------------------------------------------------------- /core/version.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | // Build variables 4 | var ( 5 | Version = "0.0.0" // Placeholder for the version 6 | BuildTime = "" // Placeholder for the build time 7 | GitSHA = "0000000" // Placeholder for the git sha 8 | ) 9 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tidwall/tile38 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | cloud.google.com/go/pubsub v1.33.0 9 | github.com/Azure/azure-event-hubs-go/v3 v3.3.16 10 | github.com/Shopify/sarama v1.36.0 11 | github.com/aws/aws-sdk-go v1.37.3 12 | github.com/eclipse/paho.mqtt.golang v1.3.1 13 | github.com/golang/protobuf v1.5.3 14 | github.com/gomodule/redigo v1.8.3 15 | github.com/iwpnd/sectr v0.1.2 16 | github.com/mmcloughlin/geohash v0.10.0 17 | github.com/nats-io/nats.go v1.31.0 18 | github.com/peterh/liner v1.2.1 19 | github.com/prometheus/client_golang v1.12.1 20 | github.com/streadway/amqp v1.0.0 21 | github.com/tidwall/assert v0.1.0 22 | github.com/tidwall/btree v1.5.0 23 | github.com/tidwall/buntdb v1.2.9 24 | github.com/tidwall/expr v0.13.0 25 | github.com/tidwall/geojson v1.4.5 26 | github.com/tidwall/gjson v1.14.3 27 | github.com/tidwall/hashmap v1.6.1 28 | github.com/tidwall/limiter v0.4.0 29 | github.com/tidwall/match v1.1.1 30 | github.com/tidwall/pretty v1.2.0 31 | github.com/tidwall/redbench v0.1.0 32 | github.com/tidwall/redcon v1.4.4 33 | github.com/tidwall/resp v0.1.1 34 | github.com/tidwall/rtree v1.9.2 35 | github.com/tidwall/sjson v1.2.4 36 | github.com/tidwall/tinylru v1.2.1 37 | github.com/xdg-go/scram v1.1.2 38 | github.com/yuin/gopher-lua v1.1.0 39 | go.uber.org/atomic v1.11.0 40 | go.uber.org/zap v1.26.0 41 | golang.org/x/net v0.38.0 42 | golang.org/x/term v0.30.0 43 | google.golang.org/api v0.151.0 44 | google.golang.org/grpc v1.59.0 45 | layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf 46 | ) 47 | 48 | require ( 49 | cloud.google.com/go v0.110.8 // indirect 50 | cloud.google.com/go/compute v1.23.1 // indirect 51 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 52 | cloud.google.com/go/iam v1.1.3 // indirect 53 | github.com/Azure/azure-amqp-common-go/v3 v3.2.1 // indirect 54 | github.com/Azure/azure-sdk-for-go v51.1.0+incompatible // indirect 55 | github.com/Azure/go-amqp v0.16.0 // indirect 56 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 57 | github.com/Azure/go-autorest/autorest v0.11.18 // indirect 58 | github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect 59 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 60 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 61 | github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect 62 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 63 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 64 | github.com/beorn7/perks v1.0.1 // indirect 65 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 66 | github.com/davecgh/go-spew v1.1.1 // indirect 67 | github.com/devigned/tab v0.1.1 // indirect 68 | github.com/eapache/go-resiliency v1.3.0 // indirect 69 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 70 | github.com/eapache/queue v1.1.0 // indirect 71 | github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect 72 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 73 | github.com/golang/snappy v0.0.4 // indirect 74 | github.com/google/s2a-go v0.1.7 // indirect 75 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 76 | github.com/googleapis/gax-go/v2 v2.12.0 // indirect 77 | github.com/gorilla/websocket v1.4.2 // indirect 78 | github.com/hashicorp/errwrap v1.0.0 // indirect 79 | github.com/hashicorp/go-multierror v1.1.1 // indirect 80 | github.com/hashicorp/go-uuid v1.0.3 // indirect 81 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 82 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 83 | github.com/jcmturner/gofork v1.7.6 // indirect 84 | github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect 85 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 86 | github.com/jmespath/go-jmespath v0.4.0 // indirect 87 | github.com/jpillora/backoff v1.0.0 // indirect 88 | github.com/klauspost/compress v1.17.2 // indirect 89 | github.com/mattn/go-runewidth v0.0.3 // indirect 90 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 91 | github.com/mitchellh/mapstructure v1.1.2 // indirect 92 | github.com/nats-io/nkeys v0.4.6 // indirect 93 | github.com/nats-io/nuid v1.0.1 // indirect 94 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 95 | github.com/prometheus/client_model v0.2.0 // indirect 96 | github.com/prometheus/common v0.32.1 // indirect 97 | github.com/prometheus/procfs v0.7.3 // indirect 98 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 99 | github.com/tidwall/conv v0.1.0 // indirect 100 | github.com/tidwall/geoindex v1.7.0 // indirect 101 | github.com/tidwall/grect v0.1.4 // indirect 102 | github.com/tidwall/rtred v0.1.2 // indirect 103 | github.com/tidwall/tinyqueue v0.1.1 // indirect 104 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 105 | github.com/xdg-go/stringprep v1.0.4 // indirect 106 | go.opencensus.io v0.24.0 // indirect 107 | go.uber.org/multierr v1.10.0 // indirect 108 | golang.org/x/crypto v0.36.0 // indirect 109 | golang.org/x/oauth2 v0.13.0 // indirect 110 | golang.org/x/sync v0.12.0 // indirect 111 | golang.org/x/sys v0.31.0 // indirect 112 | golang.org/x/text v0.23.0 // indirect 113 | golang.org/x/time v0.4.0 // indirect 114 | google.golang.org/appengine v1.6.7 // indirect 115 | google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect 116 | google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect 117 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect 118 | google.golang.org/protobuf v1.33.0 // indirect 119 | ) 120 | -------------------------------------------------------------------------------- /internal/bing/bing_test.go: -------------------------------------------------------------------------------- 1 | package bing 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestLevelFuzz(t *testing.T) { 10 | rand.Seed(time.Now().UnixNano()) 11 | for i := 0; i < 10000; i++ { 12 | level := (rand.Int() % MaxLevelOfDetail) + 1 13 | quad := "" 14 | for j := 0; j < level; j++ { 15 | quad += string(byte(rand.Int()%4) + '0') 16 | } 17 | tileX, tileY, levelOfDetail := QuadKeyToTileXY(quad) 18 | if levelOfDetail != uint64(len(quad)) { 19 | t.Fatalf("[%d,%d] levelOfDetail == %d, expect %d", i, level, levelOfDetail, len(quad)) 20 | } 21 | pixelX, pixelY := TileXYToPixelXY(tileX, tileY) 22 | latitude, longitude := PixelXYToLatLong(pixelX, pixelY, levelOfDetail) 23 | pixelX2, pixelY2 := LatLongToPixelXY(latitude, longitude, levelOfDetail) 24 | if pixelX2 != pixelX { 25 | t.Fatalf("[%d,%d] pixelX2 == %d, expect %d", i, level, pixelX2, pixelX) 26 | } 27 | if pixelY2 != pixelY { 28 | t.Fatalf("[%d,%d] pixelY2 == %d, expect %d", i, level, pixelY2, pixelY) 29 | } 30 | tileX2, tileY2 := PixelXYToTileXY(pixelX2, pixelY2) 31 | if tileX2 != tileX { 32 | t.Fatalf("[%d,%d] tileX2 == %d, expect %d", i, level, tileX2, tileX) 33 | } 34 | if tileY2 != tileY { 35 | t.Fatalf("[%d,%d] tileY2 == %d, expect %d", i, level, tileY2, tileY) 36 | } 37 | quad2 := TileXYToQuadKey(tileX2, tileY2, levelOfDetail) 38 | if quad2 != quad { 39 | t.Fatalf("[%d,%d] quad2 == %s, expect %s", i, level, quad2, quad) 40 | } 41 | } 42 | } 43 | 44 | func TestInvalidQuadKeyFuzz(t *testing.T) { 45 | rand.Seed(time.Now().UnixNano()) 46 | for i := 0; i < 10000; i++ { 47 | func() { 48 | defer func() { 49 | var s string 50 | if v := recover(); v != nil { 51 | s = v.(string) 52 | } 53 | if s != "Invalid QuadKey digit sequence." { 54 | t.Fatalf("s == '%s', expect '%s", s, "Invalid QuadKey digit sequence.") 55 | } 56 | }() 57 | level := (rand.Int() % MaxLevelOfDetail) + 1 58 | 59 | valid := true 60 | quad := "" 61 | for valid { 62 | quad = "" 63 | for j := 0; j < level; j++ { 64 | c := byte(rand.Int()%5) + '0' 65 | quad += string(c) 66 | if c < '0' || c > '3' { 67 | valid = false 68 | } 69 | } 70 | } 71 | QuadKeyToTileXY(quad) 72 | }() 73 | } 74 | } 75 | 76 | func TestLatLonClippingFuzz(t *testing.T) { 77 | rand.Seed(time.Now().UnixNano()) 78 | for i := 0; i < 10000; i++ { 79 | lat := clip(rand.Float64()*180.0-90.0, MinLatitude, MaxLatitude) 80 | lon := clip(rand.Float64()*380.0-180.0, MinLongitude, MaxLongitude) 81 | if lat < MinLatitude { 82 | t.Fatalf("lat == %f, expect < %f", lat, MinLatitude) 83 | } 84 | if lat > MaxLatitude { 85 | t.Fatalf("lat == %f, expect > %f", lat, MaxLatitude) 86 | } 87 | if lon < MinLongitude { 88 | t.Fatalf("lon == %f, expect < %f", lon, MinLongitude) 89 | } 90 | if lon > MaxLongitude { 91 | t.Fatalf("lon == %f, expect > %f", lon, MaxLongitude) 92 | } 93 | } 94 | } 95 | 96 | func TestIssue302(t *testing.T) { 97 | // Requesting tile with zoom level > 63 crashes the server #302 98 | for z := uint64(0); z < 256; z++ { 99 | tileX, tileY := PixelXYToTileXY(LatLongToPixelXY(33, -115, z)) 100 | TileXYToBounds(tileX, tileY, z) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /internal/bing/ext.go: -------------------------------------------------------------------------------- 1 | package bing 2 | 3 | import "errors" 4 | 5 | // LatLongToQuad iterates through all of the quads parts until levelOfDetail is reached. 6 | func LatLongToQuad(latitude, longitude float64, levelOfDetail uint64, iterator func(part int) bool) { 7 | pixelX, pixelY := LatLongToPixelXY(latitude, longitude, levelOfDetail) 8 | tileX, tileY := PixelXYToTileXY(pixelX, pixelY) 9 | for i := levelOfDetail; i > 0; i-- { 10 | if !iterator(partForTileXY(tileX, tileY, i)) { 11 | break 12 | } 13 | } 14 | } 15 | 16 | func partForTileXY(tileX, tileY int64, levelOfDetail uint64) int { 17 | mask := int64(1 << (levelOfDetail - 1)) 18 | if (tileX & mask) != 0 { 19 | if (tileY & mask) != 0 { 20 | return 3 21 | } 22 | return 1 23 | } else if (tileY & mask) != 0 { 24 | return 2 25 | } 26 | return 0 27 | } 28 | 29 | // TileXYToBounds returns the bounds around a tile. 30 | func TileXYToBounds(tileX, tileY int64, levelOfDetail uint64) (minLat, minLon, maxLat, maxLon float64) { 31 | size := int64(1 << levelOfDetail) 32 | pixelX, pixelY := TileXYToPixelXY(tileX, tileY) 33 | maxLat, minLon = PixelXYToLatLong(pixelX, pixelY, levelOfDetail) 34 | pixelX, pixelY = TileXYToPixelXY(tileX+1, tileY+1) 35 | minLat, maxLon = PixelXYToLatLong(pixelX, pixelY, levelOfDetail) 36 | if size == 0 || tileX%size == 0 { 37 | minLon = MinLongitude 38 | } 39 | if size == 0 || tileX%size == size-1 { 40 | maxLon = MaxLongitude 41 | } 42 | if tileY <= 0 { 43 | maxLat = MaxLatitude 44 | } 45 | if tileY >= size-1 { 46 | minLat = MinLatitude 47 | } 48 | return 49 | } 50 | 51 | // QuadKeyToBounds converts a quadkey to bounds 52 | func QuadKeyToBounds(quadkey string) (minLat, minLon, maxLat, maxLon float64, err error) { 53 | for i := 0; i < len(quadkey); i++ { 54 | switch quadkey[i] { 55 | case '0', '1', '2', '3': 56 | default: 57 | err = errors.New("invalid quadkey") 58 | return 59 | } 60 | } 61 | minLat, minLon, maxLat, maxLon = TileXYToBounds(QuadKeyToTileXY(quadkey)) 62 | return 63 | } 64 | -------------------------------------------------------------------------------- /internal/bing/ext_test.go: -------------------------------------------------------------------------------- 1 | package bing 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestIteratorFuzz(t *testing.T) { 10 | rand.Seed(time.Now().UnixNano()) 11 | for i := 0; i < 10000; i++ { 12 | latitude := rand.Float64()*180.0 - 90.0 13 | longitude := rand.Float64()*380.0 - 180.0 14 | levelOfDetail := uint64((rand.Int() % MaxLevelOfDetail) + 1) 15 | pixelX, pixelY := LatLongToPixelXY(latitude, longitude, levelOfDetail) 16 | tileX, tileY := PixelXYToTileXY(pixelX, pixelY) 17 | quad1 := TileXYToQuadKey(tileX, tileY, levelOfDetail) 18 | l := rand.Int() % len(quad1) 19 | i := 0 20 | quad2 := "" 21 | LatLongToQuad(latitude, longitude, levelOfDetail, func(part int) bool { 22 | if i == l { 23 | return false 24 | } 25 | quad2 += string(byte(part) + '0') 26 | i++ 27 | return true 28 | }) 29 | if quad2 != quad1[:l] { 30 | t.Fatalf("[%d,%d] quad2 == %s, expect %s", i, levelOfDetail, quad2, quad1[:l]) 31 | } 32 | } 33 | } 34 | 35 | func TestExt(t *testing.T) { 36 | // tileX, tileY, levelOfDetail := int64(0), int64(0), uint64(0) 37 | // parts := strings.Split(os.Getenv("TEST_TILE"), ",") 38 | // if len(parts) == 3 { 39 | // tileX, _ = strconv.ParseInt(parts[0], 10, 64) 40 | // tileY, _ = strconv.ParseInt(parts[1], 10, 64) 41 | // levelOfDetail, _ = strconv.ParseUint(parts[2], 10, 64) 42 | // } 43 | // minLat, minLon, maxLat, maxLon := TileXYToBounds(tileX, tileY, levelOfDetail) 44 | // fmt.Printf("\x1b[32m== Tile Boundaries ==\x1b[0m\n") 45 | // fmt.Printf("\x1b[31m%d,%d,%d\x1b[0m\n", tileX, tileY, levelOfDetail) 46 | // fmt.Printf("\x1b[31mWGS84 datum (longitude/latitude):\x1b[0m\n") 47 | // fmt.Printf("%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) 48 | 49 | //fmt.Printf("\x1b[32m\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) 50 | // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 0, 1) 51 | // fmt.Printf("\x1b[32m1,0\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) 52 | // minLat, minLon, maxLat, maxLon = TileXYToBounds(0, 1, 1) 53 | // fmt.Printf("\x1b[32m0,1\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) 54 | // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 1, 1) 55 | // fmt.Printf("\x1b[32m1,1\x1b[0m\n%v %v\n%v %v\n\n", minLon, minLat, maxLon, maxLat) 56 | 57 | // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 0, 1) 58 | // fmt.Printf("1,0: %f,%f %f,%f\n", minLat, minLon, maxLat, maxLon) 59 | // minLat, minLon, maxLat, maxLon = TileXYToBounds(0, 1, 1) 60 | // fmt.Printf("0,1: %f,%f %f,%f\n", minLat, minLon, maxLat, maxLon) 61 | // minLat, minLon, maxLat, maxLon = TileXYToBounds(1, 1, 1) 62 | // fmt.Printf("1,1: %f,%f %f,%f\n", minLat, minLon, maxLat, maxLon) 63 | } 64 | -------------------------------------------------------------------------------- /internal/buffer/buffer_test.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/tidwall/geojson" 7 | "github.com/tidwall/geojson/geometry" 8 | ) 9 | 10 | const lineString = `{"type":"LineString","coordinates":[ 11 | [-116.40289306640624,34.125447565116126], 12 | [-116.36444091796875,34.14818102254435], 13 | [-116.0980224609375,34.15045403191448], 14 | [-115.74920654296874,34.127721186043985], 15 | [-115.54870605468749,34.075412438417395], 16 | [-115.5267333984375,34.11407854333859], 17 | [-115.21911621093749,34.048108084909835], 18 | [-115.25207519531249,33.8339199536547], 19 | [-115.40588378906249,33.71748624018193] 20 | ]}` 21 | 22 | var lineInPoints = []geometry.Point{ 23 | {X: -115.64363479614258, Y: 34.108251327293296}, 24 | {X: -115.54355621337892, Y: 34.07199987534163}, 25 | {X: -115.21482467651367, Y: 34.051237154976164}, 26 | {X: -115.4110336303711, Y: 33.715201644740844}, 27 | {X: -116.40701293945311, Y: 34.12345809664606}, 28 | } 29 | 30 | func TestBufferLineString(t *testing.T) { 31 | g, err := geojson.Parse(lineString, nil) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | g2, err := Simple(g, 1000) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | for _, pt := range lineInPoints { 40 | ok := g2.Contains(geojson.NewPoint(pt)) 41 | if !ok { 42 | t.Fatalf("!ok") 43 | } 44 | } 45 | } 46 | 47 | const polygon = `{"type": "Polygon","coordinates":[ 48 | [ 49 | [116.46881103515624,34.277644878733824], 50 | [115.87280273437499,34.20953080048952], 51 | [115.70251464843749,34.397844946449865], 52 | [115.9881591796875,34.61286625296406], 53 | [116.46881103515624,34.277644878733824] 54 | ], 55 | [ 56 | [115.90438842773436,34.38651267795365], 57 | [116.05270385742188,34.35023911062779], 58 | [115.99914550781249,34.44655621402982], 59 | [115.90438842773436,34.38651267795365] 60 | ] 61 | ]}` 62 | 63 | var polyInPoints = []geometry.Point{ 64 | {X: 115.95837593078612, Y: 34.59887847065301}, 65 | {X: 115.98755836486816, Y: 34.61879975173954}, 66 | {X: 115.98833084106445, Y: 34.59795999847678}, 67 | {X: 116.04536533355714, Y: 34.58082509817638}, 68 | {X: 116.47567749023438, Y: 34.27651009584797}, 69 | {X: 116.42005920410155, Y: 34.32018817684490}, 70 | {X: 116.33216857910156, Y: 34.25948651450623}, 71 | {X: 115.89340209960939, Y: 34.24132422972854}, 72 | {X: 115.95588684082033, Y: 34.42786803680155}, 73 | {X: 115.97236633300783, Y: 34.42107129982385}, 74 | {X: 115.99639892578125, Y: 34.43579686485573}, 75 | {X: 116.04652404785155, Y: 34.35364042469895}, 76 | {X: 115.92155456542967, Y: 34.38877925439021}, 77 | {X: 115.96755981445311, Y: 34.37687904351907}, 78 | {X: 115.88859558105467, Y: 34.42956713470528}, 79 | {X: 115.97511291503906, Y: 34.36327673174518}, 80 | {X: 115.69564819335938, Y: 34.39784494644986}, 81 | {X: 115.87005615234375, Y: 34.20385213966983}, 82 | {X: 115.76980590820312, Y: 34.31678550602221}, 83 | } 84 | var polyOutPoints = []geometry.Point{ 85 | {X: 115.68534851074217, Y: 34.40917568058836}, 86 | {X: 115.98953247070312, Y: 34.63038297923298}, 87 | {X: 115.98541259765624, Y: 34.39671178864245}, 88 | {X: 116.31500244140626, Y: 34.22145474280257}, 89 | {X: 115.85426330566406, Y: 34.18510984477340}, 90 | } 91 | 92 | func TestBufferPolygon(t *testing.T) { 93 | g, err := geojson.Parse(polygon, nil) 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | g2, err := Simple(g, 1000) 98 | if err != nil { 99 | t.Fatal(err) 100 | } 101 | for _, pt := range polyInPoints { 102 | ok := g2.Contains(geojson.NewPoint(pt)) 103 | if !ok { 104 | t.Fatalf("!ok") 105 | } 106 | } 107 | for _, pt := range polyOutPoints { 108 | ok := g2.Contains(geojson.NewPoint(pt)) 109 | if ok { 110 | t.Fatalf("ok") 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /internal/clip/clip.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | // Clip clips the contents of a geojson object and return 9 | func Clip( 10 | obj geojson.Object, clipper geojson.Object, opts *geometry.IndexOptions, 11 | ) (clipped geojson.Object) { 12 | switch obj := obj.(type) { 13 | case *geojson.Point: 14 | return clipPoint(obj, clipper, opts) 15 | case *geojson.Rect: 16 | return clipRect(obj, clipper, opts) 17 | case *geojson.LineString: 18 | return clipLineString(obj, clipper, opts) 19 | case *geojson.Polygon: 20 | return clipPolygon(obj, clipper, opts) 21 | case *geojson.Feature: 22 | return clipFeature(obj, clipper, opts) 23 | case geojson.Collection: 24 | return clipCollection(obj, clipper, opts) 25 | } 26 | return obj 27 | } 28 | 29 | // clipSegment is Cohen-Sutherland Line Clipping 30 | // https://www.cs.helsinki.fi/group/goa/viewing/leikkaus/lineClip.html 31 | func clipSegment(seg geometry.Segment, rect geometry.Rect) ( 32 | res geometry.Segment, rejected bool, 33 | ) { 34 | startCode := getCode(rect, seg.A) 35 | endCode := getCode(rect, seg.B) 36 | if (startCode | endCode) == 0 { 37 | // trivially accept 38 | res = seg 39 | } else if (startCode & endCode) != 0 { 40 | // trivially reject 41 | rejected = true 42 | } else if startCode != 0 { 43 | // start is outside. get new start. 44 | newStart := intersect(rect, startCode, seg.A, seg.B) 45 | res, rejected = 46 | clipSegment(geometry.Segment{A: newStart, B: seg.B}, rect) 47 | } else { 48 | // end is outside. get new end. 49 | newEnd := intersect(rect, endCode, seg.A, seg.B) 50 | res, rejected = clipSegment(geometry.Segment{A: seg.A, B: newEnd}, rect) 51 | } 52 | return 53 | } 54 | 55 | // clipRing is Sutherland-Hodgman Polygon Clipping 56 | // https://www.cs.helsinki.fi/group/goa/viewing/leikkaus/intro2.html 57 | func clipRing(ring []geometry.Point, bbox geometry.Rect) ( 58 | resRing []geometry.Point, 59 | ) { 60 | if len(ring) < 4 { 61 | // under 4 elements this is not a polygon ring! 62 | return 63 | } 64 | var edge uint8 65 | var inside, prevInside bool 66 | var prev geometry.Point 67 | for edge = 1; edge <= 8; edge *= 2 { 68 | prev = ring[len(ring)-2] 69 | prevInside = (getCode(bbox, prev) & edge) == 0 70 | for _, p := range ring { 71 | inside = (getCode(bbox, p) & edge) == 0 72 | if prevInside && inside { 73 | // Staying inside 74 | resRing = append(resRing, p) 75 | } else if prevInside && !inside { 76 | // Leaving 77 | resRing = append(resRing, intersect(bbox, edge, prev, p)) 78 | } else if !prevInside && inside { 79 | // Entering 80 | resRing = append(resRing, intersect(bbox, edge, prev, p)) 81 | resRing = append(resRing, p) 82 | } /* else { 83 | // Stay outside 84 | } */ 85 | prev, prevInside = p, inside 86 | } 87 | if len(resRing) > 0 && resRing[0] != resRing[len(resRing)-1] { 88 | resRing = append(resRing, resRing[0]) 89 | } 90 | ring, resRing = resRing, []geometry.Point{} 91 | if len(ring) == 0 { 92 | break 93 | } 94 | } 95 | resRing = ring 96 | return 97 | } 98 | 99 | func getCode(bbox geometry.Rect, point geometry.Point) (code uint8) { 100 | code = 0 101 | 102 | if point.X < bbox.Min.X { 103 | code |= 1 // left 104 | } else if point.X > bbox.Max.X { 105 | code |= 2 // right 106 | } 107 | 108 | if point.Y < bbox.Min.Y { 109 | code |= 4 // bottom 110 | } else if point.Y > bbox.Max.Y { 111 | code |= 8 // top 112 | } 113 | 114 | return 115 | } 116 | 117 | func intersect(bbox geometry.Rect, code uint8, start, end geometry.Point) ( 118 | new geometry.Point, 119 | ) { 120 | if (code & 8) != 0 { // top 121 | new = geometry.Point{ 122 | X: start.X + (end.X-start.X)*(bbox.Max.Y-start.Y)/(end.Y-start.Y), 123 | Y: bbox.Max.Y, 124 | } 125 | } else if (code & 4) != 0 { // bottom 126 | new = geometry.Point{ 127 | X: start.X + (end.X-start.X)*(bbox.Min.Y-start.Y)/(end.Y-start.Y), 128 | Y: bbox.Min.Y, 129 | } 130 | } else if (code & 2) != 0 { //right 131 | new = geometry.Point{ 132 | X: bbox.Max.X, 133 | Y: start.Y + (end.Y-start.Y)*(bbox.Max.X-start.X)/(end.X-start.X), 134 | } 135 | } else if (code & 1) != 0 { // left 136 | new = geometry.Point{ 137 | X: bbox.Min.X, 138 | Y: start.Y + (end.Y-start.Y)*(bbox.Min.X-start.X)/(end.X-start.X), 139 | } 140 | } /* else { 141 | // should not call intersect with the zero code 142 | } */ 143 | 144 | return 145 | } 146 | -------------------------------------------------------------------------------- /internal/clip/clip_test.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/tidwall/geojson" 7 | "github.com/tidwall/geojson/geometry" 8 | ) 9 | 10 | func LO(points []geometry.Point) *geojson.LineString { 11 | return geojson.NewLineString(geometry.NewLine(points, nil)) 12 | } 13 | 14 | func RO(minX, minY, maxX, maxY float64) *geojson.Rect { 15 | return geojson.NewRect(geometry.Rect{ 16 | Min: geometry.Point{X: minX, Y: minY}, 17 | Max: geometry.Point{X: maxX, Y: maxY}, 18 | }) 19 | } 20 | 21 | func PPO(exterior []geometry.Point, holes [][]geometry.Point) *geojson.Polygon { 22 | return geojson.NewPolygon(geometry.NewPoly(exterior, holes, nil)) 23 | } 24 | 25 | func TestClipLineStringSimple(t *testing.T) { 26 | ls := LO([]geometry.Point{ 27 | {X: 1, Y: 1}, 28 | {X: 2, Y: 2}, 29 | {X: 3, Y: 1}}) 30 | clipped := Clip(ls, RO(1.5, 0.5, 2.5, 1.8), nil) 31 | cl, ok := clipped.(*geojson.MultiLineString) 32 | if !ok { 33 | t.Fatal("wrong type") 34 | } 35 | if len(cl.Children()) != 2 { 36 | t.Fatal("result must have two parts in MultiString") 37 | } 38 | } 39 | 40 | func TestClipPolygonSimple(t *testing.T) { 41 | exterior := []geometry.Point{ 42 | {X: 2, Y: 2}, 43 | {X: 1, Y: 2}, 44 | {X: 1.5, Y: 1.5}, 45 | {X: 1, Y: 1}, 46 | {X: 2, Y: 1}, 47 | {X: 2, Y: 2}, 48 | } 49 | holes := [][]geometry.Point{ 50 | { 51 | {X: 1.9, Y: 1.9}, 52 | {X: 1.2, Y: 1.9}, 53 | {X: 1.45, Y: 1.65}, 54 | {X: 1.9, Y: 1.5}, 55 | {X: 1.9, Y: 1.9}, 56 | }, 57 | } 58 | polygon := PPO(exterior, holes) 59 | clipped := Clip(polygon, RO(1.3, 1.3, 1.4, 2.15), nil) 60 | cp, ok := clipped.(*geojson.Polygon) 61 | if !ok { 62 | t.Fatal("wrong type") 63 | } 64 | if cp.Base().Exterior.Empty() { 65 | t.Fatal("Empty result.") 66 | } 67 | if len(cp.Base().Holes) != 1 { 68 | t.Fatal("result must be a two-ring Polygon") 69 | } 70 | } 71 | 72 | func TestClipPolygon2(t *testing.T) { 73 | exterior := []geometry.Point{ 74 | {X: 2, Y: 2}, 75 | {X: 1, Y: 2}, 76 | {X: 1.5, Y: 1.5}, 77 | {X: 1, Y: 1}, 78 | {X: 2, Y: 1}, 79 | {X: 2, Y: 2}, 80 | } 81 | holes := [][]geometry.Point{ 82 | { 83 | {X: 1.9, Y: 1.9}, 84 | {X: 1.2, Y: 1.9}, 85 | {X: 1.45, Y: 1.65}, 86 | {X: 1.9, Y: 1.5}, 87 | {X: 1.9, Y: 1.9}, 88 | }, 89 | } 90 | polygon := PPO(exterior, holes) 91 | clipped := Clip(polygon, RO(1.1, 0.8, 1.15, 2.1), nil) 92 | cp, ok := clipped.(*geojson.Polygon) 93 | if !ok { 94 | t.Fatal("wrong type") 95 | } 96 | if cp.Base().Exterior.Empty() { 97 | t.Fatal("Empty result.") 98 | } 99 | if len(cp.Base().Holes) != 0 { 100 | t.Fatal("result must be a single-ring Polygon") 101 | } 102 | } 103 | 104 | // func TestClipLineString(t *testing.T) { 105 | // featuresJSON := ` 106 | // {"type": "FeatureCollection","features": [ 107 | // {"type": "Feature","properties":{},"geometry": {"type": "LineString","coordinates": [[-71.46537780761717,42.594290856363344],[-71.37714385986328,42.600861802789524],[-71.37508392333984,42.538156868495555],[-71.43756866455078,42.535374141307415],[-71.44683837890625,42.466018925787495],[-71.334228515625,42.465005871175755],[-71.32736206054688,42.52424199254517]]}}, 108 | // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.49284362792969,42.527784255084676],[-71.35791778564453,42.527784255084676],[-71.35791778564453,42.61096959812047],[-71.49284362792969,42.61096959812047],[-71.49284362792969,42.527784255084676]]]}}, 109 | // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.47396087646484,42.48247876554176],[-71.30744934082031,42.48247876554176],[-71.30744934082031,42.576596402826894],[-71.47396087646484,42.576596402826894],[-71.47396087646484,42.48247876554176]]]}}, 110 | // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.33491516113281,42.613496290695196],[-71.29920959472656,42.613496290695196],[-71.29920959472656,42.643556064374536],[-71.33491516113281,42.643556064374536],[-71.33491516113281,42.613496290695196]]]}}, 111 | // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.37130737304686,42.530061317794775],[-71.3287353515625,42.530061317794775],[-71.3287353515625,42.60414701616359],[-71.37130737304686,42.60414701616359],[-71.37130737304686,42.530061317794775]]]}}, 112 | // {"type": "Feature","properties":{},"geometry": {"type": "Polygon","coordinates": [[[-71.52889251708984,42.564460160624115],[-71.45713806152342,42.54043355305221],[-71.53266906738281,42.49969365675931],[-71.36547088623047,42.508552415528634],[-71.43962860107422,42.58999409368092],[-71.52889251708984,42.564460160624115]]]}}, 113 | // {"type": "Feature","properties": {},"geometry": {"type": "Point","coordinates": [-71.33079528808594,42.55940269610327]}}, 114 | // {"type": "Feature","properties": {},"geometry": {"type": "Point","coordinates": [-71.27208709716797,42.53107331902133]}} 115 | // ]} 116 | // ` 117 | // rectJSON := `{"type": "Feature","properties": {},"geometry": {"type": "Polygon","coordinates": [[[-71.44065856933594,42.51740991900762],[-71.29131317138672,42.51740991900762],[-71.29131317138672,42.62663343969058],[-71.44065856933594,42.62663343969058],[-71.44065856933594,42.51740991900762]]]}}` 118 | // features := expectJSON(t, featuresJSON, nil) 119 | // rect := expectJSON(t, rectJSON, nil) 120 | // clipped := features.Clipped(rect) 121 | // println(clipped.String()) 122 | 123 | // } 124 | -------------------------------------------------------------------------------- /internal/clip/collection.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | func clipCollection( 9 | collection geojson.Collection, clipper geojson.Object, 10 | opts *geometry.IndexOptions, 11 | ) geojson.Object { 12 | var features []geojson.Object 13 | for _, feature := range collection.Children() { 14 | feature = Clip(feature, clipper, opts) 15 | if feature.Empty() { 16 | continue 17 | } 18 | if _, ok := feature.(*geojson.Feature); !ok { 19 | feature = geojson.NewFeature(feature, "") 20 | } 21 | features = append(features, feature) 22 | } 23 | return geojson.NewFeatureCollection(features) 24 | } 25 | -------------------------------------------------------------------------------- /internal/clip/feature.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | func clipFeature( 9 | feature *geojson.Feature, clipper geojson.Object, 10 | opts *geometry.IndexOptions, 11 | ) geojson.Object { 12 | newFeature := Clip(feature.Base(), clipper, opts) 13 | if _, ok := newFeature.(*geojson.Feature); !ok { 14 | newFeature = geojson.NewFeature(newFeature, feature.Members()) 15 | } 16 | return newFeature 17 | } 18 | -------------------------------------------------------------------------------- /internal/clip/linestring.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | func clipLineString( 9 | lineString *geojson.LineString, clipper geojson.Object, 10 | opts *geometry.IndexOptions, 11 | ) geojson.Object { 12 | bbox := clipper.Rect() 13 | var newPoints [][]geometry.Point 14 | var clipped geometry.Segment 15 | var rejected bool 16 | var line []geometry.Point 17 | base := lineString.Base() 18 | nSegments := base.NumSegments() 19 | for i := 0; i < nSegments; i++ { 20 | clipped, rejected = clipSegment(base.SegmentAt(i), bbox) 21 | if rejected { 22 | continue 23 | } 24 | if len(line) > 0 && line[len(line)-1] != clipped.A { 25 | newPoints = append(newPoints, line) 26 | line = []geometry.Point{clipped.A} 27 | } else if len(line) == 0 { 28 | line = append(line, clipped.A) 29 | } 30 | line = append(line, clipped.B) 31 | } 32 | if len(line) > 0 { 33 | newPoints = append(newPoints, line) 34 | } 35 | var children []*geometry.Line 36 | for _, points := range newPoints { 37 | children = append(children, 38 | geometry.NewLine(points, opts)) 39 | } 40 | if len(children) == 1 { 41 | return geojson.NewLineString(children[0]) 42 | } 43 | return geojson.NewMultiLineString(children) 44 | } 45 | -------------------------------------------------------------------------------- /internal/clip/point.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | func clipPoint( 9 | point *geojson.Point, clipper geojson.Object, opts *geometry.IndexOptions, 10 | ) geojson.Object { 11 | if point.IntersectsRect(clipper.Rect()) { 12 | return point 13 | } 14 | return geojson.NewMultiPoint(nil) 15 | } 16 | -------------------------------------------------------------------------------- /internal/clip/polygon.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | func clipPolygon( 9 | polygon *geojson.Polygon, clipper geojson.Object, 10 | opts *geometry.IndexOptions, 11 | ) geojson.Object { 12 | rect := clipper.Rect() 13 | var newPoints [][]geometry.Point 14 | base := polygon.Base() 15 | rings := []geometry.Ring{base.Exterior} 16 | rings = append(rings, base.Holes...) 17 | for _, ring := range rings { 18 | ringPoints := make([]geometry.Point, ring.NumPoints()) 19 | for i := 0; i < len(ringPoints); i++ { 20 | ringPoints[i] = ring.PointAt(i) 21 | } 22 | if clippedRing := clipRing(ringPoints, rect); len(clippedRing) > 0 { 23 | newPoints = append(newPoints, clippedRing) 24 | } 25 | } 26 | 27 | var exterior []geometry.Point 28 | var holes [][]geometry.Point 29 | if len(newPoints) > 0 { 30 | exterior = newPoints[0] 31 | } 32 | if len(newPoints) > 1 { 33 | holes = newPoints[1:] 34 | } 35 | newPoly := geojson.NewPolygon( 36 | geometry.NewPoly(exterior, holes, opts), 37 | ) 38 | if newPoly.Empty() { 39 | return geojson.NewMultiPolygon(nil) 40 | } 41 | return newPoly 42 | } 43 | -------------------------------------------------------------------------------- /internal/clip/rect.go: -------------------------------------------------------------------------------- 1 | package clip 2 | 3 | import ( 4 | "github.com/tidwall/geojson" 5 | "github.com/tidwall/geojson/geometry" 6 | ) 7 | 8 | func clipRect( 9 | rect *geojson.Rect, clipper geojson.Object, opts *geometry.IndexOptions, 10 | ) geojson.Object { 11 | base := rect.Base() 12 | points := make([]geometry.Point, base.NumPoints()) 13 | for i := 0; i < len(points); i++ { 14 | points[i] = base.PointAt(i) 15 | } 16 | poly := geometry.NewPoly(points, nil, opts) 17 | gPoly := geojson.NewPolygon(poly) 18 | return Clip(gPoly, clipper, opts) 19 | } 20 | -------------------------------------------------------------------------------- /internal/collection/geodesic.go: -------------------------------------------------------------------------------- 1 | package collection 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/tidwall/tile38/internal/object" 7 | ) 8 | 9 | func geodeticDistAlgo(center [2]float64) ( 10 | algo func(min, max [2]float64, obj *object.Object, item bool) (dist float64), 11 | ) { 12 | const earthRadius = 6371e3 13 | return func(min, max [2]float64, obj *object.Object, item bool) (dist float64) { 14 | if item { 15 | r := obj.Rect() 16 | min[0] = r.Min.X 17 | min[1] = r.Min.Y 18 | max[0] = r.Max.X 19 | max[1] = r.Max.Y 20 | } 21 | return earthRadius * pointRectDistGeodeticDeg( 22 | center[1], center[0], 23 | min[1], min[0], 24 | max[1], max[0], 25 | ) 26 | } 27 | } 28 | 29 | func pointRectDistGeodeticDeg(pLat, pLng, minLat, minLng, maxLat, maxLng float64) float64 { 30 | result := pointRectDistGeodeticRad( 31 | pLat*math.Pi/180, pLng*math.Pi/180, 32 | minLat*math.Pi/180, minLng*math.Pi/180, 33 | maxLat*math.Pi/180, maxLng*math.Pi/180, 34 | ) 35 | return result 36 | } 37 | 38 | func pointRectDistGeodeticRad(φq, λq, φl, λl, φh, λh float64) float64 { 39 | // Algorithm from: 40 | // Schubert, E., Zimek, A., & Kriegel, H.-P. (2013). 41 | // Geodetic Distance Queries on R-Trees for Indexing Geographic Data. 42 | // Lecture Notes in Computer Science, 146–164. 43 | // doi:10.1007/978-3-642-40235-7_9 44 | const ( 45 | twoΠ = 2 * math.Pi 46 | halfΠ = math.Pi / 2 47 | ) 48 | 49 | // distance on the unit sphere computed using Haversine formula 50 | distRad := func(φa, λa, φb, λb float64) float64 { 51 | if φa == φb && λa == λb { 52 | return 0 53 | } 54 | 55 | Δφ := φa - φb 56 | Δλ := λa - λb 57 | sinΔφ := math.Sin(Δφ / 2) 58 | sinΔλ := math.Sin(Δλ / 2) 59 | cosφa := math.Cos(φa) 60 | cosφb := math.Cos(φb) 61 | 62 | return 2 * math.Asin(math.Sqrt(sinΔφ*sinΔφ+sinΔλ*sinΔλ*cosφa*cosφb)) 63 | } 64 | 65 | // Simple case, point or invalid rect 66 | if φl >= φh && λl >= λh { 67 | return distRad(φl, λl, φq, λq) 68 | } 69 | 70 | if λl <= λq && λq <= λh { 71 | // q is between the bounding meridians of r 72 | // hence, q is north, south or within r 73 | if φl <= φq && φq <= φh { // Inside 74 | return 0 75 | } 76 | 77 | if φq < φl { // South 78 | return φl - φq 79 | } 80 | 81 | return φq - φh // North 82 | } 83 | 84 | // determine if q is closer to the east or west edge of r to select edge for 85 | // tests below 86 | Δλe := λl - λq 87 | Δλw := λq - λh 88 | if Δλe < 0 { 89 | Δλe += twoΠ 90 | } 91 | if Δλw < 0 { 92 | Δλw += twoΠ 93 | } 94 | var Δλ float64 // distance to closest edge 95 | var λedge float64 // longitude of closest edge 96 | if Δλe <= Δλw { 97 | Δλ = Δλe 98 | λedge = λl 99 | } else { 100 | Δλ = Δλw 101 | λedge = λh 102 | } 103 | 104 | sinΔλ, cosΔλ := math.Sincos(Δλ) 105 | tanφq := math.Tan(φq) 106 | 107 | if Δλ >= halfΠ { 108 | // If Δλ > 90 degrees (1/2 pi in radians) we're in one of the corners 109 | // (NW/SW or NE/SE depending on the edge selected). Compare against the 110 | // center line to decide which case we fall into 111 | φmid := (φh + φl) / 2 112 | if tanφq >= math.Tan(φmid)*cosΔλ { 113 | return distRad(φq, λq, φh, λedge) // North corner 114 | } 115 | return distRad(φq, λq, φl, λedge) // South corner 116 | } 117 | 118 | if tanφq >= math.Tan(φh)*cosΔλ { 119 | return distRad(φq, λq, φh, λedge) // North corner 120 | } 121 | 122 | if tanφq <= math.Tan(φl)*cosΔλ { 123 | return distRad(φq, λq, φl, λedge) // South corner 124 | } 125 | 126 | // We're to the East or West of the rect, compute distance using cross-track 127 | // Note that this is a simplification of the cross track distance formula 128 | // valid since the track in question is a meridian. 129 | return math.Asin(math.Cos(φq) * sinΔλ) 130 | } 131 | -------------------------------------------------------------------------------- /internal/collection/string.go: -------------------------------------------------------------------------------- 1 | package collection 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/tidwall/geojson" 7 | "github.com/tidwall/geojson/geometry" 8 | ) 9 | 10 | type String string 11 | 12 | var _ geojson.Object = String("") 13 | 14 | func (s String) Spatial() geojson.Spatial { 15 | return geojson.EmptySpatial{} 16 | } 17 | 18 | func (s String) ForEach(iter func(geom geojson.Object) bool) bool { 19 | return iter(s) 20 | } 21 | 22 | func (s String) Empty() bool { 23 | return true 24 | } 25 | 26 | func (s String) Valid() bool { 27 | return false 28 | } 29 | 30 | func (s String) Rect() geometry.Rect { 31 | return geometry.Rect{} 32 | } 33 | 34 | func (s String) Center() geometry.Point { 35 | return geometry.Point{} 36 | } 37 | 38 | func (s String) AppendJSON(dst []byte) []byte { 39 | data, _ := json.Marshal(string(s)) 40 | return append(dst, data...) 41 | } 42 | 43 | func (s String) String() string { 44 | return string(s) 45 | } 46 | 47 | func (s String) JSON() string { 48 | return string(s.AppendJSON(nil)) 49 | } 50 | 51 | func (s String) MarshalJSON() ([]byte, error) { 52 | return s.AppendJSON(nil), nil 53 | } 54 | 55 | func (s String) Within(obj geojson.Object) bool { 56 | return false 57 | } 58 | 59 | func (s String) Contains(obj geojson.Object) bool { 60 | return false 61 | } 62 | 63 | func (s String) Intersects(obj geojson.Object) bool { 64 | return false 65 | } 66 | 67 | func (s String) NumPoints() int { 68 | return 0 69 | } 70 | 71 | func (s String) Distance(obj geojson.Object) float64 { 72 | return 0 73 | } 74 | 75 | func (s String) Members() string { 76 | return "" 77 | } 78 | -------------------------------------------------------------------------------- /internal/deadline/deadline.go: -------------------------------------------------------------------------------- 1 | package deadline 2 | 3 | import "time" 4 | 5 | // Deadline allows for commands to expire when they run too long 6 | type Deadline struct { 7 | unixNano int64 8 | hit bool 9 | } 10 | 11 | // New returns a new deadline object 12 | func New(dl time.Time) *Deadline { 13 | return &Deadline{unixNano: dl.UnixNano()} 14 | } 15 | 16 | // Check the deadline and panic when reached 17 | // 18 | //go:noinline 19 | func (dl *Deadline) Check() { 20 | if dl == nil || dl.unixNano == 0 { 21 | return 22 | } 23 | if !dl.hit && time.Now().UnixNano() > dl.unixNano { 24 | dl.hit = true 25 | panic("deadline") 26 | } 27 | } 28 | 29 | // Hit returns true if the deadline has been hit 30 | func (dl *Deadline) Hit() bool { 31 | return dl.hit 32 | } 33 | 34 | // GetDeadlineTime returns the time object for the deadline, and an 35 | // "empty" boolean 36 | func (dl *Deadline) GetDeadlineTime() time.Time { 37 | return time.Unix(0, dl.unixNano) 38 | } 39 | -------------------------------------------------------------------------------- /internal/endpoint/amqp.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | "time" 8 | 9 | "github.com/streadway/amqp" 10 | ) 11 | 12 | const amqpExpiresAfter = time.Second * 30 13 | 14 | // AMQPConn is an endpoint connection 15 | type AMQPConn struct { 16 | mu sync.Mutex 17 | ep Endpoint 18 | conn *amqp.Connection 19 | channel *amqp.Channel 20 | ex bool 21 | t time.Time 22 | } 23 | 24 | // Expired returns true if the connection has expired 25 | func (conn *AMQPConn) Expired() bool { 26 | conn.mu.Lock() 27 | defer conn.mu.Unlock() 28 | if !conn.ex { 29 | if time.Since(conn.t) > amqpExpiresAfter { 30 | conn.close() 31 | conn.ex = true 32 | } 33 | } 34 | return conn.ex 35 | } 36 | 37 | // ExpireNow forces the connection to expire 38 | func (conn *AMQPConn) ExpireNow() { 39 | conn.mu.Lock() 40 | defer conn.mu.Unlock() 41 | conn.close() 42 | conn.ex = true 43 | } 44 | 45 | func (conn *AMQPConn) close() { 46 | if conn.conn != nil { 47 | conn.conn.Close() 48 | conn.conn = nil 49 | conn.channel = nil 50 | } 51 | } 52 | 53 | // Send sends a message 54 | func (conn *AMQPConn) Send(msg string) error { 55 | conn.mu.Lock() 56 | defer conn.mu.Unlock() 57 | 58 | if conn.ex { 59 | return errExpired 60 | } 61 | conn.t = time.Now() 62 | 63 | if conn.conn == nil { 64 | prefix := "amqp://" 65 | if conn.ep.AMQP.SSL { 66 | prefix = "amqps://" 67 | } 68 | 69 | var cfg amqp.Config 70 | cfg.Dial = func(network, addr string) (net.Conn, error) { 71 | return net.DialTimeout(network, addr, time.Second) 72 | } 73 | c, err := amqp.DialConfig(fmt.Sprintf("%s%s", prefix, conn.ep.AMQP.URI), cfg) 74 | 75 | if err != nil { 76 | return err 77 | } 78 | 79 | channel, err := c.Channel() 80 | if err != nil { 81 | return err 82 | } 83 | 84 | // Declare new exchange 85 | if err := channel.ExchangeDeclare( 86 | conn.ep.AMQP.QueueName, 87 | conn.ep.AMQP.Type, 88 | conn.ep.AMQP.Durable, 89 | conn.ep.AMQP.AutoDelete, 90 | conn.ep.AMQP.Internal, 91 | conn.ep.AMQP.NoWait, 92 | nil, 93 | ); err != nil { 94 | return err 95 | } 96 | if conn.ep.AMQP.Type != "topic" { 97 | // Create queue if queue don't exists 98 | if _, err := channel.QueueDeclare( 99 | conn.ep.AMQP.QueueName, 100 | conn.ep.AMQP.Durable, 101 | conn.ep.AMQP.AutoDelete, 102 | false, 103 | conn.ep.AMQP.NoWait, 104 | nil, 105 | ); err != nil { 106 | return err 107 | } 108 | 109 | // Binding exchange to queue 110 | if err := channel.QueueBind( 111 | conn.ep.AMQP.QueueName, 112 | conn.ep.AMQP.RouteKey, 113 | conn.ep.AMQP.QueueName, 114 | conn.ep.AMQP.NoWait, 115 | nil, 116 | ); err != nil { 117 | return err 118 | } 119 | } 120 | conn.conn = c 121 | conn.channel = channel 122 | } 123 | 124 | return conn.channel.Publish( 125 | conn.ep.AMQP.QueueName, 126 | conn.ep.AMQP.RouteKey, 127 | conn.ep.AMQP.Mandatory, 128 | conn.ep.AMQP.Immediate, 129 | amqp.Publishing{ 130 | Headers: amqp.Table{}, 131 | ContentType: "application/json", 132 | ContentEncoding: "", 133 | Body: []byte(msg), 134 | DeliveryMode: conn.ep.AMQP.DeliveryMode, 135 | Priority: conn.ep.AMQP.Priority, 136 | }, 137 | ) 138 | } 139 | 140 | func newAMQPConn(ep Endpoint) *AMQPConn { 141 | return &AMQPConn{ 142 | ep: ep, 143 | t: time.Now(), 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /internal/endpoint/disque.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/gomodule/redigo/redis" 9 | "github.com/tidwall/tile38/internal/log" 10 | ) 11 | 12 | const disqueExpiresAfter = time.Second * 30 13 | 14 | // DisqueConn is an endpoint connection 15 | type DisqueConn struct { 16 | mu sync.Mutex 17 | ep Endpoint 18 | ex bool 19 | t time.Time 20 | conn redis.Conn 21 | } 22 | 23 | func newDisqueConn(ep Endpoint) *DisqueConn { 24 | return &DisqueConn{ 25 | ep: ep, 26 | t: time.Now(), 27 | } 28 | } 29 | 30 | // Expired returns true if the connection has expired 31 | func (conn *DisqueConn) Expired() bool { 32 | conn.mu.Lock() 33 | defer conn.mu.Unlock() 34 | if !conn.ex { 35 | if time.Since(conn.t) > disqueExpiresAfter { 36 | conn.close() 37 | conn.ex = true 38 | } 39 | } 40 | return conn.ex 41 | } 42 | 43 | // ExpireNow forces the connection to expire 44 | func (conn *DisqueConn) ExpireNow() { 45 | conn.mu.Lock() 46 | defer conn.mu.Unlock() 47 | conn.close() 48 | conn.ex = true 49 | } 50 | 51 | func (conn *DisqueConn) close() { 52 | if conn.conn != nil { 53 | conn.conn.Close() 54 | conn.conn = nil 55 | } 56 | } 57 | 58 | // Send sends a message 59 | func (conn *DisqueConn) Send(msg string) error { 60 | conn.mu.Lock() 61 | defer conn.mu.Unlock() 62 | if conn.ex { 63 | return errExpired 64 | } 65 | conn.t = time.Now() 66 | if conn.conn == nil { 67 | addr := fmt.Sprintf("%s:%d", conn.ep.Disque.Host, conn.ep.Disque.Port) 68 | var err error 69 | conn.conn, err = redis.Dial("tcp", addr) 70 | if err != nil { 71 | return err 72 | } 73 | } 74 | 75 | var args []interface{} 76 | args = append(args, conn.ep.Disque.QueueName, msg, 0) 77 | if conn.ep.Disque.Options.Replicate > 0 { 78 | args = append(args, "REPLICATE", conn.ep.Disque.Options.Replicate) 79 | } 80 | 81 | reply, err := redis.String(conn.conn.Do("ADDJOB", args...)) 82 | if err != nil { 83 | conn.close() 84 | return err 85 | } 86 | log.Debugf("Disque: ADDJOB '%s'", reply) 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /internal/endpoint/eventHub.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/tidwall/gjson" 9 | 10 | eventhub "github.com/Azure/azure-event-hubs-go/v3" 11 | ) 12 | 13 | const () 14 | 15 | // HTTPConn is an endpoint connection 16 | type EvenHubConn struct { 17 | ep Endpoint 18 | } 19 | 20 | func newEventHubConn(ep Endpoint) *EvenHubConn { 21 | return &EvenHubConn{ 22 | ep: ep, 23 | } 24 | } 25 | 26 | // Expired returns true if the connection has expired 27 | func (conn *EvenHubConn) Expired() bool { 28 | return false 29 | } 30 | 31 | // ExpireNow forces the connection to expire 32 | func (conn *EvenHubConn) ExpireNow() { 33 | } 34 | 35 | // Send sends a message 36 | func (conn *EvenHubConn) Send(msg string) error { 37 | hub, err := eventhub.NewHubFromConnectionString(conn.ep.EventHub.ConnectionString) 38 | 39 | if err != nil { 40 | return err 41 | } 42 | 43 | ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) 44 | defer cancel() 45 | 46 | // parse json again to get out info for our kafka key 47 | key := gjson.Get(msg, "key") 48 | id := gjson.Get(msg, "id") 49 | keyValue := fmt.Sprintf("%s-%s", key.String(), id.String()) 50 | 51 | evtHubMsg := eventhub.NewEventFromString(msg) 52 | evtHubMsg.PartitionKey = &keyValue 53 | err = hub.Send(ctx, evtHubMsg) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /internal/endpoint/grpc.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/tidwall/tile38/internal/hservice" 10 | "golang.org/x/net/context" 11 | "google.golang.org/grpc" 12 | ) 13 | 14 | const grpcExpiresAfter = time.Second * 30 15 | 16 | // GRPCConn is an endpoint connection 17 | type GRPCConn struct { 18 | mu sync.Mutex 19 | ep Endpoint 20 | ex bool 21 | t time.Time 22 | conn *grpc.ClientConn 23 | sconn hservice.HookServiceClient 24 | } 25 | 26 | func newGRPCConn(ep Endpoint) *GRPCConn { 27 | return &GRPCConn{ 28 | ep: ep, 29 | t: time.Now(), 30 | } 31 | } 32 | 33 | // Expired returns true if the connection has expired 34 | func (conn *GRPCConn) Expired() bool { 35 | conn.mu.Lock() 36 | defer conn.mu.Unlock() 37 | if !conn.ex { 38 | if time.Since(conn.t) > grpcExpiresAfter { 39 | conn.close() 40 | conn.ex = true 41 | } 42 | } 43 | return conn.ex 44 | } 45 | 46 | // ExpireNow forces the connection to expire 47 | func (conn *GRPCConn) ExpireNow() { 48 | conn.mu.Lock() 49 | defer conn.mu.Unlock() 50 | conn.close() 51 | conn.ex = true 52 | } 53 | 54 | func (conn *GRPCConn) close() { 55 | if conn.conn != nil { 56 | conn.conn.Close() 57 | conn.conn = nil 58 | } 59 | } 60 | 61 | // Send sends a message 62 | func (conn *GRPCConn) Send(msg string) error { 63 | conn.mu.Lock() 64 | defer conn.mu.Unlock() 65 | if conn.ex { 66 | return errExpired 67 | } 68 | conn.t = time.Now() 69 | if conn.conn == nil { 70 | addr := fmt.Sprintf("%s:%d", conn.ep.GRPC.Host, conn.ep.GRPC.Port) 71 | var err error 72 | conn.conn, err = grpc.Dial(addr, grpc.WithInsecure()) 73 | if err != nil { 74 | conn.close() 75 | return err 76 | } 77 | conn.sconn = hservice.NewHookServiceClient(conn.conn) 78 | } 79 | r, err := conn.sconn.Send(context.Background(), &hservice.MessageRequest{Value: msg}) 80 | if err != nil { 81 | conn.close() 82 | return err 83 | } 84 | if !r.Ok { 85 | conn.close() 86 | return errors.New("invalid grpc reply") 87 | } 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /internal/endpoint/http.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | const ( 12 | httpExpiresAfter = time.Second * 30 13 | httpRequestTimeout = time.Second * 5 14 | httpMaxIdleConnections = 20 15 | ) 16 | 17 | // HTTPConn is an endpoint connection 18 | type HTTPConn struct { 19 | ep Endpoint 20 | client *http.Client 21 | } 22 | 23 | func newHTTPConn(ep Endpoint) *HTTPConn { 24 | return &HTTPConn{ 25 | ep: ep, 26 | client: &http.Client{ 27 | Transport: &http.Transport{ 28 | MaxIdleConnsPerHost: httpMaxIdleConnections, 29 | IdleConnTimeout: httpExpiresAfter, 30 | }, 31 | Timeout: httpRequestTimeout, 32 | }, 33 | } 34 | } 35 | 36 | // Expired returns true if the connection has expired 37 | func (conn *HTTPConn) Expired() bool { 38 | return false 39 | } 40 | 41 | // ExpireNow forces the connection to expire 42 | func (conn *HTTPConn) ExpireNow() { 43 | } 44 | 45 | // Send sends a message 46 | func (conn *HTTPConn) Send(msg string) error { 47 | req, err := http.NewRequest("POST", conn.ep.Original, bytes.NewBufferString(msg)) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | req.Header.Set("Content-Type", "application/json") 53 | resp, err := conn.client.Do(req) 54 | if err != nil { 55 | return err 56 | } 57 | // close the connection to reuse it 58 | defer resp.Body.Close() 59 | // discard response 60 | if _, err := io.Copy(io.Discard, resp.Body); err != nil { 61 | return err 62 | } 63 | // Only allow responses with status code 200, 201, and 202 64 | if resp.StatusCode != http.StatusOK && 65 | resp.StatusCode != http.StatusCreated && 66 | resp.StatusCode != http.StatusAccepted { 67 | return fmt.Errorf("invalid status: %s", resp.Status) 68 | } 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /internal/endpoint/local.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | // LocalPublisher is used to publish local notifications 4 | type LocalPublisher interface { 5 | Publish(channel string, message ...string) int 6 | } 7 | 8 | // LocalConn is an endpoint connection 9 | type LocalConn struct { 10 | ep Endpoint 11 | publisher LocalPublisher 12 | } 13 | 14 | func newLocalConn(ep Endpoint, publisher LocalPublisher) *LocalConn { 15 | return &LocalConn{ 16 | ep: ep, 17 | publisher: publisher, 18 | } 19 | } 20 | 21 | // Expired returns true if the connection has expired 22 | func (conn *LocalConn) Expired() bool { 23 | return false 24 | } 25 | 26 | // ExpireNow forces the connection to expire 27 | func (conn *LocalConn) ExpireNow() { 28 | } 29 | 30 | // Send sends a message 31 | func (conn *LocalConn) Send(msg string) error { 32 | conn.publisher.Publish(conn.ep.Local.Channel, msg) 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /internal/endpoint/mqtt.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "math/rand" 8 | "os" 9 | "sync" 10 | "time" 11 | 12 | paho "github.com/eclipse/paho.mqtt.golang" 13 | "github.com/tidwall/tile38/internal/log" 14 | ) 15 | 16 | const ( 17 | mqttExpiresAfter = time.Second * 30 18 | mqttPublishTimeout = time.Second * 5 19 | ) 20 | 21 | // MQTTConn is an endpoint connection 22 | type MQTTConn struct { 23 | mu sync.Mutex 24 | ep Endpoint 25 | conn paho.Client 26 | ex bool 27 | t time.Time 28 | } 29 | 30 | // Expired returns true if the connection has expired 31 | func (conn *MQTTConn) Expired() bool { 32 | conn.mu.Lock() 33 | defer conn.mu.Unlock() 34 | if !conn.ex { 35 | if time.Since(conn.t) > mqttExpiresAfter { 36 | conn.close() 37 | conn.ex = true 38 | } 39 | } 40 | return conn.ex 41 | } 42 | 43 | // ExpireNow forces the connection to expire 44 | func (conn *MQTTConn) ExpireNow() { 45 | conn.mu.Lock() 46 | defer conn.mu.Unlock() 47 | conn.close() 48 | conn.ex = true 49 | } 50 | 51 | func (conn *MQTTConn) close() { 52 | if conn.conn != nil { 53 | if conn.conn.IsConnected() { 54 | conn.conn.Disconnect(250) 55 | } 56 | conn.conn = nil 57 | } 58 | } 59 | 60 | // Send sends a message 61 | func (conn *MQTTConn) Send(msg string) error { 62 | conn.mu.Lock() 63 | defer conn.mu.Unlock() 64 | 65 | if conn.ex { 66 | return errExpired 67 | } 68 | conn.t = time.Now() 69 | 70 | if conn.conn == nil { 71 | uri := fmt.Sprintf("tcp://%s:%d", conn.ep.MQTT.Host, conn.ep.MQTT.Port) 72 | ops := paho.NewClientOptions() 73 | if conn.ep.MQTT.CertFile != "" || conn.ep.MQTT.KeyFile != "" || 74 | conn.ep.MQTT.CACertFile != "" { 75 | var config tls.Config 76 | if conn.ep.MQTT.CertFile != "" || conn.ep.MQTT.KeyFile != "" { 77 | cert, err := tls.LoadX509KeyPair(conn.ep.MQTT.CertFile, 78 | conn.ep.MQTT.KeyFile) 79 | if err != nil { 80 | return err 81 | } 82 | config.Certificates = append(config.Certificates, cert) 83 | } 84 | if conn.ep.MQTT.CACertFile != "" { 85 | // Load CA cert 86 | caCert, err := os.ReadFile(conn.ep.MQTT.CACertFile) 87 | if err != nil { 88 | return err 89 | } 90 | caCertPool := x509.NewCertPool() 91 | caCertPool.AppendCertsFromPEM(caCert) 92 | config.RootCAs = caCertPool 93 | } 94 | ops = ops.SetTLSConfig(&config) 95 | } 96 | //generate UUID for the client-id. 97 | b := make([]byte, 16) 98 | _, err := rand.Read(b) 99 | if err != nil { 100 | log.Debugf("Failed to generate guid for the mqtt client. The endpoint will not work") 101 | return err 102 | } 103 | uuid := fmt.Sprintf("tile38-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]) 104 | 105 | ops = ops.SetClientID(uuid).AddBroker(uri) 106 | c := paho.NewClient(ops) 107 | 108 | if token := c.Connect(); token.Wait() && token.Error() != nil { 109 | return token.Error() 110 | } 111 | 112 | conn.conn = c 113 | } 114 | 115 | t := conn.conn.Publish(conn.ep.MQTT.QueueName, conn.ep.MQTT.Qos, 116 | conn.ep.MQTT.Retained, msg) 117 | 118 | if !t.WaitTimeout(mqttPublishTimeout) || t.Error() != nil { 119 | conn.close() 120 | return t.Error() 121 | } 122 | 123 | return nil 124 | } 125 | 126 | func newMQTTConn(ep Endpoint) *MQTTConn { 127 | return &MQTTConn{ 128 | ep: ep, 129 | t: time.Now(), 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /internal/endpoint/nats.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/nats-io/nats.go" 9 | ) 10 | 11 | const natsExpiresAfter = time.Second * 30 12 | 13 | // NATSConn is an endpoint connection 14 | type NATSConn struct { 15 | mu sync.Mutex 16 | ep Endpoint 17 | ex bool 18 | t time.Time 19 | conn *nats.Conn 20 | } 21 | 22 | func newNATSConn(ep Endpoint) *NATSConn { 23 | return &NATSConn{ 24 | ep: ep, 25 | t: time.Now(), 26 | } 27 | } 28 | 29 | // Expired returns true if the connection has expired 30 | func (conn *NATSConn) Expired() bool { 31 | conn.mu.Lock() 32 | defer conn.mu.Unlock() 33 | if !conn.ex { 34 | if time.Since(conn.t) > natsExpiresAfter { 35 | conn.close() 36 | conn.ex = true 37 | } 38 | } 39 | return conn.ex 40 | } 41 | 42 | // ExpireNow forces the connection to expire 43 | func (conn *NATSConn) ExpireNow() { 44 | conn.mu.Lock() 45 | defer conn.mu.Unlock() 46 | conn.close() 47 | conn.ex = true 48 | } 49 | 50 | func (conn *NATSConn) close() { 51 | if conn.conn != nil { 52 | conn.conn.Close() 53 | conn.conn = nil 54 | } 55 | } 56 | 57 | // Send sends a message 58 | func (conn *NATSConn) Send(msg string) error { 59 | conn.mu.Lock() 60 | defer conn.mu.Unlock() 61 | if conn.ex { 62 | return errExpired 63 | } 64 | conn.t = time.Now() 65 | if conn.conn == nil { 66 | addr := fmt.Sprintf("%s:%d", conn.ep.NATS.Host, conn.ep.NATS.Port) 67 | var err error 68 | var opts []nats.Option 69 | if conn.ep.NATS.User != "" && conn.ep.NATS.Pass != "" { 70 | opts = append(opts, nats.UserInfo(conn.ep.NATS.User, conn.ep.NATS.Pass)) 71 | } 72 | if conn.ep.NATS.TLS { 73 | opts = append(opts, nats.ClientCert( 74 | conn.ep.NATS.TLSCert, conn.ep.NATS.TLSKey, 75 | )) 76 | } 77 | if conn.ep.NATS.Token != "" { 78 | opts = append(opts, nats.Token(conn.ep.NATS.Token)) 79 | } 80 | conn.conn, err = nats.Connect(addr, opts...) 81 | if err != nil { 82 | conn.close() 83 | return err 84 | } 85 | } 86 | err := conn.conn.Publish(conn.ep.NATS.Topic, []byte(msg)) 87 | if err != nil { 88 | conn.close() 89 | return err 90 | } 91 | 92 | return nil 93 | } 94 | -------------------------------------------------------------------------------- /internal/endpoint/pubsub.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "cloud.google.com/go/pubsub" 10 | "google.golang.org/api/option" 11 | ) 12 | 13 | const pubsubExpiresAfter = time.Second * 30 14 | 15 | // SQSConn is an endpoint connection 16 | type PubSubConn struct { 17 | mu sync.Mutex 18 | ep Endpoint 19 | svc *pubsub.Client 20 | topic *pubsub.Topic 21 | ex bool 22 | t time.Time 23 | } 24 | 25 | func (conn *PubSubConn) close() { 26 | if conn.svc != nil { 27 | conn.svc.Close() 28 | conn.svc = nil 29 | } 30 | } 31 | 32 | // Send sends a message 33 | func (conn *PubSubConn) Send(msg string) error { 34 | conn.mu.Lock() 35 | defer conn.mu.Unlock() 36 | 37 | if conn.ex { 38 | return errExpired 39 | } 40 | 41 | ctx := context.Background() 42 | 43 | conn.t = time.Now() 44 | 45 | if conn.svc == nil { 46 | var creds option.ClientOption 47 | var svc *pubsub.Client 48 | var err error 49 | credPath := conn.ep.PubSub.CredPath 50 | 51 | if credPath != "" { 52 | creds = option.WithCredentialsFile(credPath) 53 | svc, err = pubsub.NewClient(ctx, conn.ep.PubSub.Project, creds) 54 | } else { 55 | svc, err = pubsub.NewClient(ctx, conn.ep.PubSub.Project) 56 | } 57 | 58 | if err != nil { 59 | fmt.Println(err) 60 | return err 61 | } 62 | 63 | topic := svc.Topic(conn.ep.PubSub.Topic) 64 | 65 | conn.svc = svc 66 | conn.topic = topic 67 | } 68 | 69 | // Send message 70 | res := conn.topic.Publish(ctx, &pubsub.Message{ 71 | Data: []byte(msg), 72 | }) 73 | _, err := res.Get(ctx) 74 | if err != nil { 75 | fmt.Println(err) 76 | return err 77 | } 78 | return nil 79 | } 80 | 81 | func (conn *PubSubConn) Expired() bool { 82 | conn.mu.Lock() 83 | defer conn.mu.Unlock() 84 | if !conn.ex { 85 | if time.Since(conn.t) > pubsubExpiresAfter { 86 | conn.close() 87 | conn.ex = true 88 | } 89 | } 90 | return conn.ex 91 | } 92 | 93 | // ExpireNow forces the connection to expire 94 | func (conn *PubSubConn) ExpireNow() { 95 | conn.mu.Lock() 96 | defer conn.mu.Unlock() 97 | conn.close() 98 | conn.ex = true 99 | } 100 | 101 | func newPubSubConn(ep Endpoint) *PubSubConn { 102 | return &PubSubConn{ 103 | ep: ep, 104 | t: time.Now(), 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /internal/endpoint/redis.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/gomodule/redigo/redis" 9 | ) 10 | 11 | const redisExpiresAfter = time.Second * 30 12 | 13 | // RedisConn is an endpoint connection 14 | type RedisConn struct { 15 | mu sync.Mutex 16 | ep Endpoint 17 | ex bool 18 | t time.Time 19 | conn redis.Conn 20 | } 21 | 22 | func newRedisConn(ep Endpoint) *RedisConn { 23 | return &RedisConn{ 24 | ep: ep, 25 | t: time.Now(), 26 | } 27 | } 28 | 29 | // Expired returns true if the connection has expired 30 | func (conn *RedisConn) Expired() bool { 31 | conn.mu.Lock() 32 | defer conn.mu.Unlock() 33 | if !conn.ex { 34 | if time.Since(conn.t) > redisExpiresAfter { 35 | conn.close() 36 | conn.ex = true 37 | } 38 | } 39 | return conn.ex 40 | } 41 | 42 | // ExpireNow forces the connection to expire 43 | func (conn *RedisConn) ExpireNow() { 44 | conn.mu.Lock() 45 | defer conn.mu.Unlock() 46 | conn.close() 47 | conn.ex = true 48 | } 49 | 50 | func (conn *RedisConn) close() { 51 | if conn.conn != nil { 52 | conn.conn.Close() 53 | conn.conn = nil 54 | } 55 | } 56 | 57 | // Send sends a message 58 | func (conn *RedisConn) Send(msg string) error { 59 | conn.mu.Lock() 60 | defer conn.mu.Unlock() 61 | 62 | if conn.ex { 63 | return errExpired 64 | } 65 | conn.t = time.Now() 66 | if conn.conn == nil { 67 | addr := fmt.Sprintf("%s:%d", conn.ep.Redis.Host, conn.ep.Redis.Port) 68 | var err error 69 | conn.conn, err = redis.Dial("tcp", addr) 70 | if err != nil { 71 | conn.close() 72 | return err 73 | } 74 | } 75 | _, err := redis.Int(conn.conn.Do("PUBLISH", conn.ep.Redis.Channel, msg)) 76 | if err != nil { 77 | conn.close() 78 | return err 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /internal/endpoint/scram_client.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "crypto/sha256" 5 | "crypto/sha512" 6 | 7 | "github.com/xdg-go/scram" 8 | ) 9 | 10 | var ( 11 | SHA256 scram.HashGeneratorFcn = sha256.New 12 | SHA512 scram.HashGeneratorFcn = sha512.New 13 | ) 14 | 15 | type XDGSCRAMClient struct { 16 | *scram.Client 17 | *scram.ClientConversation 18 | scram.HashGeneratorFcn 19 | } 20 | 21 | func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) { 22 | x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID) 23 | if err != nil { 24 | return err 25 | } 26 | x.ClientConversation = x.Client.NewConversation() 27 | return nil 28 | } 29 | 30 | func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) { 31 | response, err = x.ClientConversation.Step(challenge) 32 | return 33 | } 34 | 35 | func (x *XDGSCRAMClient) Done() bool { 36 | return x.ClientConversation.Done() 37 | } 38 | -------------------------------------------------------------------------------- /internal/endpoint/sqs.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "github.com/tidwall/gjson" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/credentials" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/sqs" 15 | "github.com/tidwall/tile38/internal/log" 16 | ) 17 | 18 | const sqsExpiresAfter = time.Second * 30 19 | 20 | // SQSConn is an endpoint connection 21 | type SQSConn struct { 22 | mu sync.Mutex 23 | ep Endpoint 24 | session *session.Session 25 | svc *sqs.SQS 26 | ex bool 27 | t time.Time 28 | } 29 | 30 | func (conn *SQSConn) generateSQSURL() string { 31 | if conn.ep.SQS.PlainURL != "" { 32 | return conn.ep.SQS.PlainURL 33 | } 34 | return "https://sqs." + conn.ep.SQS.Region + ".amazonaws.com/" + 35 | conn.ep.SQS.QueueID + "/" + conn.ep.SQS.QueueName 36 | } 37 | 38 | // Expired returns true if the connection has expired 39 | func (conn *SQSConn) Expired() bool { 40 | conn.mu.Lock() 41 | defer conn.mu.Unlock() 42 | if !conn.ex { 43 | if time.Since(conn.t) > sqsExpiresAfter { 44 | conn.close() 45 | conn.ex = true 46 | } 47 | } 48 | return conn.ex 49 | } 50 | 51 | // ExpireNow forces the connection to expire 52 | func (conn *SQSConn) ExpireNow() { 53 | conn.mu.Lock() 54 | defer conn.mu.Unlock() 55 | conn.close() 56 | conn.ex = true 57 | } 58 | 59 | func (conn *SQSConn) close() { 60 | if conn.svc != nil { 61 | conn.svc = nil 62 | conn.session = nil 63 | } 64 | } 65 | 66 | // Send sends a message 67 | func (conn *SQSConn) Send(msg string) error { 68 | conn.mu.Lock() 69 | defer conn.mu.Unlock() 70 | 71 | if conn.ex { 72 | return errExpired 73 | } 74 | conn.t = time.Now() 75 | 76 | if conn.svc == nil && conn.session == nil { 77 | var creds *credentials.Credentials 78 | credPath := conn.ep.SQS.CredPath 79 | if credPath != "" { 80 | credProfile := conn.ep.SQS.CredProfile 81 | if credProfile == "" { 82 | credProfile = "default" 83 | } 84 | creds = credentials.NewSharedCredentials(credPath, credProfile) 85 | } 86 | var region string 87 | if conn.ep.SQS.Region != "" { 88 | region = conn.ep.SQS.Region 89 | } else { 90 | region = sqsRegionFromPlainURL(conn.ep.SQS.PlainURL) 91 | } 92 | sess := session.Must(session.NewSession(&aws.Config{ 93 | Region: ®ion, 94 | Credentials: creds, 95 | CredentialsChainVerboseErrors: aws.Bool(log.Level() >= 3), 96 | MaxRetries: aws.Int(5), 97 | })) 98 | svc := sqs.New(sess) 99 | if conn.ep.SQS.CreateQueue { 100 | svc.CreateQueue(&sqs.CreateQueueInput{ 101 | QueueName: aws.String(conn.ep.SQS.QueueName), 102 | Attributes: map[string]*string{ 103 | "DelaySeconds": aws.String("60"), 104 | "MessageRetentionPeriod": aws.String("86400"), 105 | }, 106 | }) 107 | } 108 | conn.session = sess 109 | conn.svc = svc 110 | } 111 | 112 | queueURL := conn.generateSQSURL() 113 | // Create message 114 | sendParams := &sqs.SendMessageInput{ 115 | MessageBody: aws.String(msg), 116 | QueueUrl: aws.String(queueURL), 117 | } 118 | if isFifoQueue(queueURL) { 119 | key := gjson.Get(msg, "key") 120 | id := gjson.Get(msg, "id") 121 | keyValue := fmt.Sprintf("%s#%s", key.String(), id.String()) 122 | sendParams.MessageGroupId = aws.String(keyValue) 123 | } 124 | _, err := conn.svc.SendMessage(sendParams) 125 | if err != nil { 126 | fmt.Println(err) 127 | return err 128 | } 129 | 130 | return nil 131 | } 132 | 133 | func newSQSConn(ep Endpoint) *SQSConn { 134 | return &SQSConn{ 135 | ep: ep, 136 | t: time.Now(), 137 | } 138 | } 139 | 140 | func probeSQS(s string) bool { 141 | // https://sqs.eu-central-1.amazonaws.com/123456789/myqueue 142 | return strings.HasPrefix(s, "https://sqs.") && 143 | strings.Contains(s, ".amazonaws.com") 144 | } 145 | 146 | func sqsRegionFromPlainURL(s string) string { 147 | parts := strings.Split(s, "https://sqs.") 148 | if len(parts) > 1 { 149 | parts = strings.Split(parts[1], ".amazonaws.com") 150 | if len(parts) > 1 { 151 | return parts[0] 152 | } 153 | } 154 | return "" 155 | } 156 | 157 | func isFifoQueue(s string) bool { 158 | return strings.HasSuffix(s, ".fifo") 159 | } 160 | -------------------------------------------------------------------------------- /internal/field/list_struct.go: -------------------------------------------------------------------------------- 1 | //go:build exclude 2 | 3 | package field 4 | 5 | type List struct { 6 | entries []Field 7 | } 8 | 9 | // bsearch searches array for value. 10 | func (fields List) bsearch(name string) (index int, found bool) { 11 | i, j := 0, len(fields.entries) 12 | for i < j { 13 | h := i + (j-i)/2 14 | if name >= fields.entries[h].name { 15 | i = h + 1 16 | } else { 17 | j = h 18 | } 19 | } 20 | if i > 0 && fields.entries[i-1].name >= name { 21 | return i - 1, true 22 | } 23 | return i, false 24 | } 25 | 26 | func (fields List) Set(field Field) List { 27 | var updated List 28 | index, found := fields.bsearch(field.name) 29 | if found { 30 | if field.value.IsZero() { 31 | // delete 32 | if len(fields.entries) > 1 { 33 | updated.entries = make([]Field, len(fields.entries)-1) 34 | copy(updated.entries, fields.entries[:index]) 35 | copy(updated.entries[index:], fields.entries[index+1:]) 36 | } 37 | } else if !fields.entries[index].value.Equals(field.value) { 38 | // update 39 | updated.entries = make([]Field, len(fields.entries)) 40 | copy(updated.entries, fields.entries) 41 | updated.entries[index].value = field.value 42 | } else { 43 | // nothing changes 44 | updated = fields 45 | } 46 | return updated 47 | } 48 | if field.Value().IsZero() { 49 | return fields 50 | } 51 | updated.entries = make([]Field, len(fields.entries)+1) 52 | copy(updated.entries, fields.entries[:index]) 53 | copy(updated.entries[index+1:], fields.entries[index:]) 54 | updated.entries[index] = field 55 | return updated 56 | } 57 | 58 | func (fields List) Get(name string) Field { 59 | index, found := fields.bsearch(name) 60 | if !found { 61 | return ZeroField 62 | } 63 | return fields.entries[index] 64 | } 65 | 66 | func (fields List) Scan(iter func(field Field) bool) { 67 | for _, f := range fields.entries { 68 | if !iter(f) { 69 | return 70 | } 71 | } 72 | } 73 | 74 | func (fields List) Len() int { 75 | return len(fields.entries) 76 | } 77 | 78 | func (fields List) Weight() int { 79 | var weight int 80 | for _, f := range fields.entries { 81 | weight += f.Weight() 82 | } 83 | return weight 84 | } 85 | -------------------------------------------------------------------------------- /internal/glob/glob.go: -------------------------------------------------------------------------------- 1 | package glob 2 | 3 | import "strings" 4 | 5 | // Glob structure for simple string matching 6 | type Glob struct { 7 | Pattern string 8 | Desc bool 9 | Limits []string 10 | IsGlob bool 11 | } 12 | 13 | // Match returns true when string matches pattern. Returns an error when the 14 | // pattern is invalid. 15 | func Match(pattern, str string) (matched bool, err error) { 16 | return wildcardMatch(pattern, str) 17 | } 18 | 19 | // IsGlob returns true when the pattern is a valid glob 20 | func IsGlob(pattern string) bool { 21 | for i := 0; i < len(pattern); i++ { 22 | switch pattern[i] { 23 | case '[', '*', '?': 24 | _, err := Match(pattern, "whatever") 25 | return err == nil 26 | } 27 | } 28 | return false 29 | } 30 | 31 | // Parse returns a glob structure from the pattern. 32 | func Parse(pattern string, desc bool) *Glob { 33 | g := &Glob{Pattern: pattern, Desc: desc, Limits: []string{"", ""}} 34 | if strings.HasPrefix(pattern, "*") { 35 | g.IsGlob = true 36 | return g 37 | } 38 | if pattern == "" { 39 | g.IsGlob = false 40 | return g 41 | } 42 | n := 0 43 | isGlob := false 44 | outer: 45 | for i := 0; i < len(pattern); i++ { 46 | switch pattern[i] { 47 | case '[', '*', '?': 48 | _, err := Match(pattern, "whatever") 49 | if err == nil { 50 | isGlob = true 51 | } 52 | break outer 53 | } 54 | n++ 55 | } 56 | if n == 0 { 57 | g.Limits = []string{pattern, pattern} 58 | g.IsGlob = false 59 | return g 60 | } 61 | var a, b string 62 | if desc { 63 | a = pattern[:n] 64 | b = a 65 | if b[n-1] == 0x00 { 66 | for len(b) > 0 && b[len(b)-1] == 0x00 { 67 | if len(b) > 1 { 68 | if b[len(b)-2] == 0x00 { 69 | b = b[:len(b)-1] 70 | } else { 71 | b = string(append([]byte(b[:len(b)-2]), b[len(b)-2]-1, 0xFF)) 72 | } 73 | } else { 74 | b = "" 75 | } 76 | } 77 | } else { 78 | b = string(append([]byte(b[:n-1]), b[n-1]-1)) 79 | } 80 | if a[n-1] == 0xFF { 81 | a = string(append([]byte(a), 0x00)) 82 | } else { 83 | a = string(append([]byte(a[:n-1]), a[n-1]+1)) 84 | } 85 | } else { 86 | a = pattern[:n] 87 | if a[n-1] == 0xFF { 88 | b = string(append([]byte(a), 0x00)) 89 | } else { 90 | b = string(append([]byte(a[:n-1]), a[n-1]+1)) 91 | } 92 | } 93 | g.Limits = []string{a, b} 94 | g.IsGlob = isGlob 95 | return g 96 | } 97 | -------------------------------------------------------------------------------- /internal/hservice/gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname "${BASH_SOURCE[0]}") 4 | protoc --go_out=plugins=grpc,import_path=hservice:. *.proto 5 | -------------------------------------------------------------------------------- /internal/hservice/hservice.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: hservice.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package hservice is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | hservice.proto 10 | 11 | It has these top-level messages: 12 | MessageRequest 13 | MessageReply 14 | */ 15 | package hservice 16 | 17 | import proto "github.com/golang/protobuf/proto" 18 | import fmt "fmt" 19 | import math "math" 20 | 21 | import ( 22 | context "golang.org/x/net/context" 23 | grpc "google.golang.org/grpc" 24 | ) 25 | 26 | // Reference imports to suppress errors if they are not otherwise used. 27 | var _ = proto.Marshal 28 | var _ = fmt.Errorf 29 | var _ = math.Inf 30 | 31 | // This is a compile-time assertion to ensure that this generated file 32 | // is compatible with the proto package it is being compiled against. 33 | // A compilation error at this line likely means your copy of the 34 | // proto package needs to be updated. 35 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 36 | 37 | // The request message containing the message value 38 | type MessageRequest struct { 39 | Value string `protobuf:"bytes,1,opt,name=value" json:"value,omitempty"` 40 | } 41 | 42 | func (m *MessageRequest) Reset() { *m = MessageRequest{} } 43 | func (m *MessageRequest) String() string { return proto.CompactTextString(m) } 44 | func (*MessageRequest) ProtoMessage() {} 45 | func (*MessageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 46 | 47 | // The response message containing an ok (true or false) 48 | type MessageReply struct { 49 | Ok bool `protobuf:"varint,1,opt,name=ok" json:"ok,omitempty"` 50 | } 51 | 52 | func (m *MessageReply) Reset() { *m = MessageReply{} } 53 | func (m *MessageReply) String() string { return proto.CompactTextString(m) } 54 | func (*MessageReply) ProtoMessage() {} 55 | func (*MessageReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } 56 | 57 | func init() { 58 | proto.RegisterType((*MessageRequest)(nil), "hservice.MessageRequest") 59 | proto.RegisterType((*MessageReply)(nil), "hservice.MessageReply") 60 | } 61 | 62 | // Reference imports to suppress errors if they are not otherwise used. 63 | var _ context.Context 64 | var _ grpc.ClientConn 65 | 66 | // This is a compile-time assertion to ensure that this generated file 67 | // is compatible with the grpc package it is being compiled against. 68 | const _ = grpc.SupportPackageIsVersion3 69 | 70 | // Client API for HookService service 71 | 72 | type HookServiceClient interface { 73 | // Sends a greeting 74 | Send(ctx context.Context, in *MessageRequest, opts ...grpc.CallOption) (*MessageReply, error) 75 | } 76 | 77 | type hookServiceClient struct { 78 | cc *grpc.ClientConn 79 | } 80 | 81 | func NewHookServiceClient(cc *grpc.ClientConn) HookServiceClient { 82 | return &hookServiceClient{cc} 83 | } 84 | 85 | func (c *hookServiceClient) Send(ctx context.Context, in *MessageRequest, opts ...grpc.CallOption) (*MessageReply, error) { 86 | out := new(MessageReply) 87 | err := grpc.Invoke(ctx, "/hservice.HookService/Send", in, out, c.cc, opts...) 88 | if err != nil { 89 | return nil, err 90 | } 91 | return out, nil 92 | } 93 | 94 | // Server API for HookService service 95 | 96 | type HookServiceServer interface { 97 | // Sends a greeting 98 | Send(context.Context, *MessageRequest) (*MessageReply, error) 99 | } 100 | 101 | func RegisterHookServiceServer(s *grpc.Server, srv HookServiceServer) { 102 | s.RegisterService(&_HookService_serviceDesc, srv) 103 | } 104 | 105 | func _HookService_Send_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 106 | in := new(MessageRequest) 107 | if err := dec(in); err != nil { 108 | return nil, err 109 | } 110 | if interceptor == nil { 111 | return srv.(HookServiceServer).Send(ctx, in) 112 | } 113 | info := &grpc.UnaryServerInfo{ 114 | Server: srv, 115 | FullMethod: "/hservice.HookService/Send", 116 | } 117 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 118 | return srv.(HookServiceServer).Send(ctx, req.(*MessageRequest)) 119 | } 120 | return interceptor(ctx, in, info, handler) 121 | } 122 | 123 | var _HookService_serviceDesc = grpc.ServiceDesc{ 124 | ServiceName: "hservice.HookService", 125 | HandlerType: (*HookServiceServer)(nil), 126 | Methods: []grpc.MethodDesc{ 127 | { 128 | MethodName: "Send", 129 | Handler: _HookService_Send_Handler, 130 | }, 131 | }, 132 | Streams: []grpc.StreamDesc{}, 133 | Metadata: fileDescriptor0, 134 | } 135 | 136 | func init() { proto.RegisterFile("hservice.proto", fileDescriptor0) } 137 | 138 | var fileDescriptor0 = []byte{ 139 | // 168 bytes of a gzipped FileDescriptorProto 140 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcb, 0x28, 0x4e, 0x2d, 141 | 0x2a, 0xcb, 0x4c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x80, 0xf1, 0x95, 0xd4, 142 | 0xb8, 0xf8, 0x7c, 0x53, 0x8b, 0x8b, 0x13, 0xd3, 0x53, 0x83, 0x52, 0x0b, 0x4b, 0x53, 0x8b, 0x4b, 143 | 0x84, 0x44, 0xb8, 0x58, 0xcb, 0x12, 0x73, 0x4a, 0x53, 0x25, 0x18, 0x15, 0x18, 0x35, 0x38, 0x83, 144 | 0x20, 0x1c, 0x25, 0x39, 0x2e, 0x1e, 0xb8, 0xba, 0x82, 0x9c, 0x4a, 0x21, 0x3e, 0x2e, 0xa6, 0xfc, 145 | 0x6c, 0xb0, 0x12, 0x8e, 0x20, 0xa6, 0xfc, 0x6c, 0x23, 0x4f, 0x2e, 0x6e, 0x8f, 0xfc, 0xfc, 0xec, 146 | 0x60, 0x88, 0xb1, 0x42, 0x56, 0x5c, 0x2c, 0xc1, 0xa9, 0x79, 0x29, 0x42, 0x12, 0x7a, 0x70, 0x9b, 147 | 0x51, 0xad, 0x91, 0x12, 0xc3, 0x22, 0x53, 0x90, 0x53, 0xa9, 0xc4, 0xe0, 0xa4, 0xc9, 0x25, 0x9c, 148 | 0x9c, 0x9f, 0xab, 0x57, 0x92, 0x99, 0x93, 0x6a, 0x6c, 0x01, 0x57, 0xe5, 0x24, 0x80, 0x64, 0x7e, 149 | 0x00, 0xc8, 0x17, 0x01, 0x8c, 0x49, 0x6c, 0x60, 0xef, 0x18, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 150 | 0x6d, 0xd0, 0x2b, 0x13, 0xe0, 0x00, 0x00, 0x00, 151 | } 152 | -------------------------------------------------------------------------------- /internal/hservice/hservice.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option java_multiple_files = true; 4 | option java_package = "com.tile38.hservice"; 5 | option java_outer_classname = "HookServiceProto"; 6 | 7 | package hservice; 8 | 9 | // The greeting service definition. 10 | service HookService { 11 | // Sends a greeting 12 | rpc Send (MessageRequest) returns (MessageReply) {} 13 | } 14 | 15 | // The request message containing the message value 16 | message MessageRequest { 17 | string value = 1; 18 | } 19 | 20 | // The response message containing an ok (true or false) 21 | message MessageReply { 22 | bool ok = 1; 23 | } 24 | -------------------------------------------------------------------------------- /internal/log/log_test.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "strings" 7 | "testing" 8 | 9 | "go.uber.org/zap" 10 | "go.uber.org/zap/zapcore" 11 | "go.uber.org/zap/zaptest/observer" 12 | ) 13 | 14 | func TestLog(t *testing.T) { 15 | f := &bytes.Buffer{} 16 | SetLogJSON(false) 17 | SetOutput(f) 18 | Printf("hello %v", "everyone") 19 | if !strings.HasSuffix(f.String(), "hello everyone\n") { 20 | t.Fatal("fail") 21 | } 22 | } 23 | 24 | func TestLogJSON(t *testing.T) { 25 | 26 | SetLogJSON(true) 27 | Build("") 28 | 29 | type tcase struct { 30 | level int 31 | format string 32 | args string 33 | ops func(...interface{}) 34 | fops func(string, ...interface{}) 35 | expMsg string 36 | expLvl zapcore.Level 37 | } 38 | 39 | fn := func(tc tcase) func(*testing.T) { 40 | return func(t *testing.T) { 41 | observedZapCore, observedLogs := observer.New(zap.DebugLevel) 42 | Set(zap.New(observedZapCore).Sugar()) 43 | SetLevel(tc.level) 44 | 45 | if tc.format != "" { 46 | tc.fops(tc.format, tc.args) 47 | } else { 48 | tc.ops(tc.args) 49 | } 50 | 51 | if observedLogs.Len() < 1 { 52 | t.Fatal("fail") 53 | } 54 | 55 | allLogs := observedLogs.All() 56 | 57 | if allLogs[0].Message != tc.expMsg { 58 | t.Fatal("fail") 59 | } 60 | 61 | if allLogs[0].Level != tc.expLvl { 62 | t.Fatal("fail") 63 | } 64 | } 65 | } 66 | 67 | tests := map[string]tcase{ 68 | "Print": { 69 | level: 1, 70 | args: "Print json logger", 71 | ops: func(args ...interface{}) { 72 | Print(args...) 73 | }, 74 | expMsg: "Print json logger", 75 | expLvl: zapcore.InfoLevel, 76 | }, 77 | "Printf": { 78 | level: 1, 79 | format: "Printf json %v", 80 | args: "logger", 81 | fops: func(format string, args ...interface{}) { 82 | Printf(format, args...) 83 | }, 84 | expMsg: "Printf json logger", 85 | expLvl: zapcore.InfoLevel, 86 | }, 87 | "Info": { 88 | level: 1, 89 | args: "Info json logger", 90 | ops: func(args ...interface{}) { 91 | Info(args...) 92 | }, 93 | expMsg: "Info json logger", 94 | expLvl: zapcore.InfoLevel, 95 | }, 96 | "Infof": { 97 | level: 1, 98 | format: "Infof json %v", 99 | args: "logger", 100 | fops: func(format string, args ...interface{}) { 101 | Infof(format, args...) 102 | }, 103 | expMsg: "Infof json logger", 104 | expLvl: zapcore.InfoLevel, 105 | }, 106 | "Debug": { 107 | level: 3, 108 | args: "Debug json logger", 109 | ops: func(args ...interface{}) { 110 | Debug(args...) 111 | }, 112 | expMsg: "Debug json logger", 113 | expLvl: zapcore.DebugLevel, 114 | }, 115 | "Debugf": { 116 | level: 3, 117 | format: "Debugf json %v", 118 | args: "logger", 119 | fops: func(format string, args ...interface{}) { 120 | Debugf(format, args...) 121 | }, 122 | expMsg: "Debugf json logger", 123 | expLvl: zapcore.DebugLevel, 124 | }, 125 | "Warn": { 126 | level: 2, 127 | args: "Warn json logger", 128 | ops: func(args ...interface{}) { 129 | Warn(args...) 130 | }, 131 | expMsg: "Warn json logger", 132 | expLvl: zapcore.WarnLevel, 133 | }, 134 | "Warnf": { 135 | level: 2, 136 | format: "Warnf json %v", 137 | args: "logger", 138 | fops: func(format string, args ...interface{}) { 139 | Warnf(format, args...) 140 | }, 141 | expMsg: "Warnf json logger", 142 | expLvl: zapcore.WarnLevel, 143 | }, 144 | "Error": { 145 | level: 1, 146 | args: "Error json logger", 147 | ops: func(args ...interface{}) { 148 | Error(args...) 149 | }, 150 | expMsg: "Error json logger", 151 | expLvl: zapcore.ErrorLevel, 152 | }, 153 | "Errorf": { 154 | level: 1, 155 | format: "Errorf json %v", 156 | args: "logger", 157 | fops: func(format string, args ...interface{}) { 158 | Errorf(format, args...) 159 | }, 160 | expMsg: "Errorf json logger", 161 | expLvl: zapcore.ErrorLevel, 162 | }, 163 | "Http": { 164 | level: 1, 165 | args: "Http json logger", 166 | ops: func(args ...interface{}) { 167 | HTTP(args...) 168 | }, 169 | expMsg: "Http json logger", 170 | expLvl: zapcore.InfoLevel, 171 | }, 172 | "Httpf": { 173 | level: 1, 174 | format: "Httpf json %v", 175 | args: "logger", 176 | fops: func(format string, args ...interface{}) { 177 | HTTPf(format, args...) 178 | }, 179 | expMsg: "Httpf json logger", 180 | expLvl: zapcore.InfoLevel, 181 | }, 182 | } 183 | 184 | for name, tc := range tests { 185 | t.Run(name, fn(tc)) 186 | } 187 | } 188 | 189 | func BenchmarkLogPrintf(t *testing.B) { 190 | SetLogJSON(false) 191 | SetLevel(1) 192 | SetOutput(io.Discard) 193 | t.ResetTimer() 194 | for i := 0; i < t.N; i++ { 195 | Printf("X %s", "Y") 196 | } 197 | } 198 | 199 | func BenchmarkLogJSONPrintf(t *testing.B) { 200 | SetLogJSON(true) 201 | SetLevel(1) 202 | 203 | ec := zap.NewProductionEncoderConfig() 204 | ec.EncodeDuration = zapcore.NanosDurationEncoder 205 | ec.EncodeTime = zapcore.EpochNanosTimeEncoder 206 | enc := zapcore.NewJSONEncoder(ec) 207 | 208 | logger := zap.New( 209 | zapcore.NewCore( 210 | enc, 211 | zapcore.AddSync(io.Discard), 212 | zap.DebugLevel, 213 | )).Sugar() 214 | 215 | Set(logger) 216 | t.ResetTimer() 217 | for i := 0; i < t.N; i++ { 218 | Printf("X %s", "Y") 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /internal/object/object_binary.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "encoding/binary" 5 | "unsafe" 6 | 7 | "github.com/tidwall/geojson" 8 | "github.com/tidwall/geojson/geometry" 9 | "github.com/tidwall/tile38/internal/field" 10 | ) 11 | 12 | type pointObject struct { 13 | base Object 14 | pt geojson.SimplePoint 15 | } 16 | 17 | type geoObject struct { 18 | base Object 19 | geo geojson.Object 20 | } 21 | 22 | const opoint = 1 23 | const ogeo = 2 24 | 25 | type Object struct { 26 | head string // tuple (kind,expires,id) 27 | fields field.List 28 | } 29 | 30 | func (o *Object) geo() geojson.Object { 31 | if o != nil { 32 | switch o.head[0] { 33 | case opoint: 34 | return &(*pointObject)(unsafe.Pointer(o)).pt 35 | case ogeo: 36 | return (*geoObject)(unsafe.Pointer(o)).geo 37 | } 38 | } 39 | return nil 40 | } 41 | 42 | // uvarint is a slightly modified version of binary.Uvarint, and it's a little 43 | // faster. But it lacks overflow checks which are not needed for our use. 44 | func uvarint(s string) (uint64, int) { 45 | var x uint64 46 | for i := 0; i < len(s); i++ { 47 | b := s[i] 48 | if b < 0x80 { 49 | return x | uint64(b)<<(i*7), i + 1 50 | } 51 | x |= uint64(b&0x7f) << (i * 7) 52 | } 53 | return 0, 0 54 | } 55 | 56 | func varint(s string) (int64, int) { 57 | ux, n := uvarint(s) 58 | x := int64(ux >> 1) 59 | if ux&1 != 0 { 60 | x = ^x 61 | } 62 | return x, n 63 | } 64 | 65 | func (o *Object) ID() string { 66 | if o.head[1] == 0 { 67 | return o.head[2:] 68 | } 69 | _, n := varint(o.head[1:]) 70 | return o.head[1+n:] 71 | } 72 | 73 | func (o *Object) Fields() field.List { 74 | return o.fields 75 | } 76 | 77 | func (o *Object) Expires() int64 { 78 | ex, _ := varint(o.head[1:]) 79 | return ex 80 | } 81 | 82 | func (o *Object) Rect() geometry.Rect { 83 | ogeo := o.geo() 84 | if ogeo == nil { 85 | return geometry.Rect{} 86 | } 87 | return ogeo.Rect() 88 | } 89 | 90 | func (o *Object) Geo() geojson.Object { 91 | return o.geo() 92 | } 93 | 94 | func (o *Object) String() string { 95 | ogeo := o.geo() 96 | if ogeo == nil { 97 | return "" 98 | } 99 | return ogeo.String() 100 | } 101 | 102 | func (o *Object) IsSpatial() bool { 103 | _, ok := o.geo().(geojson.Spatial) 104 | return ok 105 | } 106 | 107 | func (o *Object) Weight() int { 108 | var weight int 109 | weight += len(o.ID()) 110 | ogeo := o.geo() 111 | if ogeo != nil { 112 | if o.IsSpatial() { 113 | weight += ogeo.NumPoints() * 16 114 | } else { 115 | weight += len(ogeo.String()) 116 | } 117 | } 118 | weight += o.Fields().Weight() 119 | return weight 120 | } 121 | 122 | func makeHead(kind byte, id string, expires int64) string { 123 | var exb [20]byte 124 | exn := 1 125 | if expires != 0 { 126 | exn = binary.PutVarint(exb[:], expires) 127 | } 128 | n := 1 + exn + len(id) 129 | head := make([]byte, n) 130 | head[0] = kind 131 | copy(head[1:], exb[:exn]) 132 | copy(head[1+exn:], id) 133 | return *(*string)(unsafe.Pointer(&head)) 134 | } 135 | 136 | func newPoint(id string, pt geometry.Point, expires int64, fields field.List, 137 | ) *Object { 138 | return (*Object)(unsafe.Pointer(&pointObject{ 139 | Object{ 140 | head: makeHead(opoint, id, expires), 141 | fields: fields, 142 | }, 143 | geojson.SimplePoint{Point: pt}, 144 | })) 145 | } 146 | func newGeo(id string, geo geojson.Object, expires int64, fields field.List, 147 | ) *Object { 148 | return (*Object)(unsafe.Pointer(&geoObject{ 149 | Object{ 150 | head: makeHead(ogeo, id, expires), 151 | fields: fields, 152 | }, 153 | geo, 154 | })) 155 | } 156 | 157 | func New(id string, geo geojson.Object, expires int64, fields field.List, 158 | ) *Object { 159 | switch p := geo.(type) { 160 | case *geojson.SimplePoint: 161 | return newPoint(id, p.Base(), expires, fields) 162 | case *geojson.Point: 163 | if p.IsSimple() { 164 | return newPoint(id, p.Base(), expires, fields) 165 | } 166 | } 167 | return newGeo(id, geo, expires, fields) 168 | } 169 | -------------------------------------------------------------------------------- /internal/object/object_struct.go: -------------------------------------------------------------------------------- 1 | //go:build exclude 2 | 3 | package object 4 | 5 | import ( 6 | "github.com/tidwall/geojson" 7 | "github.com/tidwall/geojson/geometry" 8 | "github.com/tidwall/tile38/internal/field" 9 | ) 10 | 11 | type Object struct { 12 | id string 13 | geo geojson.Object 14 | expires int64 // unix nano expiration 15 | fields field.List 16 | } 17 | 18 | func (o *Object) ID() string { 19 | if o == nil { 20 | return "" 21 | } 22 | return o.id 23 | } 24 | 25 | func (o *Object) Fields() field.List { 26 | if o == nil { 27 | return field.List{} 28 | } 29 | return o.fields 30 | } 31 | 32 | func (o *Object) Expires() int64 { 33 | if o == nil { 34 | return 0 35 | } 36 | return o.expires 37 | } 38 | 39 | func (o *Object) Rect() geometry.Rect { 40 | if o == nil || o.geo == nil { 41 | return geometry.Rect{} 42 | } 43 | return o.geo.Rect() 44 | } 45 | 46 | func (o *Object) Geo() geojson.Object { 47 | if o == nil || o.geo == nil { 48 | return nil 49 | } 50 | return o.geo 51 | } 52 | 53 | func (o *Object) String() string { 54 | if o == nil || o.geo == nil { 55 | return "" 56 | } 57 | return o.geo.String() 58 | } 59 | 60 | func (o *Object) IsSpatial() bool { 61 | _, ok := o.geo.(geojson.Spatial) 62 | return ok 63 | } 64 | 65 | func (o *Object) Weight() int { 66 | if o == nil { 67 | return 0 68 | } 69 | var weight int 70 | weight += len(o.ID()) 71 | if o.IsSpatial() { 72 | weight += o.Geo().NumPoints() * 16 73 | } else { 74 | weight += len(o.Geo().String()) 75 | } 76 | weight += o.Fields().Weight() 77 | return weight 78 | } 79 | 80 | func New(id string, geo geojson.Object, expires int64, fields field.List, 81 | ) *Object { 82 | return &Object{ 83 | id: id, 84 | geo: geo, 85 | expires: expires, 86 | fields: fields, 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /internal/object/object_test.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/tidwall/assert" 7 | "github.com/tidwall/geojson" 8 | "github.com/tidwall/geojson/geometry" 9 | "github.com/tidwall/tile38/internal/field" 10 | ) 11 | 12 | func P(x, y float64) geojson.Object { 13 | return geojson.NewSimplePoint(geometry.Point{X: 10, Y: 20}) 14 | } 15 | func TestObject(t *testing.T) { 16 | o := New("hello", P(10, 20), 99, field.List{}) 17 | assert.Assert(o.ID() == "hello") 18 | } 19 | -------------------------------------------------------------------------------- /internal/server/aofmigrate.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "errors" 7 | "io" 8 | "os" 9 | "path" 10 | "time" 11 | 12 | "github.com/tidwall/resp" 13 | "github.com/tidwall/tile38/internal/log" 14 | ) 15 | 16 | var errCorruptedAOF = errors.New("corrupted aof file") 17 | 18 | // LegacyAOFReader represents the older AOF file reader. 19 | type LegacyAOFReader struct { 20 | r io.Reader // reader 21 | rerr error // read error 22 | chunk []byte // chunk buffer 23 | buf []byte // main buffer 24 | l int // length of valid data in buffer 25 | p int // pointer 26 | } 27 | 28 | // ReadCommand reads an old command. 29 | func (rd *LegacyAOFReader) ReadCommand() ([]byte, error) { 30 | if rd.l >= 4 { 31 | sz1 := int(binary.LittleEndian.Uint32(rd.buf[rd.p:])) 32 | if rd.l >= sz1+9 { 33 | // we have enough data for a record 34 | sz2 := int(binary.LittleEndian.Uint32(rd.buf[rd.p+4+sz1:])) 35 | if sz2 != sz1 || rd.buf[rd.p+4+sz1+4] != 0 { 36 | return nil, errCorruptedAOF 37 | } 38 | buf := rd.buf[rd.p+4 : rd.p+4+sz1] 39 | rd.p += sz1 + 9 40 | rd.l -= sz1 + 9 41 | return buf, nil 42 | } 43 | } 44 | // need more data 45 | if rd.rerr != nil { 46 | if rd.rerr == io.EOF { 47 | rd.rerr = nil // we want to return EOF, but we want to be able to try again 48 | if rd.l != 0 { 49 | return nil, io.ErrUnexpectedEOF 50 | } 51 | return nil, io.EOF 52 | } 53 | return nil, rd.rerr 54 | } 55 | if rd.p != 0 { 56 | // move p to the beginning 57 | copy(rd.buf, rd.buf[rd.p:rd.p+rd.l]) 58 | rd.p = 0 59 | } 60 | var n int 61 | n, rd.rerr = rd.r.Read(rd.chunk) 62 | if n > 0 { 63 | cbuf := rd.chunk[:n] 64 | if len(rd.buf)-rd.l < n { 65 | if len(rd.buf) == 0 { 66 | rd.buf = make([]byte, len(cbuf)) 67 | copy(rd.buf, cbuf) 68 | } else { 69 | copy(rd.buf[rd.l:], cbuf[:len(rd.buf)-rd.l]) 70 | rd.buf = append(rd.buf, cbuf[len(rd.buf)-rd.l:]...) 71 | } 72 | } else { 73 | copy(rd.buf[rd.l:], cbuf) 74 | } 75 | rd.l += n 76 | } 77 | return rd.ReadCommand() 78 | } 79 | 80 | // NewLegacyAOFReader creates a new LegacyAOFReader. 81 | func NewLegacyAOFReader(r io.Reader) *LegacyAOFReader { 82 | rd := &LegacyAOFReader{r: r, chunk: make([]byte, 0xFFFF)} 83 | return rd 84 | } 85 | 86 | func (s *Server) migrateAOF() error { 87 | _, err := os.Stat(path.Join(s.dir, "appendonly.aof")) 88 | if err == nil { 89 | return nil 90 | } 91 | if !os.IsNotExist(err) { 92 | return err 93 | } 94 | _, err = os.Stat(path.Join(s.dir, "aof")) 95 | if err != nil { 96 | if os.IsNotExist(err) { 97 | return nil 98 | } 99 | return err 100 | } 101 | log.Warn("Migrating aof to new format") 102 | newf, err := os.Create(path.Join(s.dir, "migrate.aof")) 103 | if err != nil { 104 | return err 105 | } 106 | defer newf.Close() 107 | 108 | oldf, err := os.Open(path.Join(s.dir, "aof")) 109 | if err != nil { 110 | return err 111 | } 112 | defer oldf.Close() 113 | start := time.Now() 114 | count := 0 115 | wr := bufio.NewWriter(newf) 116 | rd := NewLegacyAOFReader(oldf) 117 | for { 118 | cmdb, err := rd.ReadCommand() 119 | if err != nil { 120 | if err == io.EOF { 121 | break 122 | } 123 | return err 124 | } 125 | line := string(cmdb) 126 | var tok string 127 | values := make([]resp.Value, 0, 64) 128 | for line != "" { 129 | line, tok = token(line) 130 | if len(tok) > 0 && tok[0] == '{' { 131 | if line != "" { 132 | tok = tok + " " + line 133 | line = "" 134 | } 135 | } 136 | values = append(values, resp.StringValue(tok)) 137 | } 138 | data, err := resp.ArrayValue(values).MarshalRESP() 139 | if err != nil { 140 | return err 141 | } 142 | if _, err := wr.Write(data); err != nil { 143 | return err 144 | } 145 | if wr.Buffered() > 1024*1024 { 146 | if err := wr.Flush(); err != nil { 147 | return err 148 | } 149 | } 150 | count++ 151 | } 152 | if err := wr.Flush(); err != nil { 153 | return err 154 | } 155 | oldf.Close() 156 | newf.Close() 157 | log.Debugf("%d items: %.0f/sec", count, float64(count)/(float64(time.Since(start))/float64(time.Second))) 158 | return os.Rename(path.Join(s.dir, "migrate.aof"), path.Join(s.dir, "appendonly.aof")) 159 | } 160 | -------------------------------------------------------------------------------- /internal/server/bson.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/rand" 6 | "encoding/binary" 7 | "encoding/hex" 8 | "os" 9 | "sync/atomic" 10 | "time" 11 | ) 12 | 13 | func bsonID() string { 14 | b := make([]byte, 12) 15 | binary.BigEndian.PutUint32(b, uint32(time.Now().Unix())) 16 | copy(b[4:], bsonMachine) 17 | binary.BigEndian.PutUint32(b[8:], atomic.AddUint32(&bsonCounter, 1)) 18 | binary.BigEndian.PutUint16(b[7:], bsonProcess) 19 | return hex.EncodeToString(b) 20 | } 21 | 22 | var ( 23 | bsonProcess = uint16(os.Getpid()) 24 | bsonMachine = func() []byte { 25 | host, _ := os.Hostname() 26 | b := make([]byte, 3) 27 | Must(rand.Read(b)) 28 | host = Default(host, string(b)) 29 | hw := md5.New() 30 | hw.Write([]byte(host)) 31 | return hw.Sum(nil)[:3] 32 | }() 33 | bsonCounter = func() uint32 { 34 | b := make([]byte, 4) 35 | Must(rand.Read(b)) 36 | return binary.BigEndian.Uint32(b) 37 | }() 38 | ) 39 | -------------------------------------------------------------------------------- /internal/server/bson_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import "testing" 4 | 5 | func TestBSON(t *testing.T) { 6 | id := bsonID() 7 | if len(id) != 24 { 8 | t.Fail() 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /internal/server/dev.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math/rand" 7 | "strconv" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/tidwall/resp" 12 | "github.com/tidwall/tile38/internal/log" 13 | ) 14 | 15 | // MASSINSERT num_keys num_points [minx miny maxx maxy] 16 | 17 | func randMassInsertPosition(minLat, minLon, maxLat, maxLon float64) (float64, float64) { 18 | lat, lon := (rand.Float64()*(maxLat-minLat))+minLat, (rand.Float64()*(maxLon-minLon))+minLon 19 | return lat, lon 20 | } 21 | 22 | func (s *Server) cmdMassInsert(msg *Message) (res resp.Value, err error) { 23 | start := time.Now() 24 | vs := msg.Args[1:] 25 | 26 | minLat, minLon, maxLat, maxLon := -90.0, -180.0, 90.0, 180.0 //37.10776, -122.67145, 38.19502, -121.62775 27 | 28 | var snumCols, snumPoints string 29 | var cols, objs int 30 | var ok bool 31 | if vs, snumCols, ok = tokenval(vs); !ok || snumCols == "" { 32 | return NOMessage, errInvalidNumberOfArguments 33 | } 34 | if vs, snumPoints, ok = tokenval(vs); !ok || snumPoints == "" { 35 | return NOMessage, errInvalidNumberOfArguments 36 | } 37 | if len(vs) != 0 { 38 | var sminLat, sminLon, smaxLat, smaxLon string 39 | if vs, sminLat, ok = tokenval(vs); !ok || sminLat == "" { 40 | return NOMessage, errInvalidNumberOfArguments 41 | } 42 | if vs, sminLon, ok = tokenval(vs); !ok || sminLon == "" { 43 | return NOMessage, errInvalidNumberOfArguments 44 | } 45 | if vs, smaxLat, ok = tokenval(vs); !ok || smaxLat == "" { 46 | return NOMessage, errInvalidNumberOfArguments 47 | } 48 | if vs, smaxLon, ok = tokenval(vs); !ok || smaxLon == "" { 49 | return NOMessage, errInvalidNumberOfArguments 50 | } 51 | var err error 52 | if minLat, err = strconv.ParseFloat(sminLat, 64); err != nil { 53 | return NOMessage, err 54 | } 55 | if minLon, err = strconv.ParseFloat(sminLon, 64); err != nil { 56 | return NOMessage, err 57 | } 58 | if maxLat, err = strconv.ParseFloat(smaxLat, 64); err != nil { 59 | return NOMessage, err 60 | } 61 | if maxLon, err = strconv.ParseFloat(smaxLon, 64); err != nil { 62 | return NOMessage, err 63 | } 64 | if len(vs) != 0 { 65 | return NOMessage, errors.New("invalid number of arguments") 66 | } 67 | } 68 | n, err := strconv.ParseUint(snumCols, 10, 64) 69 | if err != nil { 70 | return NOMessage, errInvalidArgument(snumCols) 71 | } 72 | cols = int(n) 73 | n, err = strconv.ParseUint(snumPoints, 10, 64) 74 | if err != nil { 75 | return NOMessage, errInvalidArgument(snumPoints) 76 | } 77 | 78 | docmd := func(args []string) error { 79 | s.mu.Lock() 80 | defer s.mu.Unlock() 81 | nmsg := *msg 82 | nmsg._command = "" 83 | nmsg.Args = args 84 | _, d, err := s.command(&nmsg, nil) 85 | if err != nil { 86 | return err 87 | } 88 | return s.writeAOF(nmsg.Args, &d) 89 | 90 | } 91 | rand.Seed(time.Now().UnixNano()) 92 | objs = int(n) 93 | var k atomic.Uint64 94 | for i := 0; i < cols; i++ { 95 | key := "mi:" + strconv.FormatInt(int64(i), 10) 96 | func(key string) { 97 | // lock cycle 98 | for j := 0; j < objs; j++ { 99 | id := strconv.FormatInt(int64(j), 10) 100 | var values []string 101 | values = append(values, "set", key, id) 102 | fvals := []float64{ 103 | 1, // one 104 | 0, // zero 105 | -1, // negOne 106 | 14, // nibble 107 | 20.5, // tinyDiv10 108 | 120, // int8 109 | -120, // int8 110 | 20000, // int16 111 | -20000, // int16 112 | 214748300, // int32 113 | -214748300, // int32 114 | 2014748300, // float64 115 | 123.12312301, // float64 116 | } 117 | for i, fval := range fvals { 118 | values = append(values, "FIELD", 119 | fmt.Sprintf("fname:%d", i), 120 | strconv.FormatFloat(fval, 'f', -1, 64)) 121 | } 122 | if rand.Int()%2 == 0 { 123 | values = append(values, "EX", fmt.Sprint(rand.Intn(25)+5)) 124 | } 125 | 126 | if j%8 == 0 { 127 | values = append(values, "STRING", fmt.Sprintf("str%v", j)) 128 | } else { 129 | lat, lon := randMassInsertPosition(minLat, minLon, maxLat, maxLon) 130 | values = append(values, "POINT", 131 | strconv.FormatFloat(lat, 'f', -1, 64), 132 | strconv.FormatFloat(lon, 'f', -1, 64), 133 | ) 134 | } 135 | err := docmd(values) 136 | if err != nil { 137 | log.Fatal(err) 138 | return 139 | } 140 | k.Add(1) 141 | if j%1000 == 1000-1 { 142 | log.Debugf("massinsert: %s %d/%d", key, k.Load(), cols*objs) 143 | } 144 | } 145 | }(key) 146 | } 147 | log.Infof("massinsert: done %d objects", k.Load()) 148 | return OKMessage(msg, start), nil 149 | } 150 | 151 | func (s *Server) cmdSleep(msg *Message) (res resp.Value, err error) { 152 | start := time.Now() 153 | if len(msg.Args) != 2 { 154 | return NOMessage, errInvalidNumberOfArguments 155 | } 156 | d, _ := strconv.ParseFloat(msg.Args[1], 64) 157 | time.Sleep(time.Duration(float64(time.Second) * d)) 158 | return OKMessage(msg, start), nil 159 | } 160 | -------------------------------------------------------------------------------- /internal/server/expire.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/tidwall/tile38/internal/collection" 8 | "github.com/tidwall/tile38/internal/log" 9 | "github.com/tidwall/tile38/internal/object" 10 | ) 11 | 12 | const bgExpireDelay = time.Second / 10 13 | 14 | // backgroundExpiring deletes expired items from the database. 15 | // It's executes every 1/10 of a second. 16 | func (s *Server) backgroundExpiring(wg *sync.WaitGroup) { 17 | defer wg.Done() 18 | s.loopUntilServerStops(bgExpireDelay, func() { 19 | s.mu.Lock() 20 | defer s.mu.Unlock() 21 | now := time.Now() 22 | s.backgroundExpireObjects(now) 23 | s.backgroundExpireHooks(now) 24 | }) 25 | } 26 | 27 | func (s *Server) backgroundExpireObjects(now time.Time) { 28 | nano := now.UnixNano() 29 | var msgs []*Message 30 | s.cols.Scan(func(key string, col *collection.Collection) bool { 31 | col.ScanExpires(func(o *object.Object) bool { 32 | if nano < o.Expires() { 33 | return false 34 | } 35 | s.statsExpired.Add(1) 36 | msgs = append(msgs, &Message{Args: []string{"del", key, o.ID()}}) 37 | return true 38 | }) 39 | return true 40 | }) 41 | for _, msg := range msgs { 42 | _, d, err := s.cmdDEL(msg) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | if err := s.writeAOF(msg.Args, &d); err != nil { 47 | log.Fatal(err) 48 | } 49 | } 50 | if len(msgs) > 0 { 51 | log.Debugf("Expired %d objects\n", len(msgs)) 52 | } 53 | } 54 | 55 | func (s *Server) backgroundExpireHooks(now time.Time) { 56 | var msgs []*Message 57 | s.hookExpires.Ascend(nil, func(v interface{}) bool { 58 | h := v.(*Hook) 59 | if h.expires.After(now) { 60 | return false 61 | } 62 | msg := &Message{} 63 | if h.channel { 64 | msg.Args = []string{"delchan", h.Name} 65 | } else { 66 | msg.Args = []string{"delhook", h.Name} 67 | } 68 | msgs = append(msgs, msg) 69 | return true 70 | }) 71 | 72 | for _, msg := range msgs { 73 | _, d, err := s.cmdDelHook(msg) 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | if err := s.writeAOF(msg.Args, &d); err != nil { 78 | log.Fatal(err) 79 | } 80 | } 81 | if len(msgs) > 0 { 82 | log.Debugf("Expired %d hooks\n", len(msgs)) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /internal/server/expr.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/tidwall/expr" 7 | "github.com/tidwall/geojson" 8 | "github.com/tidwall/gjson" 9 | "github.com/tidwall/match" 10 | "github.com/tidwall/tile38/internal/field" 11 | "github.com/tidwall/tile38/internal/object" 12 | ) 13 | 14 | type exprPool struct { 15 | pool *sync.Pool 16 | } 17 | 18 | func typeForObject(o *object.Object) expr.Value { 19 | switch o.Geo().(type) { 20 | case *geojson.Point, *geojson.SimplePoint: 21 | return expr.String("Point") 22 | case *geojson.LineString: 23 | return expr.String("LineString") 24 | case *geojson.Polygon, *geojson.Circle, *geojson.Rect: 25 | return expr.String("Polygon") 26 | case *geojson.MultiPoint: 27 | return expr.String("MultiPoint") 28 | case *geojson.MultiLineString: 29 | return expr.String("MultiLineString") 30 | case *geojson.MultiPolygon: 31 | return expr.String("MultiPolygon") 32 | case *geojson.GeometryCollection: 33 | return expr.String("GeometryCollection") 34 | case *geojson.Feature: 35 | return expr.String("Feature") 36 | case *geojson.FeatureCollection: 37 | return expr.String("FeatureCollection") 38 | default: 39 | return expr.Undefined 40 | } 41 | } 42 | 43 | func resultToValue(r gjson.Result) expr.Value { 44 | if !r.Exists() { 45 | return expr.Undefined 46 | } 47 | switch r.Type { 48 | case gjson.String: 49 | return expr.String(r.String()) 50 | case gjson.False: 51 | return expr.Bool(false) 52 | case gjson.True: 53 | return expr.Bool(true) 54 | case gjson.Number: 55 | return expr.Number(r.Float()) 56 | case gjson.JSON: 57 | return expr.Object(r) 58 | default: 59 | return expr.Null 60 | } 61 | } 62 | 63 | func newExprPool(s *Server) *exprPool { 64 | ext := expr.NewExtender( 65 | // ref 66 | func(info expr.RefInfo, ctx *expr.Context) (expr.Value, error) { 67 | o := ctx.UserData.(*object.Object) 68 | if !info.Chain { 69 | // root 70 | if r := gjson.Get(o.Geo().Members(), info.Ident); r.Exists() { 71 | return resultToValue(r), nil 72 | } 73 | switch info.Ident { 74 | case "id": 75 | return expr.String(o.ID()), nil 76 | case "type": 77 | return typeForObject(o), nil 78 | default: 79 | var rf field.Field 80 | var ok bool 81 | o.Fields().Scan(func(f field.Field) bool { 82 | if f.Name() == info.Ident { 83 | rf = f 84 | ok = true 85 | return false 86 | } 87 | return true 88 | }) 89 | if ok { 90 | r := gjson.Parse(rf.Value().JSON()) 91 | return resultToValue(r), nil 92 | } 93 | } 94 | return expr.Number(0), nil 95 | } else { 96 | switch v := info.Value.Value().(type) { 97 | case gjson.Result: 98 | return resultToValue(v.Get(info.Ident)), nil 99 | default: 100 | // object methods 101 | switch info.Ident { 102 | case "match": 103 | return expr.Function("match"), nil 104 | } 105 | } 106 | return expr.Undefined, nil 107 | } 108 | }, 109 | // call 110 | func(info expr.CallInfo, ctx *expr.Context) (expr.Value, error) { 111 | if info.Chain { 112 | switch info.Ident { 113 | case "match": 114 | if info.Args.Len() < 0 { 115 | return expr.Undefined, nil 116 | } 117 | 118 | t := match.Match(info.Value.String(), 119 | info.Args.At(0).String()) 120 | return expr.Bool(t), nil 121 | } 122 | } 123 | return expr.Undefined, nil 124 | }, 125 | // op 126 | func(info expr.OpInfo, ctx *expr.Context) (expr.Value, error) { 127 | return expr.Undefined, nil 128 | }, 129 | ) 130 | return &exprPool{ 131 | pool: &sync.Pool{ 132 | New: func() any { 133 | ctx := &expr.Context{ 134 | Extender: ext, 135 | } 136 | return ctx 137 | }, 138 | }, 139 | } 140 | } 141 | 142 | func (p *exprPool) Get(o *object.Object) *expr.Context { 143 | ctx := p.pool.Get().(*expr.Context) 144 | ctx.UserData = o 145 | ctx.NoCase = true 146 | return ctx 147 | } 148 | 149 | func (p *exprPool) Put(ctx *expr.Context) { 150 | p.pool.Put(ctx) 151 | } 152 | 153 | func (where whereT) matchExpr(s *Server, o *object.Object) bool { 154 | ctx := s.epool.Get(o) 155 | res, _ := expr.Eval(where.name, ctx) 156 | s.epool.Put(ctx) 157 | return res.Bool() 158 | } 159 | -------------------------------------------------------------------------------- /internal/server/expression.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/tidwall/geojson" 7 | ) 8 | 9 | // BinaryOp represents various operators for expressions 10 | type BinaryOp byte 11 | 12 | // expression operator enum 13 | const ( 14 | NOOP BinaryOp = iota 15 | AND 16 | OR 17 | tokenAND = "and" 18 | tokenOR = "or" 19 | tokenNOT = "not" 20 | tokenLParen = "(" 21 | tokenRParen = ")" 22 | ) 23 | 24 | // areaExpression is (maybe negated) either an spatial object or operator + 25 | // children (other expressions). 26 | type areaExpression struct { 27 | negate bool 28 | obj geojson.Object 29 | op BinaryOp 30 | children children 31 | } 32 | 33 | type children []*areaExpression 34 | 35 | // String representation, helpful in logging. 36 | func (e *areaExpression) String() (res string) { 37 | if e.obj != nil { 38 | res = e.obj.String() 39 | } else { 40 | var chStrings []string 41 | for _, c := range e.children { 42 | chStrings = append(chStrings, c.String()) 43 | } 44 | switch e.op { 45 | case NOOP: 46 | res = "empty operator" 47 | case AND: 48 | res = "(" + strings.Join(chStrings, " "+tokenAND+" ") + ")" 49 | case OR: 50 | res = "(" + strings.Join(chStrings, " "+tokenOR+" ") + ")" 51 | default: 52 | res = "unknown operator" 53 | } 54 | } 55 | if e.negate { 56 | res = tokenNOT + " " + res 57 | } 58 | return 59 | } 60 | 61 | // Return boolean value modulo negate field of the expression. 62 | func (e *areaExpression) maybeNegate(val bool) bool { 63 | if e.negate { 64 | return !val 65 | } 66 | return val 67 | } 68 | 69 | // Methods for testing an areaExpression against the spatial object. 70 | func (e *areaExpression) testObject( 71 | o geojson.Object, 72 | objObjTest func(o1, o2 geojson.Object) bool, 73 | exprObjTest func(ae *areaExpression, ob geojson.Object) bool, 74 | ) bool { 75 | if e.obj != nil { 76 | return objObjTest(e.obj, o) 77 | } 78 | switch e.op { 79 | case AND: 80 | for _, c := range e.children { 81 | if !exprObjTest(c, o) { 82 | return false 83 | } 84 | } 85 | return true 86 | case OR: 87 | for _, c := range e.children { 88 | if exprObjTest(c, o) { 89 | return true 90 | } 91 | } 92 | return false 93 | } 94 | return false 95 | } 96 | 97 | func (e *areaExpression) rawIntersects(o geojson.Object) bool { 98 | return e.testObject(o, geojson.Object.Intersects, (*areaExpression).Intersects) 99 | } 100 | 101 | func (e *areaExpression) rawContains(o geojson.Object) bool { 102 | return e.testObject(o, geojson.Object.Contains, (*areaExpression).Contains) 103 | } 104 | 105 | func (e *areaExpression) rawWithin(o geojson.Object) bool { 106 | return e.testObject(o, geojson.Object.Within, (*areaExpression).Within) 107 | } 108 | 109 | func (e *areaExpression) Intersects(o geojson.Object) bool { 110 | return e.maybeNegate(e.rawIntersects(o)) 111 | } 112 | 113 | func (e *areaExpression) Contains(o geojson.Object) bool { 114 | return e.maybeNegate(e.rawContains(o)) 115 | } 116 | 117 | func (e *areaExpression) Within(o geojson.Object) bool { 118 | return e.maybeNegate(e.rawWithin(o)) 119 | } 120 | 121 | // Methods for testing an areaExpression against another areaExpression. 122 | func (e *areaExpression) testExpression( 123 | other *areaExpression, 124 | exprObjTest func(ae *areaExpression, ob geojson.Object) bool, 125 | rawExprExprTest func(ae1, ae2 *areaExpression) bool, 126 | exprExprTest func(ae1, ae2 *areaExpression) bool, 127 | ) bool { 128 | if other.negate { 129 | oppositeExp := &areaExpression{negate: !e.negate, obj: e.obj, op: e.op, children: e.children} 130 | nonNegateOther := &areaExpression{obj: other.obj, op: other.op, children: other.children} 131 | return exprExprTest(oppositeExp, nonNegateOther) 132 | } 133 | if other.obj != nil { 134 | return exprObjTest(e, other.obj) 135 | } 136 | switch other.op { 137 | case AND: 138 | for _, c := range other.children { 139 | if !rawExprExprTest(e, c) { 140 | return false 141 | } 142 | } 143 | return true 144 | case OR: 145 | for _, c := range other.children { 146 | if rawExprExprTest(e, c) { 147 | return true 148 | } 149 | } 150 | return false 151 | } 152 | return false 153 | } 154 | 155 | func (e *areaExpression) rawIntersectsExpr(other *areaExpression) bool { 156 | return e.testExpression( 157 | other, 158 | (*areaExpression).rawIntersects, 159 | (*areaExpression).rawIntersectsExpr, 160 | (*areaExpression).IntersectsExpr) 161 | } 162 | 163 | func (e *areaExpression) rawWithinExpr(other *areaExpression) bool { 164 | return e.testExpression( 165 | other, 166 | (*areaExpression).rawWithin, 167 | (*areaExpression).rawWithinExpr, 168 | (*areaExpression).WithinExpr) 169 | } 170 | 171 | func (e *areaExpression) rawContainsExpr(other *areaExpression) bool { 172 | return e.testExpression( 173 | other, 174 | (*areaExpression).rawContains, 175 | (*areaExpression).rawContainsExpr, 176 | (*areaExpression).ContainsExpr) 177 | } 178 | 179 | func (e *areaExpression) IntersectsExpr(other *areaExpression) bool { 180 | return e.maybeNegate(e.rawIntersectsExpr(other)) 181 | } 182 | 183 | func (e *areaExpression) WithinExpr(other *areaExpression) bool { 184 | return e.maybeNegate(e.rawWithinExpr(other)) 185 | } 186 | 187 | func (e *areaExpression) ContainsExpr(other *areaExpression) bool { 188 | return e.maybeNegate(e.rawContainsExpr(other)) 189 | } 190 | -------------------------------------------------------------------------------- /internal/server/group.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/tidwall/btree" 5 | ) 6 | 7 | func byGroupHook(va, vb interface{}) bool { 8 | a, b := va.(*groupItem), vb.(*groupItem) 9 | if a.hookName < b.hookName { 10 | return true 11 | } 12 | if a.hookName > b.hookName { 13 | return false 14 | } 15 | if a.colKey < b.colKey { 16 | return true 17 | } 18 | if a.colKey > b.colKey { 19 | return false 20 | } 21 | return a.objID < b.objID 22 | } 23 | 24 | func byGroupObject(va, vb interface{}) bool { 25 | a, b := va.(*groupItem), vb.(*groupItem) 26 | if a.colKey < b.colKey { 27 | return true 28 | } 29 | if a.colKey > b.colKey { 30 | return false 31 | } 32 | if a.objID < b.objID { 33 | return true 34 | } 35 | if a.objID > b.objID { 36 | return false 37 | } 38 | return a.hookName < b.hookName 39 | } 40 | 41 | type groupItem struct { 42 | hookName string 43 | colKey string 44 | objID string 45 | groupID string 46 | } 47 | 48 | func newGroupItem(hookName, colKey, objID string) *groupItem { 49 | groupID := bsonID() 50 | g := &groupItem{} 51 | // create a single string allocation 52 | ustr := hookName + colKey + objID + groupID 53 | var pos int 54 | g.hookName = ustr[pos : pos+len(hookName)] 55 | pos += len(hookName) 56 | g.colKey = ustr[pos : pos+len(colKey)] 57 | pos += len(colKey) 58 | g.objID = ustr[pos : pos+len(objID)] 59 | pos += len(objID) 60 | g.groupID = ustr[pos : pos+len(groupID)] 61 | pos += len(groupID) 62 | return g 63 | } 64 | 65 | func (s *Server) groupConnect(hookName, colKey, objID string) (groupID string) { 66 | g := newGroupItem(hookName, colKey, objID) 67 | s.groupHooks.Set(g) 68 | s.groupObjects.Set(g) 69 | return g.groupID 70 | } 71 | 72 | func (s *Server) groupDisconnect(hookName, colKey, objID string) { 73 | g := &groupItem{ 74 | hookName: hookName, 75 | colKey: colKey, 76 | objID: objID, 77 | } 78 | s.groupHooks.Delete(g) 79 | s.groupObjects.Delete(g) 80 | } 81 | 82 | func (s *Server) groupGet(hookName, colKey, objID string) (groupID string) { 83 | v := s.groupHooks.Get(&groupItem{ 84 | hookName: hookName, 85 | colKey: colKey, 86 | objID: objID, 87 | }) 88 | if v != nil { 89 | return v.(*groupItem).groupID 90 | } 91 | return "" 92 | } 93 | 94 | func deleteGroups(s *Server, groups []*groupItem) { 95 | var hhint btree.PathHint 96 | var ohint btree.PathHint 97 | for _, g := range groups { 98 | s.groupHooks.DeleteHint(g, &hhint) 99 | s.groupObjects.DeleteHint(g, &ohint) 100 | } 101 | } 102 | 103 | // groupDisconnectObject disconnects all hooks from provide object 104 | func (s *Server) groupDisconnectObject(colKey, objID string) { 105 | var groups []*groupItem 106 | s.groupObjects.Ascend(&groupItem{colKey: colKey, objID: objID}, 107 | func(v interface{}) bool { 108 | g := v.(*groupItem) 109 | if g.colKey != colKey || g.objID != objID { 110 | return false 111 | } 112 | groups = append(groups, g) 113 | return true 114 | }, 115 | ) 116 | deleteGroups(s, groups) 117 | } 118 | 119 | // groupDisconnectCollection disconnects all hooks from objects in provided 120 | // collection. 121 | func (s *Server) groupDisconnectCollection(colKey string) { 122 | var groups []*groupItem 123 | s.groupObjects.Ascend(&groupItem{colKey: colKey}, 124 | func(v interface{}) bool { 125 | g := v.(*groupItem) 126 | if g.colKey != colKey { 127 | return false 128 | } 129 | groups = append(groups, g) 130 | return true 131 | }, 132 | ) 133 | deleteGroups(s, groups) 134 | } 135 | 136 | // groupDisconnectHook disconnects all objects from provided hook. 137 | func (s *Server) groupDisconnectHook(hookName string) { 138 | var groups []*groupItem 139 | s.groupHooks.Ascend(&groupItem{hookName: hookName}, 140 | func(v interface{}) bool { 141 | g := v.(*groupItem) 142 | if g.hookName != hookName { 143 | return false 144 | } 145 | groups = append(groups, g) 146 | return true 147 | }, 148 | ) 149 | deleteGroups(s, groups) 150 | } 151 | -------------------------------------------------------------------------------- /internal/server/json_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkJSONString(t *testing.B) { 9 | var s = "the need for mead" 10 | for i := 0; i < t.N; i++ { 11 | jsonString(s) 12 | } 13 | } 14 | 15 | func BenchmarkJSONMarshal(t *testing.B) { 16 | var s = "the need for mead" 17 | for i := 0; i < t.N; i++ { 18 | json.Marshal(s) 19 | } 20 | } 21 | 22 | func TestIsJsonNumber(t *testing.T) { 23 | test := func(expected bool, val string) { 24 | actual := isJSONNumber(val) 25 | if expected != actual { 26 | t.Fatalf("Expected %t == isJsonNumber(\"%s\") but was %t", expected, val, actual) 27 | } 28 | } 29 | test(false, "") 30 | test(false, "-") 31 | test(false, "foo") 32 | test(false, "0123") 33 | test(false, "1.") 34 | test(false, "1.0e") 35 | test(false, "1.0e-") 36 | test(false, "1.0E10NaN") 37 | test(false, "1.0ENaN") 38 | test(true, "-1") 39 | test(true, "0") 40 | test(true, "0.0") 41 | test(true, "42") 42 | test(true, "1.0E10") 43 | test(true, "1.0e10") 44 | test(true, "1E+5") 45 | test(true, "1E-10") 46 | } 47 | -------------------------------------------------------------------------------- /internal/server/keys.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/tidwall/resp" 8 | "github.com/tidwall/tile38/internal/collection" 9 | "github.com/tidwall/tile38/internal/glob" 10 | ) 11 | 12 | // KEYS pattern 13 | func (s *Server) cmdKEYS(msg *Message) (resp.Value, error) { 14 | var start = time.Now() 15 | 16 | // >> Args 17 | 18 | args := msg.Args 19 | if len(args) != 2 { 20 | return retrerr(errInvalidNumberOfArguments) 21 | } 22 | pattern := args[1] 23 | 24 | // >> Operation 25 | 26 | keys := []string{} 27 | g := glob.Parse(pattern, false) 28 | everything := g.Limits[0] == "" && g.Limits[1] == "" 29 | if everything { 30 | s.cols.Scan( 31 | func(key string, _ *collection.Collection) bool { 32 | match, _ := glob.Match(pattern, key) 33 | if match { 34 | keys = append(keys, key) 35 | } 36 | return true 37 | }, 38 | ) 39 | } else { 40 | s.cols.Ascend(g.Limits[0], 41 | func(key string, _ *collection.Collection) bool { 42 | if key > g.Limits[1] { 43 | return false 44 | } 45 | match, _ := glob.Match(pattern, key) 46 | if match { 47 | keys = append(keys, key) 48 | } 49 | return true 50 | }, 51 | ) 52 | } 53 | 54 | // >> Response 55 | 56 | if msg.OutputType == JSON { 57 | data, _ := json.Marshal(keys) 58 | return resp.StringValue(`{"ok":true,"keys":` + string(data) + 59 | `,"elapsed":"` + time.Since(start).String() + `"}`), nil 60 | } 61 | 62 | var vals []resp.Value 63 | for _, key := range keys { 64 | vals = append(vals, resp.StringValue(key)) 65 | } 66 | return resp.ArrayValue(vals), nil 67 | } 68 | -------------------------------------------------------------------------------- /internal/server/live.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net" 9 | "sync" 10 | "time" 11 | 12 | "github.com/tidwall/redcon" 13 | "github.com/tidwall/tile38/internal/log" 14 | "go.uber.org/atomic" 15 | ) 16 | 17 | type liveBuffer struct { 18 | key string 19 | globs []string 20 | fence *liveFenceSwitches 21 | details []*commandDetails 22 | cond *sync.Cond 23 | } 24 | 25 | func (s *Server) processLives(wg *sync.WaitGroup) { 26 | defer wg.Done() 27 | var done atomic.Bool 28 | wg.Add(1) 29 | go func() { 30 | defer wg.Done() 31 | for { 32 | if done.Load() { 33 | break 34 | } 35 | s.lcond.Broadcast() 36 | time.Sleep(time.Second / 4) 37 | } 38 | }() 39 | s.lcond.L.Lock() 40 | defer s.lcond.L.Unlock() 41 | for { 42 | if s.stopServer.Load() { 43 | done.Store(true) 44 | return 45 | } 46 | for len(s.lstack) > 0 { 47 | item := s.lstack[0] 48 | s.lstack = s.lstack[1:] 49 | if len(s.lstack) == 0 { 50 | s.lstack = nil 51 | } 52 | for lb := range s.lives { 53 | lb.cond.L.Lock() 54 | if lb.key != "" && lb.key == item.key { 55 | lb.details = append(lb.details, item) 56 | lb.cond.Broadcast() 57 | } 58 | lb.cond.L.Unlock() 59 | } 60 | } 61 | s.lcond.Wait() 62 | } 63 | } 64 | 65 | func writeLiveMessage( 66 | conn net.Conn, 67 | message []byte, 68 | wrapRESP bool, 69 | connType Type, websocket bool, 70 | ) error { 71 | if len(message) == 0 { 72 | return nil 73 | } 74 | if websocket { 75 | return WriteWebSocketMessage(conn, message) 76 | } 77 | var err error 78 | switch connType { 79 | case RESP: 80 | if wrapRESP { 81 | _, err = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(message), string(message)) 82 | } else { 83 | _, err = conn.Write(message) 84 | } 85 | case Native: 86 | _, err = fmt.Fprintf(conn, "$%d %s\r\n", len(message), string(message)) 87 | } 88 | return err 89 | } 90 | 91 | func (s *Server) goLive( 92 | inerr error, conn net.Conn, rd *PipelineReader, msg *Message, websocket bool, 93 | ) error { 94 | addr := conn.RemoteAddr().String() 95 | log.Info("live " + addr) 96 | defer func() { 97 | log.Info("not live " + addr) 98 | }() 99 | switch lfs := inerr.(type) { 100 | default: 101 | return errors.New("invalid live type switches") 102 | case liveAOFSwitches: 103 | return s.liveAOF(lfs.pos, conn, rd, msg) 104 | case liveSubscriptionSwitches: 105 | return s.liveSubscription(conn, rd, msg, websocket) 106 | case liveMonitorSwitches: 107 | return s.liveMonitor(conn, rd, msg) 108 | case liveFenceSwitches: 109 | // fallthrough 110 | } 111 | 112 | // everything below is for live geofences 113 | lb := &liveBuffer{ 114 | cond: sync.NewCond(&sync.Mutex{}), 115 | } 116 | var err error 117 | var sw *scanWriter 118 | var wr bytes.Buffer 119 | lfs := inerr.(liveFenceSwitches) 120 | lb.globs = lfs.globs 121 | lb.key = lfs.key 122 | lb.fence = &lfs 123 | s.mu.RLock() 124 | sw, err = s.newScanWriter( 125 | &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false, 126 | lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields) 127 | s.mu.RUnlock() 128 | 129 | // everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS 130 | if err != nil { 131 | return err 132 | } 133 | s.lcond.L.Lock() 134 | s.lives[lb] = true 135 | s.lcond.L.Unlock() 136 | defer func() { 137 | s.lcond.L.Lock() 138 | delete(s.lives, lb) 139 | s.lcond.L.Unlock() 140 | conn.Close() 141 | }() 142 | 143 | var mustQuit bool 144 | go func() { 145 | defer func() { 146 | lb.cond.L.Lock() 147 | mustQuit = true 148 | lb.cond.Broadcast() 149 | lb.cond.L.Unlock() 150 | conn.Close() 151 | }() 152 | for { 153 | vs, err := rd.ReadMessages() 154 | if err != nil { 155 | if err != io.EOF && !(websocket && err == io.ErrUnexpectedEOF) { 156 | log.Error(err) 157 | } 158 | return 159 | } 160 | for _, v := range vs { 161 | if v == nil { 162 | continue 163 | } 164 | switch v.Command() { 165 | default: 166 | log.Error("received a live command that was not QUIT") 167 | return 168 | case "quit", "": 169 | return 170 | } 171 | } 172 | } 173 | }() 174 | outputType := msg.OutputType 175 | connType := msg.ConnType 176 | if websocket { 177 | outputType = JSON 178 | } 179 | var livemsg []byte 180 | switch outputType { 181 | case JSON: 182 | livemsg = redcon.AppendBulkString(nil, `{"ok":true,"live":true}`) 183 | case RESP: 184 | livemsg = redcon.AppendOK(nil) 185 | } 186 | if err := writeLiveMessage(conn, livemsg, false, connType, websocket); err != nil { 187 | return nil // nil return is fine here 188 | } 189 | for { 190 | lb.cond.L.Lock() 191 | if mustQuit { 192 | lb.cond.L.Unlock() 193 | return nil 194 | } 195 | for len(lb.details) > 0 { 196 | details := lb.details[0] 197 | lb.details = lb.details[1:] 198 | if len(lb.details) == 0 { 199 | lb.details = nil 200 | } 201 | fence := lb.fence 202 | lb.cond.L.Unlock() 203 | var msgs []string 204 | func() { 205 | // safely lock the fence because we are outside the main loop 206 | s.mu.RLock() 207 | defer s.mu.RUnlock() 208 | msgs = FenceMatch("", sw, fence, nil, details) 209 | }() 210 | for _, msg := range msgs { 211 | if err := writeLiveMessage(conn, []byte(msg), true, connType, websocket); err != nil { 212 | return nil // nil return is fine here 213 | } 214 | } 215 | s.statsTotalMsgsSent.Add(int64(len(msgs))) 216 | lb.cond.L.Lock() 217 | 218 | } 219 | lb.cond.Wait() 220 | lb.cond.L.Unlock() 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /internal/server/monitor.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/tidwall/resp" 12 | ) 13 | 14 | type liveMonitorSwitches struct { 15 | // no fields. everything is managed through the Message 16 | } 17 | 18 | func (sub liveMonitorSwitches) Error() string { 19 | return goingLive 20 | } 21 | 22 | func (s *Server) cmdMonitor(msg *Message) (resp.Value, error) { 23 | if len(msg.Args) != 1 { 24 | return resp.Value{}, errInvalidNumberOfArguments 25 | } 26 | return NOMessage, liveMonitorSwitches{} 27 | } 28 | 29 | func (s *Server) liveMonitor(conn net.Conn, rd *PipelineReader, msg *Message) error { 30 | s.monconnsMu.Lock() 31 | s.monconns[conn] = true 32 | s.monconnsMu.Unlock() 33 | defer func() { 34 | s.monconnsMu.Lock() 35 | delete(s.monconns, conn) 36 | s.monconnsMu.Unlock() 37 | conn.Close() 38 | }() 39 | s.monconnsMu.Lock() 40 | conn.Write([]byte("+OK\r\n")) 41 | s.monconnsMu.Unlock() 42 | msgs, err := rd.ReadMessages() 43 | if err != nil { 44 | if err == io.EOF { 45 | return nil 46 | } 47 | return err 48 | } 49 | for _, msg := range msgs { 50 | if len(msg.Args) == 1 && strings.ToLower(msg.Args[0]) == "quit" { 51 | s.monconnsMu.Lock() 52 | conn.Write([]byte("+OK\r\n")) 53 | s.monconnsMu.Unlock() 54 | return nil 55 | } 56 | } 57 | return nil 58 | } 59 | 60 | // send messages to live MONITOR clients 61 | func (s *Server) sendMonitor(err error, msg *Message, c *Client, lua bool) { 62 | s.monconnsMu.RLock() 63 | n := len(s.monconns) 64 | s.monconnsMu.RUnlock() 65 | if n == 0 { 66 | return 67 | } 68 | if (c == nil && !lua) || 69 | (err != nil && (err == errInvalidNumberOfArguments || 70 | strings.HasPrefix(err.Error(), "unknown command "))) { 71 | return 72 | } 73 | 74 | // accept all commands except for these: 75 | switch strings.ToLower(msg.Command()) { 76 | case "config", "config set", "config get", "config rewrite", 77 | "auth", "follow", "slaveof", "replconf", 78 | "aof", "aofmd5", "client", 79 | "monitor": 80 | return 81 | } 82 | 83 | var line []byte 84 | for i, arg := range msg.Args { 85 | if i > 0 { 86 | line = append(line, ' ') 87 | } 88 | line = append(line, strconv.Quote(arg)...) 89 | } 90 | tstr := fmt.Sprintf("%.6f", float64(time.Now().UnixNano())/1e9) 91 | var addr string 92 | if lua { 93 | addr = "lua" 94 | } else { 95 | addr = c.remoteAddr 96 | } 97 | s.monconnsMu.Lock() 98 | for conn := range s.monconns { 99 | fmt.Fprintf(conn, "+%s [0 %s] %s\r\n", tstr, addr, line) 100 | } 101 | s.monconnsMu.Unlock() 102 | } 103 | -------------------------------------------------------------------------------- /internal/server/must.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | func Must[T any](a T, err error) T { 4 | if err != nil { 5 | panic(err) 6 | } 7 | return a 8 | } 9 | 10 | func Default[T comparable](a, b T) T { 11 | var c T 12 | if a == c { 13 | return b 14 | } 15 | return a 16 | } 17 | -------------------------------------------------------------------------------- /internal/server/must_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func TestMust(t *testing.T) { 9 | if Must(1, nil) != 1 { 10 | t.Fail() 11 | } 12 | func() { 13 | var ended bool 14 | defer func() { 15 | if ended { 16 | t.Fail() 17 | } 18 | err, ok := recover().(error) 19 | if !ok { 20 | t.Fail() 21 | } 22 | if err.Error() != "ok" { 23 | t.Fail() 24 | } 25 | }() 26 | Must(1, errors.New("ok")) 27 | ended = true 28 | }() 29 | } 30 | 31 | func TestDefault(t *testing.T) { 32 | if Default("", "2") != "2" { 33 | t.Fail() 34 | } 35 | if Default("1", "2") != "1" { 36 | t.Fail() 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /internal/server/output.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/tidwall/resp" 8 | ) 9 | 10 | // OUTPUT [resp|json] 11 | func (s *Server) cmdOUTPUT(msg *Message) (resp.Value, error) { 12 | start := time.Now() 13 | 14 | args := msg.Args 15 | switch len(args) { 16 | case 1: 17 | if msg.OutputType == JSON { 18 | return resp.StringValue(`{"ok":true,"output":"json","elapsed":` + 19 | time.Since(start).String() + `}`), nil 20 | } 21 | return resp.StringValue("resp"), nil 22 | case 2: 23 | // Setting the original message output type will be picked up by the 24 | // server prior to the next command being executed. 25 | switch strings.ToLower(args[1]) { 26 | default: 27 | return retrerr(errInvalidArgument(args[1])) 28 | case "json": 29 | msg.OutputType = JSON 30 | case "resp": 31 | msg.OutputType = RESP 32 | } 33 | return OKMessage(msg, start), nil 34 | default: 35 | return retrerr(errInvalidNumberOfArguments) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /internal/server/pubqueue.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net" 5 | "sync" 6 | 7 | "github.com/tidwall/redcon" 8 | ) 9 | 10 | type pubQueue struct { 11 | cond *sync.Cond 12 | entries []pubQueueEntry // follower publish queue 13 | closed bool 14 | } 15 | 16 | type pubQueueEntry struct { 17 | channel string 18 | messages []string 19 | } 20 | 21 | func (s *Server) startPublishQueue(wg *sync.WaitGroup) { 22 | defer wg.Done() 23 | var buf []byte 24 | var conns []net.Conn 25 | s.pubq.cond = sync.NewCond(&sync.Mutex{}) 26 | s.pubq.cond.L.Lock() 27 | for { 28 | for len(s.pubq.entries) > 0 { 29 | entries := s.pubq.entries 30 | s.pubq.entries = nil 31 | s.pubq.cond.L.Unlock() 32 | // Get follower connections 33 | s.mu.RLock() 34 | for conn := range s.aofconnM { 35 | conns = append(conns, conn) 36 | } 37 | s.mu.RUnlock() 38 | // Buffer the PUBLISH command pipeline 39 | buf = buf[:0] 40 | for _, entry := range entries { 41 | for _, message := range entry.messages { 42 | buf = redcon.AppendArray(buf, 3) 43 | buf = redcon.AppendBulkString(buf, "PUBLISH") 44 | buf = redcon.AppendBulkString(buf, entry.channel) 45 | buf = redcon.AppendBulkString(buf, message) 46 | } 47 | } 48 | // Publish to followers 49 | for i, conn := range conns { 50 | conn.Write(buf) 51 | conns[i] = nil 52 | } 53 | conns = conns[:0] 54 | s.pubq.cond.L.Lock() 55 | } 56 | if s.pubq.closed { 57 | break 58 | } 59 | s.pubq.cond.Wait() 60 | } 61 | s.pubq.cond.L.Unlock() 62 | } 63 | 64 | func (s *Server) stopPublishQueue() { 65 | s.pubq.cond.L.Lock() 66 | s.pubq.closed = true 67 | s.pubq.cond.Broadcast() 68 | s.pubq.cond.L.Unlock() 69 | } 70 | 71 | func (s *Server) sendPublishQueue(channel string, message ...string) { 72 | s.pubq.cond.L.Lock() 73 | if !s.pubq.closed { 74 | s.pubq.entries = append(s.pubq.entries, pubQueueEntry{ 75 | channel: channel, 76 | messages: message, 77 | }) 78 | } 79 | s.pubq.cond.Broadcast() 80 | s.pubq.cond.L.Unlock() 81 | } 82 | -------------------------------------------------------------------------------- /internal/server/readonly.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/tidwall/resp" 7 | "github.com/tidwall/tile38/internal/log" 8 | ) 9 | 10 | // READONLY yes|no 11 | func (s *Server) cmdREADONLY(msg *Message) (resp.Value, error) { 12 | start := time.Now() 13 | 14 | // >> Args 15 | 16 | args := msg.Args 17 | if len(args) != 2 { 18 | return retrerr(errInvalidNumberOfArguments) 19 | } 20 | 21 | switch args[1] { 22 | case "yes", "no": 23 | default: 24 | return retrerr(errInvalidArgument(args[1])) 25 | } 26 | 27 | // >> Operation 28 | 29 | var updated bool 30 | if args[1] == "yes" { 31 | if !s.config.readOnly() { 32 | updated = true 33 | s.config.setReadOnly(true) 34 | log.Info("read only") 35 | } 36 | } else { 37 | if s.config.readOnly() { 38 | updated = true 39 | s.config.setReadOnly(false) 40 | log.Info("read write") 41 | } 42 | } 43 | if updated { 44 | s.config.write(false) 45 | } 46 | 47 | // >> Response 48 | 49 | return OKMessage(msg, start), nil 50 | } 51 | -------------------------------------------------------------------------------- /internal/server/respconn.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net" 5 | "time" 6 | 7 | "github.com/tidwall/resp" 8 | ) 9 | 10 | // RESPConn represents a simple resp connection. 11 | type RESPConn struct { 12 | conn net.Conn 13 | rd *resp.Reader 14 | wr *resp.Writer 15 | } 16 | 17 | // DialTimeout dials a resp 18 | func DialTimeout(address string, timeout time.Duration) (*RESPConn, error) { 19 | tcpconn, err := net.DialTimeout("tcp", address, timeout) 20 | if err != nil { 21 | return nil, err 22 | } 23 | conn := &RESPConn{ 24 | conn: tcpconn, 25 | rd: resp.NewReader(tcpconn), 26 | wr: resp.NewWriter(tcpconn), 27 | } 28 | return conn, nil 29 | } 30 | 31 | // Close closes the connection. 32 | func (conn *RESPConn) Close() error { 33 | conn.wr.WriteMultiBulk("quit") 34 | return conn.conn.Close() 35 | } 36 | 37 | // Do performs a command and returns a resp value. 38 | func (conn *RESPConn) Do(commandName string, args ...interface{}) ( 39 | val resp.Value, err error, 40 | ) { 41 | if err := conn.wr.WriteMultiBulk(commandName, args...); err != nil { 42 | return val, err 43 | } 44 | val, _, err = conn.rd.ReadValue() 45 | return val, err 46 | } 47 | -------------------------------------------------------------------------------- /internal/server/scan.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "time" 7 | 8 | "github.com/tidwall/resp" 9 | "github.com/tidwall/tile38/internal/object" 10 | ) 11 | 12 | func (s *Server) cmdScanArgs(vs []string) ( 13 | ls liveFenceSwitches, err error, 14 | ) { 15 | var t searchScanBaseTokens 16 | vs, t, err = s.parseSearchScanBaseTokens("scan", t, vs) 17 | if err != nil { 18 | return 19 | } 20 | ls.searchScanBaseTokens = t 21 | if len(vs) != 0 { 22 | err = errInvalidNumberOfArguments 23 | return 24 | } 25 | return 26 | } 27 | 28 | func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { 29 | start := time.Now() 30 | vs := msg.Args[1:] 31 | 32 | args, err := s.cmdScanArgs(vs) 33 | if args.usingLua() { 34 | defer args.Close() 35 | defer func() { 36 | if r := recover(); r != nil { 37 | res = NOMessage 38 | err = errors.New(r.(string)) 39 | return 40 | } 41 | }() 42 | } 43 | if err != nil { 44 | return NOMessage, err 45 | } 46 | wr := &bytes.Buffer{} 47 | sw, err := s.newScanWriter( 48 | wr, msg, args.key, args.output, args.precision, args.globs, false, 49 | args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, 50 | args.nofields) 51 | if err != nil { 52 | return NOMessage, err 53 | } 54 | if msg.OutputType == JSON { 55 | wr.WriteString(`{"ok":true`) 56 | } 57 | var ierr error 58 | if sw.col != nil { 59 | if sw.output == outputCount && len(sw.wheres) == 0 && 60 | len(sw.whereins) == 0 && len(sw.whereevals) == 0 && 61 | sw.globEverything { 62 | count := sw.col.Count() - int(args.cursor) 63 | if count < 0 { 64 | count = 0 65 | } 66 | sw.count = uint64(count) 67 | } else { 68 | limits := multiGlobParse(sw.globs, args.desc) 69 | if limits[0] == "" && limits[1] == "" { 70 | sw.col.Scan(args.desc, sw, 71 | msg.Deadline, 72 | func(o *object.Object) bool { 73 | keepGoing, err := sw.pushObject(ScanWriterParams{ 74 | obj: o, 75 | }) 76 | if err != nil { 77 | ierr = err 78 | return false 79 | } 80 | return keepGoing 81 | }, 82 | ) 83 | } else { 84 | sw.col.ScanRange(limits[0], limits[1], args.desc, sw, 85 | msg.Deadline, 86 | func(o *object.Object) bool { 87 | keepGoing, err := sw.pushObject(ScanWriterParams{ 88 | obj: o, 89 | }) 90 | if err != nil { 91 | ierr = err 92 | return false 93 | } 94 | return keepGoing 95 | }, 96 | ) 97 | } 98 | } 99 | } 100 | if ierr != nil { 101 | return retrerr(ierr) 102 | } 103 | sw.writeFoot() 104 | if msg.OutputType == JSON { 105 | wr.WriteString(`,"elapsed":"` + time.Since(start).String() + "\"}") 106 | return resp.BytesValue(wr.Bytes()), nil 107 | } 108 | return sw.respOut, nil 109 | } 110 | -------------------------------------------------------------------------------- /internal/server/scanner_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/rand" 7 | "testing" 8 | "time" 9 | 10 | "github.com/tidwall/geojson" 11 | "github.com/tidwall/geojson/geometry" 12 | "github.com/tidwall/tile38/internal/field" 13 | "github.com/tidwall/tile38/internal/object" 14 | ) 15 | 16 | type testPointItem struct { 17 | object geojson.Object 18 | fields field.List 19 | } 20 | 21 | func PO(x, y float64) *geojson.Point { 22 | return geojson.NewPoint(geometry.Point{X: x, Y: y}) 23 | } 24 | 25 | func BenchmarkFieldMatch(t *testing.B) { 26 | rand.Seed(time.Now().UnixNano()) 27 | items := make([]testPointItem, t.N) 28 | for i := 0; i < t.N; i++ { 29 | var fields field.List 30 | fields = fields.Set(field.Make("foo", fmt.Sprintf("%f", rand.Float64()*9+1))) 31 | fields = fields.Set(field.Make("bar", fmt.Sprintf("%f", math.Round(rand.Float64()*30)+1))) 32 | items[i] = testPointItem{ 33 | PO(rand.Float64()*360-180, rand.Float64()*180-90), 34 | fields, 35 | } 36 | } 37 | sw := &scanWriter{ 38 | wheres: []whereT{ 39 | {false, "foo", false, field.ValueOf("1"), false, field.ValueOf("3")}, 40 | {false, "bar", false, field.ValueOf("10"), false, field.ValueOf("30")}, 41 | }, 42 | whereins: []whereinT{ 43 | {"foo", []field.Value{field.ValueOf("1"), field.ValueOf("2")}}, 44 | {"bar", []field.Value{field.ValueOf("11"), field.ValueOf("25")}}, 45 | }, 46 | } 47 | t.ResetTimer() 48 | for i := 0; i < t.N; i++ { 49 | // one call is super fast, measurements are not reliable, let's do 100 50 | for ix := 0; ix < 100; ix++ { 51 | sw.fieldMatch(object.New("", items[i].object, 0, items[i].fields)) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/server/stats_cpu.go: -------------------------------------------------------------------------------- 1 | //go:build !linux && !darwin 2 | 3 | package server 4 | 5 | import ( 6 | "bytes" 7 | "fmt" 8 | ) 9 | 10 | func (s *Server) writeInfoCPU(w *bytes.Buffer) { 11 | fmt.Fprintf(w, 12 | "used_cpu_sys:%.2f\r\n"+ 13 | "used_cpu_user:%.2f\r\n"+ 14 | "used_cpu_sys_children:%.2f\r\n"+ 15 | "used_cpu_user_children:%.2f\r\n", 16 | 0.0, 0.0, 0.0, 0.0, 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /internal/server/stats_cpu_darlin.go: -------------------------------------------------------------------------------- 1 | //go:build linux || darwin 2 | 3 | package server 4 | 5 | import ( 6 | "bytes" 7 | "fmt" 8 | "syscall" 9 | ) 10 | 11 | func (s *Server) writeInfoCPU(w *bytes.Buffer) { 12 | var selfRu syscall.Rusage 13 | var cRu syscall.Rusage 14 | syscall.Getrusage(syscall.RUSAGE_SELF, &selfRu) 15 | syscall.Getrusage(syscall.RUSAGE_CHILDREN, &cRu) 16 | fmt.Fprintf(w, 17 | "used_cpu_sys:%.2f\r\n"+ 18 | "used_cpu_user:%.2f\r\n"+ 19 | "used_cpu_sys_children:%.2f\r\n"+ 20 | "used_cpu_user_children:%.2f\r\n", 21 | float64(selfRu.Stime.Sec)+float64(selfRu.Stime.Usec/1000000), 22 | float64(selfRu.Utime.Sec)+float64(selfRu.Utime.Usec/1000000), 23 | float64(cRu.Stime.Sec)+float64(cRu.Stime.Usec/1000000), 24 | float64(cRu.Utime.Sec)+float64(cRu.Utime.Usec/1000000), 25 | ) 26 | } 27 | -------------------------------------------------------------------------------- /internal/server/token_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/tidwall/tile38/internal/field" 8 | ) 9 | 10 | func TestLowerCompare(t *testing.T) { 11 | if !lc("hello", "hello") { 12 | t.Fatal("failed") 13 | } 14 | if !lc("Hello", "hello") { 15 | t.Fatal("failed") 16 | } 17 | if !lc("HeLLo World", "hello world") { 18 | t.Fatal("failed") 19 | } 20 | if !lc("", "") { 21 | t.Fatal("failed") 22 | } 23 | if lc("hello", "") { 24 | t.Fatal("failed") 25 | } 26 | if lc("", "hello") { 27 | t.Fatal("failed") 28 | } 29 | if lc("HeLLo World", "Hello world") { 30 | t.Fatal("failed") 31 | } 32 | } 33 | 34 | func TestParseWhereins(t *testing.T) { 35 | s := &Server{} 36 | 37 | type tcase struct { 38 | inputWhereins []whereinT 39 | expWhereins []whereinT 40 | } 41 | 42 | fn := func(tc tcase) func(t *testing.T) { 43 | return func(t *testing.T) { 44 | 45 | _, tout, err := s.parseSearchScanBaseTokens( 46 | "scan", 47 | searchScanBaseTokens{ 48 | whereins: tc.inputWhereins, 49 | }, 50 | []string{"key"}, 51 | ) 52 | got := tout.whereins 53 | exp := tc.expWhereins 54 | 55 | if err != nil { 56 | t.Fatalf("unexpected error while parsing search scan base tokens") 57 | } 58 | 59 | if len(got) != len(exp) { 60 | t.Fatalf("expected equal length whereins") 61 | } 62 | 63 | for i := range got { 64 | if got[i].name != exp[i].name { 65 | t.Fatalf("expected equal field names") 66 | } 67 | 68 | for j := range exp[i].valArr { 69 | if !got[i].match(exp[i].valArr[j]) { 70 | t.Fatalf("expected matching value arrays") 71 | } 72 | } 73 | } 74 | } 75 | } 76 | 77 | tests := map[string]tcase{ 78 | "upper case": { 79 | inputWhereins: []whereinT{ 80 | { 81 | name: "TEST", 82 | valArr: []field.Value{ 83 | field.ValueOf("1"), 84 | field.ValueOf("1"), 85 | }, 86 | }, 87 | }, 88 | expWhereins: []whereinT{ 89 | { 90 | name: "TEST", 91 | valArr: []field.Value{ 92 | field.ValueOf("1"), 93 | field.ValueOf("1"), 94 | }, 95 | }, 96 | }, 97 | }, 98 | "lower case": { 99 | inputWhereins: []whereinT{ 100 | { 101 | name: "test", 102 | valArr: []field.Value{ 103 | field.ValueOf("1"), 104 | field.ValueOf("1"), 105 | }, 106 | }, 107 | }, 108 | expWhereins: []whereinT{ 109 | { 110 | name: "test", 111 | valArr: []field.Value{ 112 | field.ValueOf("1"), 113 | field.ValueOf("1"), 114 | }, 115 | }, 116 | }, 117 | }, 118 | "mixed case": { 119 | inputWhereins: []whereinT{ 120 | { 121 | name: "teSt", 122 | valArr: []field.Value{ 123 | field.ValueOf("1"), 124 | field.ValueOf("1"), 125 | }, 126 | }, 127 | }, 128 | expWhereins: []whereinT{ 129 | { 130 | name: "teSt", 131 | valArr: []field.Value{ 132 | field.ValueOf("1"), 133 | field.ValueOf("1"), 134 | }, 135 | }, 136 | }, 137 | }, 138 | } 139 | 140 | for name, tc := range tests { 141 | t.Run(name, fn(tc)) 142 | } 143 | 144 | } 145 | 146 | // func testParseFloat(t testing.TB, s string, f float64, invalid bool) { 147 | // n, err := parseFloat(s) 148 | // if err != nil { 149 | // if invalid { 150 | // return 151 | // } 152 | // t.Fatal(err) 153 | // } 154 | // if invalid { 155 | // t.Fatalf("expecting an error for %s", s) 156 | // } 157 | // if n != f { 158 | // t.Fatalf("for '%s', expect %f, got %f", s, f, n) 159 | // } 160 | // } 161 | 162 | // func TestParseFloat(t *testing.T) { 163 | // testParseFloat(t, "100", 100, false) 164 | // testParseFloat(t, "0", 0, false) 165 | // testParseFloat(t, "-1", -1, false) 166 | // testParseFloat(t, "-0", -0, false) 167 | 168 | // testParseFloat(t, "-100", -100, false) 169 | // testParseFloat(t, "-0", -0, false) 170 | // testParseFloat(t, "+1", 1, false) 171 | // testParseFloat(t, "+0", 0, false) 172 | 173 | // testParseFloat(t, "33.102938", 33.102938, false) 174 | // testParseFloat(t, "-115.123123", -115.123123, false) 175 | 176 | // testParseFloat(t, ".1", 0.1, false) 177 | // testParseFloat(t, "0.1", 0.1, false) 178 | 179 | // testParseFloat(t, "00.1", 0.1, false) 180 | // testParseFloat(t, "01.1", 1.1, false) 181 | // testParseFloat(t, "01", 1, false) 182 | // testParseFloat(t, "-00.1", -0.1, false) 183 | // testParseFloat(t, "+00.1", 0.1, false) 184 | // testParseFloat(t, "", 0.1, true) 185 | // testParseFloat(t, " 0", 0.1, true) 186 | // testParseFloat(t, "0 ", 0.1, true) 187 | 188 | // } 189 | 190 | func BenchmarkLowerCompare(t *testing.B) { 191 | for i := 0; i < t.N; i++ { 192 | if !lc("HeLLo World", "hello world") { 193 | t.Fatal("failed") 194 | } 195 | } 196 | } 197 | 198 | func BenchmarkStringsLowerCompare(t *testing.B) { 199 | for i := 0; i < t.N; i++ { 200 | if strings.ToLower("HeLLo World") != "hello world" { 201 | t.Fatal("failed") 202 | } 203 | 204 | } 205 | } 206 | 207 | // func BenchmarkParseFloat(t *testing.B) { 208 | // s := []string{"33.10293", "-115.1203102"} 209 | // for i := 0; i < t.N; i++ { 210 | // _, err := parseFloat(s[i%2]) 211 | // if err != nil { 212 | // t.Fatal("failed") 213 | // } 214 | // } 215 | // } 216 | 217 | // func BenchmarkStrconvParseFloat(t *testing.B) { 218 | // s := []string{"33.10293", "-115.1203102"} 219 | // for i := 0; i < t.N; i++ { 220 | // _, err := strconv.ParseFloat(s[i%2], 64) 221 | // if err != nil { 222 | // t.Fatal("failed") 223 | // } 224 | // } 225 | // } 226 | -------------------------------------------------------------------------------- /internal/sstring/sstring.go: -------------------------------------------------------------------------------- 1 | // Package shared allows for 2 | package sstring 3 | 4 | import ( 5 | "sync" 6 | "unsafe" 7 | 8 | "github.com/tidwall/hashmap" 9 | ) 10 | 11 | var mu sync.Mutex 12 | var nums hashmap.Map[string, int] 13 | var strs []string 14 | 15 | // Load a shared string from its number. 16 | // Panics when there is no string assigned with that number. 17 | func Load(num int) (str string) { 18 | mu.Lock() 19 | if num >= 0 && num < len(strs) { 20 | str = strs[num] 21 | mu.Unlock() 22 | return str 23 | } 24 | mu.Unlock() 25 | panic("string not found") 26 | } 27 | 28 | // Store a shared string. 29 | // Returns a unique number that can be used to load the string later. 30 | // The number is al 31 | func Store(str string) (num int) { 32 | mu.Lock() 33 | var ok bool 34 | num, ok = nums.Get(str) 35 | if !ok { 36 | // Make a copy of the string to ensure we don't take in slices. 37 | b := make([]byte, len(str)) 38 | copy(b, str) 39 | str = *(*string)(unsafe.Pointer(&b)) 40 | num = len(strs) 41 | strs = append(strs, str) 42 | nums.Set(str, num) 43 | } 44 | mu.Unlock() 45 | return num 46 | } 47 | 48 | // Len returns the number of shared strings 49 | func Len() int { 50 | mu.Lock() 51 | n := len(strs) 52 | mu.Unlock() 53 | return n 54 | } 55 | -------------------------------------------------------------------------------- /internal/sstring/sstring_test.go: -------------------------------------------------------------------------------- 1 | package sstring 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | 8 | "github.com/tidwall/assert" 9 | ) 10 | 11 | func TestShared(t *testing.T) { 12 | for i := -1; i < 10; i++ { 13 | var str string 14 | func() { 15 | defer func() { 16 | assert.Assert(recover().(string) == "string not found") 17 | }() 18 | str = Load(i) 19 | }() 20 | assert.Assert(str == "") 21 | } 22 | assert.Assert(Store("hello") == 0) 23 | assert.Assert(Store("") == 1) 24 | assert.Assert(Store("jello") == 2) 25 | assert.Assert(Store("hello") == 0) 26 | assert.Assert(Store("") == 1) 27 | assert.Assert(Store("jello") == 2) 28 | str := Load(0) 29 | assert.Assert(str == "hello") 30 | str = Load(1) 31 | assert.Assert(str == "") 32 | str = Load(2) 33 | assert.Assert(str == "jello") 34 | 35 | assert.Assert(Len() == 3) 36 | 37 | } 38 | 39 | func randStr(n int) string { 40 | b := make([]byte, n) 41 | rand.Read(b) 42 | for i := 0; i < n; i++ { 43 | b[i] = 'a' + b[i]%26 44 | } 45 | return string(b) 46 | } 47 | 48 | func BenchmarkStore(b *testing.B) { 49 | rand.Seed(time.Now().UnixNano()) 50 | wmap := make(map[string]bool, b.N) 51 | for len(wmap) < b.N { 52 | wmap[randStr(10)] = true 53 | } 54 | words := make([]string, 0, b.N) 55 | for word := range wmap { 56 | words = append(words, word) 57 | } 58 | b.ResetTimer() 59 | for i := 0; i < b.N; i++ { 60 | Store(words[i]) 61 | } 62 | } 63 | 64 | func BenchmarkLoad(b *testing.B) { 65 | rand.Seed(time.Now().UnixNano()) 66 | wmap := make(map[string]bool, b.N) 67 | for len(wmap) < b.N { 68 | wmap[randStr(10)] = true 69 | } 70 | words := make([]string, 0, b.N) 71 | for word := range wmap { 72 | words = append(words, word) 73 | } 74 | var nums []int 75 | for i := 0; i < b.N; i++ { 76 | nums = append(nums, Store(words[i])) 77 | } 78 | rand.Shuffle(len(nums), func(i, j int) { 79 | nums[i], nums[j] = nums[j], nums[i] 80 | }) 81 | b.ResetTimer() 82 | for i := 0; i < b.N; i++ { 83 | Load(nums[i]) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /scripts/RELEASE.md: -------------------------------------------------------------------------------- 1 | **To bump a new release of Tile38** 2 | 3 | - Update CHANGELOG.md to include the newest changes. 4 | - `git commit -m $vers` changes (where `$vers` is a semver) 5 | - `git tag $vers` (where `$vers` is a semver) 6 | - `git push --tags` 7 | - `git push` 8 | - `make package` 9 | - Add a new Github Release and add the zips from packages directory. 10 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | cd $(dirname "${BASH_SOURCE[0]}")/.. 5 | 6 | if [ "$1" == "" ]; then 7 | echo "error: missing argument (binary name)" 8 | exit 1 9 | fi 10 | 11 | # Check the Go installation 12 | if [ "$(which go)" == "" ]; then 13 | echo "error: Go is not installed. Please download and follow installation"\ 14 | "instructions at https://golang.org/dl to continue." 15 | exit 1 16 | fi 17 | 18 | # Hardcode some values to the core package. 19 | if [ -d ".git" ]; then 20 | VERSION=$(git describe --tags --abbrev=0) 21 | GITSHA=$(git rev-parse --short HEAD) 22 | LDFLAGS="$LDFLAGS -X github.com/tidwall/tile38/core.Version=${VERSION}" 23 | LDFLAGS="$LDFLAGS -X github.com/tidwall/tile38/core.GitSHA=${GITSHA}" 24 | fi 25 | LDFLAGS="$LDFLAGS -X github.com/tidwall/tile38/core.BuildTime=$(date +%FT%T%z)" 26 | 27 | # Generate the core package 28 | core/gen.sh 29 | 30 | # Set final Go environment options 31 | LDFLAGS="$LDFLAGS -extldflags '-static'" 32 | export CGO_ENABLED=0 33 | 34 | # if [ "$NOMODULES" != "1" ]; then 35 | # export GO111MODULE=on 36 | # export GOFLAGS=-mod=vendor 37 | # go mod vendor 38 | # fi 39 | 40 | # Build and store objects into original directory. 41 | go build -ldflags "$LDFLAGS" -o $1 cmd/$1/*.go 42 | -------------------------------------------------------------------------------- /scripts/docker-push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | cd $(dirname "${BASH_SOURCE[0]}")/.. 5 | 6 | # GIT_BRANCH is the current branch name 7 | export GIT_BRANCH=$(git branch --show-current) 8 | # GIT_VERSION - always the last verison number, like 1.12.1. 9 | export GIT_VERSION=$(git describe --tags --abbrev=0) 10 | # GIT_COMMIT_SHORT - the short git commit number, like a718ef0. 11 | export GIT_COMMIT_SHORT=$(git rev-parse --short HEAD) 12 | # DOCKER_REPO - the base repository name to push the docker build to. 13 | export DOCKER_REPO=$DOCKER_USER/tile38 14 | 15 | if [ "$GIT_BRANCH" != "master" ]; then 16 | echo "Not pushing, not on master" 17 | elif [ "$DOCKER_USER" == "" ]; then 18 | echo "Not pushing, DOCKER_USER not set" 19 | exit 1 20 | elif [ "$DOCKER_LOGIN" == "" ]; then 21 | echo "Not pushing, DOCKER_LOGIN not set" 22 | exit 1 23 | elif [ "$DOCKER_PASSWORD" == "" ]; then 24 | echo "Not pushing, DOCKER_PASSWORD not set" 25 | exit 1 26 | else 27 | # setup cross platform builder 28 | # https://github.com/tonistiigi/binfmt 29 | docker run --privileged --rm tonistiigi/binfmt --install all 30 | docker buildx create --name multiarch --platform linux/amd64,linux/amd64/v2,linux/amd64/v3,linux/arm64,linux/386,linux/arm/v7 --use default 31 | 32 | # docker login 33 | echo $DOCKER_PASSWORD | docker login -u $DOCKER_LOGIN --password-stdin 34 | if [ "$(curl -s https://hub.docker.com/v2/repositories/$DOCKER_REPO/tags/$GIT_VERSION/ | grep "digest")" == "" ]; then 35 | # build the docker image 36 | docker buildx build \ 37 | -f Dockerfile \ 38 | --platform linux/arm64,linux/amd64 \ 39 | --build-arg VERSION=$GIT_VERSION \ 40 | --tag $DOCKER_REPO:$GIT_VERSION \ 41 | --tag $DOCKER_REPO:latest \ 42 | --tag $DOCKER_REPO:edge \ 43 | --push \ 44 | . 45 | else 46 | # build the docker image 47 | docker buildx build \ 48 | -f Dockerfile \ 49 | --platform linux/arm64,linux/amd64 \ 50 | --build-arg VERSION=$GIT_VERSION \ 51 | --tag $DOCKER_REPO:edge \ 52 | --push \ 53 | . 54 | fi 55 | fi 56 | -------------------------------------------------------------------------------- /scripts/package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | cd $(dirname "${BASH_SOURCE[0]}")/.. 5 | 6 | PLATFORM="$1" 7 | GOOS="$2" 8 | GOARCH="$3" 9 | VERSION=$(git describe --tags --abbrev=0) 10 | 11 | echo Packaging $PLATFORM Binary 12 | 13 | # Remove previous build directory, if needed. 14 | bdir=tile38-$VERSION-$GOOS-$GOARCH 15 | rm -rf packages/$bdir && mkdir -p packages/$bdir 16 | 17 | # Make the binaries. 18 | GOOS=$GOOS GOARCH=$GOARCH make all 19 | rm -f tile38-luamemtest # not needed 20 | 21 | # Copy the executable binaries. 22 | if [ "$GOOS" == "windows" ]; then 23 | mv tile38-server packages/$bdir/tile38-server.exe 24 | mv tile38-cli packages/$bdir/tile38-cli.exe 25 | mv tile38-benchmark packages/$bdir/tile38-benchmark.exe 26 | else 27 | mv tile38-server packages/$bdir 28 | mv tile38-cli packages/$bdir 29 | mv tile38-benchmark packages/$bdir 30 | fi 31 | 32 | # Copy documention and license. 33 | cp README.md packages/$bdir 34 | cp CHANGELOG.md packages/$bdir 35 | cp LICENSE packages/$bdir 36 | 37 | # Compress the package. 38 | cd packages 39 | if [ "$GOOS" == "linux" ]; then 40 | tar -zcf $bdir.tar.gz $bdir 41 | else 42 | zip -r -q $bdir.zip $bdir 43 | fi 44 | 45 | -------------------------------------------------------------------------------- /scripts/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | cd $(dirname "${BASH_SOURCE[0]}")/.. 5 | 6 | export CGO_ENABLED=0 7 | 8 | cd tests 9 | go test -coverpkg=../internal/server -coverprofile=/tmp/coverage.out $GOTEST 10 | 11 | 12 | # go test -coverpkg=../internal/... -coverprofile=/tmp/coverage.out \ 13 | # -v ./... $GOTEST 14 | 15 | go tool cover -html=/tmp/coverage.out -o /tmp/coverage.html 16 | echo "details: file:///tmp/coverage.html" 17 | cd .. 18 | 19 | if [[ "$GOTEST" == "" ]]; then 20 | go test $(go list ./... | grep -v /vendor/ | grep -v /tests) 21 | fi 22 | -------------------------------------------------------------------------------- /tests/107/.gitignore: -------------------------------------------------------------------------------- 1 | appendonly.aof 2 | log 3 | data/ 4 | -------------------------------------------------------------------------------- /tests/107/LINK: -------------------------------------------------------------------------------- 1 | https://github.com/tidwall/tile38/issues/107 2 | -------------------------------------------------------------------------------- /tests/616/main.go: -------------------------------------------------------------------------------- 1 | // Test Tile38 for Expiration Drift 2 | // Issue #616 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "math/rand" 9 | "sync" 10 | "time" 11 | 12 | "github.com/gomodule/redigo/redis" 13 | "github.com/tidwall/btree" 14 | "github.com/tidwall/gjson" 15 | "github.com/tidwall/sjson" 16 | ) 17 | 18 | const exsecs = 10 19 | const key = "__issue_616__" 20 | 21 | func makeID() string { 22 | const chars = "0123456789abcdefghijklmnopqrstuvwxyz-" 23 | var buf [10]byte 24 | rand.Read(buf[:]) 25 | for i := 0; i < len(buf); i++ { 26 | buf[i] = chars[int(buf[i])%len(chars)] 27 | } 28 | return string(buf[:]) 29 | } 30 | 31 | func main() { 32 | fmt.Printf( 33 | "The SCAN and ACTUAL values should reach about 1850 and stay\n" + 34 | "roughly the same from there on.\n") 35 | var mu sync.Mutex 36 | objs := btree.NewNonConcurrent(func(a, b interface{}) bool { 37 | ajson := a.(string) 38 | bjson := b.(string) 39 | return gjson.Get(ajson, "id").String() < gjson.Get(bjson, "id").String() 40 | }) 41 | expires := btree.NewNonConcurrent(func(a, b interface{}) bool { 42 | ajson := a.(string) 43 | bjson := b.(string) 44 | if gjson.Get(ajson, "properties.ex").Int() < gjson.Get(bjson, "properties.ex").Int() { 45 | return true 46 | } 47 | if gjson.Get(ajson, "properties.ex").Int() > gjson.Get(bjson, "properties.ex").Int() { 48 | return false 49 | } 50 | return gjson.Get(ajson, "id").String() < gjson.Get(bjson, "id").String() 51 | }) 52 | 53 | conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) 54 | must(conn.Do("DROP", key)) 55 | must(nil, conn.Close()) 56 | 57 | go func() { 58 | conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) 59 | defer conn.Close() 60 | for { 61 | ex := time.Now().UnixNano() + int64(exsecs*time.Second) 62 | for i := 0; i < 10; i++ { 63 | id := makeID() 64 | x := rand.Float64()*360 - 180 65 | y := rand.Float64()*180 - 90 66 | obj := fmt.Sprintf(`{"type":"Feature","geometry":{"type":"Point","coordinates":[%f,%f]},"properties":{}}`, x, y) 67 | obj, _ = sjson.Set(obj, "properties.ex", ex) 68 | obj, _ = sjson.Set(obj, "id", id) 69 | res := must(redis.String(conn.Do("SET", key, id, "ex", exsecs, "OBJECT", obj))).(string) 70 | if res != "OK" { 71 | panic(fmt.Sprintf("expected 'OK', got '%s'", res)) 72 | } 73 | mu.Lock() 74 | prev := objs.Set(obj) 75 | if prev != nil { 76 | expires.Delete(obj) 77 | } 78 | expires.Set(obj) 79 | mu.Unlock() 80 | } 81 | time.Sleep(time.Second / 20) 82 | } 83 | }() 84 | 85 | go func() { 86 | conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) 87 | defer conn.Close() 88 | for { 89 | time.Sleep(time.Second * 5) 90 | must(conn.Do("AOFSHRINK")) 91 | } 92 | }() 93 | 94 | go func() { 95 | conn := must(redis.Dial("tcp", ":9851")).(redis.Conn) 96 | defer conn.Close() 97 | must(conn.Do("OUTPUT", "JSON")) 98 | for { 99 | time.Sleep(time.Second / 10) 100 | var ids []string 101 | res := must(redis.String(conn.Do("SCAN", key, "LIMIT", 100000000))).(string) 102 | gjson.Get(res, "objects").ForEach(func(_, res gjson.Result) bool { 103 | ids = append(ids, res.Get("id").String()) 104 | return true 105 | }) 106 | now := time.Now().UnixNano() 107 | mu.Lock() 108 | var exobjs []string 109 | expires.Ascend(nil, func(v interface{}) bool { 110 | ex := gjson.Get(v.(string), "properties.ex").Int() 111 | if ex > now { 112 | return false 113 | } 114 | exobjs = append(exobjs, v.(string)) 115 | return true 116 | }) 117 | for _, obj := range exobjs { 118 | objs.Delete(obj) 119 | expires.Delete(obj) 120 | } 121 | fmt.Printf("\rSCAN: %d, ACTUAL: %d ", len(ids), objs.Len()) 122 | mu.Unlock() 123 | } 124 | }() 125 | select {} 126 | } 127 | 128 | func must(v interface{}, err error) interface{} { 129 | if err != nil { 130 | panic(err) 131 | } 132 | return v 133 | } 134 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | ## Tile38 Integration Testing 2 | 3 | - Uses Redis protocol 4 | - The Tile38 data is flushed before every `DoBatch` 5 | 6 | A basic test operation looks something like: 7 | 8 | ```go 9 | func keys_SET_test(mc *mockServer) error { 10 | return mc.DoBatch([][]interface{}{ 11 | {"SET", "fleet", "truck1", "POINT", 33.0001, -112.0001}, {"OK"}, 12 | {"GET", "fleet", "truck1", "POINT"}, {"[33.0001 -112.0001]"}, 13 | } 14 | } 15 | ``` 16 | 17 | Using a custom function: 18 | 19 | ```go 20 | func keys_MATCH_test(mc *mockServer) error { 21 | return mc.DoBatch([][]interface{}{ 22 | {"SET", "fleet", "truck1", "POINT", 33.0001, -112.0001}, { 23 | func(v interface{}) (resp, expect interface{}) { 24 | // v is the value as strings or slices of strings 25 | // test will pass as long as `resp` and `expect` are the same. 26 | return v, "OK" 27 | }, 28 | }, 29 | } 30 | } 31 | ``` 32 | 33 | 34 | -------------------------------------------------------------------------------- /tests/aof_legacy: -------------------------------------------------------------------------------- 1 | set 1 2 point 10 20set 2 3 point 10 20 -------------------------------------------------------------------------------- /tests/client_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/gomodule/redigo/redis" 9 | "github.com/tidwall/gjson" 10 | "github.com/tidwall/pretty" 11 | ) 12 | 13 | func subTestClient(g *testGroup) { 14 | g.regSubTest("OUTPUT", client_OUTPUT_test) 15 | g.regSubTest("CLIENT", client_CLIENT_test) 16 | } 17 | 18 | func client_OUTPUT_test(mc *mockServer) error { 19 | if err := mc.DoBatch( 20 | // tests removal of "elapsed" member. 21 | Do("OUTPUT", "json", "yaml").Err(`wrong number of arguments for 'output' command`), 22 | Do("OUTPUT", "json").Str(`{"ok":true}`), 23 | Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), 24 | Do("OUTPUT").Str(`resp`), // this is due to the internal Do test 25 | Do("OUTPUT", "resp").OK(), 26 | Do("OUTPUT", "yaml").Err(`invalid argument 'yaml'`), 27 | Do("OUTPUT").Str(`resp`), 28 | Do("OUTPUT").JSON().Str(`{"ok":true,"output":"json"}`), 29 | ); err != nil { 30 | return err 31 | } 32 | 33 | // run direct commands 34 | if _, err := mc.Do("OUTPUT", "json"); err != nil { 35 | return err 36 | } 37 | res, err := mc.Do("CLIENT", "list") 38 | if err != nil { 39 | return err 40 | } 41 | bres, ok := res.([]byte) 42 | if !ok { 43 | return errors.New("Failed to type assert CLIENT response") 44 | } 45 | sres := string(bres) 46 | if !gjson.Valid(sres) { 47 | return errors.New("CLIENT response was invalid") 48 | } 49 | info := gjson.Get(sres, "list").String() 50 | if !gjson.Valid(info) { 51 | return errors.New("CLIENT.list response was invalid") 52 | } 53 | return nil 54 | } 55 | 56 | func client_CLIENT_test(mc *mockServer) error { 57 | numConns := 20 58 | var conns []redis.Conn 59 | defer func() { 60 | for i := range conns { 61 | conns[i].Close() 62 | } 63 | }() 64 | for i := 0; i <= numConns; i++ { 65 | conn, err := redis.Dial("tcp", fmt.Sprintf(":%d", mc.port)) 66 | if err != nil { 67 | return err 68 | } 69 | conn.Do("PING") 70 | conns = append(conns, conn) 71 | } 72 | 73 | _, err := conns[1].Do("CLIENT", "setname", "cl1") 74 | if err != nil { 75 | return err 76 | } 77 | _, err = conns[2].Do("CLIENT", "setname", "cl2") 78 | if err != nil { 79 | return err 80 | } 81 | 82 | if _, err := mc.Do("OUTPUT", "JSON"); err != nil { 83 | return err 84 | } 85 | res, err := mc.Do("CLIENT", "list") 86 | if err != nil { 87 | return err 88 | } 89 | bres, ok := res.([]byte) 90 | if !ok { 91 | return errors.New("Failed to type assert CLIENT response") 92 | } 93 | sres := string(pretty.Pretty(bres)) 94 | if int(gjson.Get(sres, "list.#").Int()) < numConns { 95 | return errors.New("Invalid number of connections") 96 | } 97 | 98 | client13ID := gjson.Get(sres, "list.13.id").String() 99 | client14Addr := gjson.Get(sres, "list.14.addr").String() 100 | client15Addr := gjson.Get(sres, "list.15.addr").String() 101 | 102 | return mc.DoBatch( 103 | Do("CLIENT", "list").JSON().Func(func(s string) error { 104 | if int(gjson.Get(s, "list.#").Int()) < numConns { 105 | return errors.New("Invalid number of connections") 106 | } 107 | return nil 108 | }), 109 | Do("CLIENT", "list").Func(func(s string) error { 110 | if len(strings.Split(strings.TrimSpace(s), "\n")) < numConns { 111 | return errors.New("Invalid number of connections") 112 | } 113 | return nil 114 | }), 115 | Do("CLIENT").Err(`wrong number of arguments for 'client' command`), 116 | Do("CLIENT", "hello").Err(`Syntax error, try CLIENT (LIST | KILL | GETNAME | SETNAME)`), 117 | Do("CLIENT", "list", "arg3").Err(`wrong number of arguments for 'client' command`), 118 | Do("CLIENT", "getname", "arg3").Err(`wrong number of arguments for 'client' command`), 119 | Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":""}`), 120 | Do("CLIENT", "getname").Str(``), 121 | Do("CLIENT", "setname", "abc").OK(), 122 | Do("CLIENT", "getname").Str(`abc`), 123 | Do("CLIENT", "getname").JSON().Str(`{"ok":true,"name":"abc"}`), 124 | Do("CLIENT", "setname", "abc", "efg").Err(`wrong number of arguments for 'client' command`), 125 | Do("CLIENT", "setname", " abc ").Err(`Client names cannot contain spaces, newlines or special characters.`), 126 | Do("CLIENT", "setname", "abcd").JSON().OK(), 127 | Do("CLIENT", "kill", "name", "abcd").Err("No such client"), 128 | Do("CLIENT", "getname").Str(`abcd`), 129 | Do("CLIENT", "kill").Err(`wrong number of arguments for 'client' command`), 130 | Do("CLIENT", "kill", "").Err(`No such client`), 131 | Do("CLIENT", "kill", "abcd").Err(`No such client`), 132 | Do("CLIENT", "kill", "id", client13ID).OK(), 133 | Do("CLIENT", "kill", "id").Err("wrong number of arguments for 'client' command"), 134 | Do("CLIENT", "kill", client14Addr).OK(), 135 | Do("CLIENT", "kill", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), 136 | Do("CLIENT", "kill", "addr").Err("wrong number of arguments for 'client' command"), 137 | Do("CLIENT", "kill", "addr", client15Addr).JSON().OK(), 138 | Do("CLIENT", "kill", "addr", client14Addr, "yikes").Err("wrong number of arguments for 'client' command"), 139 | Do("CLIENT", "kill", "id", "1000").Err("No such client"), 140 | ) 141 | 142 | } 143 | -------------------------------------------------------------------------------- /tests/follower_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import "time" 4 | 5 | func subTestFollower(g *testGroup) { 6 | g.regSubTest("follow", follower_follow_test) 7 | } 8 | 9 | func follower_follow_test(mc *mockServer) error { 10 | mc2, err := mockOpenServer(MockServerOptions{ 11 | Silent: true, Metrics: false, 12 | }) 13 | if err != nil { 14 | return err 15 | } 16 | defer mc2.Close() 17 | err = mc.DoBatch( 18 | Do("SET", "mykey", "truck1", "POINT", 10, 10).OK(), 19 | Do("SET", "mykey", "truck2", "POINT", 10, 10).OK(), 20 | Do("SET", "mykey", "truck3", "POINT", 10, 10).OK(), 21 | Do("SET", "mykey", "truck4", "POINT", 10, 10).OK(), 22 | Do("SET", "mykey", "truck5", "POINT", 10, 10).OK(), 23 | Do("SET", "mykey", "truck6", "POINT", 10, 10).OK(), 24 | Do("CONFIG", "SET", "requirepass", "1234").OK(), 25 | Do("AUTH", "1234").OK(), 26 | ) 27 | if err != nil { 28 | return err 29 | } 30 | err = mc2.DoBatch( 31 | Do("SET", "mykey2", "truck1", "POINT", 10, 10).OK(), 32 | Do("SET", "mykey2", "truck2", "POINT", 10, 10).OK(), 33 | Do("GET", "mykey2", "truck1").Str(`{"type":"Point","coordinates":[10,10]}`), 34 | Do("GET", "mykey2", "truck2").Str(`{"type":"Point","coordinates":[10,10]}`), 35 | 36 | Do("CONFIG", "SET", "leaderauth", "1234").OK(), 37 | Do("FOLLOW", "localhost", mc.port).OK(), 38 | Do("GET", "mykey", "truck1").Err("catching up to leader"), 39 | Sleep(time.Second/2), 40 | 41 | Do("GET", "mykey", "truck1").Err(`{"type":"Point","coordinates":[10,10]}`), 42 | Do("GET", "mykey", "truck2").Err(`{"type":"Point","coordinates":[10,10]}`), 43 | ) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | err = mc.DoBatch( 49 | Do("SET", "mykey", "truck7", "POINT", 10, 10).OK(), 50 | Do("SET", "mykey", "truck8", "POINT", 10, 10).OK(), 51 | Do("SET", "mykey", "truck9", "POINT", 10, 10).OK(), 52 | ) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | err = mc2.DoBatch( 58 | Sleep(time.Second/2), 59 | Do("GET", "mykey", "truck7").Str(`{"type":"Point","coordinates":[10,10]}`), 60 | Do("GET", "mykey", "truck8").Str(`{"type":"Point","coordinates":[10,10]}`), 61 | Do("GET", "mykey", "truck9").Str(`{"type":"Point","coordinates":[10,10]}`), 62 | ) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /tests/json_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | func subTestJSON(g *testGroup) { 4 | g.regSubTest("basic", json_JSET_basic_test) 5 | g.regSubTest("geojson", json_JSET_geojson_test) 6 | g.regSubTest("number", json_JSET_number_test) 7 | 8 | } 9 | func json_JSET_basic_test(mc *mockServer) error { 10 | return mc.DoBatch([][]interface{}{ 11 | {"JSET", "mykey", "myid1", "hello", "world"}, {"OK"}, 12 | {"JGET", "mykey", "myid1"}, {`{"hello":"world"}`}, 13 | {"JSET", "mykey", "myid1", "hello", "planet"}, {"OK"}, 14 | {"JGET", "mykey", "myid1"}, {`{"hello":"planet"}`}, 15 | {"JSET", "mykey", "myid1", "user.name.last", "tom"}, {"OK"}, 16 | {"JSET", "mykey", "myid1", "user.name.first", "andrew"}, {"OK"}, 17 | {"JGET", "mykey", "myid1"}, {`{"hello":"planet","user":{"name":{"last":"tom","first":"andrew"}}}`}, 18 | {"JDEL", "mykey", "myid1", "user.name.last"}, {1}, 19 | {"JGET", "mykey", "myid1"}, {`{"hello":"planet","user":{"name":{"first":"andrew"}}}`}, 20 | {"JDEL", "mykey", "myid1", "user.name.last"}, {0}, 21 | {"JGET", "mykey", "myid1"}, {`{"hello":"planet","user":{"name":{"first":"andrew"}}}`}, 22 | {"JDEL", "mykey2", "myid1", "user.name.last"}, {0}, 23 | }) 24 | } 25 | 26 | func json_JSET_geojson_test(mc *mockServer) error { 27 | return mc.DoBatch([][]interface{}{ 28 | {"SET", "mykey", "myid1", "POINT", 33, -115}, {"OK"}, 29 | {"JGET", "mykey", "myid1"}, {`{"type":"Point","coordinates":[-115,33]}`}, 30 | {"JSET", "mykey", "myid1", "coordinates.1", 44}, {"OK"}, 31 | {"JGET", "mykey", "myid1"}, {`{"type":"Point","coordinates":[-115,44]}`}, 32 | {"SET", "mykey", "myid1", "OBJECT", `{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]}}`}, {"OK"}, 33 | {"JGET", "mykey", "myid1"}, {`{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]},"properties":{}}`}, 34 | {"JGET", "mykey", "myid1", "geometry.type"}, {"Point"}, 35 | {"JSET", "mykey", "myid1", "properties.tags.-1", "southwest"}, {"OK"}, 36 | {"JSET", "mykey", "myid1", "properties.tags.-1", "united states"}, {"OK"}, 37 | {"JSET", "mykey", "myid1", "properties.tags.-1", "hot"}, {"OK"}, 38 | {"JGET", "mykey", "myid1"}, {`{"type":"Feature","geometry":{"type":"Point","coordinates":[-115,44]},"properties":{"tags":["southwest","united states","hot"]}}`}, 39 | {"JDEL", "mykey", "myid1", "type"}, {"ERR missing type"}, 40 | }) 41 | } 42 | 43 | func json_JSET_number_test(mc *mockServer) error { 44 | return mc.DoBatch([][]interface{}{ 45 | {"JSET", "mykey", "myid1", "hello", "0"}, {"OK"}, 46 | {"JGET", "mykey", "myid1"}, {`{"hello":0}`}, 47 | {"JSET", "mykey", "myid1", "hello", "0123"}, {"OK"}, 48 | {"JGET", "mykey", "myid1"}, {`{"hello":"0123"}`}, 49 | {"JSET", "mykey", "myid1", "hello", "3.14"}, {"OK"}, 50 | {"JGET", "mykey", "myid1"}, {`{"hello":3.14}`}, 51 | {"JSET", "mykey", "myid1", "hello", "1.0e10"}, {"OK"}, 52 | {"JGET", "mykey", "myid1"}, {`{"hello":1.0e10}`}, 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /tests/metrics_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | func subTestMetrics(g *testGroup) { 11 | g.regSubTest("basic", metrics_basic_test) 12 | } 13 | 14 | func downloadURLWithStatusCode(u string) (int, string, error) { 15 | resp, err := http.Get(u) 16 | if err != nil { 17 | return 0, "", err 18 | } 19 | defer resp.Body.Close() 20 | body, err := io.ReadAll(resp.Body) 21 | if err != nil { 22 | return 0, "", err 23 | } 24 | return resp.StatusCode, string(body), nil 25 | } 26 | 27 | func metrics_basic_test(mc *mockServer) error { 28 | 29 | maddr := fmt.Sprintf("http://127.0.0.1:%d/", mc.metricsPort()) 30 | 31 | mc.Do("SET", "metrics_test_1", "1", "FIELD", "foo", 5.5, "POINT", 5, 5) 32 | mc.Do("SET", "metrics_test_2", "2", "FIELD", "foo", 19.19, "POINT", 19, 19) 33 | mc.Do("SET", "metrics_test_2", "3", "FIELD", "foo", 19.19, "POINT", 19, 19) 34 | mc.Do("SET", "metrics_test_2", "truck1:driver", "STRING", "John Denton") 35 | 36 | status, index, err := downloadURLWithStatusCode(maddr) 37 | if err != nil { 38 | return err 39 | } 40 | if status != 200 { 41 | return fmt.Errorf("Expected status code 200, got: %d", status) 42 | } 43 | if !strings.Contains(index, "