├── pkg └── .gitkeep ├── .dockerignore ├── version.go ├── .github ├── release.yml └── workflows │ ├── tagpr.yml │ ├── test.yml │ └── release.yml ├── .gitignore ├── .tagpr ├── docker └── Dockerfile ├── buf.yaml ├── buf.gen.yaml ├── config.go ├── aqua.yaml ├── cmd ├── katsubushi-dump │ └── main.go └── katsubushi │ └── main.go ├── converter.go ├── proto └── main.proto ├── .goreleaser.yml ├── listener.go ├── LICENSE ├── go.mod ├── benchmark.pl ├── Makefile ├── converter_test.go ├── client.go ├── CLAUDE.md ├── grpc_test.go ├── generator.go ├── generator_test.go ├── memcache.go ├── grpc.go ├── grpc ├── README.md ├── main_grpc.pb.go └── main.pb.go ├── README.md ├── client_test.go ├── http_test.go ├── http.go ├── CHANGELOG.md ├── binary_protocol.go ├── binary_protocol_test.go ├── go.sum ├── app.go └── app_test.go /pkg/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | dist/*.gz 2 | dist/*.zip 3 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | const Version = "2.2.1" 4 | -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | labels: 4 | - tagpr 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmd/katsubushi/katsubushi 2 | pkg/* 3 | !.gitkeep 4 | /vendor/ 5 | dist/* 6 | -------------------------------------------------------------------------------- /.tagpr: -------------------------------------------------------------------------------- 1 | [tagpr] 2 | vPrefix = true 3 | releaseBranch = v2 4 | versionFile = version.go 5 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # katsubushi 2 | 3 | FROM alpine:3.22 4 | ARG VERSION 5 | ARG TARGETARCH 6 | ADD dist/go-katsubushi_linux_${TARGETARCH}*/katsubushi /usr/local/bin/katsubushi 7 | EXPOSE 11212 8 | ENTRYPOINT ["/usr/local/bin/katsubushi"] 9 | -------------------------------------------------------------------------------- /buf.yaml: -------------------------------------------------------------------------------- 1 | # For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml 2 | version: v2 3 | modules: 4 | - path: grpc 5 | lint: 6 | use: 7 | - STANDARD 8 | breaking: 9 | use: 10 | - FILE 11 | -------------------------------------------------------------------------------- /buf.gen.yaml: -------------------------------------------------------------------------------- 1 | version: v2 2 | managed: 3 | enabled: true 4 | plugins: 5 | - remote: buf.build/protocolbuffers/go:v1.36.6 6 | out: grpc 7 | - remote: buf.build/grpc/go:v1.5.1 8 | out: grpc 9 | - local: protoc-gen-doc 10 | out: grpc 11 | opt: markdown,README.md 12 | inputs: 13 | - directory: proto 14 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "net" 5 | "time" 6 | ) 7 | 8 | type Config struct { 9 | IdleTimeout time.Duration 10 | LogLevel string 11 | 12 | Port int 13 | Sockpath string 14 | 15 | HTTPPort int 16 | HTTPPathPrefix string 17 | HTTPListener net.Listener 18 | 19 | GRPCPort int 20 | GRPCListener net.Listener 21 | } 22 | -------------------------------------------------------------------------------- /.github/workflows/tagpr.yml: -------------------------------------------------------------------------------- 1 | name: tagpr 2 | 3 | on: 4 | push: 5 | branches: 6 | - v2 7 | 8 | jobs: 9 | tagpr: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 13 | with: 14 | token: ${{ secrets.GITHUB_TOKEN }} 15 | - uses: Songmu/tagpr@ebb5da0cccdb47c533d4b520ebc0acd475b16614 # v1.7.0 16 | env: 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | -------------------------------------------------------------------------------- /aqua.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # yaml-language-server: $schema=https://raw.githubusercontent.com/aquaproj/aqua/main/json-schema/aqua-yaml.json 3 | # aqua - Declarative CLI Version Manager 4 | # https://aquaproj.github.io/ 5 | # checksum: 6 | # enabled: true 7 | # require_checksum: true 8 | # supported_envs: 9 | # - all 10 | registries: 11 | - type: standard 12 | ref: v4.379.0 # renovate: depName=aquaproj/aqua-registry 13 | packages: 14 | - name: bufbuild/buf@v1.55.1 15 | - name: pseudomuto/protoc-gen-doc@v1.5.1 16 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | os: [ubuntu-latest, windows-latest] 8 | go: 9 | - "1.24" 10 | - "1.23" 11 | name: Build 12 | runs-on: ${{ matrix.os }} 13 | steps: 14 | - name: Set up Go 15 | uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 16 | with: 17 | go-version: ${{ matrix.go }} 18 | id: go 19 | 20 | - name: Check out code into the Go module directory 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | 23 | - name: Build & Test 24 | run: | 25 | make test 26 | -------------------------------------------------------------------------------- /cmd/katsubushi-dump/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | katsubushi "github.com/kayac/go-katsubushi/v2" 11 | ) 12 | 13 | type Dump struct { 14 | Time time.Time `json:"time"` 15 | WorkerID uint64 `json:"worker_id"` 16 | Sequence uint64 `json:"sequence"` 17 | } 18 | 19 | func main() { 20 | if len(os.Args) < 2 { 21 | fmt.Fprintln(os.Stderr, "no id") 22 | os.Exit(1) 23 | } 24 | enc := json.NewEncoder(os.Stdout) 25 | for _, s := range os.Args[1:] { 26 | if id, err := strconv.ParseUint(s, 10, 64); err != nil { 27 | fmt.Fprintln(os.Stderr, err) 28 | os.Exit(1) 29 | } else { 30 | t, wid, seq := katsubushi.Dump(id) 31 | enc.Encode(Dump{t, wid, seq}) 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /converter.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import "time" 4 | 5 | // ToTime returns the time when id was generated. 6 | func ToTime(id uint64) time.Time { 7 | ts := id >> (WorkerIDBits + SequenceBits) 8 | d := time.Duration(int64(ts) * int64(time.Millisecond)) 9 | return Epoch.Add(d) 10 | } 11 | 12 | // ToID returns the minimum id which will be generated at time t. 13 | func ToID(t time.Time) uint64 { 14 | d := t.Sub(Epoch) 15 | ts := uint64(d.Nanoseconds()) / uint64(time.Millisecond) 16 | return ts << (WorkerIDBits + SequenceBits) 17 | } 18 | 19 | // Dump returns the structure of id. 20 | func Dump(id uint64) (t time.Time, workerID uint64, sequence uint64) { 21 | workerID = (id & (workerIDMask << SequenceBits)) >> SequenceBits 22 | sequence = id & sequenceMask 23 | return ToTime(id), workerID, sequence 24 | } 25 | -------------------------------------------------------------------------------- /proto/main.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package katsubushi; 3 | option go_package = "katsubushi/grpc"; 4 | 5 | service Generator { 6 | rpc Fetch (FetchRequest) returns (FetchResponse) {} 7 | rpc FetchMulti (FetchMultiRequest) returns (FetchMultiResponse) {} 8 | } 9 | 10 | message FetchRequest {} 11 | 12 | message FetchMultiRequest { 13 | uint32 n = 1; 14 | } 15 | 16 | message FetchResponse { 17 | uint64 id = 1; 18 | } 19 | 20 | message FetchMultiResponse { 21 | repeated uint64 ids = 1; 22 | } 23 | 24 | service Stats { 25 | rpc Get (StatsRequest) returns (StatsResponse) {} 26 | } 27 | 28 | message StatsRequest {} 29 | 30 | message StatsResponse { 31 | int32 pid = 1; 32 | int64 uptime = 2; 33 | int64 time = 3; 34 | string version = 4; 35 | int64 curr_connections = 5; 36 | int64 total_connections = 6; 37 | int64 cmd_get = 7; 38 | int64 get_hits = 8; 39 | int64 get_misses = 9; 40 | } 41 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | version: 2 4 | before: 5 | hooks: 6 | - go mod download 7 | builds: 8 | - env: 9 | - CGO_ENABLED=0 10 | main: ./cmd/katsubushi/ 11 | binary: katsubushi 12 | ldflags: 13 | - -s -w 14 | - -X github.com/kayac/go-katsubushi.Version=v{{.Version}} 15 | goos: 16 | - darwin 17 | - linux 18 | - windows 19 | goarch: 20 | - amd64 21 | - arm64 22 | archives: 23 | - name_template: "{{.ProjectName}}_v{{.Version}}_{{.Os}}_{{.Arch}}" 24 | 25 | release: 26 | prerelease: "true" 27 | checksum: 28 | name_template: "checksums.txt" 29 | snapshot: 30 | version_template: "{{ .Env.NIGHTLY_VERSION }}" 31 | changelog: 32 | sort: asc 33 | filters: 34 | exclude: 35 | - "^docs:" 36 | - "^test:" 37 | -------------------------------------------------------------------------------- /listener.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "net" 5 | "sync" 6 | "sync/atomic" 7 | ) 8 | 9 | func (app *App) wrapListener(l net.Listener) net.Listener { 10 | if _, wrapped := l.(*monitListener); wrapped { 11 | // already wrapped 12 | return l 13 | } 14 | return &monitListener{ 15 | Listener: l, 16 | app: app, 17 | } 18 | } 19 | 20 | type monitListener struct { 21 | net.Listener 22 | app *App 23 | } 24 | 25 | func (l *monitListener) Accept() (net.Conn, error) { 26 | conn, err := l.Listener.Accept() 27 | if err != nil { 28 | return nil, err 29 | } 30 | atomic.AddInt64(&l.app.currConnections, 1) 31 | atomic.AddInt64(&l.app.totalConnections, 1) 32 | return &monitConn{conn, l.app, &sync.Once{}}, nil 33 | } 34 | 35 | type monitConn struct { 36 | net.Conn 37 | app *App 38 | once *sync.Once 39 | } 40 | 41 | func (c *monitConn) Close() error { 42 | c.once.Do(func() { 43 | atomic.AddInt64(&c.app.currConnections, -1) 44 | }) 45 | return c.Conn.Close() 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2015 KAYAC Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kayac/go-katsubushi/v2 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/Songmu/retry v0.0.1 7 | github.com/bmizerany/mc v0.0.0-20180522153755-eeb3d7218919 8 | github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d 9 | github.com/fujiwara/raus v0.3.0 10 | github.com/fukata/golang-stats-api-handler v1.0.0 11 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 12 | google.golang.org/grpc v1.73.0 13 | google.golang.org/protobuf v1.36.6 14 | ) 15 | 16 | require ( 17 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect 18 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 19 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 20 | github.com/google/uuid v1.6.0 // indirect 21 | github.com/kr/pretty v0.3.0 // indirect 22 | github.com/redis/go-redis/v9 v9.11.0 // indirect 23 | github.com/stretchr/testify v1.7.0 // indirect 24 | golang.org/x/net v0.38.0 // indirect 25 | golang.org/x/sys v0.31.0 // indirect 26 | golang.org/x/text v0.23.0 // indirect 27 | google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect 28 | gopkg.in/yaml.v3 v3.0.1 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /benchmark.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | use warnings; 4 | use utf8; 5 | use feature "say"; 6 | 7 | use Test::TCP; 8 | use Proc::Guard; 9 | use Data::Dumper; 10 | use Parallel::Benchmark; 11 | use Cache::Memcached::Fast; 12 | 13 | my $katsubushi_port = Test::TCP::empty_port(); 14 | my $katsubushi = Proc::Guard->new( 15 | command => [ 16 | "go", "run", "cmd/katsubushi/main.go", 17 | "-port=${katsubushi_port}", 18 | "-worker-id=1", 19 | ], 20 | ); 21 | Test::TCP::wait_port($katsubushi_port); 22 | 23 | my $client = Cache::Memcached::Fast->new({ servers => ["localhost:${katsubushi_port}"] }); 24 | 25 | sub bench { 26 | my ($concurrency) = @_; 27 | 28 | my $pb = Parallel::Benchmark->new( 29 | time => 10, 30 | concurrency => $concurrency, 31 | benchmark => sub { 32 | $client->get("id"); 33 | return 1; 34 | }, 35 | ); 36 | 37 | $pb->run; 38 | } 39 | 40 | say "serial"; 41 | bench(1); 42 | 43 | say "parallel"; 44 | bench(4); 45 | 46 | __END__ 47 | 48 | $ perl benchmark.pl 49 | INFO[0000] Listening at [::]:50585 50 | serial 51 | 2015-04-21T15:27:11 [21263] [INFO] starting benchmark: concurrency: 1, time: 10 52 | 2015-04-21T15:27:22 [21263] [INFO] done benchmark: score 97526, elapsed 10.012 sec = 9741.033 / sec 53 | parallel 54 | 2015-04-21T15:27:22 [21263] [INFO] starting benchmark: concurrency: 4, time: 10 55 | 2015-04-21T15:27:33 [21263] [INFO] done benchmark: score 219615, elapsed 10.021 sec = 21914.999 / sec 56 | perl benchmark.pl 4.19s user 11.40s system 63% cpu 24.393 total 57 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GIT_VER := $(shell git describe --tags | sed -e 's/^v//') 2 | export GO111MODULE := on 3 | 4 | all: grpc-gen katsubushi 5 | 6 | setup: 7 | aqua i 8 | 9 | katsubushi: cmd/katsubushi/katsubushi 10 | 11 | cmd/katsubushi/katsubushi: *.go cmd/katsubushi/*.go 12 | cd cmd/katsubushi && go build -ldflags "-w -s" 13 | 14 | 15 | .PHONEY: clean test packages install 16 | install: cmd/katsubushi/katsubushi 17 | install cmd/katsubushi/katsubushi ${GOPATH}/bin 18 | 19 | clean: 20 | rm -rf cmd/katsubushi/katsubushi dist/* 21 | 22 | test: 23 | go test -race 24 | 25 | packages: 26 | goreleaser build --skip=validate --clean 27 | 28 | packages-snapshot: 29 | goreleaser build --skip=validate --clean --snapshot 30 | 31 | docker: clean packages 32 | mv dist/go-katsubushi_linux_amd64_v1 dist/go-katsubushi_linux_amd64 33 | docker buildx build \ 34 | --build-arg VERSION=v${GIT_VER} \ 35 | --platform linux/amd64,linux/arm64 \ 36 | -f docker/Dockerfile \ 37 | -t katsubushi/katsubushi:v${GIT_VER} \ 38 | -t katsubushi/katsubushi:latest \ 39 | -t ghcr.io/kayac/go-katsubushi:v${GIT_VER} \ 40 | . 41 | 42 | docker-push: 43 | mv dist/go-katsubushi_linux_amd64_v1 dist/go-katsubushi_linux_amd64 44 | docker buildx build \ 45 | --build-arg VERSION=v${GIT_VER} \ 46 | --platform linux/amd64,linux/arm64 \ 47 | -f docker/Dockerfile \ 48 | -t katsubushi/katsubushi:v${GIT_VER} \ 49 | -t katsubushi/katsubushi:latest \ 50 | -t ghcr.io/kayac/go-katsubushi:v${GIT_VER} \ 51 | --push \ 52 | . 53 | 54 | grpc-gen: proto/*.proto 55 | buf generate 56 | mv grpc/katsubushi/grpc/*.go grpc 57 | rm -fr grpc/katsubushi 58 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - "!**/*" 6 | tags: 7 | - "v*" 8 | workflow_dispatch: 9 | inputs: 10 | version: 11 | description: "Version to release" 12 | type: string 13 | required: true 14 | 15 | jobs: 16 | release: 17 | name: Release 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: Set up Go 21 | uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 22 | with: 23 | go-version: "1.24" 24 | 25 | - name: Check out code 26 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 27 | with: 28 | ref: ${{ inputs.version }} 29 | 30 | - name: setup QEMU 31 | uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 32 | 33 | - name: setup Docker Buildx 34 | uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 35 | 36 | - name: Run GoReleaser 37 | uses: goreleaser/goreleaser-action@9c156ee8a17a598857849441385a2041ef570552 # v6.3.0 38 | with: 39 | version: "~> v2" 40 | args: release 41 | env: 42 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 43 | 44 | - name: Docker Login 45 | uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 46 | with: 47 | username: fujiwara 48 | password: ${{ secrets.DOCKERHUB_TOKEN }} 49 | 50 | - name: Docker Login 51 | uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 52 | with: 53 | registry: ghcr.io 54 | username: $GITHUB_ACTOR 55 | password: ${{ secrets.GITHUB_TOKEN }} 56 | 57 | - name: docker 58 | run: | 59 | PATH=~/bin:$PATH make docker-push 60 | -------------------------------------------------------------------------------- /converter_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/kayac/go-katsubushi/v2" 8 | ) 9 | 10 | func TestConvertFixed(t *testing.T) { 11 | t1 := time.Unix(1465276650, 0) 12 | id := katsubushi.ToID(t1) 13 | if id != 189608755200000000 { 14 | t.Error("unexpected id", id) 15 | } 16 | 17 | t2 := katsubushi.ToTime(id) 18 | if !t1.Equal(t2) { 19 | t.Error("roundtrip failed") 20 | } 21 | } 22 | 23 | func TestConvertFixedSub(t *testing.T) { 24 | t1 := time.Unix(1465276650, 777000000) 25 | id := katsubushi.ToID(t1) 26 | if id != 189608758458974208 { 27 | t.Error("unexpected id", id) 28 | } 29 | 30 | t2 := katsubushi.ToTime(id) 31 | if !t1.Equal(t2) { 32 | t.Error("roundtrip failed") 33 | } 34 | } 35 | 36 | func TestConvertNow(t *testing.T) { 37 | t1 := time.Now().In(time.UTC) 38 | id := katsubushi.ToID(t1) 39 | t2 := katsubushi.ToTime(id) 40 | f := "2006-01-02T15:04:05.000" 41 | if t1.Format(f) != t2.Format(f) { 42 | t.Error("roundtrip failed", t1, t2) 43 | } 44 | } 45 | 46 | func TestDump(t *testing.T) { 47 | testCases := []struct { 48 | id uint64 49 | ts time.Time 50 | wid uint64 51 | seq uint64 52 | }{ 53 | { 54 | 354101311794212865, 55 | time.Date(2017, 9, 4, 3, 12, 11, 615000000, time.UTC), 56 | 999, 57 | 1, 58 | }, 59 | { 60 | 354103658909954052, 61 | time.Date(2017, 9, 4, 3, 21, 31, 211000000, time.UTC), 62 | 999, 63 | 4, 64 | }, 65 | } 66 | for _, tc := range testCases { 67 | ts, wid, seq := katsubushi.Dump(tc.id) 68 | if ts != tc.ts { 69 | t.Errorf("%d timestamp is not expected. got %s expected %s", tc.id, ts, tc.ts) 70 | } 71 | if wid != tc.wid { 72 | t.Errorf("%d workerID is not expected. got %d expected %d", tc.id, wid, tc.wid) 73 | } 74 | if seq != tc.seq { 75 | t.Errorf("%d sequence is not expected. got %d expected %d", tc.id, seq, tc.seq) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/Songmu/retry" 10 | ) 11 | 12 | // DefaultClientTimeout is default timeout for katsubushi client 13 | var DefaultClientTimeout = 5 * time.Second 14 | 15 | // Client is katsubushi client 16 | type Client struct { 17 | memcacheClients []*memcacheClient 18 | } 19 | 20 | // NewClient creates Client 21 | func NewClient(addrs ...string) *Client { 22 | c := &Client{ 23 | memcacheClients: make([]*memcacheClient, 0, len(addrs)), 24 | } 25 | for _, addr := range addrs { 26 | c.memcacheClients = append(c.memcacheClients, newMemcacheClient(addr)) 27 | } 28 | c.SetTimeout(DefaultClientTimeout) 29 | return c 30 | } 31 | 32 | // SetTimeout sets timeout to katsubushi servers 33 | func (c *Client) SetTimeout(t time.Duration) { 34 | for _, mc := range c.memcacheClients { 35 | mc.SetTimeout(t) 36 | } 37 | } 38 | 39 | // Fetch fetches id from katsubushi 40 | func (c *Client) Fetch(ctx context.Context) (uint64, error) { 41 | errs := fmt.Errorf("no servers available") 42 | for _, mc := range c.memcacheClients { 43 | var id uint64 44 | err := retry.Retry(2, 0, func() error { 45 | var _err error 46 | id, _err = mc.Get(ctx, "id") 47 | return _err 48 | }) 49 | if err != nil { 50 | errs = fmt.Errorf("%s: %w", err.Error(), errs) 51 | continue 52 | } 53 | return id, nil 54 | } 55 | return 0, errs 56 | } 57 | 58 | // FetchMulti fetches multiple ids from katsubushi 59 | func (c *Client) FetchMulti(ctx context.Context, n int) ([]uint64, error) { 60 | keys := make([]string, 0, n) 61 | 62 | for i := 0; i < n; i++ { 63 | keys = append(keys, strconv.Itoa(i)) 64 | } 65 | 66 | errs := fmt.Errorf("no servers available") 67 | 68 | for _, mc := range c.memcacheClients { 69 | var ids []uint64 70 | err := retry.Retry(2, 0, func() error { 71 | var _err error 72 | ids, _err = mc.GetMulti(ctx, keys) 73 | return _err 74 | }) 75 | if err != nil { 76 | errs = fmt.Errorf("%s: %w", err.Error(), errs) 77 | continue 78 | } 79 | return ids, nil 80 | } 81 | return nil, errs 82 | } 83 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # go-katsubushi Development Notes 2 | 3 | ## Project Architecture 4 | 5 | **Core Service**: Unique ID generation service using Snowflake-like algorithm 6 | - **Multi-protocol support**: memcached-compatible, HTTP API, gRPC 7 | - **Auto worker ID assignment**: Redis-based distributed worker management 8 | - **Graceful shutdown**: Context-based lifecycle management across all protocols 9 | 10 | **Key Components**: 11 | - `app.go` - Main application logic and custom slog handler 12 | - `generator.go` - Snowflake-like ID generation algorithm 13 | - `memcache.go`, `http.go`, `grpc.go` - Protocol implementations 14 | - `cmd/katsubushi/main.go` - CLI entry point 15 | 16 | ## Build & Deployment 17 | 18 | ### Build Tools 19 | - `make all` - Generate gRPC code and build binary 20 | - `make test` - Run tests with race detection 21 | - `make packages` - Cross-platform builds with GoReleaser 22 | - `make docker` - Multi-architecture Docker builds 23 | - `aqua` for CLI tool management 24 | - `buf` for Protocol Buffers generation 25 | 26 | ### Deployment Options 27 | - Standalone execution (manual worker ID) 28 | - Redis-coordinated auto worker ID assignment 29 | - Docker/Kubernetes deployment 30 | - Unix Domain Socket support 31 | 32 | ## Development Practices 33 | 34 | ### Code Patterns 35 | - **Context management**: Graceful shutdown across all servers 36 | - **Atomic operations**: `sync/atomic` for statistics counters 37 | - **Error handling**: Custom error types for domain-specific errors 38 | - **Resource management**: `defer` for cleanup 39 | - **Test injection**: `nowFunc` variable for time.Now substitution 40 | 41 | ### Logging (slog) 42 | - Custom slog.Handler maintains original log format: `2025-08-01T22:11:28.043+0900 INFO go-katsubushi/grpc.go:94 Message` 43 | - Use `slog.SetDefault()` to avoid data races, not global logger variables 44 | - Import `log/slog` in all files and use slog functions directly 45 | - Debug logs added for all protocols (memcached, gRPC, HTTP API) 46 | 47 | ### Git Workflow 48 | - Use individual file specification: `git add ` not `git add -A` 49 | - Always run `go fmt ./...` before committing 50 | - Commit messages and code comments in English 51 | 52 | ### Testing 53 | - Run `go test -race ./...` to detect data races 54 | - Protocol-specific test coverage (app_test.go, http_test.go, grpc_test.go, etc.) 55 | - Worker ID uniqueness validation 56 | - Build verification: `go build ./cmd/katsubushi/` 57 | 58 | ### Dependencies 59 | - **Core**: gRPC, Protocol Buffers, gomemcache 60 | - **Infrastructure**: fujiwara/raus (Redis auto-assignment) 61 | - **Development**: buf, aqua, goreleaser 62 | -------------------------------------------------------------------------------- /grpc_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net" 7 | "testing" 8 | "time" 9 | 10 | "github.com/kayac/go-katsubushi/v2" 11 | "github.com/kayac/go-katsubushi/v2/grpc" 12 | 13 | gogrpc "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials/insecure" 15 | ) 16 | 17 | var grpcApp *katsubushi.App 18 | var grpcPort int 19 | 20 | func init() { 21 | var err error 22 | grpcApp, err = katsubushi.New(88) 23 | if err != nil { 24 | panic(err) 25 | } 26 | listener, err := net.Listen("tcp", "localhost:0") 27 | if err != nil { 28 | panic(err) 29 | } 30 | grpcPort = listener.Addr().(*net.TCPAddr).Port 31 | go grpcApp.RunGRPCServer(context.Background(), &katsubushi.Config{GRPCListener: listener}) 32 | time.Sleep(3 * time.Second) 33 | } 34 | 35 | func newgRPCClient() (grpc.GeneratorClient, func(), error) { 36 | addr := fmt.Sprintf("localhost:%d", grpcPort) 37 | conn, err := gogrpc.Dial( 38 | addr, 39 | gogrpc.WithTransportCredentials(insecure.NewCredentials()), 40 | gogrpc.WithBlock(), 41 | ) 42 | if err != nil { 43 | return nil, func() {}, err 44 | } 45 | c := grpc.NewGeneratorClient(conn) 46 | return c, func() { conn.Close() }, nil 47 | } 48 | 49 | func TestGRPCSingle(t *testing.T) { 50 | client, close, err := newgRPCClient() 51 | defer close() 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | for i := 0; i < 10; i++ { 56 | res, err := client.Fetch(context.Background(), &grpc.FetchRequest{}) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | if res.Id == 0 { 61 | t.Fatal("id should not be 0") 62 | } 63 | t.Logf("HTTP fetched single ID: %d", res.Id) 64 | } 65 | } 66 | 67 | func TestGRPCMulti(t *testing.T) { 68 | client, close, err := newgRPCClient() 69 | defer close() 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | for i := 0; i < 10; i++ { 74 | res, err := client.FetchMulti(context.Background(), &grpc.FetchMultiRequest{N: 10}) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | if len(res.Ids) != 10 { 79 | t.Fatalf("ids should contain 10 elements %v", res.Ids) 80 | } 81 | for _, id := range res.Ids { 82 | if id == 0 { 83 | t.Fatal("id should not be 0") 84 | } 85 | } 86 | t.Logf("HTTP fetched IDs: %v", res.Ids) 87 | } 88 | } 89 | 90 | func BenchmarkGRPCClientFetch(b *testing.B) { 91 | b.ResetTimer() 92 | 93 | b.RunParallel(func(pb *testing.PB) { 94 | c, close, _ := newgRPCClient() 95 | defer close() 96 | for pb.Next() { 97 | res, err := c.Fetch(context.Background(), &grpc.FetchRequest{}) 98 | if err != nil { 99 | b.Fatal(err) 100 | } 101 | if res.Id == 0 { 102 | b.Error("could not fetch id > 0") 103 | } 104 | } 105 | }) 106 | } 107 | -------------------------------------------------------------------------------- /generator.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | var nowFunc = time.Now 10 | var nowMutex sync.RWMutex 11 | 12 | func setNowFunc(f func() time.Time) { 13 | nowMutex.Lock() 14 | defer nowMutex.Unlock() 15 | nowFunc = f 16 | } 17 | 18 | func now() time.Time { 19 | nowMutex.RLock() 20 | defer nowMutex.RUnlock() 21 | return nowFunc() 22 | } 23 | 24 | // Epoch is katsubushi epoch time (2015-01-01 00:00:00 UTC) 25 | // Generated ID includes elapsed time from Epoch. 26 | var Epoch = time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC) 27 | 28 | // for bitshift 29 | const ( 30 | WorkerIDBits = 10 31 | SequenceBits = 12 32 | workerIDMask = -1 ^ (-1 << WorkerIDBits) 33 | sequenceMask = -1 ^ (-1 << SequenceBits) 34 | ) 35 | 36 | var workerIDPool = []uint{} 37 | var newGeneratorLock sync.Mutex 38 | 39 | // errors 40 | var ( 41 | ErrInvalidWorkerID = errors.New("invalid worker id") 42 | ErrDuplicatedWorkerID = errors.New("duplicated worker") 43 | ) 44 | 45 | func checkWorkerID(id uint) error { 46 | if workerIDMask < id { 47 | return ErrInvalidWorkerID 48 | } 49 | 50 | for _, otherID := range workerIDPool { 51 | if id == otherID { 52 | return ErrDuplicatedWorkerID 53 | } 54 | } 55 | 56 | return nil 57 | } 58 | 59 | // Generator is an interface to generate unique ID. 60 | type Generator interface { 61 | NextID() (uint64, error) 62 | WorkerID() uint 63 | } 64 | 65 | type generator struct { 66 | workerID uint 67 | lastTimestamp uint64 68 | sequence uint 69 | lock sync.Mutex 70 | startedAt time.Time 71 | offset time.Duration 72 | } 73 | 74 | // NewGenerator returns new generator. 75 | func NewGenerator(workerID uint) (Generator, error) { 76 | // To keep worker ID be unique. 77 | newGeneratorLock.Lock() 78 | defer newGeneratorLock.Unlock() 79 | 80 | if err := checkWorkerID(workerID); err != nil { 81 | return nil, err 82 | } 83 | 84 | // save as already used 85 | workerIDPool = append(workerIDPool, workerID) 86 | 87 | n := now() 88 | return &generator{ 89 | workerID: workerID, 90 | startedAt: n, 91 | offset: n.Sub(Epoch), 92 | }, nil 93 | } 94 | 95 | func (g *generator) WorkerID() uint { 96 | return g.workerID 97 | } 98 | 99 | // NextID generate new ID. 100 | func (g *generator) NextID() (uint64, error) { 101 | g.lock.Lock() 102 | defer g.lock.Unlock() 103 | 104 | ts := g.timestamp() 105 | 106 | // for rewind of server clock 107 | if ts < g.lastTimestamp { 108 | return 0, errors.New("system clock was rollbacked") 109 | } 110 | 111 | if ts == g.lastTimestamp { 112 | g.sequence = (g.sequence + 1) & sequenceMask 113 | if g.sequence == 0 { 114 | // overflow 115 | ts = g.waitUntilNextTick(ts) 116 | } 117 | } else { 118 | g.sequence = 0 119 | } 120 | g.lastTimestamp = ts 121 | 122 | return (g.lastTimestamp << (WorkerIDBits + SequenceBits)) | (uint64(g.workerID) << SequenceBits) | (uint64(g.sequence)), nil 123 | } 124 | 125 | func (g *generator) timestamp() uint64 { 126 | d := now().Sub(g.startedAt) + g.offset 127 | return uint64(d.Nanoseconds()) / uint64(time.Millisecond) 128 | } 129 | 130 | func (g *generator) waitUntilNextTick(ts uint64) uint64 { 131 | next := g.timestamp() 132 | 133 | for next <= ts { 134 | next = g.timestamp() 135 | time.Sleep(50 * time.Nanosecond) 136 | } 137 | 138 | return next 139 | } 140 | -------------------------------------------------------------------------------- /generator_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "sync/atomic" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | var nextWorkerID uint32 10 | 11 | func getNextWorkerID() uint { 12 | return uint(atomic.AddUint32(&nextWorkerID, 1)) 13 | } 14 | 15 | func TestInvalidWorkerID(t *testing.T) { 16 | // workerIDMask = 10bits = 0~1023 17 | if _, err := NewGenerator(1023); err != nil { 18 | t.Errorf("unexpected error: %s", err) 19 | } 20 | 21 | if _, err := NewGenerator(1024); err != ErrInvalidWorkerID { 22 | t.Errorf("invalid error for overranged workerID: %s", err) 23 | } 24 | } 25 | 26 | func TestUniqueWorkerID(t *testing.T) { 27 | mayBeDup := getNextWorkerID() 28 | 29 | _, err := NewGenerator(mayBeDup) 30 | if err != nil { 31 | t.Fatalf("failed to create first generator: %s", err) 32 | } 33 | 34 | _, err = NewGenerator(getNextWorkerID()) 35 | if err != nil { 36 | t.Fatalf("failed to create second generator: %s", err) 37 | } 38 | 39 | g, _ := NewGenerator(mayBeDup) // duplicate!! 40 | if g != nil { 41 | t.Fatalf("worker ID must be unique") 42 | } 43 | } 44 | 45 | func TestGenerateAnID(t *testing.T) { 46 | workerID := getNextWorkerID() 47 | 48 | g, err := NewGenerator(workerID) 49 | if err != nil { 50 | t.Fatalf("failed to create new generator: %s", err) 51 | } 52 | 53 | var id uint64 54 | now := time.Now() 55 | 56 | t.Log("generate") 57 | { 58 | ident, err := g.NextID() 59 | if err != nil { 60 | t.Fatalf("failed to generate id: %s", err) 61 | } 62 | 63 | if id < 0 { 64 | t.Error("invalid id") 65 | } 66 | 67 | id = ident 68 | } 69 | 70 | t.Logf("id = %d", id) 71 | 72 | t.Log("restore timestamp") 73 | { 74 | timestampSince := uint64(Epoch.UnixNano()) / uint64(time.Millisecond) 75 | ts := (id & 0x7FFFFFFFFFC00000 >> (WorkerIDBits + SequenceBits)) + timestampSince 76 | nowMsec := uint64(now.UnixNano()) / uint64(time.Millisecond) 77 | 78 | // To avoid failure would cause by timestamp on execution. 79 | if nowMsec != ts && ts != nowMsec+1 { 80 | t.Errorf("failed to restore timestamp: %d", ts) 81 | } 82 | } 83 | 84 | t.Log("restore worker ID") 85 | { 86 | wid := uint(id & 0x3FF000 >> SequenceBits) 87 | if wid != workerID { 88 | t.Errorf("failed to restore worker ID: %d", wid) 89 | } 90 | } 91 | } 92 | 93 | func TestGenerateSomeIDs(t *testing.T) { 94 | g, _ := NewGenerator(getNextWorkerID()) 95 | ids := []uint64{} 96 | 97 | for i := 0; i < 1000; i++ { 98 | id, err := g.NextID() 99 | if err != nil { 100 | t.Fatalf("failed to generate id: %s", err) 101 | } 102 | 103 | for _, otherID := range ids { 104 | if otherID == id { 105 | t.Fatal("id duplicated!!") 106 | } 107 | } 108 | 109 | if l := len(ids); 0 < l && id < ids[l-1] { 110 | t.Fatal("generated smaller id!!") 111 | } 112 | 113 | ids = append(ids, id) 114 | } 115 | 116 | t.Logf("%d ids are tested", len(ids)) 117 | } 118 | 119 | func TestClockRollback(t *testing.T) { 120 | g, _ := NewGenerator(getNextWorkerID()) 121 | _, err := g.NextID() 122 | if err != nil { 123 | t.Fatalf("failed to generate id: %s", err) 124 | } 125 | 126 | // サーバーの時計が巻き戻った想定 127 | setNowFunc(func() time.Time { 128 | return time.Now().Add(-10 * time.Minute) 129 | }) 130 | 131 | _, err = g.NextID() 132 | if err == nil { 133 | t.Fatalf("when server clock rollback, generater must return error") 134 | } 135 | 136 | t.Log(err) 137 | } 138 | 139 | func BenchmarkGenerateID(b *testing.B) { 140 | g, _ := NewGenerator(getNextWorkerID()) 141 | b.ResetTimer() 142 | 143 | for i := 0; i < b.N; i++ { 144 | g.NextID() 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /memcache.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "io" 9 | "net" 10 | "strconv" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const memcacheDefaultTimeout = 1 * time.Second 16 | 17 | type memcacheClient struct { 18 | addr string 19 | conn net.Conn 20 | timeout time.Duration 21 | mu sync.Mutex 22 | rw *bufio.ReadWriter 23 | } 24 | 25 | func newMemcacheClient(addr string) *memcacheClient { 26 | return &memcacheClient{ 27 | addr: addr, 28 | timeout: memcacheDefaultTimeout, 29 | } 30 | } 31 | 32 | func (c *memcacheClient) SetTimeout(t time.Duration) { 33 | c.mu.Lock() 34 | defer c.mu.Unlock() 35 | c.timeout = t 36 | } 37 | 38 | func (c *memcacheClient) connect(ctx context.Context) error { 39 | var err error 40 | d := net.Dialer{Timeout: c.timeout} 41 | c.conn, err = d.DialContext(ctx, "tcp", c.addr) 42 | c.rw = bufio.NewReadWriter(bufio.NewReader(c.conn), bufio.NewWriter(c.conn)) 43 | return err 44 | } 45 | 46 | func (c *memcacheClient) close() error { 47 | defer func() { c.conn = nil }() 48 | if c.conn != nil { 49 | return c.conn.Close() 50 | } 51 | return nil 52 | } 53 | 54 | func (c *memcacheClient) Get(ctx context.Context, key string) (uint64, error) { 55 | c.mu.Lock() 56 | defer c.mu.Unlock() 57 | if c.conn == nil { 58 | if err := c.connect(ctx); err != nil { 59 | return 0, err 60 | } 61 | } 62 | c.conn.SetDeadline(time.Now().Add(c.timeout)) 63 | c.rw.Write(memdGets) 64 | c.rw.Write(memdSpc) 65 | io.WriteString(c.rw, key) 66 | c.rw.Write(memdSep) 67 | if err := c.rw.Flush(); err != nil { 68 | c.close() 69 | return 0, err 70 | } 71 | 72 | id, err := readValue(c.rw.Reader) 73 | if err != nil { 74 | c.close() 75 | return 0, err 76 | } 77 | end, _, err := c.rw.ReadLine() 78 | if err != nil { 79 | c.close() 80 | return 0, err 81 | } 82 | if !bytes.Equal(end, memdEnd) { 83 | c.close() 84 | return 0, errors.New("unexpected response. not END") 85 | } 86 | return id, nil 87 | } 88 | 89 | func (c *memcacheClient) GetMulti(ctx context.Context, keys []string) ([]uint64, error) { 90 | c.mu.Lock() 91 | defer c.mu.Unlock() 92 | if c.conn == nil { 93 | if err := c.connect(ctx); err != nil { 94 | return nil, err 95 | } 96 | } 97 | c.conn.SetDeadline(time.Now().Add(c.timeout)) 98 | c.rw.Write(memdGets) 99 | for _, key := range keys { 100 | c.rw.Write(memdSpc) 101 | io.WriteString(c.rw, key) 102 | } 103 | c.rw.Write(memdSep) 104 | if err := c.rw.Flush(); err != nil { 105 | c.close() 106 | return nil, err 107 | } 108 | 109 | ids := make([]uint64, 0, len(keys)) 110 | for i := 0; i < len(keys); i++ { 111 | id, err := readValue(c.rw.Reader) 112 | if err != nil { 113 | c.close() 114 | return nil, err 115 | } 116 | ids = append(ids, id) 117 | } 118 | end, _, err := c.rw.ReadLine() 119 | if err != nil { 120 | c.close() 121 | return nil, err 122 | } 123 | if !bytes.Equal(end, memdEnd) { 124 | c.close() 125 | return nil, errors.New("unexpected response. not END") 126 | } 127 | return ids, nil 128 | } 129 | 130 | func readValue(r *bufio.Reader) (uint64, error) { 131 | line, _, err := r.ReadLine() 132 | if err != nil { 133 | return 0, err 134 | } 135 | if len(line) == 0 { 136 | return 0, errors.New("unexpected response") 137 | } 138 | fields := bytes.Fields(line) 139 | if len(fields) != 4 || !bytes.Equal(fields[0], memdValue) { 140 | return 0, errors.New("unexpected response. not VALUE") 141 | } 142 | value, _, err := r.ReadLine() 143 | if err != nil { 144 | return 0, err 145 | } 146 | id, err := strconv.ParseUint(string(value), 10, 64) 147 | if err != nil { 148 | return 0, err 149 | } 150 | return id, nil 151 | } 152 | -------------------------------------------------------------------------------- /grpc.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log/slog" 7 | "net" 8 | "sync/atomic" 9 | 10 | "github.com/kayac/go-katsubushi/v2/grpc" 11 | 12 | grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" 13 | grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" 14 | gogrpc "google.golang.org/grpc" 15 | "google.golang.org/grpc/codes" 16 | "google.golang.org/grpc/health" 17 | "google.golang.org/grpc/health/grpc_health_v1" 18 | "google.golang.org/grpc/reflection" 19 | "google.golang.org/grpc/status" 20 | ) 21 | 22 | const ( 23 | MaxGRPCBulkSize = 1000 24 | ) 25 | 26 | type gRPCGenerator struct { 27 | grpc.GeneratorServer 28 | app *App 29 | } 30 | 31 | func (sv *gRPCGenerator) Fetch(ctx context.Context, req *grpc.FetchRequest) (*grpc.FetchResponse, error) { 32 | atomic.AddInt64(&sv.app.cmdGet, 1) 33 | slog.Debug("gRPC Fetch request") 34 | 35 | id, err := sv.app.NextID() 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to get id: %w", err) 38 | } 39 | slog.Debug("gRPC Generated ID", "id", id) 40 | res := &grpc.FetchResponse{ 41 | Id: id, 42 | } 43 | return res, nil 44 | } 45 | 46 | func (sv *gRPCGenerator) FetchMulti(ctx context.Context, req *grpc.FetchMultiRequest) (*grpc.FetchMultiResponse, error) { 47 | atomic.AddInt64(&sv.app.cmdGet, 1) 48 | n := int(req.N) 49 | slog.Debug("gRPC FetchMulti request", "n", n) 50 | if n > MaxGRPCBulkSize { 51 | return nil, fmt.Errorf("too many IDs requested: %d, n should be smaller than %d", n, MaxGRPCBulkSize) 52 | } 53 | if n == 0 { 54 | n = 1 55 | } 56 | ids := make([]uint64, 0, n) 57 | for i := 0; i < n; i++ { 58 | id, err := sv.app.NextID() 59 | if err != nil { 60 | return nil, fmt.Errorf("failed to get id: %w", err) 61 | } 62 | ids = append(ids, id) 63 | } 64 | slog.Debug("gRPC Generated IDs", "ids", ids) 65 | res := &grpc.FetchMultiResponse{ 66 | Ids: ids, 67 | } 68 | return res, nil 69 | } 70 | 71 | func (app *App) RunGRPCServer(ctx context.Context, cfg *Config) error { 72 | svGen := &gRPCGenerator{app: app} 73 | svStats := &gRPCStats{app: app} 74 | 75 | opts := []grpc_recovery.Option{ 76 | grpc_recovery.WithRecoveryHandler(grpcRecoveryFunc), 77 | } 78 | s := gogrpc.NewServer(grpc_middleware.WithUnaryServerChain( 79 | grpc_recovery.UnaryServerInterceptor(opts...), 80 | )) 81 | grpc.RegisterGeneratorServer(s, svGen) 82 | grpc.RegisterStatsServer(s, svStats) 83 | 84 | // Register health check service 85 | healthServer := health.NewServer() 86 | grpc_health_v1.RegisterHealthServer(s, healthServer) 87 | 88 | // Set health status for overall server and individual services 89 | healthServer.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING) 90 | healthServer.SetServingStatus(grpc.Generator_ServiceDesc.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING) 91 | healthServer.SetServingStatus(grpc.Stats_ServiceDesc.ServiceName, grpc_health_v1.HealthCheckResponse_SERVING) 92 | 93 | reflection.Register(s) 94 | 95 | listener := cfg.GRPCListener 96 | if listener == nil { 97 | var err error 98 | listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.GRPCPort)) 99 | if err != nil { 100 | return fmt.Errorf("failed to listen: %w", err) 101 | } 102 | } 103 | listener = app.wrapListener(listener) 104 | go func() { 105 | <-ctx.Done() 106 | slog.Info("Shutting down gRPC server") 107 | s.Stop() 108 | }() 109 | 110 | slog.Info("Listening gRPC server at " + listener.Addr().String()) 111 | return s.Serve(listener) 112 | } 113 | 114 | func grpcRecoveryFunc(p interface{}) error { 115 | slog.Error("panic", "value", p) 116 | return status.Errorf(codes.Internal, "Unexpected error") 117 | } 118 | 119 | type gRPCStats struct { 120 | grpc.StatsServer 121 | app *App 122 | } 123 | 124 | func (sv *gRPCStats) Get(ctx context.Context, req *grpc.StatsRequest) (*grpc.StatsResponse, error) { 125 | slog.Debug("gRPC Stats request") 126 | st := sv.app.GetStats() 127 | return &grpc.StatsResponse{ 128 | Pid: int32(st.Pid), 129 | Uptime: st.Uptime, 130 | Time: st.Time, 131 | Version: st.Version, 132 | CurrConnections: st.CurrConnections, 133 | TotalConnections: st.TotalConnections, 134 | CmdGet: st.CmdGet, 135 | GetHits: st.GetHits, 136 | GetMisses: st.GetMisses, 137 | }, nil 138 | } 139 | -------------------------------------------------------------------------------- /grpc/README.md: -------------------------------------------------------------------------------- 1 | # Protocol Documentation 2 | 3 | 4 | ## Table of Contents 5 | 6 | - [main.proto](#main-proto) 7 | - [FetchMultiRequest](#katsubushi-FetchMultiRequest) 8 | - [FetchMultiResponse](#katsubushi-FetchMultiResponse) 9 | - [FetchRequest](#katsubushi-FetchRequest) 10 | - [FetchResponse](#katsubushi-FetchResponse) 11 | - [StatsRequest](#katsubushi-StatsRequest) 12 | - [StatsResponse](#katsubushi-StatsResponse) 13 | 14 | - [Generator](#katsubushi-Generator) 15 | - [Stats](#katsubushi-Stats) 16 | 17 | - [Scalar Value Types](#scalar-value-types) 18 | 19 | 20 | 21 | 22 |

Top

23 | 24 | ## main.proto 25 | 26 | 27 | 28 | 29 | 30 | ### FetchMultiRequest 31 | 32 | 33 | 34 | | Field | Type | Label | Description | 35 | | ----- | ---- | ----- | ----------- | 36 | | n | [uint32](#uint32) | | | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | ### FetchMultiResponse 46 | 47 | 48 | 49 | | Field | Type | Label | Description | 50 | | ----- | ---- | ----- | ----------- | 51 | | ids | [uint64](#uint64) | repeated | | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | ### FetchRequest 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | ### FetchResponse 71 | 72 | 73 | 74 | | Field | Type | Label | Description | 75 | | ----- | ---- | ----- | ----------- | 76 | | id | [uint64](#uint64) | | | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | ### StatsRequest 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | ### StatsResponse 96 | 97 | 98 | 99 | | Field | Type | Label | Description | 100 | | ----- | ---- | ----- | ----------- | 101 | | pid | [int32](#int32) | | | 102 | | uptime | [int64](#int64) | | | 103 | | time | [int64](#int64) | | | 104 | | version | [string](#string) | | | 105 | | curr_connections | [int64](#int64) | | | 106 | | total_connections | [int64](#int64) | | | 107 | | cmd_get | [int64](#int64) | | | 108 | | get_hits | [int64](#int64) | | | 109 | | get_misses | [int64](#int64) | | | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | ### Generator 125 | 126 | 127 | | Method Name | Request Type | Response Type | Description | 128 | | ----------- | ------------ | ------------- | ------------| 129 | | Fetch | [FetchRequest](#katsubushi-FetchRequest) | [FetchResponse](#katsubushi-FetchResponse) | | 130 | | FetchMulti | [FetchMultiRequest](#katsubushi-FetchMultiRequest) | [FetchMultiResponse](#katsubushi-FetchMultiResponse) | | 131 | 132 | 133 | 134 | 135 | ### Stats 136 | 137 | 138 | | Method Name | Request Type | Response Type | Description | 139 | | ----------- | ------------ | ------------- | ------------| 140 | | Get | [StatsRequest](#katsubushi-StatsRequest) | [StatsResponse](#katsubushi-StatsResponse) | | 141 | 142 | 143 | 144 | 145 | 146 | ## Scalar Value Types 147 | 148 | | .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby | 149 | | ----------- | ----- | --- | ---- | ------ | -- | -- | --- | ---- | 150 | | double | | double | double | float | float64 | double | float | Float | 151 | | float | | float | float | float | float32 | float | float | Float | 152 | | int32 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint32 instead. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | 153 | | int64 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint64 instead. | int64 | long | int/long | int64 | long | integer/string | Bignum | 154 | | uint32 | Uses variable-length encoding. | uint32 | int | int/long | uint32 | uint | integer | Bignum or Fixnum (as required) | 155 | | uint64 | Uses variable-length encoding. | uint64 | long | int/long | uint64 | ulong | integer/string | Bignum or Fixnum (as required) | 156 | | sint32 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int32s. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | 157 | | sint64 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int64s. | int64 | long | int/long | int64 | long | integer/string | Bignum | 158 | | fixed32 | Always four bytes. More efficient than uint32 if values are often greater than 2^28. | uint32 | int | int | uint32 | uint | integer | Bignum or Fixnum (as required) | 159 | | fixed64 | Always eight bytes. More efficient than uint64 if values are often greater than 2^56. | uint64 | long | int/long | uint64 | ulong | integer/string | Bignum | 160 | | sfixed32 | Always four bytes. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | 161 | | sfixed64 | Always eight bytes. | int64 | long | int/long | int64 | long | integer/string | Bignum | 162 | | bool | | bool | boolean | boolean | bool | bool | boolean | TrueClass/FalseClass | 163 | | string | A string must always contain UTF-8 encoded or 7-bit ASCII text. | string | String | str/unicode | string | string | string | String (UTF-8) | 164 | | bytes | May contain any arbitrary sequence of bytes. | string | ByteString | str | []byte | ByteString | string | String (ASCII-8BIT) | 165 | 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # katsubushi 2 | 3 | katsubushi(鰹節) is stand alone application to generate unique ID. 4 | 5 | ## Example 6 | 7 | ``` 8 | $ telnet localhost 11212 9 | Trying ::1... 10 | Connected to localhost. 11 | Escape character is '^]'. 12 | GET new 13 | VALUE new 0 20 14 | 8070450532247928832 15 | END 16 | ``` 17 | 18 | ## Installation 19 | 20 | Download from [releases](https://github.com/kayac/go-katsubushi/releases) or build from source code. 21 | 22 | ``` 23 | $ go get github.com/kayac/go-katsubushi/v2 24 | $ cd $GOPATH/github.com/kayac/go-katsubushi 25 | make 26 | ``` 27 | 28 | ## Docker image 29 | 30 | - [katsubushi/katsubushi](https://hub.docker.com/r/katsubushi/katsubushi/) 31 | - [ghcr.io/kayac/go-katsubushi](https://github.com/kayac/go-katsubushi/pkgs/container/go-katsubushi) 32 | 33 | 34 | ``` 35 | $ docker pull katsubushi/katsubushi 36 | $ docker run -p 11212:11212 katsubushi/katsubushi -worker-id 1 37 | $ docker run -p 11212:11212 katsubushi/katsubushi -redis redis://your.redis.host:6379/0 38 | ``` 39 | 40 | ## Usage 41 | 42 | ``` 43 | $ cd $GOPATH/github.com/kayac/go-katsubushi/cmd/katsubushi 44 | ./katsubushi -worker-id=1 -port=7238 45 | ./katsubushi -worker-id=1 -sock=/path/to/unix-domain.sock 46 | ``` 47 | 48 | ## Protocol 49 | 50 | katsubushi use protocol compatible with memcached. 51 | 52 | Some commands are available with text and binary protocol. 53 | 54 | But the others are available only with text protocol. 55 | 56 | ### API 57 | 58 | #### GET, GETS 59 | 60 | Binary protocol is also available only for single key GET. 61 | 62 | ``` 63 | GET id1 id2 64 | VALUE id1 0 18 65 | 283890203179880448 66 | VALUE id2 0 18 67 | 283890203179880449 68 | END 69 | ``` 70 | 71 | VALUE(s) are unique IDs. 72 | 73 | #### STATS 74 | 75 | Returns a stats of katsubushi. 76 | 77 | Binary protocol is also available. 78 | 79 | ``` 80 | STAT pid 8018 81 | STAT uptime 17 82 | STAT time 1487754986 83 | STAT version 1.1.2 84 | STAT curr_connections 1 85 | STAT total_connections 2 86 | STAT cmd_get 2 87 | STAT get_hits 3 88 | STAT get_misses 0 89 | ``` 90 | 91 | #### VERSION 92 | 93 | Returns a version of katsubushi. 94 | 95 | Binary protocol is available, too. 96 | 97 | ``` 98 | VERSION 1.1.2 99 | ``` 100 | 101 | #### QUIT 102 | 103 | Disconnect an established connection. 104 | 105 | ## Protocol (HTTP) 106 | 107 | katsubushi also runs an HTTP server specified with `-http-port`. 108 | 109 | ### GET /id 110 | 111 | Get a single ID. 112 | 113 | When `Accept` HTTP header is 'application/json', katsubushi will return an ID as JSON format as below. 114 | 115 | ```json 116 | {"id":"1025441401866821632"} 117 | ``` 118 | 119 | Otherwise, katsubushi will return ID as text format. 120 | 121 | ``` 122 | 1025441401866821632 123 | ``` 124 | 125 | ### GET /ids?n=(number_of_ids) 126 | 127 | Get multiple IDs. 128 | 129 | When `Accept` HTTP header is 'application/json', katsubushi will return an IDs as JSON format as below. 130 | 131 | ```json 132 | {"ids":["1025442579472195584","1025442579472195585","1025442579472195586"]} 133 | ``` 134 | 135 | Otherwise, katsubushi will return ID as text format delimiterd with "\n". 136 | 137 | ``` 138 | 1025442579472195584 139 | 1025442579472195585 140 | 1025442579472195586 141 | ``` 142 | 143 | ### GET /stats 144 | 145 | Returns a stats of katsubushi. 146 | 147 | This API returns a JSON always. 148 | 149 | ```json 150 | { 151 | "pid": 1859630, 152 | "uptime": 50, 153 | "time": 1664761614, 154 | "version": "1.8.0", 155 | "curr_connections": 1, 156 | "total_connections": 5, 157 | "cmd_get": 15, 158 | "get_hits": 25, 159 | "get_misses": 0 160 | } 161 | ``` 162 | 163 | ## Protocol (gRPC) 164 | 165 | katsubushi also runs an gRPC server specified with `-grpc-port`. 166 | 167 | See [grpc/README.md](grpc/README.md). 168 | 169 | ## Algorithm 170 | 171 | katsubushi use algorithm like snowflake to generate ID. 172 | 173 | ## Commandline Options 174 | 175 | `-worker-id` or `-redis` is required. 176 | 177 | ### -worker-id 178 | 179 | ID of the worker, must be unique in your service. 180 | 181 | ### -redis 182 | 183 | URL of Redis server. e.g. `redis://example.com:6379/0` 184 | 185 | `redis://{host}:{port}/{db}?ns={namespace}` 186 | 187 | If you are using Redis Cluster, you will need to specify the URL as `rediscluster://{host}:{port}?ns={namespace}`. 188 | 189 | This option is specified, katsubushi will assign an unique worker ID via Redis. 190 | 191 | All katsubushi process for your service must use a same Redis URL. 192 | 193 | ### -min-worker-id -max-worker-id 194 | 195 | These options work with `-redis`. 196 | 197 | If we use multi katsubushi clusters, worker-id range for each clusters must not be overlapped. katsubushi can specify the worker-id range by these options. 198 | 199 | ### -port 200 | 201 | Optional. 202 | Port number used for connection. 203 | Default value is `11212`. 204 | 205 | ### -sock 206 | 207 | Optional. 208 | Path of unix doamin socket. 209 | 210 | ### -idle-timeout 211 | 212 | Optional. 213 | Connection idle timeout in seconds. 214 | `0` means infinite. 215 | Default value is `600`. 216 | 217 | ### -log-level 218 | 219 | Optional. 220 | Default value is `info`. 221 | 222 | ### -enable-pprof 223 | 224 | Optional. 225 | Boolean flag. 226 | Enable profiling API by `net/http/pprof`. 227 | Endpoint is `/debug/pprof`. 228 | 229 | ### -enable-stats 230 | 231 | Optional. 232 | Boolean flag. 233 | Enable stats API by `github.com/fukata/golang-stats-api-handler`. 234 | Endpoint is `/debug/stats`. 235 | 236 | ### -debug-port 237 | 238 | Optional. 239 | Port number for listen http used for `pprof` and `stats` API. 240 | Defalut value is `8080`. 241 | 242 | ### -http-port 243 | 244 | Optional. 245 | Port number of HTTP server. 246 | Default value is `0` (disabled). 247 | 248 | ### -grpc-port 249 | 250 | Optional. 251 | Port number of gRPC server. 252 | Default value is `0` (disabled). 253 | 254 | 255 | ## Licence 256 | 257 | [MIT](https://github.com/kayac/go-katsubushi/blob/master/LICENSE) 258 | 259 | ## Author 260 | 261 | [handlename](https://github.com/handlename) 262 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/bradfitz/gomemcache/memcache" 9 | ) 10 | 11 | func BenchmarkClientFetch(b *testing.B) { 12 | ctx, cancel := context.WithCancel(context.Background()) 13 | defer cancel() 14 | app := newTestAppAndListenTCP(ctx, b, nil) 15 | 16 | b.ResetTimer() 17 | 18 | b.RunParallel(func(pb *testing.PB) { 19 | c := NewClient(app.Listener.Addr().String()) 20 | for pb.Next() { 21 | id, err := c.Fetch(context.Background()) 22 | if err != nil { 23 | b.Fatal(err) 24 | } 25 | if id == 0 { 26 | b.Error("could not fetch id > 0") 27 | } 28 | } 29 | }) 30 | } 31 | 32 | func BenchmarkGoMemcacheFetch(b *testing.B) { 33 | ctx, cancel := context.WithCancel(context.Background()) 34 | defer cancel() 35 | app := newTestAppAndListenTCP(ctx, b, nil) 36 | 37 | b.ResetTimer() 38 | 39 | b.RunParallel(func(pb *testing.PB) { 40 | mc := memcache.New(app.Listener.Addr().String()) 41 | for pb.Next() { 42 | _, err := mc.Get("id") 43 | if err != nil { 44 | b.Fatal(err) 45 | } 46 | } 47 | }) 48 | } 49 | 50 | func TestClientFetch(t *testing.T) { 51 | ctx, cancel := context.WithCancel(context.Background()) 52 | defer cancel() 53 | app := newTestAppAndListenTCP(ctx, t, nil) 54 | c := NewClient(app.Listener.Addr().String()) 55 | 56 | id, err := c.Fetch(context.Background()) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | if id == 0 { 61 | t.Error("could not fetch id > 0") 62 | } 63 | t.Logf("fetched id %d", id) 64 | } 65 | 66 | func TestClientMulti(t *testing.T) { 67 | ctx, cancel := context.WithCancel(context.Background()) 68 | defer cancel() 69 | app := newTestAppAndListenTCP(ctx, t, nil) 70 | c := NewClient(app.Listener.Addr().String()) 71 | 72 | ids, err := c.FetchMulti(context.Background(), 3) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | if len(ids) != 3 { 77 | t.Error("FetchMulti != 3") 78 | } 79 | for _, id := range ids { 80 | if id == 0 { 81 | t.Error("could not fetch id > 0") 82 | } 83 | t.Logf("fetched id %d", id) 84 | } 85 | } 86 | 87 | func TestClientFetchRetry(t *testing.T) { 88 | ctx, cancel := context.WithCancel(context.Background()) 89 | defer cancel() 90 | to := time.Second 91 | app := newTestAppAndListenTCP(ctx, t, &to) 92 | 93 | c := NewClient(app.Listener.Addr().String()) 94 | 95 | for i := 0; i < 3; i++ { 96 | id, err := c.Fetch(context.Background()) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | if id == 0 { 101 | t.Error("could not fetch id > 0") 102 | } 103 | t.Logf("fetched id %d", id) 104 | time.Sleep(2 * time.Second) // reset by peer by idle timeout 105 | } 106 | } 107 | 108 | func TestClientFetchBackup(t *testing.T) { 109 | ctx1, cancel1 := context.WithCancel(context.Background()) 110 | defer cancel1() 111 | app1 := newTestAppAndListenTCP(ctx1, t, nil) 112 | 113 | ctx2, cancel2 := context.WithCancel(context.Background()) 114 | defer cancel2() 115 | app2 := newTestAppAndListenTCP(ctx2, t, nil) 116 | 117 | c := NewClient( 118 | app1.Listener.Addr().String(), 119 | app2.Listener.Addr().String(), 120 | ) 121 | 122 | { 123 | // fetched from app1 124 | id, err := c.Fetch(context.Background()) 125 | if err != nil { 126 | t.Fatal(err) 127 | } 128 | if id == 0 { 129 | t.Error("could not fetch id > 0") 130 | } 131 | } 132 | 133 | // shutdown app1 134 | cancelAndWait(cancel1) 135 | 136 | { 137 | // fetched from app2 138 | id, err := c.Fetch(context.Background()) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | if id == 0 { 143 | t.Error("could not fetch id > 0") 144 | } 145 | } 146 | } 147 | 148 | func TestClientFail(t *testing.T) { 149 | ctx, cancel := context.WithCancel(context.Background()) 150 | app := newTestAppAndListenTCP(ctx, t, nil) 151 | 152 | c := NewClient( 153 | app.Listener.Addr().String(), 154 | ) 155 | 156 | cancelAndWait(cancel) 157 | 158 | _, err := c.Fetch(context.Background()) 159 | if err == nil { 160 | t.Error("must be failed") 161 | } 162 | t.Logf("error: %s", err) 163 | } 164 | 165 | func TestClientFailMulti(t *testing.T) { 166 | ctx, cancel := context.WithCancel(context.Background()) 167 | app := newTestAppAndListenTCP(ctx, t, nil) 168 | 169 | c := NewClient( 170 | app.Listener.Addr().String(), 171 | ) 172 | 173 | cancelAndWait(cancel) 174 | 175 | _, err := c.FetchMulti(context.Background(), 3) 176 | if err == nil { 177 | t.Error("must be failed") 178 | } 179 | t.Logf("error: %s", err) 180 | } 181 | 182 | func TestClientFailBackup(t *testing.T) { 183 | ctx1, cancel1 := context.WithCancel(context.Background()) 184 | app1 := newTestAppAndListenTCP(ctx1, t, nil) 185 | 186 | ctx2, cancel2 := context.WithCancel(context.Background()) 187 | app2 := newTestAppAndListenTCP(ctx2, t, nil) 188 | 189 | c := NewClient( 190 | app1.Listener.Addr().String(), 191 | app2.Listener.Addr().String(), 192 | ) 193 | 194 | cancelAndWait(cancel1) 195 | cancelAndWait(cancel2) 196 | 197 | _, err := c.Fetch(context.Background()) 198 | if err == nil { 199 | t.Error("must be failed") 200 | } 201 | t.Logf("error: %s", err) 202 | } 203 | 204 | func TestClientFailBackupMulti(t *testing.T) { 205 | ctx1, cancel1 := context.WithCancel(context.Background()) 206 | app1 := newTestAppAndListenTCP(ctx1, t, nil) 207 | 208 | ctx2, cancel2 := context.WithCancel(context.Background()) 209 | app2 := newTestAppAndListenTCP(ctx2, t, nil) 210 | 211 | c := NewClient( 212 | app1.Listener.Addr().String(), 213 | app2.Listener.Addr().String(), 214 | ) 215 | 216 | cancelAndWait(cancel1) 217 | cancelAndWait(cancel2) 218 | 219 | _, err := c.FetchMulti(context.Background(), 3) 220 | if err == nil { 221 | t.Error("must be failed") 222 | } 223 | t.Logf("error: %s", err) 224 | } 225 | 226 | func TestClientTimeout(t *testing.T) { 227 | ctx, cancel := context.WithCancel(context.Background()) 228 | defer cancel() 229 | 230 | app := newTestAppDelayed(t, time.Second) 231 | l, _ := app.ListenerTCP("localhost:0") 232 | go app.Serve(ctx, l) 233 | <-app.Ready() 234 | 235 | c := NewClient(app.Listener.Addr().String()) 236 | c.SetTimeout(500 * time.Millisecond) 237 | 238 | _, err := c.Fetch(context.Background()) 239 | if err == nil { 240 | t.Error("timeout expected but err is nil") 241 | } 242 | t.Logf("client timeout: %s", err) 243 | } 244 | 245 | func cancelAndWait(cancel context.CancelFunc) { 246 | cancel() 247 | time.Sleep(100 * time.Millisecond) 248 | } 249 | -------------------------------------------------------------------------------- /http_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "net" 10 | "net/http/httptest" 11 | "strconv" 12 | "testing" 13 | "time" 14 | 15 | "github.com/kayac/go-katsubushi/v2" 16 | ) 17 | 18 | var httpApp *katsubushi.App 19 | var httpPort int 20 | 21 | func init() { 22 | var err error 23 | httpApp, err = katsubushi.New(80) 24 | if err != nil { 25 | panic(err) 26 | } 27 | listener, err := net.Listen("tcp", "localhost:0") 28 | if err != nil { 29 | panic(err) 30 | } 31 | httpPort = listener.Addr().(*net.TCPAddr).Port 32 | go httpApp.RunHTTPServer(context.Background(), &katsubushi.Config{HTTPListener: listener}) 33 | time.Sleep(3 * time.Second) 34 | } 35 | 36 | func TestHTTPSingle(t *testing.T) { 37 | req := httptest.NewRequest("GET", "/id", nil) 38 | w := httptest.NewRecorder() 39 | 40 | httpApp.HTTPGetSingleID(w, req) 41 | if w.Code != 200 { 42 | t.Errorf("status code should be 200 but %d", w.Code) 43 | } 44 | b := new(bytes.Buffer) 45 | if _, err := io.Copy(b, w.Body); err != nil { 46 | t.Errorf("failed to read body: %v", err) 47 | } 48 | if id, err := strconv.ParseUint(b.String(), 10, 64); err != nil { 49 | t.Errorf("body should be a number uint64: %v", err) 50 | } else { 51 | t.Logf("HTTP fetched single ID: %d", id) 52 | } 53 | } 54 | 55 | func TestHTTPSingleJSON(t *testing.T) { 56 | req := httptest.NewRequest("GET", "/id", nil) 57 | req.Header.Set("Accept", "application/json") 58 | w := httptest.NewRecorder() 59 | httpApp.HTTPGetSingleID(w, req) 60 | if w.Code != 200 { 61 | t.Errorf("status code should be 200 but %d", w.Code) 62 | } 63 | v := struct { 64 | ID string `json:"id"` 65 | }{} 66 | if err := json.NewDecoder(w.Body).Decode(&v); err != nil { 67 | t.Errorf("failed to decode body: %v", err) 68 | } 69 | if id, err := strconv.ParseUint(v.ID, 10, 64); err != nil { 70 | t.Errorf("body should be a number uint64: %v", err) 71 | } else { 72 | t.Logf("HTTP fetched single ID as JSON: %d", id) 73 | } 74 | } 75 | 76 | func TestHTTPMulti(t *testing.T) { 77 | req := httptest.NewRequest("GET", "/ids?n=10", nil) 78 | w := httptest.NewRecorder() 79 | 80 | httpApp.HTTPGetMultiID(w, req) 81 | if w.Code != 200 { 82 | t.Errorf("status code should be 200 but %d", w.Code) 83 | } 84 | b := new(bytes.Buffer) 85 | if _, err := io.Copy(b, w.Body); err != nil { 86 | t.Errorf("failed to read body: %v", err) 87 | } 88 | bs := bytes.Split(b.Bytes(), []byte("\n")) 89 | if len(bs) != 10 { 90 | t.Errorf("body should contain 10 lines but %d", len(bs)) 91 | } 92 | for _, b := range bs { 93 | if id, err := strconv.ParseUint(string(b), 10, 64); err != nil { 94 | t.Errorf("body should be a number uint64: %v", err) 95 | } else { 96 | t.Logf("HTTP fetched ID: %d", id) 97 | } 98 | } 99 | } 100 | 101 | func TestHTTPMultiJSON(t *testing.T) { 102 | req := httptest.NewRequest("GET", "/ids?n=10", nil) 103 | req.Header.Set("Accept", "application/json") 104 | w := httptest.NewRecorder() 105 | 106 | httpApp.HTTPGetMultiID(w, req) 107 | if w.Code != 200 { 108 | t.Errorf("status code should be 200 but %d", w.Code) 109 | } 110 | v := struct { 111 | IDs []string `json:"ids"` 112 | }{} 113 | if err := json.NewDecoder(w.Body).Decode(&v); err != nil { 114 | t.Errorf("failed to decode body: %v", err) 115 | } 116 | if len(v.IDs) != 10 { 117 | t.Errorf("body should contain 10 lines but %d", len(v.IDs)) 118 | } 119 | for _, id := range v.IDs { 120 | if i, err := strconv.ParseUint(id, 10, 64); err != nil { 121 | t.Errorf("body should be a number uint64: %v", err) 122 | } else { 123 | t.Logf("HTTP fetched single ID as JSON: %d", i) 124 | } 125 | } 126 | } 127 | 128 | func testHTTPStats(t *testing.T) *katsubushi.MemdStats { 129 | req := httptest.NewRequest("GET", "/stats", nil) 130 | req.Header.Set("Accept", "application/json") 131 | w := httptest.NewRecorder() 132 | httpApp.HTTPGetStats(w, req) 133 | if w.Code != 200 { 134 | t.Errorf("status code should be 200 but %d", w.Code) 135 | } 136 | var s katsubushi.MemdStats 137 | if err := json.NewDecoder(w.Body).Decode(&s); err != nil { 138 | t.Errorf("failed to read body: %v", err) 139 | } 140 | t.Logf("%#v", s) 141 | return &s 142 | } 143 | 144 | func TestHTTPStats(t *testing.T) { 145 | TestHTTPSingle(t) 146 | s1 := testHTTPStats(t) 147 | 148 | TestHTTPSingle(t) 149 | s2 := testHTTPStats(t) 150 | if s2.CmdGet != s1.CmdGet+1 { 151 | t.Errorf("cmd_get should be incremented by 1 but %d", s2.CmdGet-s1.CmdGet) 152 | } 153 | if s2.GetHits != s1.GetHits+1 { 154 | t.Errorf("get_hits should be incremented by 1 but %d", s2.GetHits-s1.GetHits) 155 | } 156 | 157 | TestHTTPMulti(t) 158 | s3 := testHTTPStats(t) 159 | if s3.CmdGet != s2.CmdGet+1 { 160 | t.Errorf("cmd_get should be incremented by 10 but %d", s3.CmdGet-s2.CmdGet) 161 | } 162 | if s3.GetHits != s2.GetHits+10 { 163 | t.Errorf("get_hits should be incremented by 10 but %d", s3.GetHits-s2.GetHits) 164 | } 165 | } 166 | 167 | func TestHTTPSingleCS(t *testing.T) { 168 | u := fmt.Sprintf("http://localhost:%d", httpPort) 169 | client, err := katsubushi.NewHTTPClient([]string{u}, "") 170 | if err != nil { 171 | t.Fatal(err) 172 | } 173 | for i := 0; i < 10; i++ { 174 | id, err := client.Fetch(context.Background()) 175 | if err != nil { 176 | t.Fatal(err) 177 | } 178 | if id == 0 { 179 | t.Fatal("id should not be 0") 180 | } 181 | t.Logf("HTTP fetched single ID: %d", id) 182 | } 183 | } 184 | 185 | func TestHTTPMultiCS(t *testing.T) { 186 | u := fmt.Sprintf("http://localhost:%d", httpPort) 187 | client, err := katsubushi.NewHTTPClient([]string{u}, "") 188 | if err != nil { 189 | t.Fatal(err) 190 | } 191 | for i := 0; i < 10; i++ { 192 | ids, err := client.FetchMulti(context.Background(), 10) 193 | if err != nil { 194 | t.Fatal(err) 195 | } 196 | if len(ids) != 10 { 197 | t.Fatalf("ids should contain 10 elements %v", ids) 198 | } 199 | for _, id := range ids { 200 | if id == 0 { 201 | t.Fatal("id should not be 0") 202 | } 203 | } 204 | t.Logf("HTTP fetched IDs: %v", ids) 205 | } 206 | } 207 | 208 | func BenchmarkHTTPClientFetch(b *testing.B) { 209 | b.ResetTimer() 210 | 211 | b.RunParallel(func(pb *testing.PB) { 212 | u := fmt.Sprintf("http://localhost:%d", httpPort) 213 | c, _ := katsubushi.NewHTTPClient([]string{u}, "") 214 | for pb.Next() { 215 | id, err := c.Fetch(context.Background()) 216 | if err != nil { 217 | b.Fatal(err) 218 | } 219 | if id == 0 { 220 | b.Error("could not fetch id > 0") 221 | } 222 | } 223 | }) 224 | } 225 | -------------------------------------------------------------------------------- /cmd/katsubushi/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "log/slog" 9 | "net" 10 | "net/http" 11 | "net/http/pprof" 12 | "os" 13 | "os/signal" 14 | "strings" 15 | "sync" 16 | "syscall" 17 | "time" 18 | 19 | "github.com/fujiwara/raus" 20 | stats_api "github.com/fukata/golang-stats-api-handler" 21 | "github.com/kayac/go-katsubushi/v2" 22 | ) 23 | 24 | type profConfig struct { 25 | enablePprof bool 26 | enableStats bool 27 | debugPort int 28 | } 29 | 30 | func (pc profConfig) enabled() bool { 31 | return pc.enablePprof || pc.enableStats 32 | } 33 | 34 | func init() { 35 | raus.LockExpires = 600 * time.Second 36 | } 37 | 38 | func main() { 39 | var ( 40 | showVersion bool 41 | redisURL string 42 | minWorkerID uint 43 | maxWorkerID uint 44 | workerID uint 45 | ) 46 | pc := &profConfig{} 47 | kc := &katsubushi.Config{} 48 | 49 | flag.UintVar(&workerID, "worker-id", 0, "worker id. must be unique.") 50 | flag.IntVar(&kc.Port, "port", 11212, "port to listen.") 51 | flag.StringVar(&kc.Sockpath, "sock", "", "unix domain socket to listen. ignore port option when set this.") 52 | flag.DurationVar(&kc.IdleTimeout, "idle-timeout", katsubushi.DefaultIdleTimeout, "connection will be closed if there are no packets over the seconds. 0 means infinite.") 53 | flag.StringVar(&kc.LogLevel, "log-level", "info", "log level (panic, fatal, error, warn, info = Default, debug)") 54 | flag.IntVar(&kc.HTTPPort, "http-port", 0, "port to listen http server. 0 means disable.") 55 | flag.IntVar(&kc.GRPCPort, "grpc-port", 0, "port to listen grpc server. 0 means disable.") 56 | 57 | flag.BoolVar(&pc.enablePprof, "enable-pprof", false, "") 58 | flag.BoolVar(&pc.enableStats, "enable-stats", false, "") 59 | flag.IntVar(&pc.debugPort, "debug-port", 8080, "port to listen for debug") 60 | 61 | flag.BoolVar(&showVersion, "version", false, "show version number") 62 | flag.StringVar(&redisURL, "redis", "", "URL of Redis for automated worker id allocation") 63 | flag.UintVar(&minWorkerID, "min-worker-id", 0, "minimum automated worker id") 64 | flag.UintVar(&maxWorkerID, "max-worker-id", 0, "maximum automated worker id") 65 | flag.VisitAll(envToFlag) 66 | flag.Parse() 67 | 68 | if showVersion { 69 | fmt.Println("katsubushi version:", katsubushi.Version) 70 | return 71 | } 72 | 73 | if err := katsubushi.SetLogLevel(kc.LogLevel); err != nil { 74 | slog.Error("failed to set log level", "level", kc.LogLevel, "error", err) 75 | os.Exit(1) 76 | } 77 | 78 | var wg sync.WaitGroup 79 | ctx, cancel := context.WithCancel(context.Background()) 80 | 81 | wg.Add(1) 82 | go signalHandler(ctx, cancel, &wg) 83 | 84 | if workerID == 0 { 85 | if redisURL == "" { 86 | fmt.Println("please set -worker-id or -redis") 87 | os.Exit(1) 88 | } 89 | var err error 90 | wg.Add(1) 91 | workerID, err = assignWorkerID(ctx, &wg, redisURL, minWorkerID, maxWorkerID) 92 | if err != nil { 93 | slog.Error("failed to assign worker-id", "error", err) 94 | os.Exit(1) 95 | } 96 | } 97 | 98 | // for profiling 99 | if pc.enabled() { 100 | slog.Info("Enabling profiler") 101 | wg.Add(1) 102 | go profiler(ctx, cancel, &wg, pc) 103 | } 104 | 105 | app, err := katsubushi.New(workerID) 106 | if err != nil { 107 | slog.Error("failed to create app", "error", err) 108 | os.Exit(1) 109 | } 110 | 111 | // main server 112 | var errs []error 113 | wg.Add(1) 114 | go func() { 115 | defer wg.Done() 116 | if err := app.RunServer(ctx, kc); err != nil { 117 | errs = append(errs, err) 118 | cancel() 119 | } 120 | }() 121 | 122 | // http server 123 | if kc.HTTPPort != 0 { 124 | wg.Add(1) 125 | go func() { 126 | defer wg.Done() 127 | if err := app.RunHTTPServer(ctx, kc); err != nil { 128 | errs = append(errs, err) 129 | cancel() 130 | } 131 | }() 132 | } 133 | 134 | if kc.GRPCPort != 0 { 135 | wg.Add(1) 136 | go func() { 137 | defer wg.Done() 138 | if err := app.RunGRPCServer(ctx, kc); err != nil { 139 | errs = append(errs, err) 140 | cancel() 141 | } 142 | }() 143 | } 144 | 145 | wg.Wait() 146 | code := 0 147 | if len(errs) > 0 { 148 | for _, err := range errs { 149 | if errors.Is(err, context.Canceled) || errors.Is(err, http.ErrServerClosed) { 150 | continue 151 | } 152 | slog.Error("server error", "error", err) 153 | code = 1 154 | } 155 | } 156 | slog.Info("Shutdown completed") 157 | os.Exit(code) 158 | } 159 | 160 | func profiler(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, pc *profConfig) { 161 | defer wg.Done() 162 | 163 | mux := http.NewServeMux() 164 | if pc.enablePprof { 165 | mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 166 | mux.HandleFunc("/debug/pprof/profile", pprof.Profile) 167 | mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 168 | mux.HandleFunc("/debug/pprof/", pprof.Index) 169 | slog.Info("EnablePprof on /debug/pprof") 170 | } 171 | if pc.enableStats { 172 | mux.HandleFunc("/debug/stats", stats_api.Handler) 173 | slog.Info("EnableStats on /debug/stats") 174 | } 175 | addr := fmt.Sprintf("localhost:%d", pc.debugPort) 176 | slog.Info("Listening debugger on", "addr", addr) 177 | ln, err := net.Listen("tcp", addr) 178 | if err != nil { 179 | slog.Error("failed to listen", "addr", addr, "error", err) 180 | return 181 | } 182 | 183 | go func() { 184 | <-ctx.Done() 185 | ln.Close() 186 | }() 187 | 188 | if err := http.Serve(ln, mux); err != nil { 189 | slog.Error("failed to serve", "error", err) 190 | return 191 | } 192 | } 193 | 194 | func signalHandler(ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup) { 195 | defer wg.Done() 196 | trapSignals := []os.Signal{ 197 | syscall.SIGHUP, 198 | syscall.SIGINT, 199 | syscall.SIGTERM, 200 | syscall.SIGQUIT, 201 | } 202 | sigCh := make(chan os.Signal, 1) 203 | signal.Notify(sigCh, trapSignals...) 204 | select { 205 | case sig := <-sigCh: 206 | slog.Info("Got signal", "signal", sig) 207 | cancel() 208 | case <-ctx.Done(): 209 | } 210 | } 211 | 212 | func assignWorkerID(ctx context.Context, wg *sync.WaitGroup, redisURL string, min, max uint) (uint, error) { 213 | defer wg.Done() 214 | defaultMax := uint((1 << katsubushi.WorkerIDBits) - 1) 215 | if min == 0 { 216 | min = 1 217 | } 218 | if max == 0 { 219 | max = defaultMax 220 | } 221 | if min > max { 222 | return 0, errors.New("max-worker-id must be larger than min-worker-id") 223 | } 224 | if max > defaultMax { 225 | return 0, fmt.Errorf("max-worker-id must be smaller than %d", defaultMax) 226 | } 227 | slog.Info("Waiting for worker-id automated assignment", "min", min, "max", max, "redisURL", redisURL) 228 | r, err := raus.New(redisURL, min, max) 229 | if err != nil { 230 | slog.Error("failed to assign worker-id", "error", err) 231 | return 0, err 232 | } 233 | r.SetSlogLogger(katsubushi.SlogLogger()) 234 | id, ch, err := r.Get(ctx) 235 | if err != nil { 236 | return 0, err 237 | } 238 | slog.Info("Assigned worker-id", "id", id) 239 | 240 | wg.Add(1) 241 | go func() { 242 | defer wg.Done() 243 | err, more := <-ch 244 | if err != nil { 245 | panic(err) 246 | } 247 | if !more { 248 | // shutdown 249 | } 250 | }() 251 | return id, nil 252 | } 253 | 254 | func envToFlag(f *flag.Flag) { 255 | names := []string{ 256 | strings.ToUpper(strings.Replace(f.Name, "-", "_", -1)), 257 | strings.ToLower(strings.Replace(f.Name, "-", "_", -1)), 258 | } 259 | for _, name := range names { 260 | if s := os.Getenv(name); s != "" { 261 | f.Value.Set(s) 262 | break 263 | } 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /http.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "log/slog" 10 | "net" 11 | "net/http" 12 | "net/url" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "sync/atomic" 17 | "time" 18 | ) 19 | 20 | const ( 21 | MaxHTTPBulkSize = 1000 22 | ) 23 | 24 | func (app *App) RunHTTPServer(ctx context.Context, cfg *Config) error { 25 | mux := http.NewServeMux() 26 | mux.HandleFunc(fmt.Sprintf("/%sid", cfg.HTTPPathPrefix), app.HTTPGetSingleID) 27 | mux.HandleFunc(fmt.Sprintf("/%sids", cfg.HTTPPathPrefix), app.HTTPGetMultiID) 28 | mux.HandleFunc(fmt.Sprintf("/%sstats", cfg.HTTPPathPrefix), app.HTTPGetStats) 29 | s := &http.Server{ 30 | Handler: mux, 31 | } 32 | // shutdown 33 | go func() { 34 | <-ctx.Done() 35 | slog.Info("Shutting down HTTP server") 36 | s.Shutdown(ctx) 37 | }() 38 | 39 | listener := cfg.HTTPListener 40 | if listener == nil { 41 | var err error 42 | listener, err = net.Listen("tcp", fmt.Sprintf(":%d", cfg.HTTPPort)) 43 | if err != nil { 44 | return fmt.Errorf("failed to listen: %w", err) 45 | } 46 | } 47 | listener = app.wrapListener(listener) 48 | slog.Info("Listening HTTP server at " + listener.Addr().String()) 49 | return s.Serve(listener) 50 | } 51 | 52 | func (app *App) HTTPGetSingleID(w http.ResponseWriter, req *http.Request) { 53 | if req.Method != "GET" { 54 | w.WriteHeader(http.StatusMethodNotAllowed) 55 | return 56 | } 57 | atomic.AddInt64(&app.cmdGet, 1) 58 | slog.Debug("HTTP GetSingleID request", "remote", req.RemoteAddr) 59 | id, err := app.NextID() 60 | if err != nil { 61 | slog.Error("Failed to generate ID", "error", err) 62 | w.WriteHeader(http.StatusInternalServerError) 63 | return 64 | } 65 | slog.Debug("HTTP Generated ID", "id", id) 66 | if strings.Contains(req.Header.Get("Accept"), "application/json") { 67 | w.Header().Set("Content-Type", "application/json") 68 | fmt.Fprintf(w, `{"id":"%d"}`, id) 69 | } else { 70 | w.Header().Set("Content-Type", "text/plain") 71 | fmt.Fprintf(w, "%d", id) 72 | } 73 | } 74 | 75 | func (app *App) HTTPGetMultiID(w http.ResponseWriter, req *http.Request) { 76 | if req.Method != "GET" { 77 | w.WriteHeader(http.StatusMethodNotAllowed) 78 | return 79 | } 80 | atomic.AddInt64(&app.cmdGet, 1) 81 | slog.Debug("HTTP GetMultiID request", "remote", req.RemoteAddr) 82 | var n int64 83 | if ns := req.FormValue("n"); ns == "" { 84 | n = 1 85 | } else { 86 | var err error 87 | n, err = strconv.ParseInt(ns, 10, 64) 88 | if err != nil { 89 | slog.Error("Failed to parse n parameter", "error", err) 90 | w.WriteHeader(http.StatusBadRequest) 91 | return 92 | } 93 | } 94 | if n > MaxHTTPBulkSize { 95 | msg := fmt.Sprintf("too many IDs requested: %d, n should be smaller than %d", n, MaxHTTPBulkSize) 96 | slog.Error(msg) 97 | w.WriteHeader(http.StatusBadRequest) 98 | w.Write([]byte(msg)) 99 | return 100 | } 101 | ids := make([]string, 0, n) 102 | for i := int64(0); i < n; i++ { 103 | id, err := app.NextID() 104 | if err != nil { 105 | slog.Error("Failed to generate ID", "error", err) 106 | w.WriteHeader(http.StatusInternalServerError) 107 | return 108 | } 109 | ids = append(ids, strconv.FormatUint(id, 10)) 110 | } 111 | slog.Debug("HTTP Generated IDs", "ids", ids, "count", len(ids)) 112 | if strings.Contains(req.Header.Get("Accept"), "application/json") { 113 | w.Header().Set("Content-Type", "application/json") 114 | json.NewEncoder(w).Encode(struct { 115 | IDs []string `json:"ids"` 116 | }{ids}) 117 | } else { 118 | w.Header().Set("Content-Type", "text/plain") 119 | fmt.Fprint(w, strings.Join(ids, "\n")) 120 | } 121 | } 122 | 123 | func (app *App) HTTPGetStats(w http.ResponseWriter, req *http.Request) { 124 | if req.Method != "GET" { 125 | w.WriteHeader(http.StatusMethodNotAllowed) 126 | return 127 | } 128 | slog.Debug("HTTP GetStats request", "remote", req.RemoteAddr) 129 | s := app.GetStats() 130 | w.Header().Set("Content-Type", "application/json") 131 | enc := json.NewEncoder(w) 132 | enc.SetIndent("", " ") 133 | if err := enc.Encode(s); err != nil { 134 | slog.Error("Failed to encode stats", "error", err) 135 | w.WriteHeader(http.StatusInternalServerError) 136 | return 137 | } 138 | } 139 | 140 | type HTTPClient struct { 141 | client *http.Client 142 | urls []*url.URL 143 | pathPrefix string 144 | pool *sync.Pool 145 | } 146 | 147 | // NewHTTPClient creates HTTPClient 148 | func NewHTTPClient(urls []string, pathPrefix string) (*HTTPClient, error) { 149 | c := &HTTPClient{ 150 | client: &http.Client{ 151 | Timeout: DefaultClientTimeout, 152 | }, 153 | pool: &sync.Pool{ 154 | New: func() interface{} { 155 | return new(bytes.Buffer) 156 | }, 157 | }, 158 | } 159 | for _, _u := range urls { 160 | u, err := url.Parse(_u) 161 | if err != nil { 162 | return nil, fmt.Errorf("failed to parse URL: %s: %w", _u, err) 163 | } 164 | if u.Scheme != "http" && u.Scheme != "https" { 165 | return nil, fmt.Errorf("invalid URL scheme: %s", u.Scheme) 166 | } 167 | c.urls = append(c.urls, u) 168 | } 169 | return c, nil 170 | } 171 | 172 | // SetTimeout sets timeout to katsubushi servers 173 | func (c *HTTPClient) SetTimeout(t time.Duration) { 174 | c.client.Timeout = t 175 | } 176 | 177 | // Fetch fetches id from katsubushi via HTTP 178 | func (c *HTTPClient) Fetch(ctx context.Context) (uint64, error) { 179 | errs := fmt.Errorf("no servers available") 180 | for _, u := range c.urls { 181 | id, err := func(u *url.URL) (uint64, error) { 182 | u.Path = fmt.Sprintf("/%sid", c.pathPrefix) 183 | req, _ := http.NewRequestWithContext(ctx, "GET", u.String(), nil) 184 | resp, err := c.client.Do(req) 185 | if err != nil { 186 | return 0, err 187 | } 188 | defer resp.Body.Close() 189 | if resp.StatusCode != http.StatusOK { 190 | return 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode) 191 | } 192 | b := c.pool.Get().(*bytes.Buffer) 193 | defer func() { 194 | b.Reset() 195 | c.pool.Put(b) 196 | }() 197 | if _, err := io.Copy(b, resp.Body); err != nil { 198 | return 0, err 199 | } 200 | if id, err := strconv.ParseUint(b.String(), 10, 64); err != nil { 201 | return 0, err 202 | } else { 203 | return id, nil 204 | } 205 | }(u) 206 | if err != nil { 207 | errs = fmt.Errorf("failed to fetch id from %s: %w", u, err) 208 | } 209 | return id, nil 210 | } 211 | return 0, errs 212 | } 213 | 214 | // FetchMulti fetches multiple ids from katsubushi via HTTP 215 | func (c *HTTPClient) FetchMulti(ctx context.Context, n int) ([]uint64, error) { 216 | errs := fmt.Errorf("no servers available") 217 | ids := make([]uint64, 0, n) 218 | for _, u := range c.urls { 219 | ids, err := func(u *url.URL) ([]uint64, error) { 220 | u.Path = fmt.Sprintf("/%sids", c.pathPrefix) 221 | u.RawQuery = fmt.Sprintf("n=%d", n) 222 | req, _ := http.NewRequestWithContext(ctx, "GET", u.String(), nil) 223 | resp, err := c.client.Do(req) 224 | if err != nil { 225 | return nil, err 226 | } 227 | defer resp.Body.Close() 228 | if resp.StatusCode != http.StatusOK { 229 | return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) 230 | } 231 | 232 | b := c.pool.Get().(*bytes.Buffer) 233 | defer func() { 234 | b.Reset() 235 | c.pool.Put(b) 236 | }() 237 | if _, err := io.Copy(b, resp.Body); err != nil { 238 | return nil, err 239 | 240 | } 241 | bs := bytes.Split(b.Bytes(), []byte("\n")) 242 | if len(bs) != n { 243 | return nil, err 244 | } 245 | for _, b := range bs { 246 | if id, err := strconv.ParseUint(string(b), 10, 64); err != nil { 247 | return nil, err 248 | } else { 249 | ids = append(ids, id) 250 | } 251 | } 252 | return ids, nil 253 | }(u) 254 | if err != nil { 255 | errs = fmt.Errorf("failed to fetch ids from %s: %w", u, errs) 256 | } 257 | return ids, nil 258 | } 259 | return nil, errs 260 | } 261 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [v2.2.1](https://github.com/kayac/go-katsubushi/compare/v2.2.0...v2.2.1) - 2025-08-02 4 | - Improve logging performance and consistency by @fujiwara in https://github.com/kayac/go-katsubushi/pull/78 5 | 6 | ## [v2.2.0](https://github.com/kayac/go-katsubushi/compare/v2.1.0...v2.2.0) - 2025-08-01 7 | - Replace zap with slog and maintain original log format by @fujiwara in https://github.com/kayac/go-katsubushi/pull/73 8 | - Replace pkg/errors with fmt.Errorf and update raus to v0.2.0 by @fujiwara in https://github.com/kayac/go-katsubushi/pull/75 9 | - Update Alpine Linux base image to 3.22 by @fujiwara in https://github.com/kayac/go-katsubushi/pull/76 10 | - Add gRPC health check endpoint by @fujiwara in https://github.com/kayac/go-katsubushi/pull/77 11 | 12 | ## [v2.1.0](https://github.com/kayac/go-katsubushi/compare/v2.0.4...v2.1.0) - 2025-06-22 13 | - Migrate to goreleaser v2 by @handlename in https://github.com/kayac/go-katsubushi/pull/66 14 | - Pin Action versions by @handlename in https://github.com/kayac/go-katsubushi/pull/67 15 | - Introduce tagpr by @handlename in https://github.com/kayac/go-katsubushi/pull/68 16 | - Build with go1.24 by @handlename in https://github.com/kayac/go-katsubushi/pull/70 17 | - Setup tools with aqua, generate codes using buf by @handlename in https://github.com/kayac/go-katsubushi/pull/71 18 | - Push latest tag by @handlename in https://github.com/kayac/go-katsubushi/pull/72 19 | 20 | ## [v2.0.4](https://github.com/kayac/go-katsubushi/compare/v2.0.3...v2.0.4) - 2024-03-14 21 | - Bump google.golang.org/grpc from 1.49.0 to 1.56.3 by @dependabot in https://github.com/kayac/go-katsubushi/pull/63 22 | - Bump google.golang.org/protobuf from 1.30.0 to 1.33.0 by @dependabot in https://github.com/kayac/go-katsubushi/pull/64 23 | - bump Go and actions versions by @fujiwara in https://github.com/kayac/go-katsubushi/pull/65 24 | 25 | ## [v2.0.3](https://github.com/kayac/go-katsubushi/compare/v2.0.2...v2.0.3) - 2023-10-17 26 | - Bump golang.org/x/net from 0.7.0 to 0.17.0 by @dependabot in https://github.com/kayac/go-katsubushi/pull/61 27 | 28 | ## [v2.0.2](https://github.com/kayac/go-katsubushi/compare/v2.0.1...v2.0.2) - 2023-09-06 29 | - add document of redis cluster by @soh335 in https://github.com/kayac/go-katsubushi/pull/59 30 | - Bump golang.org/x/net from 0.0.0-20221002022538-bcab6841153b to 0.7.0 by @dependabot in https://github.com/kayac/go-katsubushi/pull/60 31 | 32 | ## [v2.0.1](https://github.com/kayac/go-katsubushi/compare/v2.0.0...v2.0.1) - 2022-11-16 33 | - Fix major version module name by @tamiflu in https://github.com/kayac/go-katsubushi/pull/58 34 | 35 | ## [v2.0.0](https://github.com/kayac/go-katsubushi/compare/1.7.0...v2.0.0) - 2022-10-11 36 | - add http-server by @fujiwara in https://github.com/kayac/go-katsubushi/pull/50 37 | - Add katsubushi.HTTPClient by @fujiwara in https://github.com/kayac/go-katsubushi/pull/51 38 | - add gRPC server by @fujiwara in https://github.com/kayac/go-katsubushi/pull/52 39 | - refactor listeners by @fujiwara in https://github.com/kayac/go-katsubushi/pull/53 40 | - add ctx to client.Fetch, httpClient.Fetch by @fujiwara in https://github.com/kayac/go-katsubushi/pull/54 41 | - add gRPC stats service. by @fujiwara in https://github.com/kayac/go-katsubushi/pull/56 42 | - use goreleaser for release workflow by @fujiwara in https://github.com/kayac/go-katsubushi/pull/55 43 | - V2 by @fujiwara in https://github.com/kayac/go-katsubushi/pull/57 44 | 45 | ## [v1.7.0](https://github.com/kayac/go-katsubushi/compare/1.6.2...v1.7.0) - 2022-08-16 46 | - Redis cluster support by @fujiwara in https://github.com/kayac/go-katsubushi/pull/49 47 | 48 | ## [1.7.0](https://github.com/kayac/go-katsubushi/compare/1.6.2...1.7.0) - 2022-08-16 49 | - Redis cluster support by @fujiwara in https://github.com/kayac/go-katsubushi/pull/49 50 | 51 | ## [1.6.2](https://github.com/kayac/go-katsubushi/compare/v1.6.1.1...1.6.2) - 2022-01-28 52 | - bump actions/checkout v2 by @shogo82148 in https://github.com/kayac/go-katsubushi/pull/46 53 | 54 | ## [v1.6.2](https://github.com/kayac/go-katsubushi/compare/v1.6.1.1...v1.6.2) - 2022-01-28 55 | - bump actions/checkout v2 by @shogo82148 in https://github.com/kayac/go-katsubushi/pull/46 56 | 57 | ## [v1.6.1](https://github.com/kayac/go-katsubushi/compare/1.6.0.1...v1.6.1) - 2022-01-21 58 | - build with go v1.17 by @handlename in https://github.com/kayac/go-katsubushi/pull/44 59 | - docker image support apple m1 by @ken39arg in https://github.com/kayac/go-katsubushi/pull/43 60 | - fix the typo `katubushi => katsubushi` by @d-tsuji in https://github.com/kayac/go-katsubushi/pull/42 61 | - bump Go version by @fujiwara in https://github.com/kayac/go-katsubushi/pull/45 62 | 63 | ## [1.6.1](https://github.com/kayac/go-katsubushi/compare/1.6.0.1...1.6.1) - 2022-01-21 64 | - build with go v1.17 by @handlename in https://github.com/kayac/go-katsubushi/pull/44 65 | - docker image support apple m1 by @ken39arg in https://github.com/kayac/go-katsubushi/pull/43 66 | - fix the typo `katubushi => katsubushi` by @d-tsuji in https://github.com/kayac/go-katsubushi/pull/42 67 | - bump Go version by @fujiwara in https://github.com/kayac/go-katsubushi/pull/45 68 | 69 | ## [v1.6.0](https://github.com/kayac/go-katsubushi/compare/v1.5.4...v1.6.0) - 2020-03-18 70 | - implement STAT via binary protocol by @Songmu in https://github.com/kayac/go-katsubushi/pull/40 71 | 72 | ## [v1.5.4](https://github.com/kayac/go-katsubushi/compare/v1.5.3...v1.5.4) - 2019-03-20 73 | - fix the typo `sequense => sequence` by @hbin in https://github.com/kayac/go-katsubushi/pull/38 74 | - migrate to go module by @fujiwara in https://github.com/kayac/go-katsubushi/pull/39 75 | 76 | ## [v1.5.3](https://github.com/kayac/go-katsubushi/compare/v1.5.2...v1.5.3) - 2018-06-22 77 | - Ignore errors after a server shutdown. by @fujiwara in https://github.com/kayac/go-katsubushi/pull/37 78 | 79 | ## [v1.5.2](https://github.com/kayac/go-katsubushi/compare/v1.5.1...v1.5.2) - 2018-06-19 80 | - Suppress error output by warning at deadline exceeded. by @fujiwara in https://github.com/kayac/go-katsubushi/pull/36 81 | - Client timeout by @fujiwara in https://github.com/kayac/go-katsubushi/pull/35 82 | 83 | ## [v1.5.1](https://github.com/kayac/go-katsubushi/compare/v1.5.0.1...v1.5.1) - 2018-03-23 84 | - Suppress error logs on EOF just after connected. by @fujiwara in https://github.com/kayac/go-katsubushi/pull/34 85 | 86 | ## [v1.5.0](https://github.com/kayac/go-katsubushi/compare/v1.4.3...v1.5.0) - 2018-02-22 87 | - implement binary protocol (minimum) by @xorphitus in https://github.com/kayac/go-katsubushi/pull/29 88 | - Output scanner.Err() after a request read. by @fujiwara in https://github.com/kayac/go-katsubushi/pull/30 89 | - Remove SetIdleTimeout(). by @fujiwara in https://github.com/kayac/go-katsubushi/pull/27 90 | - supported various options when `docker run` by @tkuchiki in https://github.com/kayac/go-katsubushi/pull/32 91 | 92 | ## [v1.4.3](https://github.com/kayac/go-katsubushi/compare/v1.4.2.1...v1.4.3) - 2017-12-18 93 | - Fix goroutine leak. by @fujiwara in https://github.com/kayac/go-katsubushi/pull/28 94 | 95 | ## [v1.4.2](https://github.com/kayac/go-katsubushi/compare/v1.4.1...v1.4.2) - 2017-11-20 96 | - Reduce memory allocation by @fujiwara in https://github.com/kayac/go-katsubushi/pull/22 97 | - vendoring by dep by @fujiwara in https://github.com/kayac/go-katsubushi/pull/23 98 | - travis: update go versions by @dvrkps in https://github.com/kayac/go-katsubushi/pull/24 99 | - Fix race condition of STATS command by @shogo82148 in https://github.com/kayac/go-katsubushi/pull/25 100 | - Fix/fail on 1.8.x by @fujiwara in https://github.com/kayac/go-katsubushi/pull/26 101 | 102 | ## [v1.4.1](https://github.com/kayac/go-katsubushi/compare/v1.4.0...v1.4.1) - 2017-09-07 103 | - add katsubushi-dump command by @fujiwara in https://github.com/kayac/go-katsubushi/pull/20 104 | - use Monotonic time by @fujiwara in https://github.com/kayac/go-katsubushi/pull/21 105 | 106 | ## [v1.4.0](https://github.com/kayac/go-katsubushi/compare/v1.3.0...v1.4.0) - 2017-08-23 107 | - fix generate algorithm by @fujiwara in https://github.com/kayac/go-katsubushi/pull/18 108 | - replace memcache client with own implementation by @fujiwara in https://github.com/kayac/go-katsubushi/pull/19 109 | -------------------------------------------------------------------------------- /binary_protocol.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "log/slog" 9 | "net" 10 | "reflect" 11 | "strconv" 12 | "sync/atomic" 13 | ) 14 | 15 | const ( 16 | headerSize = 24 17 | magicRequest = 0x80 18 | magicResponse = 0x81 19 | opcodeGet = 0x00 20 | opcodeVersion = 0x0b 21 | opcodeStat = 0x10 22 | ) 23 | 24 | type bRequest struct { 25 | magic byte 26 | opcode byte 27 | dataType byte 28 | vBucket [2]byte 29 | opaque [4]byte 30 | cas [8]byte 31 | extras []byte 32 | key string 33 | value string 34 | } 35 | 36 | func newBRequest(r io.Reader) (req *bRequest, err error) { 37 | req = &bRequest{} 38 | buf := make([]byte, headerSize) 39 | n, e := io.ReadFull(r, buf) 40 | if n == 0 { 41 | return nil, io.EOF 42 | } else if n < headerSize { 43 | return nil, fmt.Errorf("binary request header is shorter than %d: %x", headerSize, buf) 44 | } 45 | if e != nil { 46 | return nil, fmt.Errorf("failed to read binary request header: %s", e) 47 | } 48 | 49 | req.magic = buf[0] 50 | if req.magic == 0x00 { 51 | return nil, io.EOF 52 | } else if req.magic != magicRequest { 53 | return nil, fmt.Errorf("invalid request magic: %x", req.magic) 54 | } 55 | 56 | req.opcode = buf[1] 57 | req.dataType = buf[5] 58 | req.vBucket[0] = buf[6] 59 | req.vBucket[1] = buf[7] 60 | req.opaque[0] = buf[12] 61 | req.opaque[1] = buf[13] 62 | req.opaque[2] = buf[14] 63 | req.opaque[3] = buf[15] 64 | req.cas[0] = buf[16] 65 | req.cas[1] = buf[17] 66 | req.cas[2] = buf[18] 67 | req.cas[3] = buf[19] 68 | req.cas[4] = buf[20] 69 | req.cas[5] = buf[21] 70 | req.cas[6] = buf[22] 71 | req.cas[7] = buf[23] 72 | 73 | keyLen := binary.BigEndian.Uint16(buf[2:4]) 74 | extraLen := uint8(buf[4]) 75 | bodyLen := binary.BigEndian.Uint32(buf[8:12]) 76 | 77 | if bodyLen < uint32(keyLen)+uint32(extraLen) { 78 | return nil, fmt.Errorf("total body %d is too small. key length: %d, extra length %d", bodyLen, keyLen, extraLen) 79 | } 80 | 81 | bodyBuf := make([]byte, bodyLen) 82 | _, e2 := io.ReadFull(r, bodyBuf) 83 | if e2 != nil { 84 | return nil, fmt.Errorf("failed to read binary request body: %s", e2) 85 | } 86 | 87 | req.extras = bodyBuf[0:extraLen] 88 | req.key = string(bodyBuf[extraLen : uint16(extraLen)+keyLen]) 89 | req.value = string(bodyBuf[uint16(extraLen)+keyLen : bodyLen]) 90 | 91 | return 92 | } 93 | 94 | type bResponse struct { 95 | magic byte 96 | opcode byte 97 | dataType byte 98 | status [2]byte 99 | opaque [4]byte 100 | cas [8]byte 101 | extras []byte 102 | key string 103 | value string 104 | } 105 | 106 | type bResponseConfig struct { 107 | status [2]byte 108 | cas [8]byte 109 | extras []byte 110 | key string 111 | value string 112 | } 113 | 114 | func newBResponse(opcode byte, opaque [4]byte, resConf bResponseConfig) *bResponse { 115 | var extras []byte 116 | if resConf.extras == nil { 117 | extras = []byte{} 118 | } else { 119 | extras = resConf.extras 120 | } 121 | 122 | return &bResponse{ 123 | magic: magicResponse, 124 | opcode: opcode, 125 | dataType: 0x00, // data type: reserved for future 126 | status: resConf.status, 127 | opaque: opaque, 128 | cas: resConf.cas, 129 | extras: extras, 130 | key: resConf.key, 131 | value: resConf.value, 132 | } 133 | } 134 | 135 | func (res bResponse) Bytes() []byte { 136 | extraLen := len(res.extras) 137 | keyLen := len(res.key) 138 | valueLen := len(res.value) 139 | totalLen := headerSize + extraLen + keyLen + valueLen 140 | 141 | keyLenBytes := make([]byte, 2) 142 | binary.BigEndian.PutUint16(keyLenBytes, uint16(keyLen)) 143 | 144 | extraLenByte := byte(extraLen) 145 | 146 | bodyLenBytes := make([]byte, 4) 147 | binary.BigEndian.PutUint32(bodyLenBytes, uint32(extraLen+keyLen+valueLen)) 148 | 149 | data := make([]byte, totalLen) 150 | data[0] = res.magic 151 | data[1] = res.opcode 152 | data[2] = keyLenBytes[0] 153 | data[3] = keyLenBytes[1] 154 | data[4] = extraLenByte 155 | data[5] = res.dataType 156 | data[6] = res.status[0] 157 | data[7] = res.status[1] 158 | data[8] = bodyLenBytes[0] 159 | data[9] = bodyLenBytes[1] 160 | data[10] = bodyLenBytes[2] 161 | data[11] = bodyLenBytes[3] 162 | data[12] = res.opaque[0] 163 | data[13] = res.opaque[1] 164 | data[14] = res.opaque[2] 165 | data[15] = res.opaque[3] 166 | data[16] = res.cas[0] 167 | data[17] = res.cas[1] 168 | data[18] = res.cas[2] 169 | data[19] = res.cas[3] 170 | data[20] = res.cas[4] 171 | data[21] = res.cas[5] 172 | data[22] = res.cas[6] 173 | data[23] = res.cas[7] 174 | 175 | copy(data[headerSize:], res.extras) 176 | copy(data[headerSize+extraLen:], res.key) 177 | copy(data[headerSize+extraLen+keyLen:], res.value) 178 | 179 | return data 180 | } 181 | 182 | // IsBinaryProtocol judges whether a protocol is binary or text 183 | func (app *App) IsBinaryProtocol(r *bufio.Reader) (bool, error) { 184 | firstByte, err := r.Peek(1) 185 | if err != nil { 186 | return false, err 187 | } 188 | return firstByte[0] == magicRequest, nil 189 | } 190 | 191 | // RespondToBinary responds to a binary request with a binary response. 192 | // A request should be read from r, not conn. 193 | // Because the request reader might be buffered. 194 | func (app *App) RespondToBinary(r io.Reader, conn net.Conn) { 195 | for { 196 | app.extendDeadline(conn) 197 | 198 | req, err := newBRequest(r) 199 | if err != nil { 200 | if err != io.EOF { 201 | slog.Warn("Failed to read binary request", "error", err) 202 | } 203 | return 204 | } 205 | 206 | cmd, err := app.BytesToBinaryCmd(*req) 207 | if err != nil { 208 | if err := app.writeBinaryError(conn); err != nil { 209 | slog.Warn("error on write error", "error", err) 210 | return 211 | } 212 | continue 213 | } 214 | w := bufio.NewWriter(conn) 215 | if err := cmd.Execute(app, w); err != nil { 216 | slog.Warn("error on execute cmd", "cmd", fmt.Sprintf("%v", cmd), "error", err) 217 | return 218 | } 219 | if err := w.Flush(); err != nil { 220 | if err != io.EOF { 221 | slog.Warn("error on cmd write", "cmd", fmt.Sprintf("%v", cmd), "error", err) 222 | } 223 | return 224 | } 225 | } 226 | } 227 | 228 | func (app *App) writeBinaryError(w io.Writer) error { 229 | // TODO: Opcode, Opaque and Status are static and not accurate. It's better to make them dynamic 230 | // opcode: GET, it should be a requested opcode 231 | // opaque: zero padding, it should be a requested opaque 232 | res := newBResponse(opcodeGet, [4]byte{0x00, 0x00, 0x00, 0x00}, bResponseConfig{ 233 | // status: Internal Error, it should be determined by a request or server condition 234 | status: [2]byte{0x00, 0x84}, 235 | }) 236 | 237 | n, err := w.Write(res.Bytes()) 238 | if n < len(respError) { 239 | return fmt.Errorf("failed to write error response") 240 | } 241 | return err 242 | } 243 | 244 | // BytesToCmd converts byte array to a MemdBCmd and returns it. 245 | func (app *App) BytesToBinaryCmd(req bRequest) (cmd MemdCmd, err error) { 246 | switch req.opcode { 247 | case opcodeGet: 248 | atomic.AddInt64(&(app.cmdGet), 1) 249 | cmd = &MemdBCmdGet{ 250 | Name: "GET", 251 | Key: req.key, 252 | Opaque: req.opaque, 253 | } 254 | case opcodeVersion: 255 | cmd = &MemdBCmdVersion{ 256 | Opaque: req.opaque, 257 | } 258 | case opcodeStat: 259 | cmd = &MemdBCmdStat{ 260 | Key: req.key, 261 | Opaque: req.opaque, 262 | } 263 | default: 264 | err = fmt.Errorf("unknown binary command: %x", req.opcode) 265 | } 266 | return 267 | } 268 | 269 | // MemdCmdGet defines binary Get command. 270 | type MemdBCmdGet struct { 271 | Name string 272 | Key string 273 | Opaque [4]byte 274 | } 275 | 276 | // Execute generates new ID. 277 | func (cmd *MemdBCmdGet) Execute(app *App, w io.Writer) error { 278 | id, err := app.NextID() 279 | if err != nil { 280 | slog.Warn("Failed to generate ID", "error", err) 281 | if err = app.writeError(w); err != nil { 282 | slog.Warn("error on write error", "error", err) 283 | return err 284 | } 285 | return nil 286 | } 287 | slog.Debug("Generated ID", "id", id) 288 | 289 | res := newBResponse(opcodeGet, cmd.Opaque, bResponseConfig{ 290 | // fixed 4bytes flags is given to GET response 291 | extras: []byte{0x00, 0x00, 0x00, 0x00}, 292 | value: strconv.FormatUint(id, 10), 293 | }) 294 | 295 | _, err2 := w.Write(res.Bytes()) 296 | return err2 297 | } 298 | 299 | // MemdBCmdVersion defines binary VERSION command. 300 | type MemdBCmdVersion struct { 301 | Opaque [4]byte 302 | } 303 | 304 | // Execute writes binary Version number. 305 | func (cmd MemdBCmdVersion) Execute(app *App, w io.Writer) error { 306 | res := newBResponse(opcodeVersion, cmd.Opaque, bResponseConfig{ 307 | value: Version, 308 | }) 309 | 310 | _, err := w.Write(res.Bytes()) 311 | return err 312 | } 313 | 314 | // MemdCmdStat defines binary Stat command. 315 | type MemdBCmdStat struct { 316 | Key string 317 | Opaque [4]byte 318 | } 319 | 320 | // Execute writes binary stat 321 | // ref. https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#stat 322 | func (cmd *MemdBCmdStat) Execute(app *App, w io.Writer) error { 323 | // ignore optional key (items, slabs) for now 324 | s := app.GetStats() 325 | return s.writeBinaryTo(w, cmd.Opaque) 326 | } 327 | 328 | func (s MemdStats) writeBinaryTo(w io.Writer, opaque [4]byte) error { 329 | statsValue := reflect.ValueOf(s) 330 | statsType := reflect.TypeOf(s) 331 | for i := 0; i < statsType.NumField(); i++ { 332 | field := statsType.Field(i) 333 | tag := field.Tag.Get("memd") 334 | if tag == "" { 335 | continue 336 | } 337 | var val string 338 | v := statsValue.FieldByIndex(field.Index).Interface() 339 | switch _v := v.(type) { 340 | case int: 341 | val = strconv.Itoa(_v) 342 | case int64: 343 | val = strconv.FormatInt(int64(_v), 10) 344 | case string: 345 | val = string(_v) 346 | } 347 | res := newBResponse(opcodeStat, opaque, bResponseConfig{ 348 | key: tag, 349 | value: val, 350 | }) 351 | if _, err := w.Write(res.Bytes()); err != nil { 352 | return err 353 | } 354 | } 355 | // to teminate the sequence 356 | emptyRes := newBResponse(opcodeStat, opaque, bResponseConfig{}) 357 | _, err := w.Write(emptyRes.Bytes()) 358 | return err 359 | } 360 | -------------------------------------------------------------------------------- /grpc/main_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.5.1 4 | // - protoc (unknown) 5 | // source: main.proto 6 | 7 | package grpc 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file 17 | // is compatible with the grpc package it is being compiled against. 18 | // Requires gRPC-Go v1.64.0 or later. 19 | const _ = grpc.SupportPackageIsVersion9 20 | 21 | const ( 22 | Generator_Fetch_FullMethodName = "/katsubushi.Generator/Fetch" 23 | Generator_FetchMulti_FullMethodName = "/katsubushi.Generator/FetchMulti" 24 | ) 25 | 26 | // GeneratorClient is the client API for Generator service. 27 | // 28 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 29 | type GeneratorClient interface { 30 | Fetch(ctx context.Context, in *FetchRequest, opts ...grpc.CallOption) (*FetchResponse, error) 31 | FetchMulti(ctx context.Context, in *FetchMultiRequest, opts ...grpc.CallOption) (*FetchMultiResponse, error) 32 | } 33 | 34 | type generatorClient struct { 35 | cc grpc.ClientConnInterface 36 | } 37 | 38 | func NewGeneratorClient(cc grpc.ClientConnInterface) GeneratorClient { 39 | return &generatorClient{cc} 40 | } 41 | 42 | func (c *generatorClient) Fetch(ctx context.Context, in *FetchRequest, opts ...grpc.CallOption) (*FetchResponse, error) { 43 | cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) 44 | out := new(FetchResponse) 45 | err := c.cc.Invoke(ctx, Generator_Fetch_FullMethodName, in, out, cOpts...) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return out, nil 50 | } 51 | 52 | func (c *generatorClient) FetchMulti(ctx context.Context, in *FetchMultiRequest, opts ...grpc.CallOption) (*FetchMultiResponse, error) { 53 | cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) 54 | out := new(FetchMultiResponse) 55 | err := c.cc.Invoke(ctx, Generator_FetchMulti_FullMethodName, in, out, cOpts...) 56 | if err != nil { 57 | return nil, err 58 | } 59 | return out, nil 60 | } 61 | 62 | // GeneratorServer is the server API for Generator service. 63 | // All implementations must embed UnimplementedGeneratorServer 64 | // for forward compatibility. 65 | type GeneratorServer interface { 66 | Fetch(context.Context, *FetchRequest) (*FetchResponse, error) 67 | FetchMulti(context.Context, *FetchMultiRequest) (*FetchMultiResponse, error) 68 | mustEmbedUnimplementedGeneratorServer() 69 | } 70 | 71 | // UnimplementedGeneratorServer must be embedded to have 72 | // forward compatible implementations. 73 | // 74 | // NOTE: this should be embedded by value instead of pointer to avoid a nil 75 | // pointer dereference when methods are called. 76 | type UnimplementedGeneratorServer struct{} 77 | 78 | func (UnimplementedGeneratorServer) Fetch(context.Context, *FetchRequest) (*FetchResponse, error) { 79 | return nil, status.Errorf(codes.Unimplemented, "method Fetch not implemented") 80 | } 81 | func (UnimplementedGeneratorServer) FetchMulti(context.Context, *FetchMultiRequest) (*FetchMultiResponse, error) { 82 | return nil, status.Errorf(codes.Unimplemented, "method FetchMulti not implemented") 83 | } 84 | func (UnimplementedGeneratorServer) mustEmbedUnimplementedGeneratorServer() {} 85 | func (UnimplementedGeneratorServer) testEmbeddedByValue() {} 86 | 87 | // UnsafeGeneratorServer may be embedded to opt out of forward compatibility for this service. 88 | // Use of this interface is not recommended, as added methods to GeneratorServer will 89 | // result in compilation errors. 90 | type UnsafeGeneratorServer interface { 91 | mustEmbedUnimplementedGeneratorServer() 92 | } 93 | 94 | func RegisterGeneratorServer(s grpc.ServiceRegistrar, srv GeneratorServer) { 95 | // If the following call pancis, it indicates UnimplementedGeneratorServer was 96 | // embedded by pointer and is nil. This will cause panics if an 97 | // unimplemented method is ever invoked, so we test this at initialization 98 | // time to prevent it from happening at runtime later due to I/O. 99 | if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { 100 | t.testEmbeddedByValue() 101 | } 102 | s.RegisterService(&Generator_ServiceDesc, srv) 103 | } 104 | 105 | func _Generator_Fetch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 106 | in := new(FetchRequest) 107 | if err := dec(in); err != nil { 108 | return nil, err 109 | } 110 | if interceptor == nil { 111 | return srv.(GeneratorServer).Fetch(ctx, in) 112 | } 113 | info := &grpc.UnaryServerInfo{ 114 | Server: srv, 115 | FullMethod: Generator_Fetch_FullMethodName, 116 | } 117 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 118 | return srv.(GeneratorServer).Fetch(ctx, req.(*FetchRequest)) 119 | } 120 | return interceptor(ctx, in, info, handler) 121 | } 122 | 123 | func _Generator_FetchMulti_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 124 | in := new(FetchMultiRequest) 125 | if err := dec(in); err != nil { 126 | return nil, err 127 | } 128 | if interceptor == nil { 129 | return srv.(GeneratorServer).FetchMulti(ctx, in) 130 | } 131 | info := &grpc.UnaryServerInfo{ 132 | Server: srv, 133 | FullMethod: Generator_FetchMulti_FullMethodName, 134 | } 135 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 136 | return srv.(GeneratorServer).FetchMulti(ctx, req.(*FetchMultiRequest)) 137 | } 138 | return interceptor(ctx, in, info, handler) 139 | } 140 | 141 | // Generator_ServiceDesc is the grpc.ServiceDesc for Generator service. 142 | // It's only intended for direct use with grpc.RegisterService, 143 | // and not to be introspected or modified (even as a copy) 144 | var Generator_ServiceDesc = grpc.ServiceDesc{ 145 | ServiceName: "katsubushi.Generator", 146 | HandlerType: (*GeneratorServer)(nil), 147 | Methods: []grpc.MethodDesc{ 148 | { 149 | MethodName: "Fetch", 150 | Handler: _Generator_Fetch_Handler, 151 | }, 152 | { 153 | MethodName: "FetchMulti", 154 | Handler: _Generator_FetchMulti_Handler, 155 | }, 156 | }, 157 | Streams: []grpc.StreamDesc{}, 158 | Metadata: "main.proto", 159 | } 160 | 161 | const ( 162 | Stats_Get_FullMethodName = "/katsubushi.Stats/Get" 163 | ) 164 | 165 | // StatsClient is the client API for Stats service. 166 | // 167 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 168 | type StatsClient interface { 169 | Get(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error) 170 | } 171 | 172 | type statsClient struct { 173 | cc grpc.ClientConnInterface 174 | } 175 | 176 | func NewStatsClient(cc grpc.ClientConnInterface) StatsClient { 177 | return &statsClient{cc} 178 | } 179 | 180 | func (c *statsClient) Get(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error) { 181 | cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) 182 | out := new(StatsResponse) 183 | err := c.cc.Invoke(ctx, Stats_Get_FullMethodName, in, out, cOpts...) 184 | if err != nil { 185 | return nil, err 186 | } 187 | return out, nil 188 | } 189 | 190 | // StatsServer is the server API for Stats service. 191 | // All implementations must embed UnimplementedStatsServer 192 | // for forward compatibility. 193 | type StatsServer interface { 194 | Get(context.Context, *StatsRequest) (*StatsResponse, error) 195 | mustEmbedUnimplementedStatsServer() 196 | } 197 | 198 | // UnimplementedStatsServer must be embedded to have 199 | // forward compatible implementations. 200 | // 201 | // NOTE: this should be embedded by value instead of pointer to avoid a nil 202 | // pointer dereference when methods are called. 203 | type UnimplementedStatsServer struct{} 204 | 205 | func (UnimplementedStatsServer) Get(context.Context, *StatsRequest) (*StatsResponse, error) { 206 | return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") 207 | } 208 | func (UnimplementedStatsServer) mustEmbedUnimplementedStatsServer() {} 209 | func (UnimplementedStatsServer) testEmbeddedByValue() {} 210 | 211 | // UnsafeStatsServer may be embedded to opt out of forward compatibility for this service. 212 | // Use of this interface is not recommended, as added methods to StatsServer will 213 | // result in compilation errors. 214 | type UnsafeStatsServer interface { 215 | mustEmbedUnimplementedStatsServer() 216 | } 217 | 218 | func RegisterStatsServer(s grpc.ServiceRegistrar, srv StatsServer) { 219 | // If the following call pancis, it indicates UnimplementedStatsServer was 220 | // embedded by pointer and is nil. This will cause panics if an 221 | // unimplemented method is ever invoked, so we test this at initialization 222 | // time to prevent it from happening at runtime later due to I/O. 223 | if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { 224 | t.testEmbeddedByValue() 225 | } 226 | s.RegisterService(&Stats_ServiceDesc, srv) 227 | } 228 | 229 | func _Stats_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 230 | in := new(StatsRequest) 231 | if err := dec(in); err != nil { 232 | return nil, err 233 | } 234 | if interceptor == nil { 235 | return srv.(StatsServer).Get(ctx, in) 236 | } 237 | info := &grpc.UnaryServerInfo{ 238 | Server: srv, 239 | FullMethod: Stats_Get_FullMethodName, 240 | } 241 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 242 | return srv.(StatsServer).Get(ctx, req.(*StatsRequest)) 243 | } 244 | return interceptor(ctx, in, info, handler) 245 | } 246 | 247 | // Stats_ServiceDesc is the grpc.ServiceDesc for Stats service. 248 | // It's only intended for direct use with grpc.RegisterService, 249 | // and not to be introspected or modified (even as a copy) 250 | var Stats_ServiceDesc = grpc.ServiceDesc{ 251 | ServiceName: "katsubushi.Stats", 252 | HandlerType: (*StatsServer)(nil), 253 | Methods: []grpc.MethodDesc{ 254 | { 255 | MethodName: "Get", 256 | Handler: _Stats_Get_Handler, 257 | }, 258 | }, 259 | Streams: []grpc.StreamDesc{}, 260 | Metadata: "main.proto", 261 | } 262 | -------------------------------------------------------------------------------- /binary_protocol_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "reflect" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestBResponseBytes(t *testing.T) { 12 | res := bResponse{ 13 | magic: 0x81, 14 | opcode: 0x00, 15 | dataType: 0x00, 16 | status: [2]byte{0x00, 0x00}, 17 | opaque: [4]byte{0x00, 0x00, 0x00, 0x00}, 18 | cas: [8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}, 19 | extras: []byte{0xde, 0xad, 0xbe, 0xef}, 20 | key: "", 21 | value: "World", 22 | } 23 | actual := res.Bytes() 24 | 25 | expected := []byte{ 26 | 0x81, 0x00, 0x00, 0x00, 27 | 0x04, 0x00, 0x00, 0x00, 28 | 0x00, 0x00, 0x00, 0x09, 29 | 0x00, 0x00, 0x00, 0x00, 30 | 0x00, 0x00, 0x00, 0x00, 31 | 0x00, 0x00, 0x00, 0x01, 32 | 0xde, 0xad, 0xbe, 0xef, 33 | 0x57, 0x6f, 0x72, 0x6c, 34 | 0x64, 35 | } 36 | 37 | if bytes.Compare(actual, expected) != 0 { 38 | t.Errorf("Unextected byte slice: %s", actual) 39 | } 40 | } 41 | 42 | func TestNewBRequest(t *testing.T) { 43 | { 44 | input := []byte{ 45 | 0x80, 0x02, 0x00, 0x05, 46 | 0x08, 0x00, 0x00, 0x00, 47 | 0x00, 0x00, 0x00, 0x12, 48 | 0x00, 0x00, 0x00, 0x00, 49 | 0x00, 0x00, 0x00, 0x00, 50 | 0x00, 0x00, 0x00, 0x00, 51 | 0xde, 0xad, 0xbe, 0xef, 52 | 0x00, 0x00, 0x1c, 0x20, 53 | 0x48, 0x65, 0x6c, 0x6c, 54 | 0x6f, 0x57, 0x6f, 0x72, 55 | 0x6c, 0x64, 56 | } 57 | br := bytes.NewReader(input) 58 | req, err := newBRequest(br) 59 | 60 | if err != nil { 61 | t.Errorf("Failed to parse request: %s", err) 62 | } 63 | 64 | if req.magic != 0x80 { 65 | t.Errorf("Unexpected magic: %x", req.magic) 66 | } 67 | 68 | if req.opcode != 0x02 { 69 | t.Errorf("Unexpected opcode: %x", req.opcode) 70 | } 71 | 72 | if req.dataType != 0x00 { 73 | t.Errorf("Unexpected data type: %x", req.dataType) 74 | } 75 | 76 | if bytes.Compare(req.vBucket[:], []byte{0x00, 0x00}) != 0 { 77 | t.Errorf("Unexpected VBucket: %x", req.vBucket) 78 | } 79 | 80 | if bytes.Compare(req.opaque[:], []byte{0x00, 0x00, 0x00, 0x00}) != 0 { 81 | t.Errorf("Unexpected opaque: %x", req.opaque) 82 | } 83 | 84 | if bytes.Compare(req.cas[:], []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}) != 0 { 85 | t.Errorf("Unexpected cas: %x", req.cas) 86 | } 87 | 88 | if bytes.Compare(req.extras, []byte{0xde, 0xad, 0xbe, 0xef, 0x00, 0x00, 0x1c, 0x20}) != 0 { 89 | t.Errorf("Unexpected extras: %x", req.extras) 90 | } 91 | 92 | if req.key != "Hello" { 93 | t.Errorf("Unexpected kes: %s", req.key) 94 | } 95 | 96 | if req.value != "World" { 97 | t.Errorf("Unexpected value: %x", req.value) 98 | } 99 | } 100 | 101 | { 102 | input := []byte{ 103 | 0x80, 0x00, 0x00, 0x00, 104 | 0x00, 0x00, 0x00, 0x00, 105 | 0x00, 0x00, 0x00, 0x00, 106 | 0x00, 0x00, 0x00, 0x00, 107 | 0x00, 0x00, 0x00, 0x00, 108 | 0x00, 0x00, 0x00, 109 | } 110 | br := bytes.NewReader(input) 111 | _, err := newBRequest(br) 112 | 113 | if err == nil { 114 | t.Error("too short header is not detected") 115 | } 116 | } 117 | 118 | { 119 | input := []byte{ 120 | 0x82, 0x00, 0x00, 0x00, 121 | // extra length = 2 122 | 0x02, 0x00, 0x00, 0x00, 123 | // total body = 1 124 | 0x00, 0x00, 0x00, 0x02, 125 | 0x00, 0x00, 0x00, 0x00, 126 | 0x00, 0x00, 0x00, 0x00, 127 | 0x00, 0x00, 0x00, 0x00, 128 | // body = "aa" 129 | 0x61, 0x61, 130 | } 131 | br := bytes.NewReader(input) 132 | _, err := newBRequest(br) 133 | 134 | if err == nil { 135 | t.Error("length inconsistency is not detected") 136 | } 137 | } 138 | 139 | { 140 | input := []byte{ 141 | 0x82, 0x00, 0x00, 0x00, 142 | // extra length = 2 143 | 0x02, 0x00, 0x00, 0x00, 144 | // total body = 2 145 | 0x00, 0x00, 0x00, 0x02, 146 | 0x00, 0x00, 0x00, 0x00, 147 | 0x00, 0x00, 0x00, 0x00, 148 | 0x00, 0x00, 0x00, 0x00, 149 | // body = "a" 150 | 0x61, 151 | } 152 | br := bytes.NewReader(input) 153 | _, err := newBRequest(br) 154 | 155 | if err == nil { 156 | t.Error("too short body is not detected") 157 | } 158 | } 159 | 160 | { 161 | input := []byte{ 162 | 0x80, 0x00, 0x00, 0x00, 163 | 0x00, 0x00, 0x00, 0x00, 164 | // total body = 2 165 | 0x00, 0x00, 0x00, 0x02, 166 | 0x00, 0x00, 0x00, 0x00, 167 | 0x00, 0x00, 0x00, 0x00, 168 | 0x00, 0x00, 0x00, 0x00, 169 | // value = "a" 170 | 0x61, 171 | } 172 | br := bytes.NewReader(input) 173 | _, err := newBRequest(br) 174 | 175 | if err == nil { 176 | t.Error("too short body is not detected") 177 | } 178 | } 179 | 180 | { 181 | input := []byte{ 182 | 0x80, 0x00, 0x00, 0x00, 183 | // extra length = 1 184 | 0x01, 0x00, 0x00, 0x00, 185 | 0x00, 0x00, 0x00, 0x00, 186 | // total body = 2 187 | 0x00, 0x00, 0x00, 0x01, 188 | 0x00, 0x00, 0x00, 0x00, 189 | 0x00, 0x00, 0x00, 0x00, 190 | // value = "a" 191 | 0x61, 192 | } 193 | br := bytes.NewReader(input) 194 | _, err := newBRequest(br) 195 | 196 | if err == nil { 197 | t.Error("too short body is not detected") 198 | } 199 | } 200 | } 201 | 202 | func TestIsBinaryProtocol(t *testing.T) { 203 | app := newTestApp(t, nil) 204 | 205 | binCmd := []byte{ 206 | 0x80, 0x0b, 0x00, 0x00, 207 | 0x00, 0x00, 0x00, 0x00, 208 | 0x00, 0x00, 0x00, 0x00, 209 | 0x00, 0x00, 0x00, 0x00, 210 | 0x00, 0x00, 0x00, 0x00, 211 | 0x00, 0x00, 0x00, 0x00, 212 | } 213 | 214 | br := bytes.NewReader(binCmd) 215 | bufBr := bufio.NewReader(br) 216 | { 217 | isBin, err := app.IsBinaryProtocol(bufBr) 218 | if err != nil { 219 | t.Fatal(err) 220 | } 221 | if !isBin { 222 | t.Errorf("Binary protocol request is regarded as text protocol") 223 | } 224 | } 225 | 226 | sr := strings.NewReader("VERSION") 227 | bufSr := bufio.NewReader(sr) 228 | { 229 | isBin, err := app.IsBinaryProtocol(bufSr) 230 | if err != nil { 231 | t.Fatal(err) 232 | } 233 | if isBin { 234 | t.Errorf("Text protocol request is regarded as binary protocol") 235 | } 236 | } 237 | } 238 | 239 | func TestMemdStats_writeBinaryTo(t *testing.T) { 240 | s := MemdStats{ 241 | Pid: 12, 242 | Uptime: 134, 243 | Time: 999999, 244 | Version: "v1.5.7", 245 | CurrConnections: 1023, 246 | TotalConnections: 12345, 247 | CmdGet: 5312, 248 | GetHits: 5311, 249 | GetMisses: 1, 250 | } 251 | w := &bytes.Buffer{} 252 | s.writeBinaryTo(w, [4]byte{0x00, 0x00, 0x00, 0x00}) 253 | 254 | expect := []byte{ 255 | 0x81, 0x10, // response Magic, Opcode 256 | 0x00, 0x03, // Key length 257 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 258 | 0x00, 0x00, 0x00, 0x05, // Total body 259 | 0x00, 0x00, 0x00, 0x00, // Opaque 260 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 261 | 0x70, 0x69, 0x64, // Key 262 | 0x31, 0x32, // Value 263 | // Next field 264 | 0x81, 0x10, // response Magic, Opcode 265 | 0x00, 0x06, // Key length 266 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 267 | 0x00, 0x00, 0x00, 0x09, // Total Body 268 | 0x00, 0x00, 0x00, 0x00, // Opaque 269 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 270 | 0x75, 0x70, 0x74, 0x69, 0x6d, 0x65, // Key 271 | 0x31, 0x33, 0x34, // Value 272 | // Next field 273 | 0x81, 0x10, // response Magic, Opcode 274 | 0x00, 0x04, // Key length 275 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 276 | 0x00, 0x00, 0x00, 0x0a, // Total Body 277 | 0x00, 0x00, 0x00, 0x00, // Opaque 278 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 279 | 0x74, 0x69, 0x6d, 0x65, // Key 280 | 0x39, 0x39, 0x39, 0x39, 0x39, 0x39, // Value 281 | // Next field 282 | 0x81, 0x10, // response Magic, Opcode 283 | 0x00, 0x07, // Key length 284 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 285 | 0x00, 0x00, 0x00, 0x0d, // Total Body 286 | 0x00, 0x00, 0x00, 0x00, // Opaque 287 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 288 | 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, // Key 289 | 0x76, 0x31, 0x2e, 0x35, 0x2e, 0x37, // Value 290 | // Next field 291 | 0x81, 0x10, // response Magic, Opcode 292 | 0x00, 0x10, // Key length 293 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 294 | 0x00, 0x00, 0x00, 0x14, // Total body 295 | 0x00, 0x00, 0x00, 0x00, // Opaque 296 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 297 | 0x63, 0x75, 0x72, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, // Key 298 | 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x31, 0x30, 0x32, 0x33, // Value 299 | // Next field 300 | 0x81, 0x10, // response Magic, Opcode 301 | 0x00, 0x11, // Key length 302 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 303 | 0x00, 0x00, 0x00, 0x16, // Total body 304 | 0x00, 0x00, 0x00, 0x00, // Opaque 305 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 306 | 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, // Key 307 | 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x31, 0x32, 0x33, 0x34, 0x35, // Value 308 | // Next field 309 | 0x81, 0x10, // respones Magic, Opcode 310 | 0x00, 0x07, // Key length 311 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 312 | 0x00, 0x00, 0x00, 0x0b, // Total body 313 | 0x00, 0x00, 0x00, 0x00, // Opaque 314 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 315 | 0x63, 0x6d, 0x64, 0x5f, 0x67, 0x65, 0x74, // Key 316 | 0x35, 0x33, 0x31, 0x32, // Value 317 | // Next field 318 | 0x81, 0x10, // response Mgic, Opcode 319 | 0x00, 0x08, // Key length 320 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 321 | 0x00, 0x00, 0x00, 0x0c, // Total body 322 | 0x00, 0x00, 0x00, 0x00, // Opaque 323 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 324 | 0x67, 0x65, 0x74, 0x5f, 0x68, 0x69, 0x74, 0x73, // Key 325 | 0x35, 0x33, 0x31, 0x31, // Value 326 | // Next field 327 | 0x81, 0x10, // response Magic, Opcode 328 | 0x00, 0x0a, // Key length 329 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 330 | 0x00, 0x00, 0x00, 0x0b, // Total body 331 | 0x00, 0x00, 0x00, 0x00, // Opaque 332 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 333 | 0x67, 0x65, 0x74, 0x5f, 0x6d, 0x69, 0x73, 0x73, 0x65, 0x73, // Key 334 | 0x31, // Value 335 | // Last empty field 336 | 0x81, 0x10, // response Magic, Opcode 337 | 0x00, 0x00, // Key length 338 | 0x00, 0x00, 0x00, 0x00, // Extra Length(1), Data type(1), VBucket(2) 339 | 0x00, 0x00, 0x00, 0x00, // Total body 340 | 0x00, 0x00, 0x00, 0x00, // Opaque 341 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // CAS 342 | } 343 | if g, e := w.Bytes(), expect; !reflect.DeepEqual(g, e) { 344 | t.Errorf("got: \n%#v,\nexpect: %#v\n", g, e) 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /grpc/main.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.36.6 4 | // protoc (unknown) 5 | // source: main.proto 6 | 7 | package grpc 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | unsafe "unsafe" 15 | ) 16 | 17 | const ( 18 | // Verify that this generated code is sufficiently up-to-date. 19 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 20 | // Verify that runtime/protoimpl is sufficiently up-to-date. 21 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 22 | ) 23 | 24 | type FetchRequest struct { 25 | state protoimpl.MessageState `protogen:"open.v1"` 26 | unknownFields protoimpl.UnknownFields 27 | sizeCache protoimpl.SizeCache 28 | } 29 | 30 | func (x *FetchRequest) Reset() { 31 | *x = FetchRequest{} 32 | mi := &file_main_proto_msgTypes[0] 33 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 34 | ms.StoreMessageInfo(mi) 35 | } 36 | 37 | func (x *FetchRequest) String() string { 38 | return protoimpl.X.MessageStringOf(x) 39 | } 40 | 41 | func (*FetchRequest) ProtoMessage() {} 42 | 43 | func (x *FetchRequest) ProtoReflect() protoreflect.Message { 44 | mi := &file_main_proto_msgTypes[0] 45 | if x != nil { 46 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 47 | if ms.LoadMessageInfo() == nil { 48 | ms.StoreMessageInfo(mi) 49 | } 50 | return ms 51 | } 52 | return mi.MessageOf(x) 53 | } 54 | 55 | // Deprecated: Use FetchRequest.ProtoReflect.Descriptor instead. 56 | func (*FetchRequest) Descriptor() ([]byte, []int) { 57 | return file_main_proto_rawDescGZIP(), []int{0} 58 | } 59 | 60 | type FetchMultiRequest struct { 61 | state protoimpl.MessageState `protogen:"open.v1"` 62 | N uint32 `protobuf:"varint,1,opt,name=n,proto3" json:"n,omitempty"` 63 | unknownFields protoimpl.UnknownFields 64 | sizeCache protoimpl.SizeCache 65 | } 66 | 67 | func (x *FetchMultiRequest) Reset() { 68 | *x = FetchMultiRequest{} 69 | mi := &file_main_proto_msgTypes[1] 70 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 71 | ms.StoreMessageInfo(mi) 72 | } 73 | 74 | func (x *FetchMultiRequest) String() string { 75 | return protoimpl.X.MessageStringOf(x) 76 | } 77 | 78 | func (*FetchMultiRequest) ProtoMessage() {} 79 | 80 | func (x *FetchMultiRequest) ProtoReflect() protoreflect.Message { 81 | mi := &file_main_proto_msgTypes[1] 82 | if x != nil { 83 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 84 | if ms.LoadMessageInfo() == nil { 85 | ms.StoreMessageInfo(mi) 86 | } 87 | return ms 88 | } 89 | return mi.MessageOf(x) 90 | } 91 | 92 | // Deprecated: Use FetchMultiRequest.ProtoReflect.Descriptor instead. 93 | func (*FetchMultiRequest) Descriptor() ([]byte, []int) { 94 | return file_main_proto_rawDescGZIP(), []int{1} 95 | } 96 | 97 | func (x *FetchMultiRequest) GetN() uint32 { 98 | if x != nil { 99 | return x.N 100 | } 101 | return 0 102 | } 103 | 104 | type FetchResponse struct { 105 | state protoimpl.MessageState `protogen:"open.v1"` 106 | Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` 107 | unknownFields protoimpl.UnknownFields 108 | sizeCache protoimpl.SizeCache 109 | } 110 | 111 | func (x *FetchResponse) Reset() { 112 | *x = FetchResponse{} 113 | mi := &file_main_proto_msgTypes[2] 114 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 115 | ms.StoreMessageInfo(mi) 116 | } 117 | 118 | func (x *FetchResponse) String() string { 119 | return protoimpl.X.MessageStringOf(x) 120 | } 121 | 122 | func (*FetchResponse) ProtoMessage() {} 123 | 124 | func (x *FetchResponse) ProtoReflect() protoreflect.Message { 125 | mi := &file_main_proto_msgTypes[2] 126 | if x != nil { 127 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 128 | if ms.LoadMessageInfo() == nil { 129 | ms.StoreMessageInfo(mi) 130 | } 131 | return ms 132 | } 133 | return mi.MessageOf(x) 134 | } 135 | 136 | // Deprecated: Use FetchResponse.ProtoReflect.Descriptor instead. 137 | func (*FetchResponse) Descriptor() ([]byte, []int) { 138 | return file_main_proto_rawDescGZIP(), []int{2} 139 | } 140 | 141 | func (x *FetchResponse) GetId() uint64 { 142 | if x != nil { 143 | return x.Id 144 | } 145 | return 0 146 | } 147 | 148 | type FetchMultiResponse struct { 149 | state protoimpl.MessageState `protogen:"open.v1"` 150 | Ids []uint64 `protobuf:"varint,1,rep,packed,name=ids,proto3" json:"ids,omitempty"` 151 | unknownFields protoimpl.UnknownFields 152 | sizeCache protoimpl.SizeCache 153 | } 154 | 155 | func (x *FetchMultiResponse) Reset() { 156 | *x = FetchMultiResponse{} 157 | mi := &file_main_proto_msgTypes[3] 158 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 159 | ms.StoreMessageInfo(mi) 160 | } 161 | 162 | func (x *FetchMultiResponse) String() string { 163 | return protoimpl.X.MessageStringOf(x) 164 | } 165 | 166 | func (*FetchMultiResponse) ProtoMessage() {} 167 | 168 | func (x *FetchMultiResponse) ProtoReflect() protoreflect.Message { 169 | mi := &file_main_proto_msgTypes[3] 170 | if x != nil { 171 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 172 | if ms.LoadMessageInfo() == nil { 173 | ms.StoreMessageInfo(mi) 174 | } 175 | return ms 176 | } 177 | return mi.MessageOf(x) 178 | } 179 | 180 | // Deprecated: Use FetchMultiResponse.ProtoReflect.Descriptor instead. 181 | func (*FetchMultiResponse) Descriptor() ([]byte, []int) { 182 | return file_main_proto_rawDescGZIP(), []int{3} 183 | } 184 | 185 | func (x *FetchMultiResponse) GetIds() []uint64 { 186 | if x != nil { 187 | return x.Ids 188 | } 189 | return nil 190 | } 191 | 192 | type StatsRequest struct { 193 | state protoimpl.MessageState `protogen:"open.v1"` 194 | unknownFields protoimpl.UnknownFields 195 | sizeCache protoimpl.SizeCache 196 | } 197 | 198 | func (x *StatsRequest) Reset() { 199 | *x = StatsRequest{} 200 | mi := &file_main_proto_msgTypes[4] 201 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 202 | ms.StoreMessageInfo(mi) 203 | } 204 | 205 | func (x *StatsRequest) String() string { 206 | return protoimpl.X.MessageStringOf(x) 207 | } 208 | 209 | func (*StatsRequest) ProtoMessage() {} 210 | 211 | func (x *StatsRequest) ProtoReflect() protoreflect.Message { 212 | mi := &file_main_proto_msgTypes[4] 213 | if x != nil { 214 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 215 | if ms.LoadMessageInfo() == nil { 216 | ms.StoreMessageInfo(mi) 217 | } 218 | return ms 219 | } 220 | return mi.MessageOf(x) 221 | } 222 | 223 | // Deprecated: Use StatsRequest.ProtoReflect.Descriptor instead. 224 | func (*StatsRequest) Descriptor() ([]byte, []int) { 225 | return file_main_proto_rawDescGZIP(), []int{4} 226 | } 227 | 228 | type StatsResponse struct { 229 | state protoimpl.MessageState `protogen:"open.v1"` 230 | Pid int32 `protobuf:"varint,1,opt,name=pid,proto3" json:"pid,omitempty"` 231 | Uptime int64 `protobuf:"varint,2,opt,name=uptime,proto3" json:"uptime,omitempty"` 232 | Time int64 `protobuf:"varint,3,opt,name=time,proto3" json:"time,omitempty"` 233 | Version string `protobuf:"bytes,4,opt,name=version,proto3" json:"version,omitempty"` 234 | CurrConnections int64 `protobuf:"varint,5,opt,name=curr_connections,json=currConnections,proto3" json:"curr_connections,omitempty"` 235 | TotalConnections int64 `protobuf:"varint,6,opt,name=total_connections,json=totalConnections,proto3" json:"total_connections,omitempty"` 236 | CmdGet int64 `protobuf:"varint,7,opt,name=cmd_get,json=cmdGet,proto3" json:"cmd_get,omitempty"` 237 | GetHits int64 `protobuf:"varint,8,opt,name=get_hits,json=getHits,proto3" json:"get_hits,omitempty"` 238 | GetMisses int64 `protobuf:"varint,9,opt,name=get_misses,json=getMisses,proto3" json:"get_misses,omitempty"` 239 | unknownFields protoimpl.UnknownFields 240 | sizeCache protoimpl.SizeCache 241 | } 242 | 243 | func (x *StatsResponse) Reset() { 244 | *x = StatsResponse{} 245 | mi := &file_main_proto_msgTypes[5] 246 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 247 | ms.StoreMessageInfo(mi) 248 | } 249 | 250 | func (x *StatsResponse) String() string { 251 | return protoimpl.X.MessageStringOf(x) 252 | } 253 | 254 | func (*StatsResponse) ProtoMessage() {} 255 | 256 | func (x *StatsResponse) ProtoReflect() protoreflect.Message { 257 | mi := &file_main_proto_msgTypes[5] 258 | if x != nil { 259 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 260 | if ms.LoadMessageInfo() == nil { 261 | ms.StoreMessageInfo(mi) 262 | } 263 | return ms 264 | } 265 | return mi.MessageOf(x) 266 | } 267 | 268 | // Deprecated: Use StatsResponse.ProtoReflect.Descriptor instead. 269 | func (*StatsResponse) Descriptor() ([]byte, []int) { 270 | return file_main_proto_rawDescGZIP(), []int{5} 271 | } 272 | 273 | func (x *StatsResponse) GetPid() int32 { 274 | if x != nil { 275 | return x.Pid 276 | } 277 | return 0 278 | } 279 | 280 | func (x *StatsResponse) GetUptime() int64 { 281 | if x != nil { 282 | return x.Uptime 283 | } 284 | return 0 285 | } 286 | 287 | func (x *StatsResponse) GetTime() int64 { 288 | if x != nil { 289 | return x.Time 290 | } 291 | return 0 292 | } 293 | 294 | func (x *StatsResponse) GetVersion() string { 295 | if x != nil { 296 | return x.Version 297 | } 298 | return "" 299 | } 300 | 301 | func (x *StatsResponse) GetCurrConnections() int64 { 302 | if x != nil { 303 | return x.CurrConnections 304 | } 305 | return 0 306 | } 307 | 308 | func (x *StatsResponse) GetTotalConnections() int64 { 309 | if x != nil { 310 | return x.TotalConnections 311 | } 312 | return 0 313 | } 314 | 315 | func (x *StatsResponse) GetCmdGet() int64 { 316 | if x != nil { 317 | return x.CmdGet 318 | } 319 | return 0 320 | } 321 | 322 | func (x *StatsResponse) GetGetHits() int64 { 323 | if x != nil { 324 | return x.GetHits 325 | } 326 | return 0 327 | } 328 | 329 | func (x *StatsResponse) GetGetMisses() int64 { 330 | if x != nil { 331 | return x.GetMisses 332 | } 333 | return 0 334 | } 335 | 336 | var File_main_proto protoreflect.FileDescriptor 337 | 338 | const file_main_proto_rawDesc = "" + 339 | "\n" + 340 | "\n" + 341 | "main.proto\x12\n" + 342 | "katsubushi\"\x0e\n" + 343 | "\fFetchRequest\"!\n" + 344 | "\x11FetchMultiRequest\x12\f\n" + 345 | "\x01n\x18\x01 \x01(\rR\x01n\"\x1f\n" + 346 | "\rFetchResponse\x12\x0e\n" + 347 | "\x02id\x18\x01 \x01(\x04R\x02id\"&\n" + 348 | "\x12FetchMultiResponse\x12\x10\n" + 349 | "\x03ids\x18\x01 \x03(\x04R\x03ids\"\x0e\n" + 350 | "\fStatsRequest\"\x92\x02\n" + 351 | "\rStatsResponse\x12\x10\n" + 352 | "\x03pid\x18\x01 \x01(\x05R\x03pid\x12\x16\n" + 353 | "\x06uptime\x18\x02 \x01(\x03R\x06uptime\x12\x12\n" + 354 | "\x04time\x18\x03 \x01(\x03R\x04time\x12\x18\n" + 355 | "\aversion\x18\x04 \x01(\tR\aversion\x12)\n" + 356 | "\x10curr_connections\x18\x05 \x01(\x03R\x0fcurrConnections\x12+\n" + 357 | "\x11total_connections\x18\x06 \x01(\x03R\x10totalConnections\x12\x17\n" + 358 | "\acmd_get\x18\a \x01(\x03R\x06cmdGet\x12\x19\n" + 359 | "\bget_hits\x18\b \x01(\x03R\agetHits\x12\x1d\n" + 360 | "\n" + 361 | "get_misses\x18\t \x01(\x03R\tgetMisses2\x9a\x01\n" + 362 | "\tGenerator\x12>\n" + 363 | "\x05Fetch\x12\x18.katsubushi.FetchRequest\x1a\x19.katsubushi.FetchResponse\"\x00\x12M\n" + 364 | "\n" + 365 | "FetchMulti\x12\x1d.katsubushi.FetchMultiRequest\x1a\x1e.katsubushi.FetchMultiResponse\"\x002E\n" + 366 | "\x05Stats\x12<\n" + 367 | "\x03Get\x12\x18.katsubushi.StatsRequest\x1a\x19.katsubushi.StatsResponse\"\x00Bt\n" + 368 | "\x0ecom.katsubushiB\tMainProtoP\x01Z\x0fkatsubushi/grpc\xa2\x02\x03KXX\xaa\x02\n" + 369 | "Katsubushi\xca\x02\n" + 370 | "Katsubushi\xe2\x02\x16Katsubushi\\GPBMetadata\xea\x02\n" + 371 | "Katsubushib\x06proto3" 372 | 373 | var ( 374 | file_main_proto_rawDescOnce sync.Once 375 | file_main_proto_rawDescData []byte 376 | ) 377 | 378 | func file_main_proto_rawDescGZIP() []byte { 379 | file_main_proto_rawDescOnce.Do(func() { 380 | file_main_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_main_proto_rawDesc), len(file_main_proto_rawDesc))) 381 | }) 382 | return file_main_proto_rawDescData 383 | } 384 | 385 | var file_main_proto_msgTypes = make([]protoimpl.MessageInfo, 6) 386 | var file_main_proto_goTypes = []any{ 387 | (*FetchRequest)(nil), // 0: katsubushi.FetchRequest 388 | (*FetchMultiRequest)(nil), // 1: katsubushi.FetchMultiRequest 389 | (*FetchResponse)(nil), // 2: katsubushi.FetchResponse 390 | (*FetchMultiResponse)(nil), // 3: katsubushi.FetchMultiResponse 391 | (*StatsRequest)(nil), // 4: katsubushi.StatsRequest 392 | (*StatsResponse)(nil), // 5: katsubushi.StatsResponse 393 | } 394 | var file_main_proto_depIdxs = []int32{ 395 | 0, // 0: katsubushi.Generator.Fetch:input_type -> katsubushi.FetchRequest 396 | 1, // 1: katsubushi.Generator.FetchMulti:input_type -> katsubushi.FetchMultiRequest 397 | 4, // 2: katsubushi.Stats.Get:input_type -> katsubushi.StatsRequest 398 | 2, // 3: katsubushi.Generator.Fetch:output_type -> katsubushi.FetchResponse 399 | 3, // 4: katsubushi.Generator.FetchMulti:output_type -> katsubushi.FetchMultiResponse 400 | 5, // 5: katsubushi.Stats.Get:output_type -> katsubushi.StatsResponse 401 | 3, // [3:6] is the sub-list for method output_type 402 | 0, // [0:3] is the sub-list for method input_type 403 | 0, // [0:0] is the sub-list for extension type_name 404 | 0, // [0:0] is the sub-list for extension extendee 405 | 0, // [0:0] is the sub-list for field type_name 406 | } 407 | 408 | func init() { file_main_proto_init() } 409 | func file_main_proto_init() { 410 | if File_main_proto != nil { 411 | return 412 | } 413 | type x struct{} 414 | out := protoimpl.TypeBuilder{ 415 | File: protoimpl.DescBuilder{ 416 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 417 | RawDescriptor: unsafe.Slice(unsafe.StringData(file_main_proto_rawDesc), len(file_main_proto_rawDesc)), 418 | NumEnums: 0, 419 | NumMessages: 6, 420 | NumExtensions: 0, 421 | NumServices: 2, 422 | }, 423 | GoTypes: file_main_proto_goTypes, 424 | DependencyIndexes: file_main_proto_depIdxs, 425 | MessageInfos: file_main_proto_msgTypes, 426 | }.Build() 427 | File_main_proto = out.File 428 | file_main_proto_goTypes = nil 429 | file_main_proto_depIdxs = nil 430 | } 431 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Songmu/retry v0.0.1 h1:1qvwUmo87XGkrUTo42ZtVC+1tF4QWShNE7C7Mn3WVYY= 4 | github.com/Songmu/retry v0.0.1/go.mod h1:7sXIW7eseB9fq0FUvigRcQMVLR9tuHI0Scok+rkpAuA= 5 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= 6 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= 7 | github.com/bmizerany/mc v0.0.0-20180522153755-eeb3d7218919 h1:UEJyWXBXnY+R6z63tZnrRfi9P3Vq6nSTo3ORhMTtgk8= 8 | github.com/bmizerany/mc v0.0.0-20180522153755-eeb3d7218919/go.mod h1:ELaoyfvY8sW5J6I1pehzCBIPmKVDOdyZNS331TLZjcI= 9 | github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d h1:7IjN4QP3c38xhg6wz8R3YjoU+6S9e7xBc0DAVLLIpHE= 10 | github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= 11 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 12 | github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= 13 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 14 | github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= 15 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 16 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 17 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 18 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 19 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 20 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 21 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 23 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 25 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 26 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 27 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 28 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 29 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 30 | github.com/fujiwara/raus v0.3.0 h1:tsR2/J1wuW6XQgU0IQI/MoUJkf9dOPvGPe2O10JAV28= 31 | github.com/fujiwara/raus v0.3.0/go.mod h1:nVs6zZUfDVAfMGPSQ7GW3tONbpuBLjLkXfIlUkOrY5w= 32 | github.com/fukata/golang-stats-api-handler v1.0.0 h1:N6M25vhs1yAvwGBpFY6oBmMOZeJdcWnvA+wej8pKeko= 33 | github.com/fukata/golang-stats-api-handler v1.0.0/go.mod h1:1sIi4/rHq6s/ednWMZqTmRq3765qTUSs/c3xF6lj8J8= 34 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 35 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 36 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 37 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 38 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 39 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 40 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 41 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 42 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 43 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 44 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 45 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 46 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 47 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 48 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 49 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 50 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 51 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 52 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 53 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 54 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= 55 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= 56 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 57 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 58 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 59 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 60 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 61 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 62 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 63 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 64 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 65 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 66 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 67 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 68 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 69 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 70 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 71 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 72 | github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs= 73 | github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= 74 | github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= 75 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 76 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 77 | github.com/soh335/go-test-redisserver v0.1.0 h1:FZYs/CVmUFP1uHVq7avxU+HpRoFIv2JWzhdV/g2Hyk4= 78 | github.com/soh335/go-test-redisserver v0.1.0/go.mod h1:vofbm8mr+As7DkCPPNA9Jy/KOA6bBl50xd0VUkisMzY= 79 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 80 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 81 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 82 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 83 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 84 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 85 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 86 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 87 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 88 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 89 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 90 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 91 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 92 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 93 | go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= 94 | go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= 95 | go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= 96 | go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= 97 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 98 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 99 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 100 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 101 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 102 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 103 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 104 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 105 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 106 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 107 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 108 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 109 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 110 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 111 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 112 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 113 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 114 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 115 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 116 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 117 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 118 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 119 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 120 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 121 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 122 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 123 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 124 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 125 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 126 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 127 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 128 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 129 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 130 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 131 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 132 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 133 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 134 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 135 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 136 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 137 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 138 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 139 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 140 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 141 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 142 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 143 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 144 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 145 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 146 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 147 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 148 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 149 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 150 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 151 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 152 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 153 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 154 | google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 155 | google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= 156 | google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= 157 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 158 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 159 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 160 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 161 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 162 | google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= 163 | google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= 164 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 165 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 166 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 167 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 168 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 169 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 170 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 171 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 172 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 173 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 174 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 175 | -------------------------------------------------------------------------------- /app.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "log/slog" 10 | "net" 11 | "os" 12 | "path/filepath" 13 | "reflect" 14 | "runtime" 15 | "strconv" 16 | "strings" 17 | "sync" 18 | "sync/atomic" 19 | "time" 20 | ) 21 | 22 | // customHandler implements slog.Handler with custom formatting 23 | type customHandler struct { 24 | h slog.Handler 25 | w io.Writer 26 | cache sync.Map // Cache for source file paths 27 | } 28 | 29 | func newCustomHandler(w io.Writer, level slog.Level) *customHandler { 30 | opts := &slog.HandlerOptions{ 31 | Level: level, 32 | AddSource: true, 33 | ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { 34 | // Remove default keys as we'll format them ourselves 35 | if a.Key == slog.TimeKey || a.Key == slog.LevelKey || a.Key == slog.SourceKey || a.Key == slog.MessageKey { 36 | return slog.Attr{} 37 | } 38 | return a 39 | }, 40 | } 41 | return &customHandler{ 42 | h: slog.NewTextHandler(w, opts), 43 | w: w, 44 | } 45 | } 46 | 47 | func (h *customHandler) Enabled(ctx context.Context, level slog.Level) bool { 48 | return h.h.Enabled(ctx, level) 49 | } 50 | 51 | // Extract just the package/file.go from the full path with caching 52 | func (h *customHandler) getFilePath(path string) []byte { 53 | if cached, ok := h.cache.Load(path); ok { 54 | return cached.([]byte) 55 | } 56 | dir := filepath.Dir(path) // /path/to/bar 57 | base := filepath.Base(path) // baz.txt 58 | parentDir := filepath.Base(dir) // bar 59 | result := []byte(filepath.Join(parentDir, base)) // bar/baz.txt 60 | h.cache.Store(path, result) 61 | return result 62 | } 63 | 64 | func (h *customHandler) Handle(ctx context.Context, r slog.Record) error { 65 | // Format: 2025-08-01T22:11:28.043+0900 INFO go-katsubushi/grpc.go:94 Message 66 | // Pre-allocate buffer with estimated size 67 | // Time(29) + \t + Level(5) + \t + Source(~30) + \t + Message + Attrs 68 | estimatedSize := 128 + len(r.Message) 69 | buf := make([]byte, 0, estimatedSize) 70 | 71 | // Time 72 | buf = r.Time.AppendFormat(buf, "2006-01-02T15:04:05.000-0700") 73 | buf = append(buf, '\t') 74 | 75 | // Level - optimize common cases 76 | switch r.Level { 77 | case slog.LevelDebug: 78 | buf = append(buf, "DEBUG"...) 79 | case slog.LevelInfo: 80 | buf = append(buf, "INFO"...) 81 | case slog.LevelWarn: 82 | buf = append(buf, "WARN"...) 83 | case slog.LevelError: 84 | buf = append(buf, "ERROR"...) 85 | default: 86 | buf = append(buf, r.Level.String()...) 87 | } 88 | buf = append(buf, '\t') 89 | 90 | // Source location 91 | if r.PC != 0 { 92 | fs := runtime.CallersFrames([]uintptr{r.PC}) 93 | f, _ := fs.Next() 94 | file := h.getFilePath(f.File) 95 | buf = append(buf, file...) 96 | buf = append(buf, ':') 97 | buf = strconv.AppendInt(buf, int64(f.Line), 10) 98 | buf = append(buf, '\t') 99 | } 100 | 101 | // Message 102 | buf = append(buf, r.Message...) 103 | 104 | // Attributes - optimize common types 105 | r.Attrs(func(a slog.Attr) bool { 106 | buf = append(buf, ' ') 107 | buf = append(buf, a.Key...) 108 | buf = append(buf, '=') 109 | 110 | // Optimize common value types 111 | switch v := a.Value.Any().(type) { 112 | case string: 113 | buf = append(buf, v...) 114 | case int: 115 | buf = strconv.AppendInt(buf, int64(v), 10) 116 | case int64: 117 | buf = strconv.AppendInt(buf, v, 10) 118 | case uint64: 119 | buf = strconv.AppendUint(buf, v, 10) 120 | case bool: 121 | if v { 122 | buf = append(buf, "true"...) 123 | } else { 124 | buf = append(buf, "false"...) 125 | } 126 | case float64: 127 | buf = strconv.AppendFloat(buf, v, 'f', -1, 64) 128 | default: 129 | buf = append(buf, fmt.Sprint(v)...) 130 | } 131 | return true 132 | }) 133 | 134 | buf = append(buf, '\n') 135 | 136 | _, err := h.w.Write(buf) 137 | return err 138 | } 139 | 140 | func (h *customHandler) WithAttrs(attrs []slog.Attr) slog.Handler { 141 | return &customHandler{h: h.h.WithAttrs(attrs), w: h.w} 142 | } 143 | 144 | func (h *customHandler) WithGroup(name string) slog.Handler { 145 | return &customHandler{h: h.h.WithGroup(name), w: h.w} 146 | } 147 | 148 | var ( 149 | respError = []byte("ERROR\r\n") 150 | memdSep = []byte("\r\n") 151 | memdSepLen = len(memdSep) 152 | memdSpc = []byte(" ") 153 | memdGets = []byte("GETS") 154 | memdValue = []byte("VALUE") 155 | memdEnd = []byte("END") 156 | memdValHeader = []byte("VALUE ") 157 | memdValFooter = []byte("END\r\n") 158 | memdStatHeader = []byte("STAT ") 159 | memdVersionHeader = []byte("VERSION ") 160 | 161 | // DefaultIdleTimeout is the default idle timeout. 162 | DefaultIdleTimeout = 600 * time.Second 163 | 164 | // InfiniteIdleTimeout means that idle timeout is disabled. 165 | InfiniteIdleTimeout = time.Duration(0) 166 | ) 167 | 168 | // App is main struct of the Application. 169 | type App struct { 170 | Listener net.Listener 171 | 172 | gen Generator 173 | readyCh chan interface{} 174 | 175 | // App will disconnect connection if there are no commands until idleTimeout. 176 | idleTimeout time.Duration 177 | 178 | startedAt time.Time 179 | 180 | // these values are accessed atomically 181 | currConnections int64 182 | totalConnections int64 183 | cmdGet int64 184 | getHits int64 185 | getMisses int64 186 | } 187 | 188 | // New create and returns new App instance. 189 | func New(workerID uint) (*App, error) { 190 | gen, err := NewGenerator(workerID) 191 | if err != nil { 192 | return nil, err 193 | } 194 | return &App{ 195 | gen: gen, 196 | startedAt: time.Now(), 197 | readyCh: make(chan interface{}), 198 | }, nil 199 | } 200 | 201 | // NewAppWithGenerator create and returns new App instance with specified Generator. 202 | func NewAppWithGenerator(gen Generator, workerID uint) (*App, error) { 203 | return &App{ 204 | gen: gen, 205 | startedAt: time.Now(), 206 | readyCh: make(chan interface{}), 207 | }, nil 208 | } 209 | 210 | func init() { 211 | handler := newCustomHandler(os.Stderr, slog.LevelInfo) 212 | l := slog.New(handler) 213 | slog.SetDefault(l) 214 | } 215 | 216 | // SetLogLevel sets log level. 217 | // Log level must be one of debug, info, warning, error, fatal and panic. 218 | func SetLogLevel(str string) error { 219 | var level slog.Level 220 | switch str { 221 | case "debug": 222 | level = slog.LevelDebug 223 | case "info": 224 | level = slog.LevelInfo 225 | case "warning": 226 | level = slog.LevelWarn 227 | case "error": 228 | level = slog.LevelError 229 | case "fatal", "panic": 230 | // slog doesn't have fatal/panic levels, use error 231 | level = slog.LevelError 232 | default: 233 | return fmt.Errorf("invalid log level %s", str) 234 | } 235 | handler := newCustomHandler(os.Stderr, level) 236 | l := slog.New(handler) 237 | slog.SetDefault(l) 238 | return nil 239 | } 240 | 241 | // SlogLogger returns the default slog logger. 242 | func SlogLogger() *slog.Logger { 243 | return slog.Default() 244 | } 245 | 246 | func (app *App) RunServer(ctx context.Context, kc *Config) error { 247 | var l net.Listener 248 | var err error 249 | if kc.Sockpath != "" { 250 | l, err = app.ListenerSock(kc.Sockpath) 251 | if err != nil { 252 | return err 253 | } 254 | } else { 255 | l, err = app.ListenerTCP(fmt.Sprintf(":%d", kc.Port)) 256 | if err != nil { 257 | return err 258 | } 259 | } 260 | app.idleTimeout = kc.IdleTimeout 261 | return app.Serve(ctx, l) 262 | } 263 | 264 | // ListenerSock starts listen Unix Domain Socket on sockpath. 265 | func (app *App) ListenerSock(sockpath string) (net.Listener, error) { 266 | // NOTE: gomemcache expect filepath contains slashes. 267 | l, err := net.Listen("unix", filepath.ToSlash(sockpath)) 268 | if err != nil { 269 | return nil, err 270 | } 271 | return app.wrapListener(l), nil 272 | } 273 | 274 | // ListenerTCP starts listen on host:port. 275 | func (app *App) ListenerTCP(addr string) (net.Listener, error) { 276 | l, err := net.Listen("tcp", addr) 277 | if err != nil { 278 | return nil, err 279 | } 280 | return app.wrapListener(l), nil 281 | } 282 | 283 | // Serve starts a server. 284 | func (app *App) Serve(ctx context.Context, l net.Listener) error { 285 | slog.Info("Listening server at " + l.Addr().String()) 286 | slog.Info("Worker ID = " + strconv.FormatUint(uint64(app.gen.WorkerID()), 10)) 287 | 288 | app.Listener = l 289 | close(app.readyCh) 290 | 291 | go func() { 292 | <-ctx.Done() 293 | if err := l.Close(); err != nil { 294 | slog.Warn("Failed to close listener", "error", err) 295 | } 296 | }() 297 | 298 | for { 299 | conn, err := l.Accept() 300 | if err != nil { 301 | select { 302 | case <-ctx.Done(): 303 | slog.Info("Shutting down server") 304 | return nil 305 | default: 306 | slog.Warn("Error on accept connection", "error", err) 307 | return err 308 | } 309 | } 310 | slog.Debug("Connected from " + conn.RemoteAddr().String()) 311 | 312 | go app.handleConn(ctx, conn) 313 | } 314 | } 315 | 316 | // Ready returns a channel which become readable when the app can accept connections. 317 | func (app *App) Ready() chan interface{} { 318 | return app.readyCh 319 | } 320 | 321 | func (app *App) handleConn(ctx context.Context, conn net.Conn) { 322 | ctx2, cancel := context.WithCancel(ctx) 323 | defer cancel() 324 | go func() { 325 | <-ctx2.Done() 326 | conn.Close() 327 | slog.Debug("Closed " + conn.RemoteAddr().String()) 328 | }() 329 | 330 | app.extendDeadline(conn) 331 | 332 | bufReader := bufio.NewReader(conn) 333 | isBin, err := app.IsBinaryProtocol(bufReader) 334 | if err != nil { 335 | if errors.Is(err, io.EOF) || strings.Contains(err.Error(), "i/o timeout") { 336 | slog.Debug("Connection closed", "remote", conn.RemoteAddr().String(), "error", err) 337 | return 338 | } 339 | slog.Error("error on read first byte to decide binary protocol or not", "error", err) 340 | return 341 | } 342 | if isBin { 343 | slog.Debug("binary protocol") 344 | app.RespondToBinary(bufReader, conn) 345 | return 346 | } 347 | 348 | scanner := bufio.NewScanner(bufReader) 349 | w := bufio.NewWriter(conn) 350 | var deadline time.Time 351 | for scanner.Scan() { 352 | deadline, err = app.extendDeadline(conn) 353 | if err != nil { 354 | slog.Warn("error on set deadline", "error", err) 355 | return 356 | } 357 | cmd, err := app.BytesToCmd(scanner.Bytes()) 358 | if err != nil { 359 | if err := app.writeError(conn); err != nil { 360 | slog.Warn("error on write error", "error", err) 361 | return 362 | } 363 | continue 364 | } 365 | if err := cmd.Execute(app, w); err != nil { 366 | if err != io.EOF { 367 | slog.Warn("error on execute cmd", "cmd", fmt.Sprintf("%v", cmd), "error", err) 368 | } 369 | return 370 | } 371 | if err := w.Flush(); err != nil { 372 | if err != io.EOF { 373 | slog.Warn("error on cmd write to conn", "cmd", fmt.Sprintf("%v", cmd), "error", err) 374 | } 375 | return 376 | } 377 | } 378 | if err := scanner.Err(); err != nil { 379 | select { 380 | case <-ctx.Done(): 381 | // shutting down 382 | return 383 | default: 384 | } 385 | if !deadline.IsZero() && time.Now().After(deadline) { 386 | slog.Debug("deadline exceeded", "error", err) 387 | } else { 388 | slog.Warn("error on scanning request", "error", err) 389 | } 390 | } 391 | } 392 | 393 | // GetStats returns MemdStats of app 394 | func (app *App) GetStats() MemdStats { 395 | now := time.Now() 396 | return MemdStats{ 397 | Pid: os.Getpid(), 398 | Uptime: int64(now.Sub(app.startedAt).Seconds()), 399 | Time: time.Now().Unix(), 400 | Version: Version, 401 | CurrConnections: atomic.LoadInt64(&app.currConnections), 402 | TotalConnections: atomic.LoadInt64(&app.totalConnections), 403 | CmdGet: atomic.LoadInt64(&app.cmdGet), 404 | GetHits: atomic.LoadInt64(&app.getHits), 405 | GetMisses: atomic.LoadInt64(&app.getMisses), 406 | } 407 | } 408 | 409 | func (app *App) writeError(conn io.Writer) (err error) { 410 | _, err = conn.Write(respError) 411 | if err != nil { 412 | slog.Warn("Failed to write error response", "error", err) 413 | } 414 | 415 | return 416 | } 417 | 418 | // NextID generates new ID. 419 | func (app *App) NextID() (uint64, error) { 420 | id, err := app.gen.NextID() 421 | if err != nil { 422 | atomic.AddInt64(&(app.getMisses), 1) 423 | } else { 424 | atomic.AddInt64(&(app.getHits), 1) 425 | } 426 | return id, err 427 | } 428 | 429 | // BytesToCmd converts byte array to a MemdCmd and returns it. 430 | func (app *App) BytesToCmd(data []byte) (cmd MemdCmd, err error) { 431 | if len(data) == 0 { 432 | return nil, fmt.Errorf("no command") 433 | } 434 | 435 | fields := strings.Fields(string(data)) 436 | switch name := strings.ToUpper(fields[0]); name { 437 | case "GET", "GETS": 438 | atomic.AddInt64(&(app.cmdGet), 1) 439 | if len(fields) < 2 { 440 | err = fmt.Errorf("GET command needs key as second parameter") 441 | return 442 | } 443 | cmd = &MemdCmdGet{ 444 | Name: name, 445 | Keys: fields[1:], 446 | } 447 | case "QUIT": 448 | cmd = MemdCmdQuit(0) 449 | case "STATS": 450 | cmd = MemdCmdStats(0) 451 | case "VERSION": 452 | cmd = MemdCmdVersion(0) 453 | default: 454 | err = fmt.Errorf("unknown command: %s", name) 455 | } 456 | return 457 | } 458 | 459 | func (app *App) extendDeadline(conn net.Conn) (time.Time, error) { 460 | if app.idleTimeout == InfiniteIdleTimeout { 461 | return time.Time{}, nil 462 | } 463 | d := time.Now().Add(app.idleTimeout) 464 | return d, conn.SetDeadline(d) 465 | } 466 | 467 | // MemdCmd defines a command. 468 | type MemdCmd interface { 469 | Execute(*App, io.Writer) error 470 | } 471 | 472 | // MemdCmdGet defines Get command. 473 | type MemdCmdGet struct { 474 | Name string 475 | Keys []string 476 | } 477 | 478 | // Execute generates new ID. 479 | func (cmd *MemdCmdGet) Execute(app *App, conn io.Writer) error { 480 | values := make([]string, len(cmd.Keys)) 481 | for i := range cmd.Keys { 482 | id, err := app.NextID() 483 | if err != nil { 484 | slog.Warn("Failed to generate ID", "error", err) 485 | if err = app.writeError(conn); err != nil { 486 | slog.Warn("error on write error", "error", err) 487 | return err 488 | } 489 | return nil 490 | } 491 | slog.Debug("Generated ID", "id", id) 492 | values[i] = strconv.FormatUint(id, 10) 493 | } 494 | _, err := MemdValue{ 495 | Keys: cmd.Keys, 496 | Flags: 0, 497 | Values: values, 498 | }.WriteTo(conn) 499 | return err 500 | } 501 | 502 | // MemdCmdQuit defines QUIT command. 503 | type MemdCmdQuit int 504 | 505 | // Execute disconnect by server. 506 | func (cmd MemdCmdQuit) Execute(app *App, conn io.Writer) error { 507 | return io.EOF 508 | } 509 | 510 | // MemdCmdStats defines STATS command. 511 | type MemdCmdStats int 512 | 513 | // Execute writes STATS response. 514 | func (cmd MemdCmdStats) Execute(app *App, conn io.Writer) error { 515 | _, err := app.GetStats().WriteTo(conn) 516 | return err 517 | } 518 | 519 | // MemdCmdVersion defines VERSION command. 520 | type MemdCmdVersion int 521 | 522 | // Execute writes Version number. 523 | func (cmd MemdCmdVersion) Execute(app *App, w io.Writer) error { 524 | w.Write(memdVersionHeader) 525 | io.WriteString(w, Version) 526 | _, err := w.Write(memdSep) 527 | return err 528 | } 529 | 530 | // MemdValue defines return value for client. 531 | type MemdValue struct { 532 | Keys []string 533 | Flags int 534 | Values []string 535 | } 536 | 537 | // MemdStats defines result of STATS command. 538 | type MemdStats struct { 539 | Pid int `memd:"pid" json:"pid"` 540 | Uptime int64 `memd:"uptime" json:"uptime"` 541 | Time int64 `memd:"time" json:"time"` 542 | Version string `memd:"version" json:"version"` 543 | CurrConnections int64 `memd:"curr_connections" json:"curr_connections"` 544 | TotalConnections int64 `memd:"total_connections" json:"total_connections"` 545 | CmdGet int64 `memd:"cmd_get" json:"cmd_get"` 546 | GetHits int64 `memd:"get_hits" json:"get_hits"` 547 | GetMisses int64 `memd:"get_misses" json:"get_misses"` 548 | } 549 | 550 | // WriteTo writes content of MemdValue to io.Writer. 551 | // Its format is compatible to memcached protocol. 552 | func (v MemdValue) WriteTo(w io.Writer) (int64, error) { 553 | for i, key := range v.Keys { 554 | w.Write(memdValHeader) 555 | io.WriteString(w, key) 556 | w.Write(memdSpc) 557 | io.WriteString(w, strconv.Itoa(v.Flags)) 558 | w.Write(memdSpc) 559 | io.WriteString(w, strconv.Itoa(len(v.Values[i]))) 560 | w.Write(memdSep) 561 | io.WriteString(w, v.Values[i]) 562 | w.Write(memdSep) 563 | } 564 | n, err := w.Write(memdValFooter) 565 | return int64(n), err 566 | } 567 | 568 | // WriteTo writes result of STATS command to io.Writer. 569 | func (s MemdStats) WriteTo(w io.Writer) (int64, error) { 570 | statsValue := reflect.ValueOf(s) 571 | statsType := reflect.TypeOf(s) 572 | for i := 0; i < statsType.NumField(); i++ { 573 | w.Write(memdStatHeader) 574 | field := statsType.Field(i) 575 | if tag := field.Tag.Get("memd"); tag != "" { 576 | io.WriteString(w, tag) 577 | } else { 578 | io.WriteString(w, strings.ToUpper(field.Name)) 579 | } 580 | w.Write(memdSpc) 581 | v := statsValue.FieldByIndex(field.Index).Interface() 582 | switch _v := v.(type) { 583 | case int: 584 | io.WriteString(w, strconv.Itoa(_v)) 585 | case int64: 586 | io.WriteString(w, strconv.FormatInt(int64(_v), 10)) 587 | case string: 588 | io.WriteString(w, string(_v)) 589 | } 590 | w.Write(memdSep) 591 | } 592 | n, err := w.Write(memdValFooter) 593 | return int64(n), err 594 | } 595 | -------------------------------------------------------------------------------- /app_test.go: -------------------------------------------------------------------------------- 1 | package katsubushi 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "flag" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net" 12 | "os" 13 | "path/filepath" 14 | "regexp" 15 | "strconv" 16 | "strings" 17 | "sync" 18 | "testing" 19 | "time" 20 | 21 | "encoding/hex" 22 | 23 | "github.com/bmizerany/mc" 24 | "github.com/bradfitz/gomemcache/memcache" 25 | ) 26 | 27 | func TestMain(m *testing.M) { 28 | flag.Parse() 29 | if testing.Verbose() { 30 | SetLogLevel("debug") 31 | } else { 32 | SetLogLevel("panic") 33 | } 34 | os.Exit(m.Run()) 35 | } 36 | 37 | type delayedGenerator struct { 38 | gen Generator 39 | workerID uint 40 | delay time.Duration 41 | } 42 | 43 | func newDelayedGenerator(workerID uint, delay time.Duration) (*delayedGenerator, error) { 44 | g, err := NewGenerator(workerID) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return &delayedGenerator{ 49 | gen: g, 50 | workerID: workerID, 51 | delay: delay, 52 | }, nil 53 | } 54 | 55 | func (g *delayedGenerator) NextID() (uint64, error) { 56 | time.Sleep(g.delay) 57 | return g.gen.NextID() 58 | } 59 | 60 | func (g *delayedGenerator) WorkerID() uint { 61 | return g.workerID 62 | } 63 | 64 | func newTestApp(t testing.TB, timeout *time.Duration) *App { 65 | app, err := New(getNextWorkerID()) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | return app 70 | } 71 | 72 | func newTestAppDelayed(t testing.TB, delay time.Duration) *App { 73 | workerID := getNextWorkerID() 74 | gen, err := newDelayedGenerator(workerID, delay) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | app, err := NewAppWithGenerator(gen, workerID) 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | return app 83 | } 84 | 85 | func newTestAppAndListenTCP(ctx context.Context, t testing.TB, timeout *time.Duration) *App { 86 | app := newTestApp(t, timeout) 87 | 88 | l, _ := app.ListenerTCP("localhost:0") 89 | if timeout != nil { 90 | app.idleTimeout = *timeout 91 | } 92 | go app.Serve(ctx, l) 93 | <-app.Ready() 94 | 95 | return app 96 | } 97 | 98 | func newTestAppAndListenSock(ctx context.Context, t testing.TB) (*App, string) { 99 | app := newTestApp(t, nil) 100 | 101 | tmpDir, _ := ioutil.TempDir("", "go-katsubushi-") 102 | 103 | l, _ := app.ListenerSock(filepath.Join(tmpDir, "katsubushi.sock")) 104 | go app.Serve(ctx, l) 105 | <-app.Ready() 106 | 107 | return app, tmpDir 108 | } 109 | 110 | func TestApp(t *testing.T) { 111 | ctx := context.Background() 112 | app := newTestAppAndListenTCP(ctx, t, nil) 113 | mc := memcache.New(app.Listener.Addr().String()) 114 | 115 | item, err := mc.Get("hoge") 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | 120 | t.Logf("key = %s", item.Key) 121 | t.Logf("flags = %d", item.Flags) 122 | t.Logf("id = %s", item.Value) 123 | 124 | if k := item.Key; k != "hoge" { 125 | t.Errorf("Unexpected key: %s", k) 126 | } 127 | 128 | if f := item.Flags; f != 0 { 129 | t.Errorf("Unexpected flags: %d", f) 130 | } 131 | 132 | if _, err := strconv.ParseInt(string(item.Value), 10, 64); err != nil { 133 | t.Errorf("Invalid id: %s", err) 134 | } 135 | } 136 | 137 | func TestAppMulti(t *testing.T) { 138 | ctx := context.Background() 139 | app := newTestAppAndListenTCP(ctx, t, nil) 140 | mc := memcache.New(app.Listener.Addr().String()) 141 | keys := []string{"foo", "bar", "baz"} 142 | items, err := mc.GetMulti(keys) 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | for _, key := range keys { 148 | item := items[key] 149 | if k := item.Key; k != key { 150 | t.Errorf("Unexpected key: %s", k) 151 | } 152 | 153 | if f := item.Flags; f != 0 { 154 | t.Errorf("Unexpected flags: %d", f) 155 | } 156 | 157 | if _, err := strconv.ParseInt(string(item.Value), 10, 64); err != nil { 158 | t.Errorf("Invalid id: %s", err) 159 | } 160 | } 161 | } 162 | 163 | func TestAppSock(t *testing.T) { 164 | ctx := context.Background() 165 | app, tmpDir := newTestAppAndListenSock(ctx, t) 166 | mc := memcache.New(app.Listener.Addr().String()) 167 | defer os.RemoveAll(tmpDir) 168 | 169 | item, err := mc.Get("hoge") 170 | if err != nil { 171 | t.Fatal(err) 172 | } 173 | 174 | t.Logf("key = %s", item.Key) 175 | t.Logf("flags = %d", item.Flags) 176 | t.Logf("id = %s", item.Value) 177 | 178 | if k := item.Key; k != "hoge" { 179 | t.Errorf("Unexpected key: %s", k) 180 | } 181 | 182 | if f := item.Flags; f != 0 { 183 | t.Errorf("Unexpected flags: %d", f) 184 | } 185 | 186 | if _, err := strconv.ParseInt(string(item.Value), 10, 64); err != nil { 187 | t.Errorf("Invalid id: %s", err) 188 | } 189 | } 190 | 191 | func TestAppError(t *testing.T) { 192 | ctx := context.Background() 193 | app := newTestAppAndListenTCP(ctx, t, nil) 194 | mc := memcache.New(app.Listener.Addr().String()) 195 | 196 | err := mc.Set(&memcache.Item{ 197 | Key: "hoge", 198 | Value: []byte("fuga"), 199 | }) 200 | 201 | if err == nil { 202 | t.Fatal("Must be error") 203 | } 204 | 205 | if r := regexp.MustCompile(`ERROR`); !r.MatchString(err.Error()) { 206 | t.Errorf("Unexpected error: %s", err) 207 | } 208 | } 209 | 210 | func TestAppIdleTimeout(t *testing.T) { 211 | ctx := context.Background() 212 | to := time.Second 213 | app := newTestAppAndListenTCP(ctx, t, &to) 214 | 215 | mc := memcache.New(app.Listener.Addr().String()) 216 | 217 | t.Log("Before timeout") 218 | { 219 | item, err := mc.Get("hoge") 220 | if err != nil { 221 | t.Fatal(err) 222 | } 223 | 224 | if _, err := strconv.ParseInt(string(item.Value), 10, 64); err != nil { 225 | t.Errorf("Invalid id: %s", err) 226 | } 227 | } 228 | 229 | time.Sleep(2 * time.Second) 230 | 231 | t.Log("After timeout") 232 | { 233 | _, err := mc.Get("hoge") 234 | if err == nil { 235 | t.Fatal("Connection must be disconnected") 236 | } 237 | } 238 | } 239 | 240 | func BenchmarkApp(b *testing.B) { 241 | app, _ := New(getNextWorkerID()) 242 | l, _ := app.ListenerTCP(":0") 243 | go app.Serve(context.Background(), l) 244 | <-app.Ready() 245 | 246 | errorPattern := regexp.MustCompile(`ERROR`) 247 | 248 | b.ResetTimer() 249 | 250 | b.RunParallel(func(pb *testing.PB) { 251 | client, err := newTestClient(app.Listener.Addr().String()) 252 | if err != nil { 253 | b.Fatalf("Failed to connect to app: %s", err) 254 | } 255 | for pb.Next() { 256 | resp, err := client.Command("GET hoge") 257 | if err != nil { 258 | b.Fatalf("Error on write: %s", err) 259 | } 260 | if errorPattern.Match(resp) { 261 | b.Fatalf("Got ERROR") 262 | } 263 | } 264 | }) 265 | } 266 | 267 | func BenchmarkAppSock(b *testing.B) { 268 | app, _ := New(getNextWorkerID()) 269 | tmpDir, _ := ioutil.TempDir("", "go-katsubushi-") 270 | defer os.RemoveAll(tmpDir) 271 | 272 | l, _ := app.ListenerSock(filepath.Join(tmpDir, "katsubushi.sock")) 273 | go app.Serve(context.Background(), l) 274 | <-app.Ready() 275 | 276 | errorPattern := regexp.MustCompile(`ERROR`) 277 | 278 | b.ResetTimer() 279 | 280 | b.RunParallel(func(pb *testing.PB) { 281 | client, err := newTestClientSock(filepath.Join(tmpDir, "katsubushi.sock")) 282 | if err != nil { 283 | b.Fatalf("Failed to connect to app: %s", err) 284 | } 285 | for pb.Next() { 286 | resp, err := client.Command("GET hoge") 287 | if err != nil { 288 | b.Fatalf("Error on write: %s", err) 289 | } 290 | if errorPattern.Match(resp) { 291 | b.Fatalf("Got ERROR") 292 | } 293 | } 294 | }) 295 | } 296 | 297 | func TestStats(t *testing.T) { 298 | s := MemdStats{ 299 | Pid: 12345, 300 | Uptime: 10, 301 | Time: 1432714475, 302 | Version: "0.0.1", 303 | CurrConnections: 10, 304 | TotalConnections: 123, 305 | CmdGet: 399, 306 | GetHits: 396, 307 | GetMisses: 3, 308 | } 309 | var b bytes.Buffer 310 | buf := bufio.NewWriter(&b) 311 | s.WriteTo(buf) 312 | buf.Flush() 313 | expected := `STAT pid 12345 314 | STAT uptime 10 315 | STAT time 1432714475 316 | STAT version 0.0.1 317 | STAT curr_connections 10 318 | STAT total_connections 123 319 | STAT cmd_get 399 320 | STAT get_hits 396 321 | STAT get_misses 3 322 | END 323 | ` 324 | expected = strings.Replace(expected, "\n", "\r\n", -1) 325 | if res := b.String(); res != expected { 326 | t.Error("unexpected STATS output", res, expected) 327 | } 328 | } 329 | 330 | type testClient struct { 331 | conn net.Conn 332 | } 333 | 334 | func (c *testClient) Command(str string) ([]byte, error) { 335 | resp := make([]byte, 1024) 336 | _, err := c.conn.Write([]byte(str + "\r\n")) 337 | if err != nil { 338 | return nil, err 339 | } 340 | n, err := c.conn.Read(resp) 341 | if err != nil { 342 | return nil, err 343 | } 344 | return resp[0:n], nil 345 | } 346 | 347 | func newTestClient(addr string) (*testClient, error) { 348 | conn, err := net.DialTimeout("tcp", addr, 1*time.Second) 349 | if err != nil { 350 | return nil, err 351 | } 352 | return &testClient{conn}, nil 353 | } 354 | 355 | func newTestClientSock(path string) (*testClient, error) { 356 | conn, err := net.DialTimeout("unix", path, 1*time.Second) 357 | if err != nil { 358 | return nil, err 359 | } 360 | return &testClient{conn}, nil 361 | } 362 | 363 | func TestAppVersion(t *testing.T) { 364 | ctx := context.Background() 365 | app := newTestAppAndListenTCP(ctx, t, nil) 366 | client, err := newTestClient(app.Listener.Addr().String()) 367 | if err != nil { 368 | t.Fatal(err) 369 | } 370 | _resp, err := client.Command("VERSION") 371 | if string(_resp) != "VERSION "+Version+"\r\n" { 372 | t.Error("invalid version", string(_resp)) 373 | } 374 | } 375 | 376 | func TestAppQuit(t *testing.T) { 377 | ctx := context.Background() 378 | app := newTestAppAndListenTCP(ctx, t, nil) 379 | client, err := newTestClient(app.Listener.Addr().String()) 380 | if err != nil { 381 | t.Fatal(err) 382 | } 383 | _, err = client.Command("QUIT") 384 | // quitしたら切断されるのでreadしたらEOFがくるはず 385 | if err != io.EOF { 386 | t.Error("QUIT failed", err) 387 | } 388 | } 389 | 390 | func TestAppStats(t *testing.T) { 391 | ctx := context.Background() 392 | app := newTestAppAndListenTCP(ctx, t, nil) 393 | client, err := newTestClient(app.Listener.Addr().String()) 394 | if err != nil { 395 | t.Fatalf("Failed to connect to app: %s", err) 396 | } 397 | { 398 | _resp, err := client.Command("STATS") 399 | if err != nil { 400 | t.Fatal(err) 401 | } 402 | stats, err := parseStats(string(_resp)) 403 | if err != nil { 404 | t.Fatal(err) 405 | } 406 | if stats["total_connections"] != 1 || 407 | stats["curr_connections"] != 1 || 408 | stats["cmd_get"] != 0 || 409 | stats["get_hits"] != 0 || 410 | stats["get_misses"] != 0 { 411 | t.Error("invalid stats", stats) 412 | } 413 | } 414 | 415 | _, _ = client.Command("GET id") 416 | { 417 | // get したあとは get_hits, cmd_get が増えてる 418 | _resp, err := client.Command("STATS") 419 | if err != nil { 420 | t.Fatal(err) 421 | } 422 | stats, err := parseStats(string(_resp)) 423 | if err != nil { 424 | t.Fatal(err) 425 | } 426 | if stats["total_connections"] != 1 || 427 | stats["curr_connections"] != 1 || 428 | stats["cmd_get"] != 1 || 429 | stats["get_hits"] != 1 || 430 | stats["get_misses"] != 0 { 431 | t.Error("invalid stats", stats) 432 | } 433 | } 434 | 435 | time.Sleep(2 * time.Second) 436 | { 437 | // uptimeが増えてるはず 438 | _resp, err := client.Command("STATS") 439 | if err != nil { 440 | t.Fatal(err) 441 | } 442 | stats, err := parseStats(string(_resp)) 443 | if err != nil { 444 | t.Fatal(err) 445 | } 446 | if stats["uptime"] < 2 { 447 | t.Error("invalid stats", stats) 448 | } 449 | } 450 | } 451 | 452 | func parseStats(str string) (map[string]int64, error) { 453 | lines := strings.Split(str, "\r\n") 454 | stats := make(map[string]int64, len(lines)) 455 | for _, line := range lines { 456 | col := strings.Split(line, " ") 457 | if len(col) < 3 { 458 | continue 459 | } 460 | if col[0] == "STAT" { 461 | stats[col[1]], _ = strconv.ParseInt(col[2], 10, 64) 462 | } 463 | } 464 | if lines[len(lines)-2] != "END" { 465 | return nil, fmt.Errorf("end of result != END %#v", lines) 466 | } 467 | if len(lines)-2 != len(stats) { 468 | return nil, fmt.Errorf("includes invalid line %#v", stats) 469 | } 470 | return stats, nil 471 | } 472 | 473 | func TestAppEmptyCommand(t *testing.T) { 474 | ctx := context.Background() 475 | app := newTestAppAndListenTCP(ctx, t, nil) 476 | client, err := newTestClient(app.Listener.Addr().String()) 477 | if err != nil { 478 | t.Fatal(err) 479 | } 480 | _resp, err := client.Command("") // empty string 481 | if err != nil { 482 | t.Fatal(err) 483 | } 484 | if !strings.HasPrefix(string(_resp), "ERROR") { 485 | t.Errorf("expected ERROR got %s", _resp) 486 | } 487 | } 488 | 489 | func TestAppStatsRaceCondition(t *testing.T) { 490 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 491 | defer cancel() 492 | app := newTestAppAndListenTCP(ctx, t, nil) 493 | 494 | var wg sync.WaitGroup 495 | wg.Add(1) 496 | go func() { 497 | defer wg.Done() 498 | 499 | client, err := newTestClient(app.Listener.Addr().String()) 500 | if err != nil { 501 | t.Fatalf("Failed to connect to app: %s", err) 502 | } 503 | for { 504 | select { 505 | case <-ctx.Done(): 506 | return 507 | default: 508 | } 509 | client.Command("GET id") 510 | } 511 | }() 512 | 513 | wg.Add(1) 514 | go func() { 515 | defer wg.Done() 516 | 517 | client, err := newTestClient(app.Listener.Addr().String()) 518 | if err != nil { 519 | t.Fatalf("Failed to connect to app: %s", err) 520 | } 521 | for { 522 | select { 523 | case <-ctx.Done(): 524 | return 525 | default: 526 | } 527 | client.Command("STATS") 528 | } 529 | }() 530 | 531 | wg.Wait() 532 | } 533 | 534 | func TestAppCancel(t *testing.T) { 535 | ctx, cancel := context.WithCancel(context.Background()) 536 | app := newTestAppAndListenTCP(ctx, t, nil) 537 | { 538 | client, err := newTestClient(app.Listener.Addr().String()) 539 | if err != nil { 540 | t.Fatal(err) 541 | } 542 | _, err = client.Command("VERSION") 543 | if err != nil { 544 | t.Fatal(err) 545 | } 546 | cancelAndWait(cancel) 547 | // disconnect by peer after canceled 548 | res, err := client.Command("VERSION") 549 | if err == nil && len(res) > 0 { // response returned 550 | t.Fatal(err, res) 551 | } 552 | t.Log(res, err) 553 | } 554 | { 555 | // failed to conenct after canceled 556 | _, err := newTestClient(app.Listener.Addr().String()) 557 | if err == nil { 558 | t.Fatal(err) 559 | } 560 | } 561 | } 562 | 563 | type testClientBinary struct { 564 | conn net.Conn 565 | } 566 | 567 | func (c *testClientBinary) Command(cmd []byte) ([]byte, error) { 568 | resp := make([]byte, 1024) 569 | _, err := c.conn.Write(cmd) 570 | if err != nil { 571 | return nil, err 572 | } 573 | n, err := c.conn.Read(resp) 574 | if err != nil { 575 | return nil, err 576 | } 577 | return resp[0:n], nil 578 | } 579 | 580 | func newTestClientBinary(addr string) (*testClientBinary, error) { 581 | conn, err := net.DialTimeout("tcp", addr, 1*time.Second) 582 | if err != nil { 583 | return nil, err 584 | } 585 | return &testClientBinary{conn}, nil 586 | } 587 | 588 | func newTestClientBinarySock(path string) (*testClientBinary, error) { 589 | conn, err := net.DialTimeout("unix", path, 1*time.Second) 590 | if err != nil { 591 | return nil, err 592 | } 593 | return &testClientBinary{conn}, nil 594 | } 595 | 596 | func TestAppBinary(t *testing.T) { 597 | ctx := context.Background() 598 | app := newTestAppAndListenTCP(ctx, t, nil) 599 | cn, err := mc.Dial("tcp", app.Listener.Addr().String()) 600 | if err != nil { 601 | t.Fatal(err) 602 | } 603 | 604 | val, cas, flags, err := cn.Get("hoge") 605 | if err != nil { 606 | t.Fatal(err) 607 | } 608 | 609 | t.Logf("cas = %d", cas) 610 | t.Logf("flags = %d", flags) 611 | t.Logf("id = %s", val) 612 | 613 | if cas != 0 { 614 | t.Errorf("Unexpected cas: %d", cas) 615 | } 616 | 617 | if flags != 0 { 618 | t.Errorf("Unexpected flags: %d", flags) 619 | } 620 | 621 | if _, err := strconv.ParseInt(string(val), 10, 64); err != nil { 622 | t.Errorf("Invalid id: %s", err) 623 | } 624 | } 625 | 626 | func TestAppBinarySock(t *testing.T) { 627 | ctx := context.Background() 628 | app, tmpDir := newTestAppAndListenSock(ctx, t) 629 | cn, err := mc.Dial("unix", app.Listener.Addr().String()) 630 | defer os.RemoveAll(tmpDir) 631 | if err != nil { 632 | t.Fatal(err) 633 | } 634 | 635 | value, cas, flags, err := cn.Get("hoge") 636 | if err != nil { 637 | t.Fatal(err) 638 | } 639 | 640 | t.Logf("cas = %d", cas) 641 | t.Logf("flags = %d", flags) 642 | t.Logf("id = %s", value) 643 | 644 | if cas != 0 { 645 | t.Errorf("Unexpected cas: %d", cas) 646 | } 647 | 648 | if flags != 0 { 649 | t.Errorf("Unexpected flags: %d", flags) 650 | } 651 | 652 | if _, err := strconv.ParseInt(string(value), 10, 64); err != nil { 653 | t.Errorf("Invalid id: %s", err) 654 | } 655 | } 656 | 657 | func TestAppBinaryError(t *testing.T) { 658 | ctx := context.Background() 659 | app := newTestAppAndListenTCP(ctx, t, nil) 660 | client, err := newTestClientBinary(app.Listener.Addr().String()) 661 | if err != nil { 662 | t.Fatal(err) 663 | } 664 | 665 | // add-command 666 | // key: "Hello" 667 | // value: "World" 668 | // flags: 0xdeadbeef 669 | // expiry: in two hours 670 | cmd := []byte{ 671 | 0x80, 0x02, 0x00, 0x05, 672 | 0x08, 0x00, 0x00, 0x00, 673 | 0x00, 0x00, 0x00, 0x12, 674 | 0x00, 0x00, 0x00, 0x00, 675 | 0x00, 0x00, 0x00, 0x00, 676 | 0x00, 0x00, 0x00, 0x00, 677 | 0xde, 0xad, 0xbe, 0xef, 678 | 0x00, 0x00, 0x1c, 0x20, 679 | 0x48, 0x65, 0x6c, 0x6c, 680 | 0x6f, 0x57, 0x6f, 0x72, 681 | 0x6c, 0x64, 682 | } 683 | 684 | expected := []byte{ 685 | 0x81, 0x00, 0x00, 0x00, 686 | // status: Internal Error 687 | 0x00, 0x00, 0x00, 0x84, 688 | 0x00, 0x00, 0x00, 0x00, 689 | 0x00, 0x00, 0x00, 0x00, 690 | 0x00, 0x00, 0x00, 0x00, 691 | 0x00, 0x00, 0x00, 0x00, 692 | } 693 | 694 | resp, err := client.Command(cmd) 695 | if bytes.Compare(resp, expected) != 0 { 696 | t.Errorf("invalid error response: %s", hex.Dump(resp)) 697 | } 698 | } 699 | 700 | func TestAppBinaryIdleTimeout(t *testing.T) { 701 | ctx := context.Background() 702 | timeout := 1 * time.Second 703 | app := newTestAppAndListenTCP(ctx, t, &timeout) 704 | 705 | cn, err := mc.Dial("tcp", app.Listener.Addr().String()) 706 | if err != nil { 707 | t.Fatal(err) 708 | } 709 | 710 | t.Log("Before timeout") 711 | { 712 | val, _, _, err := cn.Get("hoge") 713 | if err != nil { 714 | t.Fatal(err) 715 | } 716 | 717 | if _, err := strconv.ParseInt(string(val), 10, 64); err != nil { 718 | t.Errorf("Invalid id: %s", err) 719 | } 720 | } 721 | 722 | time.Sleep(2 * time.Second) 723 | 724 | t.Log("After timeout") 725 | { 726 | _, _, _, err := cn.Get("hoge") 727 | if err == nil { 728 | t.Fatal("Connection must be disconnected") 729 | } 730 | } 731 | } 732 | 733 | func BenchmarkAppBinary(b *testing.B) { 734 | app, _ := New(getNextWorkerID()) 735 | l, _ := app.ListenerTCP(":0") 736 | go app.Serve(context.Background(), l) 737 | <-app.Ready() 738 | 739 | // GET Hello 740 | cmd := []byte{ 741 | 0x80, 0x00, 0x00, 0x05, 742 | 0x00, 0x00, 0x00, 0x00, 743 | 0x00, 0x00, 0x00, 0x05, 744 | 0x00, 0x00, 0x00, 0x00, 745 | 0x00, 0x00, 0x00, 0x00, 746 | 0x00, 0x00, 0x00, 0x00, 747 | 0x48, 0x65, 0x6c, 0x6c, 748 | 0x6f, 749 | } 750 | 751 | b.ResetTimer() 752 | 753 | b.RunParallel(func(pb *testing.PB) { 754 | client, err := newTestClientBinary(app.Listener.Addr().String()) 755 | if err != nil { 756 | b.Fatalf("Failed to connect to app: %s", err) 757 | } 758 | for pb.Next() { 759 | resp, err := client.Command(cmd) 760 | if err != nil { 761 | b.Fatalf("Error on write: %s", err) 762 | } 763 | if resp[0] != 0x81 || resp[1] != 0x00 { 764 | b.Fatalf("Got ERROR") 765 | } 766 | } 767 | }) 768 | } 769 | 770 | func BenchmarkAppBinarySock(b *testing.B) { 771 | app, _ := New(getNextWorkerID()) 772 | tmpDir, _ := ioutil.TempDir("", "go-katsubushi-") 773 | defer os.RemoveAll(tmpDir) 774 | 775 | l, _ := app.ListenerSock(filepath.Join(tmpDir, "katsubushi.sock")) 776 | go app.Serve(context.Background(), l) 777 | <-app.Ready() 778 | 779 | // GET Hello 780 | cmd := []byte{ 781 | 0x80, 0x00, 0x00, 0x05, 782 | 0x00, 0x00, 0x00, 0x00, 783 | 0x00, 0x00, 0x00, 0x05, 784 | 0x00, 0x00, 0x00, 0x00, 785 | 0x00, 0x00, 0x00, 0x00, 786 | 0x00, 0x00, 0x00, 0x00, 787 | 0x48, 0x65, 0x6c, 0x6c, 788 | 0x6f, 789 | } 790 | 791 | b.ResetTimer() 792 | 793 | b.RunParallel(func(pb *testing.PB) { 794 | client, err := newTestClientBinarySock(filepath.Join(tmpDir, "katsubushi.sock")) 795 | if err != nil { 796 | b.Fatalf("Failed to connect to app: %s", err) 797 | } 798 | for pb.Next() { 799 | resp, err := client.Command(cmd) 800 | if err != nil { 801 | b.Fatalf("Error on write: %s", err) 802 | } 803 | if resp[0] != 0x81 || resp[1] != 0x00 { 804 | b.Fatalf("Got ERROR") 805 | } 806 | } 807 | }) 808 | } 809 | 810 | func TestAppBinaryVersion(t *testing.T) { 811 | ctx := context.Background() 812 | app := newTestAppAndListenTCP(ctx, t, nil) 813 | client, err := newTestClientBinary(app.Listener.Addr().String()) 814 | if err != nil { 815 | t.Fatal(err) 816 | } 817 | versionBytes := []byte(Version) 818 | cmd := []byte{ 819 | 0x80, 0x0b, 0x00, 0x00, 820 | 0x00, 0x00, 0x00, 0x00, 821 | 0x00, 0x00, 0x00, 0x00, 822 | 0x00, 0x00, 0x00, 0x00, 823 | 0x00, 0x00, 0x00, 0x00, 824 | 0x00, 0x00, 0x00, 0x00, 825 | } 826 | expected := []byte{ 827 | 0x81, 0x0b, 0x00, 0x00, 828 | 0x00, 0x00, 0x00, 0x00, 829 | // length of body 830 | 0x00, 0x00, 0x00, byte(len(versionBytes)), 831 | 0x00, 0x00, 0x00, 0x00, 832 | 0x00, 0x00, 0x00, 0x00, 833 | 0x00, 0x00, 0x00, 0x00, 834 | } 835 | expected = append(expected, versionBytes...) 836 | 837 | resp, err := client.Command(cmd) 838 | if bytes.Compare(resp, expected) != 0 { 839 | t.Errorf("invalid version response: %s", hex.Dump(resp)) 840 | } 841 | } 842 | 843 | func TestAppBinaryCancel(t *testing.T) { 844 | versionCmd := []byte{ 845 | 0x80, 0x0b, 0x00, 0x00, 846 | 0x00, 0x00, 0x00, 0x00, 847 | 0x00, 0x00, 0x00, 0x00, 848 | 0x00, 0x00, 0x00, 0x00, 849 | 0x00, 0x00, 0x00, 0x00, 850 | 0x00, 0x00, 0x00, 0x00, 851 | } 852 | 853 | ctx, cancel := context.WithCancel(context.Background()) 854 | app := newTestAppAndListenTCP(ctx, t, nil) 855 | { 856 | client, err := newTestClientBinary(app.Listener.Addr().String()) 857 | if err != nil { 858 | t.Fatal(err) 859 | } 860 | _, err = client.Command(versionCmd) 861 | if err != nil { 862 | t.Fatal(err) 863 | } 864 | cancelAndWait(cancel) 865 | // disconnect by peer after canceled 866 | res, err := client.Command(versionCmd) 867 | if err == nil && len(res) > 24 && res[0] == 0x81 { // response returned 868 | t.Fatal(err, res) 869 | } 870 | t.Log(res, err) 871 | } 872 | { 873 | // failed to conenct after canceled 874 | _, err := newTestClientBinary(app.Listener.Addr().String()) 875 | if err == nil { 876 | t.Fatal(err) 877 | } 878 | } 879 | } 880 | --------------------------------------------------------------------------------