├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── cron │ ├── README.md │ ├── cmd.go │ └── cron.go ├── http │ ├── README.md │ ├── cmd.go │ ├── hooks │ │ ├── hooks.go │ │ ├── log.go │ │ ├── metrics.go │ │ └── traceid.go │ └── http.go └── sniper │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── new │ └── cmd.go │ ├── rpc │ ├── cmd.go │ ├── proto.go │ ├── route.go │ ├── server.go │ ├── tpl.go │ └── util.go │ └── twirp │ ├── generator.go │ └── templates │ ├── field.go │ ├── file.go │ ├── msg.go │ ├── register.go │ └── rule │ ├── contains.go │ ├── default.go │ ├── eq.go │ ├── gt.go │ ├── gte.go │ ├── in.go │ ├── len.go │ ├── lt.go │ ├── lte.go │ ├── max-items.go │ ├── max-len.go │ ├── min-items.go │ ├── min-len.go │ ├── not-contains.go │ ├── not-in.go │ ├── pattern.go │ ├── prefix.go │ ├── range.go │ ├── register.go │ ├── suffix.go │ ├── type.go │ └── unique.go ├── dao └── README.md ├── go.mod ├── go.sum ├── main.go ├── pkg ├── README.md ├── conf │ ├── README.md │ └── conf.go ├── go.mod ├── go.sum ├── http │ ├── http.go │ └── metrics.go ├── log │ ├── README.md │ ├── helper.go │ └── log.go ├── memdb │ ├── README.md │ ├── memdb.go │ ├── memdb_test.go │ ├── metrics.go │ ├── observer.go │ ├── utils.go │ └── utils_test.go ├── pkg.go ├── sqldb │ ├── README.md │ ├── metrics.go │ ├── model.go │ ├── observer.go │ ├── sqldb.go │ ├── sqldb_test.go │ ├── utils.go │ └── utils_test.go ├── trace │ ├── README.md │ └── trace.go └── twirp │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── NOTICE │ ├── PROTOCOL.md │ ├── THIRD_PARTY │ ├── client.go │ ├── context.go │ ├── ctxsetters.go │ ├── docs │ ├── best_practices.md │ ├── command_line.md │ ├── curl.md │ ├── errors.md │ ├── example.md │ ├── headers.md │ ├── hooks.md │ ├── install.md │ ├── intro.md │ ├── mux.md │ ├── protobuf_and_json.md │ ├── routing.md │ ├── spec_changelog.md │ ├── spec_v5.md │ └── spec_v6.md │ ├── errors.go │ ├── errors_test.go │ ├── hooks.go │ ├── hooks_test.go │ └── server.go ├── rpc └── README.md ├── sniper.toml └── svc └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | /.idea 3 | /coverage.out 4 | /sniper 5 | /rpc/**/*.md 6 | /rpc/**/*.pb.go 7 | /rpc/**/*.twirp.go 8 | /rpc/**/*.validate.go 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 go-kiss 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # https://stackoverflow.com/a/12959694 2 | rwildcard=$(wildcard $1$2) $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2)) 3 | 4 | RPC_PROTOS := $(call rwildcard,rpc/,*.proto) 5 | PKG_PROTOS := $(call rwildcard,pkg/,*.proto) 6 | 7 | RPC_PBGENS := $(RPC_PROTOS:.proto=.twirp.go) 8 | PKG_PBGENS := $(PKG_PROTOS:.proto=.pb.go) 9 | 10 | .PRECIOUS: $(RPC_PBGENS) $(PKG_PBGENS) 11 | 12 | # 参数 Mfoo.proto=bar/foo 表示 foo.proto 生成的 go 文件所对应的包名是 bar/foo。 13 | # 14 | # 如是在 proto 中引用了其他 proto，生成的 go 文件需要导入对应的包。 15 | # 但 protoc 和 proto-gen-go 无法单独从 proto 文件获取当前项目的包名， 16 | # 最好的办法就是通过 go_package 手工指定，但这样写起来太丑了，所以改用 M 参数。 17 | # 18 | # 如果你自己写了包供别人导入使用，则一定要在 proto 中设置 go_package 选项。 19 | # 20 | # 更多讨论请参考 21 | # https://github.com/golang/protobuf/issues/1158#issuecomment-650694184 22 | # 23 | # $(...) 中的神奇代码是为实现以下替换 24 | # pkg/kv/taishan/taishan.proto => sniper/pkg/taishan 25 | %.pb.go: %.proto 26 | protoc --go_out=M$<=$(patsubst %/,%,$(dir $<)):. $< 27 | 28 | # $(...) 中的神奇代码是为实现以下替换 29 | # rpc/util/v0/kv.proto => rpc/util/v0;util_v0 30 | %.twirp.go: %.proto 31 | $(eval m=$<=$(join \ 32 | $(patsubst %/,%\;,\ 33 | $(dir $<)\ 34 | ),\ 35 | $(subst /v,_v,\ 36 | $(patsubst rpc/%,%,\ 37 | $(patsubst %/,%,$(dir $<))\ 38 | )\ 39 | )\ 40 | )) 41 | protoc --plugin=protoc-gen-twirp=$(shell which sniper) \ 42 | --twirp_out=M$m:. \ 43 | --go_out=M$m:. \ 44 | $< 45 | 46 | default: rpc pkg 47 | go build -trimpath -mod=readonly 48 | 49 | rpc: $(RPC_PBGENS) 50 | @exit 51 | 52 | pkg: $(PKG_PBGENS) 53 | @exit 54 | 55 | clean: 56 | git clean -x -f -d 57 | 58 | .PHONY: clean rpc pkg 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sniper 轻量级业务框架 2 | 3 | Sniper 是一套轻量级但又很现代化的业务框架。轻量体现在只集成了最必要的功能，现代 4 | 则体现在接口描述IDL、可观测、强大的脚手架等方面。 5 | 6 | Sniper 框架从 2018 年开发并开源，在我们业务生产环境平稳运行，至少可以应对五百万 7 | DAU量级的业务。我们也不断把多年的生产实践经验固化到 Sniper 框架，希望能帮助更多 8 | 的朋友。 9 | 10 | 有兴趣的同学也可以加我的微信`taoshu-in`我拉大家进群讨论。 11 | 12 | ## 系统要求 13 | 14 | Sniper 仅支持 UNIX 环境。Windows 用户需要在 WSL 下使用。 15 | 16 | 环境准备好之后，需要安装以下工具的最新版本： 17 | 18 | - go 19 | - git 20 | - make 21 | - [protoc](https://github.com/google/protobuf) 22 | 23 | ## 快速入门 24 | 25 | 安装 sniper 脚手架： 26 | 27 | ```bash 28 | go install github.com/go-kiss/sniper/cmd/sniper@latest 29 | ``` 30 | 31 | 创建一个新项目： 32 | 33 | ```bash 34 | sniper new --pkg helloworld 35 | ``` 36 | 37 | 切换到 helloworld 目录。 38 | 39 | 运行服务： 40 | 41 | ```bash 42 | CONF_PATH=`pwd` go run main.go http 43 | ``` 44 | 45 | 使用 [httpie](https://httpie.io) 调用示例接口： 46 | 47 | ```bash 48 | http :8080/api/foo.v1.Bar/Echo msg=hello 49 | ``` 50 | 51 | 应该会收到如下响应内容： 52 | 53 | ``` 54 | HTTP/1.1 200 OK 55 | Content-Length: 15 56 | Content-Type: application/json 57 | Date: Thu, 14 Oct 2021 09:49:16 GMT 58 | X-Trace-Id: 08c408b0a4cd12c0 59 | 60 | { 61 | "msg": "hello" 62 | } 63 | ``` 64 | 65 | ## 深入理解 66 | 67 | Sniper 框架几乎每一个目录下都有 README.md 文件，建议仔细阅读。 68 | 69 | 如需了解 Sniper 框架的工作原理和设计原则，请移步我的[博客](https://taoshu.in/go/sniper.html)。 70 | -------------------------------------------------------------------------------- /cmd/cron/README.md: -------------------------------------------------------------------------------- 1 | # cmd/cron 2 | 3 | 注册定时任务请参考 [cron.go](./cron.go)。 4 | 5 | 查看所有定时任务 6 | ```bash 7 | go main.go cron list 8 | ``` 9 | 10 | 执行一次某个任务 11 | ```bash 12 | go main.go cron once foo 13 | ``` 14 | 15 | 调度所有定时任务 16 | ```bash 17 | go main.go cron 18 | ``` 19 | -------------------------------------------------------------------------------- /cmd/cron/cmd.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "runtime/debug" 12 | "sync" 13 | "syscall" 14 | "time" 15 | 16 | "github.com/go-kiss/sniper/pkg" 17 | "github.com/go-kiss/sniper/pkg/conf" 18 | "github.com/go-kiss/sniper/pkg/log" 19 | "github.com/go-kiss/sniper/pkg/trace" 20 | "github.com/opentracing/opentracing-go" 21 | "github.com/prometheus/client_golang/prometheus/promhttp" 22 | crond "github.com/robfig/cron/v3" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | type jobInfo struct { 27 | Name string `json:"name"` 28 | Spec string `json:"spec"` 29 | Tasks []string `json:"tasks"` 30 | job func(ctx context.Context) error 31 | } 32 | 33 | func (j *jobInfo) Run() { 34 | j.job(context.Background()) 35 | } 36 | 37 | var c = crond.New() 38 | 39 | var jobs = map[string]*jobInfo{} 40 | var httpJobs = map[string]*jobInfo{} 41 | 42 | var port int 43 | 44 | func init() { 45 | Cmd.Flags().IntVar(&port, "port", 8080, "metrics listen port") 46 | } 47 | 48 | // Cmd run job once or periodically 49 | var Cmd = &cobra.Command{ 50 | Use: "cron", 51 | Short: "Run cron job", 52 | Long: `You can list all jobs and run certain one once. 53 | If you run job cmd WITHOUT any sub cmd, job will be sheduled like cron.`, 54 | Run: func(cmd *cobra.Command, args []string) { 55 | // 不指定 handler 则会使用默认 handler 56 | server := &http.Server{Addr: fmt.Sprintf(":%d", port)} 57 | go func() { 58 | http.Handle("/metrics", promhttp.Handler()) 59 | 60 | http.HandleFunc("/ListTasks", func(w http.ResponseWriter, r *http.Request) { 61 | ctx := context.Background() 62 | span, ctx := opentracing.StartSpanFromContext(ctx, "ListTasks") 63 | defer span.Finish() 64 | 65 | w.Header().Set("x-trace-id", trace.GetTraceID(ctx)) 66 | w.Header().Set("content-type", "application/json") 67 | 68 | buf, err := json.Marshal(httpJobs) 69 | if err != nil { 70 | w.WriteHeader(http.StatusInternalServerError) 71 | w.Write([]byte(err.Error())) 72 | return 73 | } 74 | 75 | w.Write(buf) 76 | }) 77 | 78 | http.HandleFunc("/RunTask", func(w http.ResponseWriter, r *http.Request) { 79 | ctx := context.Background() 80 | span, ctx := opentracing.StartSpanFromContext(ctx, "RunTask") 81 | defer span.Finish() 82 | 83 | w.Header().Set("x-trace-id", trace.GetTraceID(ctx)) 84 | 85 | name := r.FormValue("name") 86 | job, ok := httpJobs[name] 87 | if !ok { 88 | w.WriteHeader(http.StatusNotFound) 89 | w.Write([]byte("job " + name + " not found\n")) 90 | return 91 | } 92 | 93 | if err := job.job(ctx); err != nil { 94 | w.WriteHeader(http.StatusInternalServerError) 95 | w.Write([]byte(fmt.Sprintf("%+v", err))) 96 | return 97 | } 98 | 99 | w.Write([]byte("run job " + name + " done\n")) 100 | }) 101 | 102 | http.HandleFunc("/monitor/ping", func(w http.ResponseWriter, r *http.Request) { 103 | w.Write([]byte("pong")) 104 | }) 105 | 106 | if err := server.ListenAndServe(); err != nil { 107 | panic(err) 108 | } 109 | }() 110 | 111 | go func() { 112 | conf.OnConfigChange(func() { pkg.Reset() }) 113 | conf.WatchConfig() 114 | 115 | c.Run() 116 | }() 117 | 118 | stop := make(chan os.Signal, 1) 119 | signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) 120 | <-stop 121 | 122 | var wg sync.WaitGroup 123 | go func() { 124 | wg.Add(1) 125 | defer wg.Done() 126 | 127 | c.Stop() 128 | }() 129 | go func() { 130 | wg.Add(1) 131 | defer wg.Done() 132 | 133 | err := server.Shutdown(context.Background()) 134 | if err != nil { 135 | panic(err) 136 | } 137 | }() 138 | wg.Wait() 139 | }, 140 | } 141 | 142 | var cmdList = &cobra.Command{ 143 | Use: "list", 144 | Short: "List all cron jobs", 145 | Long: `List all cron jobs.`, 146 | Run: func(cmd *cobra.Command, args []string) { 147 | for k, v := range jobs { 148 | fmt.Printf("%s [%s]\n", k, v.Spec) 149 | } 150 | for k, v := range httpJobs { 151 | fmt.Printf("%s [%s]\n", k, v.Spec) 152 | } 153 | }, 154 | } 155 | 156 | // once 命令参数，可以在 cron 中使用 157 | // sniper cron once foo bar 则 onceArgs = []string{"bar"} 158 | // sniper cron once foo 1 2 3 则 onceArgs = []string{"1", "2", "3"} 159 | var onceArgs []string 160 | 161 | var cmdOnce = &cobra.Command{ 162 | Use: "once job", 163 | Short: "Run job once", 164 | Long: `Run job once.`, 165 | Args: cobra.MinimumNArgs(1), 166 | Run: func(cmd *cobra.Command, args []string) { 167 | name := args[0] 168 | onceArgs = args[1:] 169 | job := jobs[name] 170 | if job != nil { 171 | job.job(context.Background()) 172 | } 173 | }, 174 | } 175 | 176 | func init() { 177 | Cmd.AddCommand( 178 | cmdList, 179 | cmdOnce, 180 | ) 181 | } 182 | 183 | // sepc 参数请参考 https://pkg.go.dev/github.com/robfig/cron/v3 184 | func cron(name string, spec string, job func(ctx context.Context) error) { 185 | if _, ok := jobs[name]; ok { 186 | panic(name + " is used") 187 | } 188 | 189 | j := regjob(name, spec, job, []string{}) 190 | jobs[name] = j 191 | 192 | if spec == "@manual" { 193 | return 194 | } 195 | 196 | if _, err := c.AddJob(spec, j); err != nil { 197 | panic(err) 198 | } 199 | } 200 | 201 | func manual(name string, job func(ctx context.Context) error) { 202 | cron(name, "@manual", job) 203 | } 204 | 205 | func regjob(name string, spec string, job func(ctx context.Context) error, tasks []string) (ji *jobInfo) { 206 | j := func(ctx context.Context) (err error) { 207 | span, ctx := opentracing.StartSpanFromContext(ctx, "Cron") 208 | defer span.Finish() 209 | 210 | span.SetTag("name", name) 211 | 212 | logger := log.Get(ctx) 213 | 214 | defer func() { 215 | if r := recover(); r != nil { 216 | err = errors.New(fmt.Sprintf("%+v stack: %s", r, string(debug.Stack()))) 217 | logger.Error(err) 218 | } 219 | }() 220 | 221 | if conf.GetBool("JOB_PAUSE") { 222 | logger.Errorf("skip cron job %s[%s]", name, spec) 223 | return 224 | } 225 | 226 | t := time.Now() 227 | if err = job(ctx); err != nil { 228 | logger.Errorf("cron job error: %+v", err) 229 | } 230 | d := time.Since(t) 231 | 232 | logger.WithField("cost", d.Seconds()).Infof("cron job %s[%s]", name, spec) 233 | return 234 | } 235 | 236 | ji = &jobInfo{Name: name, Spec: spec, job: j, Tasks: tasks} 237 | return 238 | } 239 | -------------------------------------------------------------------------------- /cmd/cron/cron.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | func init() { 10 | manual("foo", func(ctx context.Context) error { 11 | fmt.Printf("manual run foo with args: %+v\n", onceArgs) 12 | return nil 13 | }) 14 | 15 | cron("bar", "@every 1m", func(ctx context.Context) error { 16 | fmt.Printf("run bar @%v\n", time.Now()) 17 | return nil 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /cmd/http/README.md: -------------------------------------------------------------------------------- 1 | # cmd/server 2 | 3 | ## 注册服务 4 | 5 | 自动注册服务请参考 [rpc/README.md](../../rpc/README.md#自动注册)。 6 | 注册外部服务请参考 `initMux` 方法，内部服务参考 `initInternalMux` 方法。 7 | 8 | 实现服务接口请参考 [rpc/README.md](../../rpc/README.md)。 9 | 10 | ## 启动服务 11 | 12 | ```bash 13 | go run main.go http --port=8080 14 | ``` 15 | -------------------------------------------------------------------------------- /cmd/http/cmd.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "net" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "runtime/debug" 12 | "strings" 13 | "sync" 14 | "syscall" 15 | "time" 16 | 17 | "github.com/go-kiss/sniper/pkg" 18 | "github.com/go-kiss/sniper/pkg/conf" 19 | "github.com/go-kiss/sniper/pkg/log" 20 | "github.com/opentracing/opentracing-go" 21 | "github.com/opentracing/opentracing-go/ext" 22 | "github.com/prometheus/client_golang/prometheus/promhttp" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | var port int 27 | 28 | // Cmd run http server 29 | var Cmd = &cobra.Command{ 30 | Use: "http", 31 | Short: "Run http server", 32 | Long: `Run http server`, 33 | Run: func(cmd *cobra.Command, args []string) { 34 | main() 35 | }, 36 | } 37 | 38 | func init() { 39 | Cmd.Flags().IntVar(&port, "port", 8080, "listen port") 40 | } 41 | 42 | var server *http.Server 43 | 44 | type panicHandler struct { 45 | handler http.Handler 46 | } 47 | 48 | // 从 http 标准库搬来的 49 | type tcpKeepAliveListener struct { 50 | *net.TCPListener 51 | } 52 | 53 | func (ln tcpKeepAliveListener) Accept() (net.Conn, error) { 54 | tc, err := ln.AcceptTCP() 55 | if err != nil { 56 | return nil, err 57 | } 58 | tc.SetKeepAlive(true) 59 | tc.SetKeepAlivePeriod(3 * time.Minute) 60 | return tc, nil 61 | } 62 | 63 | var logger = log.Get(context.Background()) 64 | 65 | func startSpan(r *http.Request) (*http.Request, opentracing.Span) { 66 | operation := "ServerHTTP" 67 | 68 | ctx := r.Context() 69 | var span opentracing.Span 70 | 71 | tracer := opentracing.GlobalTracer() 72 | carrier := opentracing.HTTPHeadersCarrier(r.Header) 73 | 74 | if spanCtx, err := tracer.Extract(opentracing.HTTPHeaders, carrier); err == nil { 75 | span = opentracing.StartSpan(operation, ext.RPCServerOption(spanCtx)) 76 | ctx = opentracing.ContextWithSpan(ctx, span) 77 | } else { 78 | span, ctx = opentracing.StartSpanFromContext(ctx, operation) 79 | } 80 | 81 | ext.SpanKindRPCServer.Set(span) 82 | span.SetTag(string(ext.HTTPUrl), r.URL.Path) 83 | 84 | return r.WithContext(ctx), span 85 | } 86 | 87 | func (s panicHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 88 | r, span := startSpan(r) 89 | 90 | defer func() { 91 | if rec := recover(); rec != nil { 92 | ctx := r.Context() 93 | log.Get(ctx).Error(rec, string(debug.Stack())) 94 | } 95 | span.Finish() 96 | }() 97 | 98 | origin := r.Header.Get("Origin") 99 | suffix := conf.Get("CORS_ORIGIN_SUFFIX") 100 | 101 | if origin != "" && suffix != "" && strings.HasSuffix(origin, suffix) { 102 | w.Header().Add("Access-Control-Allow-Origin", origin) 103 | w.Header().Add("Access-Control-Allow-Methods", "GET,POST,OPTIONS") 104 | w.Header().Add("Access-Control-Allow-Credentials", "true") 105 | w.Header().Add("Access-Control-Allow-Headers", "Origin,No-Cache,X-Requested-With,If-Modified-Since,Pragma,Last-Modified,Cache-Control,Expires,Content-Type,Access-Control-Allow-Credentials,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Cache-Webcdn,Content-Length") 106 | } 107 | 108 | if r.Method == http.MethodOptions { 109 | return 110 | } 111 | 112 | s.handler.ServeHTTP(w, r) 113 | } 114 | 115 | func main() { 116 | reload := make(chan int, 1) 117 | stop := make(chan os.Signal, 1) 118 | 119 | conf.OnConfigChange(func() { reload <- 1 }) 120 | conf.WatchConfig() 121 | signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) 122 | 123 | startServer() 124 | 125 | for { 126 | select { 127 | case <-reload: 128 | pkg.Reset() 129 | case sg := <-stop: 130 | stopServer() 131 | // 仿 nginx 使用 HUP 信号重载配置 132 | if sg == syscall.SIGHUP { 133 | startServer() 134 | } else { 135 | pkg.Stop() 136 | return 137 | } 138 | } 139 | } 140 | } 141 | 142 | func startServer() { 143 | logger.Info("start server") 144 | 145 | rand.Seed(int64(time.Now().Nanosecond())) 146 | 147 | mux := http.NewServeMux() 148 | 149 | initMux(mux) 150 | 151 | var handler http.Handler 152 | 153 | handler = panicHandler{handler: mux} 154 | 155 | if prefix := conf.Get("RPC_PREFIX"); prefix != "" && prefix != "/" { 156 | handler = http.StripPrefix(prefix, handler) 157 | } 158 | 159 | http.Handle("/", handler) 160 | http.Handle("/metrics", promhttp.Handler()) 161 | 162 | http.HandleFunc("/monitor/ping", func(w http.ResponseWriter, r *http.Request) { 163 | w.Write([]byte("pong")) 164 | }) 165 | 166 | addr := fmt.Sprintf(":%d", port) 167 | server = &http.Server{ 168 | IdleTimeout: 60 * time.Second, 169 | } 170 | 171 | // 配置下发可能会多次触发重启，必须等待 Listen() 调用成功 172 | var wg sync.WaitGroup 173 | 174 | wg.Add(1) 175 | go func() { 176 | // 本段代码基本搬自 http 标准库 177 | ln, err := net.Listen("tcp", addr) 178 | if err != nil { 179 | panic(err) 180 | } 181 | wg.Done() 182 | 183 | err = server.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)}) 184 | if err != http.ErrServerClosed { 185 | panic(err) 186 | } 187 | }() 188 | 189 | wg.Wait() 190 | } 191 | 192 | func stopServer() { 193 | logger.Info("stop server") 194 | 195 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 196 | defer cancel() 197 | 198 | if err := server.Shutdown(ctx); err != nil { 199 | logger.Fatal(err) 200 | } 201 | 202 | pkg.Reset() 203 | } 204 | -------------------------------------------------------------------------------- /cmd/http/hooks/hooks.go: -------------------------------------------------------------------------------- 1 | package hooks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-kiss/sniper/pkg/twirp" 7 | ) 8 | 9 | type ServerHooker interface { 10 | Hooks() map[string]*twirp.ServerHooks 11 | } 12 | 13 | func ServerHooks(server interface{}) *twirp.ServerHooks { 14 | hooker, ok := server.(ServerHooker) 15 | if !ok { 16 | return nil 17 | } 18 | 19 | hooks := hooker.Hooks() 20 | if len(hooks) == 0 { 21 | return nil 22 | } 23 | 24 | serverHooks := hooks[""] 25 | 26 | return &twirp.ServerHooks{ 27 | RequestReceived: func(ctx context.Context) (context.Context, error) { 28 | if serverHooks != nil { 29 | return serverHooks.CallRequestReceived(ctx) 30 | } 31 | return ctx, nil 32 | }, 33 | RequestRouted: func(ctx context.Context) (context.Context, error) { 34 | method, _ := twirp.MethodName(ctx) 35 | if hooks, ok := hooks[method]; ok { 36 | return hooks.CallRequestRouted(ctx) 37 | } else if serverHooks != nil { 38 | return serverHooks.CallRequestRouted(ctx) 39 | } 40 | return ctx, nil 41 | }, 42 | ResponsePrepared: func(ctx context.Context) context.Context { 43 | method, _ := twirp.MethodName(ctx) 44 | if hooks, ok := hooks[method]; ok { 45 | return hooks.CallResponsePrepared(ctx) 46 | } else if serverHooks != nil { 47 | return serverHooks.CallResponsePrepared(ctx) 48 | } 49 | return ctx 50 | }, 51 | ResponseSent: func(ctx context.Context) { 52 | method, _ := twirp.MethodName(ctx) 53 | if hooks, ok := hooks[method]; ok { 54 | hooks.CallResponseSent(ctx) 55 | } else if serverHooks != nil { 56 | serverHooks.CallResponseSent(ctx) 57 | } 58 | }, 59 | Error: func(ctx context.Context, twerr twirp.Error) context.Context { 60 | method, _ := twirp.MethodName(ctx) 61 | if hooks, ok := hooks[method]; ok { 62 | return hooks.CallError(ctx, twerr) 63 | } else if serverHooks != nil { 64 | return serverHooks.CallError(ctx, twerr) 65 | } 66 | return ctx 67 | }, 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /cmd/http/hooks/log.go: -------------------------------------------------------------------------------- 1 | package hooks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-kiss/sniper/pkg/log" 7 | "github.com/go-kiss/sniper/pkg/trace" 8 | "github.com/go-kiss/sniper/pkg/twirp" 9 | "github.com/opentracing/opentracing-go" 10 | ) 11 | 12 | type bizResponse interface { 13 | GetCode() int32 14 | GetMsg() string 15 | } 16 | 17 | var Log = &twirp.ServerHooks{ 18 | ResponseSent: func(ctx context.Context) { 19 | var bizCode int32 20 | var bizMsg string 21 | resp, _ := twirp.Response(ctx) 22 | if br, ok := resp.(bizResponse); ok { 23 | bizCode = br.GetCode() 24 | bizMsg = br.GetMsg() 25 | } 26 | 27 | span := opentracing.SpanFromContext(ctx) 28 | duration := trace.GetDuration(span) 29 | 30 | status, _ := twirp.StatusCode(ctx) 31 | if _, ok := ctx.Deadline(); ok { 32 | if ctx.Err() != nil { 33 | status = "503" 34 | } 35 | } 36 | 37 | hreq, _ := twirp.HttpRequest(ctx) 38 | path := hreq.URL.Path 39 | 40 | // 外部爬接口脚本会请求任意 API 41 | // 导致 prometheus 无法展示数据 42 | if status != "404" { 43 | rpcDurations.WithLabelValues( 44 | path, 45 | status, 46 | ).Observe(duration.Seconds()) 47 | } 48 | 49 | form := hreq.Form 50 | // 新版本采用json/protobuf形式，公共参数需要读取query 51 | if len(form) == 0 { 52 | form = hreq.URL.Query() 53 | } 54 | 55 | log.Get(ctx).WithFields(log.Fields{ 56 | "ip": hreq.RemoteAddr, 57 | "path": path, 58 | "status": status, 59 | "params": form.Encode(), 60 | "cost": duration.Seconds(), 61 | "biz_code": bizCode, 62 | "biz_msg": bizMsg, 63 | }).Info("new rpc") 64 | }, 65 | Error: func(ctx context.Context, err twirp.Error) context.Context { 66 | c := twirp.ServerHTTPStatusFromErrorCode(err.Code()) 67 | 68 | if c >= 500 { 69 | log.Get(ctx).Errorf("%+v", err) 70 | } else if c >= 400 { 71 | log.Get(ctx).Warn(err) 72 | } 73 | 74 | return ctx 75 | }, 76 | } 77 | -------------------------------------------------------------------------------- /cmd/http/hooks/metrics.go: -------------------------------------------------------------------------------- 1 | package hooks 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var defBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1} 8 | var rpcDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 9 | Namespace: "sniper", 10 | Subsystem: "rpc", 11 | Name: "server_durations_seconds", 12 | Help: "RPC latency distributions", 13 | Buckets: defBuckets, 14 | }, []string{"path", "code"}) 15 | 16 | func init() { 17 | prometheus.MustRegister(rpcDurations) 18 | } 19 | -------------------------------------------------------------------------------- /cmd/http/hooks/traceid.go: -------------------------------------------------------------------------------- 1 | package hooks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-kiss/sniper/pkg/trace" 7 | "github.com/go-kiss/sniper/pkg/twirp" 8 | "github.com/opentracing/opentracing-go" 9 | ) 10 | 11 | var TraceID = &twirp.ServerHooks{ 12 | RequestReceived: func(ctx context.Context) (context.Context, error) { 13 | traceID := trace.GetTraceID(ctx) 14 | twirp.SetHTTPResponseHeader(ctx, "x-trace-id", traceID) 15 | 16 | return ctx, nil 17 | }, 18 | RequestRouted: func(ctx context.Context) (context.Context, error) { 19 | pkg, _ := twirp.PackageName(ctx) 20 | service, _ := twirp.ServiceName(ctx) 21 | method, _ := twirp.MethodName(ctx) 22 | 23 | api := "/" + pkg + "." + service + "/" + method 24 | 25 | _, ctx = opentracing.StartSpanFromContext(ctx, api) 26 | 27 | return ctx, nil 28 | }, 29 | ResponseSent: func(ctx context.Context) { 30 | if span := opentracing.SpanFromContext(ctx); span != nil { 31 | span.Finish() 32 | } 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /cmd/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | 6 | "sniper/cmd/http/hooks" 7 | 8 | "github.com/go-kiss/sniper/pkg/twirp" 9 | ) 10 | 11 | var commonHooks = twirp.ChainHooks(hooks.TraceID, hooks.Log) 12 | 13 | func initMux(mux *http.ServeMux) { 14 | } 15 | -------------------------------------------------------------------------------- /cmd/sniper/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-kiss/sniper/cmd/sniper 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/dave/dst v0.26.2 7 | github.com/fatih/color v1.13.0 8 | github.com/spf13/cobra v1.2.1 9 | golang.org/x/mod v0.5.1 10 | google.golang.org/protobuf v1.26.0 11 | ) 12 | 13 | require ( 14 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 15 | github.com/mattn/go-colorable v0.1.9 // indirect 16 | github.com/mattn/go-isatty v0.0.14 // indirect 17 | github.com/sergi/go-diff v1.1.0 // indirect 18 | github.com/spf13/pflag v1.0.5 // indirect 19 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect 20 | golang.org/x/tools v0.1.2 // indirect 21 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 22 | ) 23 | -------------------------------------------------------------------------------- /cmd/sniper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "github.com/go-kiss/sniper/cmd/sniper/new" 8 | "github.com/go-kiss/sniper/cmd/sniper/rpc" 9 | "github.com/go-kiss/sniper/cmd/sniper/twirp" 10 | "github.com/spf13/cobra" 11 | "google.golang.org/protobuf/compiler/protogen" 12 | ) 13 | 14 | var version bool 15 | var protocHelp bool 16 | 17 | // Cmd 脚手架命令 18 | var Cmd = &cobra.Command{ 19 | Use: "sniper", 20 | Short: "sniper 脚手架", 21 | Long: ``, 22 | Run: func(cmd *cobra.Command, args []string) { 23 | if version { 24 | fmt.Println(twirp.Version) 25 | return 26 | } 27 | 28 | g := twirp.NewGenerator() 29 | 30 | var flags flag.FlagSet 31 | 32 | flags.StringVar(&g.OptionPrefix, "option_prefix", "sniper", "legacy option prefix") 33 | flags.StringVar(&g.RootPackage, "root_package", "github.com/go-kiss/sniper", "root package of pkg") 34 | flags.BoolVar(&g.ValidateEnable, "validate_enable", false, "generate *.validate.go") 35 | 36 | if protocHelp { 37 | fmt.Println("protoc-gen-twirp " + twirp.Version) 38 | flags.PrintDefaults() 39 | return 40 | } 41 | 42 | protogen.Options{ 43 | ParamFunc: flags.Set, 44 | }.Run(g.Generate) 45 | }, 46 | } 47 | 48 | func init() { 49 | Cmd.Flags().BoolVar(&version, "version", false, "工具版本") 50 | Cmd.Flags().BoolVar(&protocHelp, "protoc-help", false, "查看 protoc-gen-twirp 帮助") 51 | } 52 | 53 | func main() { 54 | Cmd.AddCommand(rpc.Cmd) 55 | Cmd.AddCommand(new.Cmd) 56 | Cmd.Execute() 57 | } 58 | -------------------------------------------------------------------------------- /cmd/sniper/new/cmd.go: -------------------------------------------------------------------------------- 1 | package new 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "strings" 7 | 8 | "github.com/fatih/color" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var pkg, branch string 13 | 14 | func init() { 15 | Cmd.Flags().StringVar(&pkg, "pkg", "", "项目包名") 16 | Cmd.Flags().StringVar(&branch, "branch", "master", "项目模板分支") 17 | 18 | Cmd.MarkFlagRequired("pkg") 19 | } 20 | 21 | // Cmd 项目初始化工具 22 | var Cmd = &cobra.Command{ 23 | Use: "new", 24 | Short: "创建 sniper 项目", 25 | Long: `默认包名为 sniper`, 26 | Run: func(cmd *cobra.Command, args []string) { 27 | color.White(strings.TrimLeft(` 28 | ███████ ███ ██ ██ ██████ ███████ ██████ 29 | ██ ████ ██ ██ ██ ██ ██ ██ ██ 30 | ███████ ██ ██ ██ ██ ██████ █████ ██████ 31 | ██ ██ ██ ██ ██ ██ ██ ██ ██ 32 | ███████ ██ ████ ██ ██ ███████ ██ ██ 33 | https://github.com/go-kiss/sniper 34 | `, "\n")) 35 | 36 | fail := false 37 | if err := exec.Command("git", "--version").Run(); err != nil { 38 | color.Red("git is not found") 39 | fail = true 40 | } 41 | 42 | if err := exec.Command("make", "--version").Run(); err != nil { 43 | color.Red("make is not found") 44 | fail = true 45 | } 46 | 47 | if err := exec.Command("protoc", "--version").Run(); err != nil { 48 | color.Red("protoc is not found") 49 | fail = true 50 | } 51 | 52 | if fail { 53 | os.Exit(110) 54 | } 55 | 56 | run("go", "install", "google.golang.org/protobuf/cmd/protoc-gen-go@latest") 57 | 58 | parts := strings.Split(pkg, "/") 59 | path := parts[len(parts)-1] 60 | run("git", "clone", "https://github.com/go-kiss/sniper.git", 61 | "--quiet", "--depth=1", "--branch="+branch, path) 62 | 63 | if err := os.Chdir(path); err != nil { 64 | panic(err) 65 | } 66 | 67 | if pkg == "sniper" { 68 | return 69 | } 70 | 71 | color.Cyan("rename sniper to " + pkg) 72 | replace("go.mod", "module sniper", "module "+pkg, 1) 73 | for _, p := range []string{"main.go", "cmd/http/http.go"} { 74 | replace(p, `"sniper`, `"`+pkg, -1) 75 | } 76 | 77 | color.Cyan("register foo service") 78 | run("sniper", "rpc", "--server=foo", "--version=1", "--service=bar") 79 | 80 | color.Cyan("you can run service by") 81 | color.Yellow("CONF_PATH=`pwd` go run main.go http") 82 | color.Cyan("you can use the httpie to call api by") 83 | color.Yellow("http :8080/api/foo.v1.Bar/Echo msg=hello") 84 | }, 85 | } 86 | 87 | func replace(path, old, new string, n int) { 88 | b, err := os.ReadFile(path) 89 | if err != nil { 90 | panic(err) 91 | } 92 | 93 | s := string(b) 94 | s = strings.Replace(s, old, new, n) 95 | 96 | if err := os.WriteFile(path, []byte(s), 0); err != nil { 97 | panic(err) 98 | } 99 | } 100 | 101 | func run(name string, args ...string) { 102 | color.Cyan(name + " " + strings.Join(args, " ")) 103 | cmd := exec.Command(name, args...) 104 | cmd.Stdout = os.Stdout 105 | cmd.Stderr = os.Stderr 106 | if err := cmd.Run(); err != nil { 107 | panic(err) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /cmd/sniper/rpc/cmd.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/fatih/color" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var ( 11 | server, service, version string 12 | ) 13 | 14 | func init() { 15 | Cmd.Flags().StringVar(&server, "server", "", "服务") 16 | Cmd.Flags().StringVar(&service, "service", "", "子服务") 17 | Cmd.Flags().StringVar(&version, "version", "1", "版本") 18 | 19 | Cmd.MarkFlagRequired("server") 20 | } 21 | 22 | // Cmd 接口生成工具 23 | var Cmd = &cobra.Command{ 24 | Use: "rpc", 25 | Short: "生成 rpc 接口", 26 | Long: `脚手架功能： 27 | - 生成 rpc/**/*.proto 模版 28 | - 生成 rpc/**/*.go 29 | - 生成 rpc/**/*.pb.go 30 | - 生成 rpc/**/*.twirp.go 31 | - 注册接口到 http server`, 32 | Run: func(cmd *cobra.Command, args []string) { 33 | if !isSniperDir() { 34 | color.Red("只能在 sniper 项目根目录运行!") 35 | os.Exit(1) 36 | } 37 | 38 | if service == "" { 39 | service = server 40 | } 41 | 42 | genProto() 43 | genOrUpdateServer() 44 | registerServer() 45 | }, 46 | } 47 | -------------------------------------------------------------------------------- /cmd/sniper/rpc/proto.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | ) 8 | 9 | func genProto() { 10 | path := fmt.Sprintf("rpc/%s/v%s/%s.proto", server, version, service) 11 | if !fileExists(path) { 12 | tpl := &protoTpl{ 13 | Server: server, 14 | Version: version, 15 | Service: upper1st(service), 16 | } 17 | 18 | save(path, tpl) 19 | } 20 | 21 | cmd := exec.Command("make", "rpc") 22 | cmd.Stdout = os.Stdout 23 | cmd.Stderr = os.Stderr 24 | if err := cmd.Run(); err != nil { 25 | panic(err) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /cmd/sniper/rpc/route.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "strings" 9 | "text/template" 10 | 11 | "github.com/dave/dst" 12 | "github.com/dave/dst/decorator" 13 | "github.com/dave/dst/decorator/resolver/goast" 14 | "github.com/dave/dst/decorator/resolver/simple" 15 | ) 16 | 17 | func serverRegistered(gen *dst.FuncDecl) bool { 18 | has := false 19 | deleted := map[int]bool{} 20 | for i, s := range gen.Body.List { 21 | bs, ok := s.(*dst.BlockStmt) 22 | if !ok { 23 | continue 24 | } 25 | // 提取 s := &foo_v1.FooServer{} 的 foo_v1.FooServer 26 | // 保存到 id 变量 27 | ue, ok := bs.List[0].(*dst.AssignStmt).Rhs[0].(*dst.UnaryExpr) 28 | if !ok { 29 | continue 30 | } 31 | // id.Name 保存 FooServer 32 | // id.Path 保存 sniper/rpc/bar/v1 33 | id, ok := ue.X.(*dst.CompositeLit).Type.(*dst.Ident) 34 | if !ok { 35 | continue 36 | } 37 | 38 | if !hasProto(id) { 39 | deleted[i] = true 40 | } 41 | 42 | if !strings.HasSuffix(id.Path, "/"+server+"/v"+version) { 43 | continue 44 | } 45 | 46 | if id.Name != upper1st(service)+"Server" { 47 | continue 48 | } 49 | 50 | has = true 51 | } 52 | 53 | stmts := []dst.Stmt{} 54 | for i, s := range gen.Body.List { 55 | if !deleted[i] { 56 | stmts = append(stmts, s) 57 | } 58 | } 59 | gen.Body.List = stmts 60 | 61 | return has 62 | } 63 | 64 | func hasProto(id *dst.Ident) bool { 65 | parts := strings.Split(id.Path, "/") 66 | proto := strings.ToLower(id.Name[:len(id.Name)-6]) + ".proto" 67 | proto = strings.Join(parts[1:], "/") + "/" + proto 68 | 69 | return fileExists(proto) 70 | } 71 | 72 | func genServerRoute(initMux *dst.FuncDecl) { 73 | if serverRegistered(initMux) { 74 | return 75 | } 76 | 77 | args := ®SrvTpl{ 78 | Package: module(), 79 | Server: server, 80 | Version: version, 81 | Service: upper1st(service), 82 | } 83 | t, err := template.New("sniper").Parse(args.tpl()) 84 | if err != nil { 85 | panic(err) 86 | } 87 | buf := &bytes.Buffer{} 88 | if err := t.Execute(buf, args); err != nil { 89 | panic(err) 90 | } 91 | 92 | d := decorator.NewDecoratorWithImports(nil, "http", goast.New()) 93 | f, err := d.Parse(buf) 94 | if err != nil { 95 | panic(err) 96 | } 97 | 98 | for _, d := range f.Decls { 99 | if fd, ok := d.(*dst.FuncDecl); ok { 100 | stmt := fd.Body.List[0].(*dst.BlockStmt) 101 | initMux.Body.List = append(initMux.Body.List, stmt) 102 | return 103 | } 104 | } 105 | } 106 | 107 | func registerServer() { 108 | routeFile := "cmd/http/http.go" 109 | b, err := os.ReadFile(routeFile) 110 | if err != nil { 111 | panic(err) 112 | } 113 | d := decorator.NewDecoratorWithImports(nil, "http", goast.New()) 114 | routeAst, err := d.Parse(b) 115 | if err != nil { 116 | panic(err) 117 | } 118 | 119 | // 处理注册路由 120 | for _, decl := range routeAst.Decls { 121 | f, ok := decl.(*dst.FuncDecl) 122 | if ok && f.Name.Name == "initMux" { 123 | genServerRoute(f) 124 | break 125 | } 126 | } 127 | 128 | f, err := os.OpenFile(routeFile, os.O_WRONLY|os.O_TRUNC, 0644) 129 | if err != nil { 130 | return 131 | } 132 | defer f.Close() 133 | 134 | alias := server + "_v" + version 135 | path := fmt.Sprintf(`%s/rpc/%s/v%s`, module(), server, version) 136 | rr := simple.RestorerResolver{path: alias} 137 | for _, i := range routeAst.Imports { 138 | alias := "" 139 | path, _ := strconv.Unquote(i.Path.Value) 140 | if i.Name != nil { 141 | alias = i.Name.Name 142 | } else { 143 | parts := strings.Split(path, "/") 144 | alias = parts[len(parts)-1] 145 | } 146 | rr[path] = alias 147 | } 148 | r := decorator.NewRestorerWithImports("http", rr) 149 | fr := r.FileRestorer() 150 | fr.Alias[path] = alias 151 | if err := fr.Fprint(f, routeAst); err != nil { 152 | panic(err) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /cmd/sniper/rpc/server.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "go/token" 7 | "os" 8 | "strconv" 9 | "text/template" 10 | 11 | "github.com/dave/dst" 12 | "github.com/dave/dst/decorator" 13 | "github.com/dave/dst/decorator/resolver/goast" 14 | "github.com/dave/dst/decorator/resolver/simple" 15 | ) 16 | 17 | func genOrUpdateServer() { 18 | serverPkg := server + "_v" + version 19 | serverPath := fmt.Sprintf("rpc/%s/v%s/%s.go", server, version, service) 20 | twirpPath := fmt.Sprintf("rpc/%s/v%s/%s.twirp.go", server, version, service) 21 | 22 | serverAst := parseAst(serverPath, serverPkg) 23 | twirpAst := parseAst(twirpPath, serverPkg) 24 | 25 | for _, d := range twirpAst.Decls { 26 | if it, ok := isInterfaceType(d); ok { 27 | imports := twirpAst.Imports 28 | 29 | appendFuncs(serverAst, it, imports) 30 | updateComments(serverAst, it) 31 | 32 | saveCode(serverAst, imports, serverPath, serverPkg) 33 | 34 | return // 只处理第一个服务 35 | } 36 | } 37 | } 38 | 39 | func isInterfaceType(d dst.Decl) (*dst.InterfaceType, bool) { 40 | gd, ok := d.(*dst.GenDecl) 41 | if !ok || gd.Tok != token.TYPE { 42 | return nil, false 43 | } 44 | 45 | it, ok := gd.Specs[0].(*dst.TypeSpec).Type.(*dst.InterfaceType) 46 | if !ok { 47 | return nil, false 48 | } 49 | 50 | return it, true 51 | } 52 | 53 | func parseAst(path, pkg string) *dst.File { 54 | d := decorator.NewDecoratorWithImports(nil, pkg, goast.New()) 55 | ast, err := d.Parse(readCode(path)) 56 | if err != nil { 57 | panic(err) 58 | } 59 | return ast 60 | } 61 | 62 | func readCode(serverFile string) []byte { 63 | var code []byte 64 | if fileExists(serverFile) { 65 | var err error 66 | code, err = os.ReadFile(serverFile) 67 | if err != nil { 68 | panic(err) 69 | } 70 | } else { 71 | t := &srvTpl{ 72 | Server: server, 73 | Version: version, 74 | Service: upper1st(service), 75 | } 76 | 77 | buf := &bytes.Buffer{} 78 | 79 | tmpl, err := template.New("sniper").Parse(t.tpl()) 80 | if err != nil { 81 | panic(err) 82 | } 83 | 84 | if err := tmpl.Execute(buf, t); err != nil { 85 | panic(err) 86 | } 87 | code = buf.Bytes() 88 | } 89 | return code 90 | } 91 | 92 | func saveCode(ast *dst.File, imports []*dst.ImportSpec, file, pkg string) { 93 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 94 | if err != nil { 95 | panic(err) 96 | } 97 | defer f.Close() 98 | 99 | rr := simple.RestorerResolver{} 100 | for _, i := range imports { 101 | alias := i.Name.Name 102 | path, _ := strconv.Unquote(i.Path.Value) 103 | rr[path] = alias 104 | } 105 | r := decorator.NewRestorerWithImports(pkg, rr) 106 | if err := r.Fprint(f, ast); err != nil { 107 | panic(err) 108 | } 109 | } 110 | 111 | func updateComments(serverAst *dst.File, twirp *dst.InterfaceType) { 112 | comments := getComments(twirp) 113 | 114 | decls := make([]dst.Decl, 0, len(serverAst.Decls)) 115 | for _, decl := range serverAst.Decls { 116 | decls = append(decls, decl) 117 | 118 | switch d := decl.(type) { 119 | case *dst.GenDecl: // 服务注释 120 | if d.Tok != token.TYPE { 121 | continue 122 | } 123 | ts, ok := d.Specs[0].(*dst.TypeSpec) 124 | if !ok || ts.Name.Name != upper1st(service)+"Server" { 125 | continue 126 | } 127 | 128 | api := fmt.Sprintf( 129 | "%sServer 实现 /%s.v%s.%s 服务", 130 | upper1st(service), 131 | server, 132 | version, 133 | upper1st(service), 134 | ) 135 | if c := comments[upper1st(service)]; c != nil { 136 | d.Decs.Start.Replace("// " + api + "\n") 137 | d.Decs.Start.Append(c...) 138 | } 139 | case *dst.FuncDecl: // 函数注释 140 | api := fmt.Sprintf( 141 | "%s 实现 /%s.v%s.%s/%s 接口", 142 | d.Name.Name, 143 | server, 144 | version, 145 | upper1st(service), 146 | d.Name.Name, 147 | ) 148 | 149 | if c, ok := comments[d.Name.Name]; c != nil { 150 | d.Decs.Start.Replace("// " + api + "\n") 151 | d.Decs.Start.Append(c...) 152 | } else if !ok { 153 | if d.Recv != nil && d.Name.IsExported() && d.Name.Name != "Hooks" { 154 | // 删除 proto 中不存在的方法 155 | st, ok := d.Recv.List[0].Type.(*dst.StarExpr) 156 | if ok { 157 | x, ok := st.X.(*dst.Ident) 158 | if ok && x.Name == upper1st(service)+"Server" { 159 | decls = decls[:len(decls)-1] 160 | } 161 | } 162 | } 163 | } 164 | } 165 | } 166 | serverAst.Decls = decls 167 | } 168 | 169 | func getComments(d *dst.InterfaceType) map[string]dst.Decorations { 170 | comments := map[string]dst.Decorations{} 171 | // rpc service注释单独添加 172 | comments[upper1st(service)] = d.Decs.Interface 173 | 174 | for _, method := range d.Methods.List { 175 | name := method.Names[0].Name 176 | 177 | if isTwirpFunc(name) { 178 | continue 179 | } 180 | 181 | comments[name] = method.Decs.Start 182 | } 183 | 184 | return comments 185 | } 186 | 187 | func appendFuncs(serverAst *dst.File, twirp *dst.InterfaceType, imports []*dst.ImportSpec) { 188 | buf := &bytes.Buffer{} 189 | buf.WriteString("package main\n") 190 | 191 | for _, i := range imports { 192 | alias := i.Name.Name 193 | // twirp 文件导入的包都有别名 194 | fmt.Fprintf(buf, "import %s %s\n", alias, i.Path.Value) 195 | } 196 | 197 | definedFuncs := scanDefinedFuncs(serverAst) 198 | 199 | for _, m := range twirp.Methods.List { 200 | name := m.Names[0].Name 201 | 202 | if isTwirpFunc(name) { 203 | continue 204 | } 205 | 206 | ft := m.Type.(*dst.FuncType) 207 | 208 | // 接口定义没有指定参数名 209 | ft.Params.List[0].Names = []*dst.Ident{{Name: "ctx"}} 210 | ft.Params.List[1].Names = []*dst.Ident{{Name: "req"}} 211 | ft.Results.List[0].Names = []*dst.Ident{{Name: "resp"}} 212 | ft.Results.List[1].Names = []*dst.Ident{{Name: "err"}} 213 | 214 | if f, ok := definedFuncs[name]; ok { 215 | f.Type = ft 216 | continue 217 | } 218 | 219 | in := ft.Params.List[1].Type.(*dst.StarExpr).X 220 | out := ft.Results.List[0].Type.(*dst.StarExpr).X 221 | 222 | appendFunc(buf, name, getType(in), getType(out)) 223 | } 224 | 225 | pkg := server + "_v" + version 226 | d := decorator.NewDecoratorWithImports(nil, pkg, goast.New()) 227 | f, err := d.Parse(buf.Bytes()) 228 | if err != nil { 229 | panic(err) 230 | } 231 | 232 | for _, d := range f.Decls { 233 | if v, ok := d.(*dst.FuncDecl); ok { 234 | name := v.Name.Name 235 | if _, ok := definedFuncs[name]; !ok { 236 | serverAst.Decls = append(serverAst.Decls, v) 237 | } 238 | } 239 | } 240 | 241 | for _, d := range serverAst.Decls { 242 | if v, ok := d.(*dst.FuncDecl); ok { 243 | name := v.Name.Name 244 | if f, ok := definedFuncs[name]; ok { 245 | v.Type = f.Type 246 | } 247 | } 248 | } 249 | } 250 | 251 | func isTwirpFunc(name string) bool { 252 | return name == "Do" || name == "ServiceDescriptor" || 253 | name == "ProtocGenTwirpVersion" 254 | } 255 | 256 | func getType(e dst.Expr) string { 257 | switch v := e.(type) { 258 | case *dst.Ident: 259 | return v.Name 260 | case *dst.SelectorExpr: 261 | return v.X.(*dst.Ident).Name + "." + v.Sel.Name 262 | } 263 | return "" 264 | } 265 | 266 | func appendFunc(buf *bytes.Buffer, name, reqType, respType string) { 267 | args := &funcTpl{ 268 | Name: name, 269 | ReqType: reqType, 270 | RespType: respType, 271 | Service: upper1st(service), 272 | } 273 | 274 | t, err := template.New("server").Parse(args.tpl()) 275 | if err != nil { 276 | panic(err) 277 | } 278 | 279 | if err := t.Execute(buf, args); err != nil { 280 | panic(err) 281 | } 282 | } 283 | 284 | func scanDefinedFuncs(file *dst.File) map[string]*dst.FuncDecl { 285 | fs := make(map[string]*dst.FuncDecl) 286 | 287 | for _, decl := range file.Decls { 288 | if f, ok := decl.(*dst.FuncDecl); ok { 289 | fs[f.Name.Name] = f 290 | } 291 | } 292 | 293 | return fs 294 | } 295 | -------------------------------------------------------------------------------- /cmd/sniper/rpc/tpl.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type tpl interface { 8 | tpl() string 9 | } 10 | 11 | type srvTpl struct { 12 | Server string // 服务 13 | Version string // 版本 14 | Service string // 子服务 15 | } 16 | 17 | func (t *srvTpl) tpl() string { 18 | return strings.TrimLeft(` 19 | package {{.Server}}_v{{.Version}} 20 | 21 | import ( 22 | "context" 23 | 24 | "github.com/go-kiss/sniper/pkg/twirp" 25 | ) 26 | 27 | type {{.Service}}Server struct{} 28 | 29 | // Hooks 返回 server 和 method 对应的 hooks 30 | // 如果设定了 method 的 hooks，则不再执行 server 一级的 hooks 31 | func (s *{{.Service}}Server) Hooks() map[string]*twirp.ServerHooks { 32 | return map[string]*twirp.ServerHooks { 33 | // "": nil, // Server 一级 hooks 34 | // "Echo": nil, // Echo 方法的 hooks 35 | } 36 | } 37 | `, "\n") 38 | } 39 | 40 | type funcTpl struct { 41 | Service string // 服务名 42 | Name string // 函数名 43 | ReqType string // 请求消息类型 44 | RespType string // 返回消息类型 45 | } 46 | 47 | func (t *funcTpl) tpl() string { 48 | return ` 49 | func (s *{{.Service}}Server) {{.Name}}(ctx context.Context, req *{{.ReqType}}) (resp *{{.RespType}}, err error) { 50 | {{if eq .Name "Echo"}} 51 | return &{{.Service}}EchoResp{Msg: req.Msg}, nil 52 | {{else}} 53 | // FIXME 请开始你的表演 54 | return 55 | {{end}} 56 | } 57 | ` 58 | } 59 | 60 | type regSrvTpl struct { 61 | Package string // 包名 62 | Server string // 服务 63 | Version string // 版本 64 | Service string // 子服务 65 | } 66 | 67 | func (t *regSrvTpl) tpl() string { 68 | return strings.TrimLeft(` 69 | package main 70 | import {{.Server}}_v{{.Version}} "{{.Package}}/rpc/{{.Server}}/v{{.Version}}" 71 | func main() { 72 | { 73 | s := &{{.Server}}_v{{.Version}}.{{.Service}}Server{} 74 | hooks := twirp.ChainHooks(commonHooks, hooks.ServerHooks(s)) 75 | handler := {{.Server}}_v{{.Version}}.New{{.Service}}Server(s,hooks) 76 | mux.Handle({{.Server}}_v{{.Version}}.{{.Service}}PathPrefix, handler) 77 | } 78 | } 79 | `, "\n") 80 | } 81 | 82 | type protoTpl struct { 83 | Server string // 服务 84 | Version string // 版本 85 | Service string // 子服务 86 | } 87 | 88 | func (t *protoTpl) tpl() string { 89 | return strings.TrimLeft(` 90 | syntax = "proto3"; 91 | 92 | package {{.Server}}.v{{.Version}}; 93 | 94 | // FIXME 服务必须写注释 95 | service {{.Service}} { 96 | // FIXME 接口必须写注释 97 | rpc Echo({{.Service}}EchoReq) returns ({{.Service}}EchoResp); 98 | } 99 | 100 | message {{.Service}}EchoReq { 101 | // FIXME 请求字段必须写注释 102 | string msg = 1; 103 | } 104 | 105 | message {{.Service}}EchoResp { 106 | // FIXME 响应字段必须写注释 107 | string msg = 1; 108 | } 109 | `, "\n") 110 | } 111 | -------------------------------------------------------------------------------- /cmd/sniper/rpc/util.go: -------------------------------------------------------------------------------- 1 | package rpc 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "path/filepath" 7 | "text/template" 8 | 9 | "golang.org/x/mod/modfile" 10 | ) 11 | 12 | func module() string { 13 | b, err := os.ReadFile("go.mod") 14 | if err != nil { 15 | panic(err) 16 | } 17 | 18 | f, err := modfile.Parse("", b, nil) 19 | if err != nil { 20 | panic(err) 21 | } 22 | 23 | return f.Module.Mod.Path 24 | } 25 | 26 | func isSniperDir() bool { 27 | dirs, err := os.ReadDir(".") 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | // 检查 sniper 项目目录结构 33 | // sniper 项目依赖 cmd/pkg/rpc 三个目录 34 | sniperDirs := map[string]bool{"cmd": true, "pkg": true, "rpc": true} 35 | 36 | c := 0 37 | for _, d := range dirs { 38 | if sniperDirs[d.Name()] { 39 | c++ 40 | } 41 | } 42 | 43 | return c == len(sniperDirs) 44 | } 45 | 46 | func upper1st(s string) string { 47 | if len(s) == 0 { 48 | return s 49 | } 50 | 51 | r := []rune(s) 52 | 53 | if r[0] >= 97 && r[0] <= 122 { 54 | r[0] -= 32 // 大小写字母ASCII值相差32位 55 | } 56 | 57 | return string(r) 58 | } 59 | 60 | func save(path string, t tpl) { 61 | buf := &bytes.Buffer{} 62 | 63 | tmpl, err := template.New("sniper").Parse(t.tpl()) 64 | if err != nil { 65 | panic(err) 66 | } 67 | 68 | err = tmpl.Execute(buf, t) 69 | if err != nil { 70 | panic(err) 71 | } 72 | dir := filepath.Dir(path) 73 | if err := os.MkdirAll(dir, 0755); err != nil { 74 | panic(err) 75 | } 76 | 77 | if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil { 78 | panic(err) 79 | } 80 | } 81 | 82 | func fileExists(file string) bool { 83 | fd, err := os.Open(file) 84 | defer fd.Close() 85 | 86 | if err != nil && os.IsNotExist(err) { 87 | return false 88 | } 89 | return true 90 | } 91 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/field.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | const fieldTpl = ` 4 | {{ range validate . }} 5 | {{ . }} 6 | {{ end }} 7 | 8 | {{ message . }} 9 | ` 10 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/file.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | const fileTpl = ` 4 | package {{ pkg . }} 5 | 6 | import ( 7 | "fmt" 8 | "net" 9 | "net/url" 10 | "regexp" 11 | "strings" 12 | "unicode/utf8" 13 | ) 14 | 15 | // ensure the imports are used 16 | var ( 17 | _ = fmt.Print 18 | _ = utf8.UTFMax 19 | _ = (*regexp.Regexp)(nil) 20 | _ = (*strings.Reader)(nil) 21 | _ = net.IPv4len 22 | _ = (*url.URL)(nil) 23 | ) 24 | 25 | {{ range .Messages }} 26 | {{ template "msg" . }} 27 | {{ end }} 28 | ` 29 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/msg.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | const msgTpl = ` 4 | func (m *{{ msgTyp . }}) validate() error { 5 | if m == nil { return nil } 6 | 7 | {{ range .Fields }} 8 | {{ template "field" . }} 9 | {{ end }} 10 | 11 | return nil 12 | } 13 | 14 | type {{ errname . }} struct { 15 | field string 16 | reason string 17 | } 18 | 19 | // Error satisfies the builtin error interface 20 | func (e {{ errname . }}) Error() string { 21 | return fmt.Sprintf( 22 | "invalid {{ (msgTyp .) }}.%s: %s", 23 | e.field, 24 | e.reason) 25 | } 26 | ` 27 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/register.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | import ( 4 | "text/template" 5 | ) 6 | 7 | // Register 注册模版 8 | func Register(tpl *template.Template) { 9 | template.Must(tpl.New("field").Parse(fieldTpl)) 10 | template.Must(tpl.New("msg").Parse(msgTpl)) 11 | template.Must(tpl.Parse(fileTpl)) 12 | } 13 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/contains.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const containsTpl = ` 4 | if !strings.Contains({{ .Key }}, {{ .Value }}) { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value not contains {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/default.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const defaultTpl = ` 4 | // 未完成的validate类型 5 | ` 6 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/eq.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const eqTpl = ` 4 | if {{ .Key }} != {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must equal {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/gt.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const gtTpl = ` 4 | if {{ .Key }} <= {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must greater than {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/gte.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const gteTpl = ` 4 | if {{ .Key }} < {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must greater than or equal to {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/in.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const inTpl = ` 4 | var {{ .Field.GoIdent.GoName }}_In = map[{{ goType .Field.Desc.Kind }}]struct{}{ 5 | {{ range slice .Value }} 6 | {{ . }}:{}, 7 | {{ end }} 8 | } 9 | 10 | if _, ok := {{ .Field.GoIdent.GoName }}_In[{{ .Key }}]; !ok { 11 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 12 | field: "{{ .Field.GoName }}", 13 | reason: "value must be in list {{ escape .Value }}", 14 | } 15 | } 16 | ` 17 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/len.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const lenTpl = ` 4 | if utf8.RuneCountInString({{ .Key }}) != {{ .Value }}{ 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value length must be {{ .Value }} runes", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/lt.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const ltTpl = ` 4 | if {{ .Key }} >= {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must less than {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/lte.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const lteTpl = ` 4 | if {{ .Key }} > {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must less than or equal to {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/max-items.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const maxItemsTpl = ` 4 | if len({{ .Key }}) > {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must contain at most {{ .Value }} item(s)", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/max-len.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const maxLenTpl = ` 4 | if utf8.RuneCountInString({{ .Key }}) > {{ .Value }}{ 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value length must be at most {{ .Value }} runes", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/min-items.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const minItemsTpl = ` 4 | if len({{ .Key }}) < {{ .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must contain at least {{ .Value }} item(s)", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/min-len.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const minLenTpl = ` 4 | if utf8.RuneCountInString({{ .Key }}) < {{ .Value }}{ 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value length must be at least {{ .Value }} runes", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/not-contains.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const notContainsTpl = ` 4 | if strings.Contains({{ .Key }}, {{ .Value }}) { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value contains {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/not-in.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const notInTpl = ` 4 | var {{ .Field.GoIdent.GoName }}_NotIn = map[{{ goType .Field.Desc.Kind }}]struct{}{ 5 | {{ range slice .Value }} 6 | {{ . }}:{}, 7 | {{ end }} 8 | } 9 | 10 | if _, ok := {{ .Field.GoIdent.GoName }}_NotIn[{{ .Key }}]; ok { 11 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 12 | field: "{{ .Field.GoName }}", 13 | reason: "value must be not in list {{ escape .Value }}", 14 | } 15 | } 16 | ` 17 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/pattern.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const patternTpl = ` 4 | var {{ .Field.GoIdent.GoName }}_Pattern = regexp.MustCompile({{ .Value }}) 5 | 6 | if !{{ .Field.GoIdent.GoName }}_Pattern.MatchString({{ .Key }}){ 7 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 8 | field: "{{ .Field.GoName }}", 9 | reason: "value does not match regex pattern {{ escape .Value }}", 10 | } 11 | } 12 | ` 13 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/prefix.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const prefixTpl = ` 4 | if !strings.HasPrefix({{ .Key }}, {{ .Value }}) { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value does not have prefix {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/range.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const rangeTpl = ` 4 | if {{ rangeRule .Key .Value }} { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value must in range {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/register.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | "text/template" 9 | 10 | "google.golang.org/protobuf/compiler/protogen" 11 | "google.golang.org/protobuf/reflect/protoreflect" 12 | ) 13 | 14 | // 类型 15 | const float32Typ = "float32" 16 | const float64Typ = "float64" 17 | const int32Typ = "int32" 18 | const int64Typ = "int64" 19 | const uint32Typ = "uint32" 20 | const uint64Typ = "uint64" 21 | const stringTyp = "string" 22 | const boolTyp = "bool" 23 | const enumTyp = "enum" 24 | const byteTyp = "byte" 25 | const messageTyp = "message" 26 | 27 | // 规则 28 | const eqTyp = "eq" 29 | const ltTyp = "lt" 30 | const gtTyp = "gt" 31 | const gteTyp = "gte" 32 | const lteTyp = "lte" 33 | const inTyp = "in" 34 | const notInTyp = "not_in" 35 | const lenTyp = "len" 36 | const minLenTyp = "min_len" 37 | const maxLenTyp = "max_len" 38 | const patternTyp = "pattern" 39 | const prefixTyp = "prefix" 40 | const suffixTyp = "suffix" 41 | const containsTyp = "contains" 42 | const notContainsTyp = "not_contains" 43 | const minItemsTyp = "min_items" 44 | const maxItemsTyp = "max_items" 45 | const uniqueTyp = "unique" 46 | const typeTyp = "type" 47 | const rangeTyp = "range" 48 | 49 | var tienum = map[string]string{ 50 | eqTyp: eqTpl, 51 | ltTyp: ltTpl, 52 | gtTyp: gtTpl, 53 | gteTyp: gteTpl, 54 | lteTyp: lteTpl, 55 | inTyp: inTpl, 56 | notInTyp: notInTpl, 57 | lenTyp: lenTpl, 58 | minLenTyp: minLenTpl, 59 | maxLenTyp: maxLenTpl, 60 | patternTyp: patternTpl, 61 | prefixTyp: prefixTpl, 62 | suffixTyp: suffixTpl, 63 | containsTyp: containsTpl, 64 | notContainsTyp: notContainsTpl, 65 | minItemsTyp: minItemsTpl, 66 | maxItemsTyp: maxItemsTpl, 67 | uniqueTyp: uniqueTpl, 68 | typeTyp: typeTpl, 69 | rangeTyp: rangeTpl, 70 | } 71 | 72 | // TemplateInfo 用以生成最终的 rule 模版 73 | type TemplateInfo struct { 74 | Field protogen.Field // field 内容 75 | Key string // key 名 正常情况为 m.GetX() repeated 情况为 item 76 | Value string // value 77 | } 78 | 79 | // Rule 获取规则 目前从注释中正则获取 80 | type Rule struct { 81 | Key string // 规则类型 82 | Value string // 规则内容 83 | } 84 | 85 | // RegisterFunctions 注册方法 86 | func RegisterFunctions(tpl *template.Template) { 87 | tpl.Funcs(map[string]interface{}{ 88 | "msgTyp": msgTyp, 89 | "errname": errName, 90 | "pkg": pkgName, 91 | "slice": slicefunc, 92 | "accessor": accessor, 93 | "escape": escape, 94 | "goType": protoTypeToGoType, 95 | "rangeRule": rangeRulefunc, 96 | "validate": validatefunc, 97 | "message": messagefunc, 98 | }) 99 | } 100 | 101 | // msgTyp 返回 msg 名 102 | func msgTyp(message protogen.Message) string { 103 | return message.GoIdent.GoName 104 | } 105 | 106 | // errName 返回 err 名 107 | func errName(message protogen.Message) string { 108 | return msgTyp(message) + "ValidationError" 109 | } 110 | 111 | // pkgName 返回包名 112 | func pkgName(file protogen.File) string { 113 | return string(file.GoPackageName) 114 | } 115 | 116 | // slicefunc [1,2,3] 解析成数组 117 | func slicefunc(s string) (r []string) { 118 | re := regexp.MustCompile(`^\[(.*)\]$`) 119 | matched := re.FindStringSubmatch(s) 120 | 121 | if len(matched) <= 1 { 122 | return 123 | } 124 | ss := strings.Split(matched[1], ",") 125 | for _, v := range ss { 126 | r = append(r, v) 127 | } 128 | return 129 | } 130 | 131 | // accessor 获取 m.GetField 字符串 132 | func accessor(field protogen.Field) string { 133 | return fmt.Sprintf("m.Get%s()", field.GoName) 134 | } 135 | 136 | // escape 转义字符串中的"并返回 137 | func escape(s string) string { 138 | return strings.Replace(s, "\"", "", -1) 139 | } 140 | 141 | // protoTypeToGoType 转化 proto 数据类型为 go 数据类型 142 | func protoTypeToGoType(kind protoreflect.Kind) (typ string) { 143 | switch kind { 144 | case protoreflect.BoolKind: 145 | return boolTyp 146 | case protoreflect.EnumKind: 147 | return enumTyp 148 | case protoreflect.Int32Kind: 149 | return int32Typ 150 | case protoreflect.Sint32Kind: 151 | return int32Typ 152 | case protoreflect.Uint32Kind: 153 | return uint32Typ 154 | case protoreflect.Int64Kind: 155 | return int64Typ 156 | case protoreflect.Sint64Kind: 157 | return int64Typ 158 | case protoreflect.Uint64Kind: 159 | return uint64Typ 160 | case protoreflect.Sfixed32Kind: 161 | return int32Typ 162 | case protoreflect.Fixed32Kind: 163 | return uint32Typ 164 | case protoreflect.FloatKind: 165 | return float32Typ 166 | case protoreflect.Sfixed64Kind: 167 | return int64Typ 168 | case protoreflect.Fixed64Kind: 169 | return uint64Typ 170 | case protoreflect.DoubleKind: 171 | return float64Typ 172 | case protoreflect.StringKind: 173 | return stringTyp 174 | case protoreflect.BytesKind: 175 | return byteTyp 176 | case protoreflect.MessageKind: 177 | return messageTyp 178 | case protoreflect.GroupKind: 179 | return "" 180 | default: 181 | return "" 182 | } 183 | } 184 | 185 | // rangeRulefunc 返回对 range 规则的判断 186 | func rangeRulefunc(key string, value string) string { 187 | 188 | matched := regexp.MustCompile(`(\(|\[)(.+),(.+)(\)|\])`).FindStringSubmatch(value) 189 | if len(matched) < 5 { 190 | panic(key + "range value 不规范") 191 | } 192 | 193 | faultRule := map[string]string{ 194 | "(": " <= ", 195 | "[": " < ", 196 | ")": " >= ", 197 | "]": " > ", 198 | } 199 | 200 | v1 := faultRule[matched[1]] 201 | v2 := matched[2] 202 | v3 := matched[3] 203 | v4 := faultRule[matched[4]] 204 | 205 | return key + v1 + v2 + "&&" + key + v4 + v3 206 | } 207 | 208 | // validatefunc 返回 field 校验规则 209 | func validatefunc(field protogen.Field) (ss []string) { 210 | rs := getRules(field.Comments) // 获取所有规则 211 | 212 | for _, v := range rs { 213 | s := getTemplateInfo(field, v) 214 | ss = append(ss, s) 215 | } 216 | return 217 | } 218 | 219 | // messagefunc 处理 message 间的互相调用 repeated 需要增加循环 220 | func messagefunc(field protogen.Field) (str string) { 221 | if field.Desc.Kind() != protoreflect.MessageKind { 222 | return 223 | } 224 | 225 | str = ` 226 | if v, ok := interface{}(` + accessor(field) + `).(interface{ validate() error }); ok { 227 | if err := v.validate(); err != nil { 228 | return ` + field.Parent.GoIdent.GoName + `ValidationError { 229 | field: "` + field.GoName + `", 230 | reason: "embedded message failed validation " + err.Error(), 231 | } 232 | } 233 | } 234 | ` 235 | 236 | if field.Desc.IsList() { 237 | str = ` 238 | for _, item := range ` + accessor(field) + ` { 239 | if v, ok := interface{}(item).(interface{ validate() error }); ok { 240 | if err := v.validate(); err != nil { 241 | return ` + field.Parent.GoIdent.GoName + `ValidationError { 242 | field: "` + field.GoName + `", 243 | reason: "embedded message failed validation " + err.Error(), 244 | } 245 | } 246 | } 247 | } 248 | ` 249 | } 250 | return 251 | } 252 | 253 | func getTemplateInfo(field protogen.Field, r Rule) (s string) { 254 | ti := TemplateInfo{ 255 | Field: field, 256 | Key: accessor(field), 257 | Value: r.Value, 258 | } 259 | if v, ok := tienum[r.Key]; ok { 260 | s = v 261 | } 262 | 263 | if field.Desc.IsList() && r.Key != minItemsTyp && r.Key != maxItemsTyp && r.Key != uniqueTyp { 264 | s = ` 265 | for _, item := range ` + accessor(field) + ` { 266 | ` + s + ` 267 | } 268 | ` 269 | ti.Key = "item" 270 | } 271 | 272 | if s == "" { 273 | s = defaultTpl 274 | } 275 | 276 | tpl := template.New("rule") 277 | RegisterFunctions(tpl) 278 | template.Must(tpl.Parse(s)) 279 | 280 | buf := &bytes.Buffer{} 281 | 282 | if err := tpl.Execute(buf, ti); err != nil { 283 | panic(err) 284 | } 285 | 286 | return buf.String() 287 | } 288 | 289 | // getRules 返回了每行符合正则的 rules 数组 290 | func getRules(cs protogen.CommentSet) (rs []Rule) { 291 | ops := make([]string, 0, len(tienum)) 292 | for op, _ := range tienum { 293 | ops = append(ops, op) 294 | } 295 | 296 | r := "@(" + strings.Join(ops, "|") + "):\\s*(.+)\\s*" 297 | re := regexp.MustCompile(r) 298 | 299 | for _, line := range strings.Split(string(cs.Leading), "\n") { 300 | matched := re.FindStringSubmatch(line) 301 | 302 | if len(matched) < 3 { 303 | continue 304 | } 305 | 306 | r := Rule{ 307 | Key: matched[1], 308 | Value: matched[2], 309 | } 310 | 311 | rs = append(rs, r) 312 | } 313 | 314 | return 315 | } 316 | 317 | func inKinds(item protoreflect.Kind, items []protoreflect.Kind) bool { 318 | for _, v := range items { 319 | if v == item { 320 | return true 321 | } 322 | } 323 | return false 324 | } 325 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/suffix.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const suffixTpl = ` 4 | if !strings.HasSuffix({{ .Key }}, {{ .Value }}) { 5 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 6 | field: "{{ .Field.GoName }}", 7 | reason: "value does not have suffix {{ escape .Value }}", 8 | } 9 | } 10 | ` 11 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/type.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const typeTpl = ` 4 | {{ if eq .Value "url" }} 5 | if _, err := url.Parse({{ .Key }}); err != nil { 6 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 7 | field: "{{ .Field.GoName }}", 8 | reason: "value must be a valid URL", 9 | } 10 | } 11 | {{ else if eq .Value "ip" }} 12 | if ip := net.ParseIP({{ .Key }}); ip == nil { 13 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 14 | field: "{{ .Field.GoName }}", 15 | reason: "value must be a valid IP address", 16 | } 17 | } 18 | {{ else if eq .Value "phone" }} 19 | var {{ .Field.GoIdent.GoName }}_Pattern = regexp.MustCompile("1[3-9]\\d{9}") 20 | 21 | if !{{ .Field.GoIdent.GoName }}_Pattern.MatchString({{ .Key }}){ 22 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 23 | field: "{{ .Field.GoName }}", 24 | reason: "value does not match regex pattern {{ escape .Value }}", 25 | } 26 | } 27 | {{ else if eq .Value "email" }} 28 | var {{ .Field.GoIdent.GoName }}_Pattern = regexp.MustCompile("[a-zA-Z0-9_-]+@[a-zA-Z0-9_-]+(\\.[a-zA-Z0-9_-]+)+") 29 | 30 | if !{{ .Field.GoIdent.GoName }}_Pattern.MatchString({{ .Key }}){ 31 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 32 | field: "{{ .Field.GoName }}", 33 | reason: "value does not match regex pattern {{ escape .Value }}", 34 | } 35 | } 36 | {{ else }} 37 | // undefined type 38 | {{ end }} 39 | ` 40 | -------------------------------------------------------------------------------- /cmd/sniper/twirp/templates/rule/unique.go: -------------------------------------------------------------------------------- 1 | package rule 2 | 3 | const uniqueTpl = ` 4 | {{ .Field.GoIdent.GoName }}_Unique := make(map[{{ goType .Field.Desc.Kind }}]struct{}, len({{ .Key }})) 5 | 6 | for idx, item := range {{ .Key }} { 7 | _, _ = idx, item 8 | if _, exists :={{ .Field.GoIdent.GoName }}_Unique[item]; exists { 9 | return {{ .Field.Parent.GoIdent.GoName }}ValidationError { 10 | field: "{{ .Field.GoName }}", 11 | reason: "repeated value must contain unique items", 12 | } 13 | } 14 | {{ .Field.GoIdent.GoName }}_Unique[item] = struct{}{} 15 | } 16 | ` 17 | -------------------------------------------------------------------------------- /dao/README.md: -------------------------------------------------------------------------------- 1 | # dao 2 | 3 | 数据访问层，负责访问 DB、MC、外部 HTTP 等接口，对上层屏蔽数据访问细节。 4 | 5 | 具体职责有： 6 | - SQL 拼接和 DB 访问逻辑 7 | - DB 的拆库折表逻辑 8 | - DB 的缓存读写逻辑 9 | - HTTP 接口调用逻辑 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module sniper 2 | 3 | go 1.17 4 | 5 | replace github.com/go-kiss/sniper/pkg => ./pkg 6 | 7 | require ( 8 | github.com/go-kiss/sniper/pkg v0.0.0-00010101000000-000000000000 9 | github.com/opentracing/opentracing-go v1.2.0 10 | github.com/prometheus/client_golang v1.11.0 11 | github.com/robfig/cron/v3 v3.0.1 12 | github.com/spf13/cobra v1.3.0 13 | go.uber.org/automaxprocs v1.4.0 14 | google.golang.org/protobuf v1.27.1 15 | ) 16 | 17 | require ( 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 20 | github.com/fsnotify/fsnotify v1.5.1 // indirect 21 | github.com/golang/protobuf v1.5.2 // indirect 22 | github.com/hashicorp/hcl v1.0.0 // indirect 23 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 24 | github.com/k0kubun/pp/v3 v3.1.0 // indirect 25 | github.com/kr/text v0.2.0 // indirect 26 | github.com/magiconair/properties v1.8.5 // indirect 27 | github.com/mattn/go-colorable v0.1.12 // indirect 28 | github.com/mattn/go-isatty v0.0.14 // indirect 29 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 30 | github.com/mitchellh/mapstructure v1.4.3 // indirect 31 | github.com/pelletier/go-toml v1.9.4 // indirect 32 | github.com/pkg/errors v0.9.1 // indirect 33 | github.com/prometheus/client_model v0.2.0 // indirect 34 | github.com/prometheus/common v0.32.1 // indirect 35 | github.com/prometheus/procfs v0.7.3 // indirect 36 | github.com/sirupsen/logrus v1.8.1 // indirect 37 | github.com/spf13/afero v1.8.0 // indirect 38 | github.com/spf13/cast v1.4.1 // indirect 39 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 40 | github.com/spf13/pflag v1.0.5 // indirect 41 | github.com/spf13/viper v1.10.1 // indirect 42 | github.com/subosito/gotenv v1.2.0 // indirect 43 | github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect 44 | github.com/uber/jaeger-lib v2.4.1+incompatible // indirect 45 | go.uber.org/atomic v1.9.0 // indirect 46 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect 47 | golang.org/x/text v0.3.7 // indirect 48 | gopkg.in/ini.v1 v1.66.2 // indirect 49 | gopkg.in/yaml.v2 v2.4.0 // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "net/http/pprof" // 注册 pprof 接口 5 | 6 | "sniper/cmd/cron" 7 | "sniper/cmd/http" 8 | 9 | "github.com/spf13/cobra" 10 | _ "go.uber.org/automaxprocs" // 根据容器配额设置 maxprocs 11 | ) 12 | 13 | func main() { 14 | root := cobra.Command{Use: "sniper"} 15 | 16 | root.AddCommand( 17 | cron.Cmd, 18 | http.Cmd, 19 | ) 20 | 21 | root.Execute() 22 | } 23 | -------------------------------------------------------------------------------- /pkg/README.md: -------------------------------------------------------------------------------- 1 | # Sniper 工具包 2 | 3 | 非 sniper 项目也可以使用 4 | 5 | ``` 6 | go get github.com/go-kiss/sniper/pkg@latest 7 | ``` 8 | 9 | 独立使用的时候需要通过导入 pkg 包完成初始化 10 | 11 | ```go 12 | import _ "github.com/go-kiss/sniper/pkg" 13 | ``` 14 | -------------------------------------------------------------------------------- /pkg/conf/README.md: -------------------------------------------------------------------------------- 1 | # conf 2 | 3 | 默认从 sniper.toml 加载配置。虽然 sniper.toml 支持复杂的数据结构， 4 | 但框架要求只能设置 k-v 型配置。目的是为了跟环境变量相兼容。 5 | 6 | sniper.toml 中的所有配置项都可以使用环境变量覆写。 7 | 8 | 如果配置文件不在项目根目录，则可以通过环境变量`CONF_PATH`指定。 9 | 10 | 框架还会自动监听`CONF_PATH`目录下所有 toml 内容变更，发现变更会自动加载。 11 | 12 | 最后，配置名跟环境变量一样，不区分大小写字母。 13 | 14 | # 示例 15 | ```go 16 | import "github.com/go-kiss/sniper/pkg/conf" 17 | 18 | a := conf.Get("LOG_LEVEL") 19 | 20 | b := conf.File("foo").GetInt32("WORKER_NUM") 21 | ``` 22 | 23 | Sniper 的 memdb/sqldb 等组件依赖 conf 组件。如果不想通过文件的方式加载配置， 24 | 则可以覆盖`conf.Get`方法实现新的配置加载逻辑。 25 | -------------------------------------------------------------------------------- /pkg/conf/conf.go: -------------------------------------------------------------------------------- 1 | // Package conf 提供最基础的配置加载功能 2 | package conf 3 | 4 | import ( 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/fsnotify/fsnotify" 11 | "github.com/spf13/viper" 12 | ) 13 | 14 | // Get 查询配置/环境变量 15 | var Get func(string) string 16 | 17 | var ( 18 | // Host 主机名 19 | Host = "localhost" 20 | // App 服务标识 21 | App = "localapp" 22 | // Env 运行环境 23 | Env = "dev" 24 | // Zone 服务区域 25 | Zone = "sh001" 26 | 27 | files = map[string]*Conf{} 28 | 29 | defaultFile = "sniper" 30 | ) 31 | 32 | func init() { 33 | Host, _ = os.Hostname() 34 | if appID := os.Getenv("APP_ID"); appID != "" { 35 | App = appID 36 | } 37 | 38 | if env := os.Getenv("ENV"); env != "" { 39 | Env = env 40 | } 41 | 42 | if zone := os.Getenv("ZONE"); zone != "" { 43 | Zone = zone 44 | } 45 | 46 | if name := os.Getenv("CONF_NAME"); name != "" { 47 | defaultFile = name 48 | } 49 | 50 | path := os.Getenv("CONF_PATH") 51 | if path == "" { 52 | var err error 53 | if path, err = os.Getwd(); err != nil { 54 | panic(err) 55 | } 56 | } 57 | 58 | fs, err := os.ReadDir(path) 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | for _, f := range fs { 64 | if !strings.HasSuffix(f.Name(), ".toml") { 65 | continue 66 | } 67 | 68 | v := viper.New() 69 | v.SetConfigFile(filepath.Join(path, f.Name())) 70 | if err := v.ReadInConfig(); err != nil { 71 | panic(err) 72 | } 73 | v.AutomaticEnv() 74 | 75 | name := strings.TrimSuffix(f.Name(), ".toml") 76 | files[name] = &Conf{v} 77 | } 78 | 79 | Get = GetString 80 | } 81 | 82 | type Conf struct { 83 | *viper.Viper 84 | } 85 | 86 | // File 根据文件名获取对应配置对象 87 | // 目前仅支持 toml 文件，不用传扩展名 88 | // 如果要读取 foo.toml 配置，可以 File("foo").Get("bar") 89 | func File(name string) *Conf { 90 | return files[name] 91 | } 92 | 93 | // OnConfigChange 注册配置文件变更回调 94 | // 需要在 WatchConfig 之前调用 95 | func OnConfigChange(run func()) { 96 | for _, v := range files { 97 | v.OnConfigChange(func(in fsnotify.Event) { run() }) 98 | } 99 | } 100 | 101 | // WatchConfig 启动配置变更监听，业务代码不要调用。 102 | func WatchConfig() { 103 | for _, v := range files { 104 | v.WatchConfig() 105 | } 106 | } 107 | 108 | // Set 设置配置，仅用于测试 109 | func Set(key string, value interface{}) { File(defaultFile).Set(key, value) } 110 | 111 | func GetBool(key string) bool { return File(defaultFile).GetBool(key) } 112 | func GetDuration(key string) time.Duration { return File(defaultFile).GetDuration(key) } 113 | func GetFloat64(key string) float64 { return File(defaultFile).GetFloat64(key) } 114 | func GetInt(key string) int { return File(defaultFile).GetInt(key) } 115 | func GetInt32(key string) int32 { return File(defaultFile).GetInt32(key) } 116 | func GetInt64(key string) int64 { return File(defaultFile).GetInt64(key) } 117 | func GetIntSlice(key string) []int { return File(defaultFile).GetIntSlice(key) } 118 | func GetSizeInBytes(key string) uint { return File(defaultFile).GetSizeInBytes(key) } 119 | func GetString(key string) string { return File(defaultFile).GetString(key) } 120 | func GetStringSlice(key string) []string { return File(defaultFile).GetStringSlice(key) } 121 | func GetTime(key string) time.Time { return File(defaultFile).GetTime(key) } 122 | func GetUint(key string) uint { return File(defaultFile).GetUint(key) } 123 | func GetUint32(key string) uint32 { return File(defaultFile).GetUint32(key) } 124 | func GetUint64(key string) uint64 { return File(defaultFile).GetUint64(key) } 125 | 126 | func GetStringMap(key string) map[string]interface{} { return File(defaultFile).GetStringMap(key) } 127 | func GetStringMapString(key string) map[string]string { 128 | return File(defaultFile).GetStringMapString(key) 129 | } 130 | func GetStringMapStringSlice(key string) map[string][]string { 131 | return File(defaultFile).GetStringMapStringSlice(key) 132 | } 133 | -------------------------------------------------------------------------------- /pkg/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-kiss/sniper/pkg 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/dlmiddlecote/sqlstats v1.0.2 7 | github.com/fsnotify/fsnotify v1.5.1 8 | github.com/go-redis/redis/extra/rediscmd/v8 v8.11.4 9 | github.com/go-redis/redis/v8 v8.11.4 10 | github.com/go-sql-driver/mysql v1.6.0 11 | github.com/jmoiron/sqlx v1.3.4 12 | github.com/k0kubun/pp/v3 v3.1.0 13 | github.com/mattn/go-isatty v0.0.14 14 | github.com/ngrok/sqlmw v0.0.0-20210819213940-241da6c2def4 15 | github.com/opentracing/opentracing-go v1.2.0 16 | github.com/prometheus/client_golang v1.11.0 17 | github.com/sirupsen/logrus v1.8.1 18 | github.com/spf13/viper v1.10.1 19 | github.com/uber/jaeger-client-go v2.30.0+incompatible 20 | github.com/uber/jaeger-lib v2.4.1+incompatible 21 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 22 | google.golang.org/protobuf v1.27.1 23 | modernc.org/sqlite v1.13.1 24 | ) 25 | 26 | require ( 27 | github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect 28 | github.com/beorn7/perks v1.0.1 // indirect 29 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 30 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 31 | github.com/golang/protobuf v1.5.2 // indirect 32 | github.com/hashicorp/hcl v1.0.0 // indirect 33 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect 34 | github.com/magiconair/properties v1.8.5 // indirect 35 | github.com/mattn/go-colorable v0.1.12 // indirect 36 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 37 | github.com/mitchellh/mapstructure v1.4.3 // indirect 38 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 39 | github.com/pelletier/go-toml v1.9.4 // indirect 40 | github.com/pkg/errors v0.9.1 // indirect 41 | github.com/prometheus/client_model v0.2.0 // indirect 42 | github.com/prometheus/common v0.32.1 // indirect 43 | github.com/prometheus/procfs v0.7.3 // indirect 44 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect 45 | github.com/spf13/afero v1.8.0 // indirect 46 | github.com/spf13/cast v1.4.1 // indirect 47 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 48 | github.com/spf13/pflag v1.0.5 // indirect 49 | github.com/subosito/gotenv v1.2.0 // indirect 50 | go.uber.org/atomic v1.9.0 // indirect 51 | golang.org/x/mod v0.4.2 // indirect 52 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect 53 | golang.org/x/text v0.3.7 // indirect 54 | golang.org/x/tools v0.1.5 // indirect 55 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 56 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 57 | gopkg.in/ini.v1 v1.66.2 // indirect 58 | gopkg.in/yaml.v2 v2.4.0 // indirect 59 | lukechampine.com/uint128 v1.1.1 // indirect 60 | modernc.org/cc/v3 v3.34.0 // indirect 61 | modernc.org/ccgo/v3 v3.11.2 // indirect 62 | modernc.org/libc v1.11.3 // indirect 63 | modernc.org/mathutil v1.4.1 // indirect 64 | modernc.org/memory v1.0.5 // indirect 65 | modernc.org/opt v0.1.1 // indirect 66 | modernc.org/strutil v1.1.1 // indirect 67 | modernc.org/token v1.0.0 // indirect 68 | ) 69 | -------------------------------------------------------------------------------- /pkg/http/http.go: -------------------------------------------------------------------------------- 1 | // Package http 提供基础 http 客户端组件 2 | // 3 | // 本包通过替换 http.DefaultTransport 实现以下功能： 4 | // - 日志(logging) 5 | // - 链路追踪(tracing) 6 | // - 指标监控(metrics) 7 | // 8 | // 请务必使用 http.NewRequestWithContext 构造 req 对象，这样才能传递 ctx 信息。 9 | // 10 | // 如果希望使用自定义 Transport，需要将 RoundTrip 的逻辑 11 | // 最终委托给 http.DefaultTransport 12 | // 13 | // 使用示例： 14 | // req, _ := http.NewRequestWithContext(ctx, method, url, body) 15 | // c := &http.Client{ 16 | // Timeout: 1 * time.Second, 17 | // } 18 | // resp, err := c.Do(req) 19 | package http 20 | 21 | import ( 22 | "fmt" 23 | "net/http" 24 | "regexp" 25 | "time" 26 | 27 | "github.com/go-kiss/sniper/pkg/log" 28 | "github.com/opentracing/opentracing-go" 29 | "github.com/opentracing/opentracing-go/ext" 30 | ) 31 | 32 | func init() { 33 | http.DefaultTransport = &roundTripper{ 34 | r: http.DefaultTransport, 35 | } 36 | } 37 | 38 | type roundTripper struct { 39 | r http.RoundTripper 40 | } 41 | 42 | var digitsRE = regexp.MustCompile(`\b\d+\b`) 43 | 44 | func (r *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 45 | ctx := req.Context() 46 | span, ctx := opentracing.StartSpanFromContext(ctx, "DoHTTP") 47 | defer span.Finish() 48 | 49 | opentracing.GlobalTracer().Inject( 50 | span.Context(), 51 | opentracing.HTTPHeaders, 52 | opentracing.HTTPHeadersCarrier(req.Header), 53 | ) 54 | 55 | start := time.Now() 56 | resp, err := r.r.RoundTrip(req) 57 | duration := time.Since(start) 58 | 59 | url := fmt.Sprintf("%s%s", req.URL.Host, req.URL.Path) 60 | 61 | status := http.StatusOK 62 | if err != nil { 63 | status = http.StatusInternalServerError 64 | } else { 65 | status = resp.StatusCode 66 | } 67 | 68 | log.Get(ctx).Debugf( 69 | "[HTTP] method:%s url:%s status:%d query:%s", 70 | req.Method, 71 | url, 72 | status, 73 | req.URL.RawQuery, 74 | ) 75 | 76 | span.SetTag(string(ext.Component), "http") 77 | span.SetTag(string(ext.HTTPUrl), url) 78 | span.SetTag(string(ext.HTTPMethod), req.Method) 79 | span.SetTag(string(ext.HTTPStatusCode), status) 80 | 81 | // 在 url 附带参数会产生大量 metrics 指标，影响 prometheus 性能。 82 | // 默认会把 url 中带有的纯数字替换成 %d 83 | // /v123/4/56/foo => /v123/%d/%d/foo 84 | url = digitsRE.ReplaceAllString(url, "%d") 85 | 86 | httpDurations.WithLabelValues( 87 | url, 88 | fmt.Sprint(status), 89 | ).Observe(duration.Seconds()) 90 | 91 | return resp, err 92 | } 93 | -------------------------------------------------------------------------------- /pkg/http/metrics.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var defBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1} 8 | 9 | var httpDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 10 | Namespace: "sniper", 11 | Subsystem: "http", 12 | Name: "req_durations_seconds", 13 | Help: "HTTP latency distributions", 14 | Buckets: defBuckets, 15 | }, []string{"url", "status"}) 16 | 17 | func init() { 18 | prometheus.MustRegister(httpDurations) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/log/README.md: -------------------------------------------------------------------------------- 1 | # log 2 | 3 | log 目前最低级别是 debug，可以通过 LOG_LEVEL 环境变量或者配置项指定。 4 | 5 | log 会记录上下文信息，所以需要传入一个 ctx 才能获取 log 实例。 6 | 7 | ## 示例 8 | 9 | ```go 10 | import "github.com/go-kiss/sniper/pkg/log" 11 | 12 | log.Get(ctx).Errorf("1 + 2 = %d", 1 + 2) 13 | log.Errorf(ctx, "1 + 2 = %d", 1 + 2) 14 | ``` 15 | -------------------------------------------------------------------------------- /pkg/log/helper.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import "context" 4 | 5 | func Trace(ctx context.Context, args ...interface{}) { 6 | Get(ctx).Trace(args...) 7 | } 8 | 9 | func Debug(ctx context.Context, args ...interface{}) { 10 | Get(ctx).Debug(args...) 11 | } 12 | 13 | func Info(ctx context.Context, args ...interface{}) { 14 | Get(ctx).Info(args...) 15 | } 16 | 17 | func Warn(ctx context.Context, args ...interface{}) { 18 | Get(ctx).Warn(args...) 19 | } 20 | 21 | func Error(ctx context.Context, args ...interface{}) { 22 | Get(ctx).Error(args...) 23 | } 24 | 25 | func Fatal(ctx context.Context, args ...interface{}) { 26 | Get(ctx).Fatal(args...) 27 | } 28 | 29 | func Panic(ctx context.Context, args ...interface{}) { 30 | Get(ctx).Panic(args...) 31 | } 32 | 33 | func Tracef(ctx context.Context, format string, args ...interface{}) { 34 | Get(ctx).Tracef(format, args...) 35 | } 36 | 37 | func Debugf(ctx context.Context, format string, args ...interface{}) { 38 | Get(ctx).Debugf(format, args...) 39 | } 40 | 41 | func Infof(ctx context.Context, format string, args ...interface{}) { 42 | Get(ctx).Infof(format, args...) 43 | } 44 | 45 | func Warnf(ctx context.Context, format string, args ...interface{}) { 46 | Get(ctx).Warnf(format, args...) 47 | } 48 | 49 | func Errorf(ctx context.Context, format string, args ...interface{}) { 50 | Get(ctx).Errorf(format, args...) 51 | } 52 | 53 | func Fatalf(ctx context.Context, format string, args ...interface{}) { 54 | Get(ctx).Fatalf(format, args...) 55 | } 56 | 57 | func Panicf(ctx context.Context, format string, args ...interface{}) { 58 | Get(ctx).Panicf(format, args...) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | // Package log 基础日志组件 2 | package log 3 | 4 | import ( 5 | "context" 6 | "os" 7 | 8 | "github.com/go-kiss/sniper/pkg/conf" 9 | "github.com/go-kiss/sniper/pkg/trace" 10 | "github.com/k0kubun/pp/v3" 11 | "github.com/mattn/go-isatty" 12 | "github.com/sirupsen/logrus" 13 | ) 14 | 15 | func init() { 16 | setLevel() 17 | initPP() 18 | } 19 | 20 | func initPP() { 21 | out := os.Stdout 22 | pp.SetDefaultOutput(out) 23 | 24 | if !isatty.IsTerminal(out.Fd()) { 25 | pp.ColoringEnabled = false 26 | } 27 | } 28 | 29 | // Logger logger 30 | type Logger = *logrus.Entry 31 | 32 | // Fields fields 33 | type Fields = logrus.Fields 34 | 35 | var levels = map[string]logrus.Level{ 36 | "panic": logrus.PanicLevel, 37 | "fatal": logrus.FatalLevel, 38 | "error": logrus.ErrorLevel, 39 | "warn": logrus.WarnLevel, 40 | "info": logrus.InfoLevel, 41 | "debug": logrus.DebugLevel, 42 | } 43 | 44 | func setLevel() { 45 | levelConf := conf.Get("LOG_LEVEL_" + conf.Host) 46 | 47 | if levelConf == "" { 48 | levelConf = conf.Get("LOG_LEVEL") 49 | } 50 | 51 | if level, ok := levels[levelConf]; ok { 52 | logrus.SetLevel(level) 53 | } else { 54 | logrus.SetLevel(logrus.DebugLevel) 55 | } 56 | } 57 | 58 | // Get 获取日志实例 59 | func Get(ctx context.Context) Logger { 60 | return logrus.WithFields(logrus.Fields{ 61 | "env": conf.Env, 62 | "app": conf.App, 63 | "host": conf.Host, 64 | "trace_id": trace.GetTraceID(ctx), 65 | }) 66 | } 67 | 68 | // Reset 使用最新配置重置日志级别 69 | func Reset() { 70 | setLevel() 71 | } 72 | 73 | // PP 类似 PHP 的 var_dump 74 | func PP(args ...interface{}) { 75 | pp.Println(args...) 76 | } 77 | -------------------------------------------------------------------------------- /pkg/memdb/README.md: -------------------------------------------------------------------------------- 1 | # memdb 2 | 3 | memdb 主要解决以下问题： 4 | 5 | - 加载 redis 配置 6 | - 记录 redis 执行日志 7 | - 上报 opentracing 追踪数据 8 | - 汇总 prometheus 监控指标 9 | 10 | 核心思想是调用`AddHook`添加回调，拦截所有缓存操作进行观察。 11 | 12 | ## 配置 13 | 14 | 框架默认只支持 redis。 15 | 16 | 每个数据库的配置需要指定一个名字，并添加`MEMDB_DSN_`前缀。 17 | 18 | 配置内容使用 url 格式，参数使用 query 字符串传递。 19 | 20 | ```yaml 21 | MEMDB_DSN_BAR = "redis://name:password@localhost:6379?DB=1" 22 | ``` 23 | 24 | 除了 hostname 之外，支持所有类型为`int/bool/time.Duration`的配置。 25 | 26 | 配置列表参考官方文档： 27 | 28 | ## 使用 29 | 30 | 框架通过`memdb.Get(name)`函数获取缓存实例，入参是配置名（去掉前缀）， 31 | 返回的是`*redis.Client`对象。 32 | 33 | ```go 34 | import "github.com/go-kiss/sniper/pkg/sqldb" 35 | 36 | db := Get("foo") 37 | db.Set(ctx, "a", "123", 0) 38 | ``` 39 | -------------------------------------------------------------------------------- /pkg/memdb/memdb.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/go-kiss/sniper/pkg/conf" 7 | "github.com/go-redis/redis/v8" 8 | "golang.org/x/sync/singleflight" 9 | ) 10 | 11 | var ( 12 | sfg singleflight.Group 13 | rwl sync.RWMutex 14 | 15 | dbs = map[string]*Client{} 16 | ) 17 | 18 | type nameKey struct{} 19 | 20 | // Client redis 客户端 21 | type Client struct { 22 | redis.UniversalClient 23 | } 24 | 25 | // Get 获取缓存实例 26 | // 27 | // db := Get("foo") 28 | // db.Set(ctx, "a", "123", 0) 29 | func Get(name string) *Client { 30 | rwl.RLock() 31 | if db, ok := dbs[name]; ok { 32 | rwl.RUnlock() 33 | return db 34 | } 35 | rwl.RUnlock() 36 | 37 | v, _, _ := sfg.Do(name, func() (interface{}, error) { 38 | opts := &redis.UniversalOptions{} 39 | 40 | dsn := conf.Get("MEMDB_DSN_" + name) 41 | setOptions(opts, dsn) 42 | 43 | rdb := redis.NewUniversalClient(opts) 44 | 45 | rdb.AddHook(observer{name: name}) 46 | 47 | registerStats(name, rdb) 48 | 49 | db := &Client{rdb} 50 | 51 | rwl.Lock() 52 | defer rwl.Unlock() 53 | dbs[name] = db 54 | 55 | return db, nil 56 | }) 57 | 58 | return v.(*Client) 59 | } 60 | -------------------------------------------------------------------------------- /pkg/memdb/memdb_test.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/go-kiss/sniper/pkg/conf" 8 | ) 9 | 10 | func TestMemDb(t *testing.T) { 11 | conf.Set("MEMDB_DSN_foo", "redis://localhost:6379/") 12 | 13 | ctx := context.Background() 14 | db := Get("foo") 15 | 16 | s := db.Set(ctx, "a", "123", 0) 17 | if err := s.Err(); err != nil { 18 | t.Fatal(err) 19 | } 20 | 21 | sc := db.Get(ctx, "a") 22 | if v, err := sc.Result(); err != nil { 23 | t.Fatal(err) 24 | } else if v != "123" { 25 | t.Fatal("invalid string: " + v) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /pkg/memdb/metrics.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "github.com/go-redis/redis/v8" 5 | "github.com/prometheus/client_golang/prometheus" 6 | ) 7 | 8 | var defBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1} 9 | 10 | var redisDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 11 | Namespace: "sniper", 12 | Subsystem: "memdb", 13 | Name: "commands_duration_seconds", 14 | Help: "commands latency distributions", 15 | Buckets: defBuckets, 16 | }, []string{"db_name", "cmd"}) 17 | 18 | func init() { 19 | prometheus.MustRegister(redisDurations) 20 | } 21 | 22 | type StatsCollector struct { 23 | db redis.UniversalClient 24 | 25 | // descriptions of exported metrics 26 | hitDesc *prometheus.Desc 27 | missDesc *prometheus.Desc 28 | timeoutDesc *prometheus.Desc 29 | totalDesc *prometheus.Desc 30 | idleDesc *prometheus.Desc 31 | staleDesc *prometheus.Desc 32 | } 33 | 34 | const ( 35 | namespace = "sniper" 36 | subsystem = "memdb_connections" 37 | ) 38 | 39 | func registerStats(dbName string, db redis.UniversalClient) { 40 | labels := prometheus.Labels{"db_name": dbName} 41 | s := &StatsCollector{ 42 | db: db, 43 | hitDesc: prometheus.NewDesc( 44 | prometheus.BuildFQName(namespace, subsystem, "hit"), 45 | "The number number of times free connection was NOT found in the pool.", 46 | nil, 47 | labels, 48 | ), 49 | missDesc: prometheus.NewDesc( 50 | prometheus.BuildFQName(namespace, subsystem, "miss"), 51 | "The number of times free connection was found in the pool.", 52 | nil, 53 | labels, 54 | ), 55 | timeoutDesc: prometheus.NewDesc( 56 | prometheus.BuildFQName(namespace, subsystem, "timeout"), 57 | "The number of times a wait timeout occurred.", 58 | nil, 59 | labels, 60 | ), 61 | totalDesc: prometheus.NewDesc( 62 | prometheus.BuildFQName(namespace, subsystem, "total"), 63 | "The number of total connections in the pool.", 64 | nil, 65 | labels, 66 | ), 67 | idleDesc: prometheus.NewDesc( 68 | prometheus.BuildFQName(namespace, subsystem, "idle"), 69 | "The number of idle connections in the pool.", 70 | nil, 71 | labels, 72 | ), 73 | staleDesc: prometheus.NewDesc( 74 | prometheus.BuildFQName(namespace, subsystem, "stale"), 75 | "The number of stale connections in the pool.", 76 | nil, 77 | labels, 78 | ), 79 | } 80 | 81 | prometheus.MustRegister(s) 82 | return 83 | } 84 | 85 | // Describe implements the prometheus.Collector interface. 86 | func (c StatsCollector) Describe(ch chan<- *prometheus.Desc) { 87 | ch <- c.hitDesc 88 | ch <- c.missDesc 89 | ch <- c.timeoutDesc 90 | ch <- c.totalDesc 91 | ch <- c.idleDesc 92 | ch <- c.staleDesc 93 | } 94 | 95 | // Collect implements the prometheus.Collector interface. 96 | func (c StatsCollector) Collect(ch chan<- prometheus.Metric) { 97 | stats := c.db.PoolStats() 98 | 99 | ch <- prometheus.MustNewConstMetric( 100 | c.hitDesc, 101 | prometheus.CounterValue, 102 | float64(stats.Hits), 103 | ) 104 | ch <- prometheus.MustNewConstMetric( 105 | c.missDesc, 106 | prometheus.CounterValue, 107 | float64(stats.Misses), 108 | ) 109 | ch <- prometheus.MustNewConstMetric( 110 | c.timeoutDesc, 111 | prometheus.CounterValue, 112 | float64(stats.Timeouts), 113 | ) 114 | ch <- prometheus.MustNewConstMetric( 115 | c.totalDesc, 116 | prometheus.GaugeValue, 117 | float64(stats.TotalConns), 118 | ) 119 | ch <- prometheus.MustNewConstMetric( 120 | c.idleDesc, 121 | prometheus.GaugeValue, 122 | float64(stats.IdleConns), 123 | ) 124 | ch <- prometheus.MustNewConstMetric( 125 | c.staleDesc, 126 | prometheus.GaugeValue, 127 | float64(stats.StaleConns), 128 | ) 129 | } 130 | -------------------------------------------------------------------------------- /pkg/memdb/observer.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/go-kiss/sniper/pkg/log" 7 | "github.com/go-kiss/sniper/pkg/trace" 8 | "github.com/go-redis/redis/extra/rediscmd/v8" 9 | "github.com/go-redis/redis/v8" 10 | "github.com/opentracing/opentracing-go" 11 | "github.com/opentracing/opentracing-go/ext" 12 | ) 13 | 14 | // 观察所有 redis 命令执行情况 15 | type observer struct { 16 | name string 17 | } 18 | 19 | func (o observer) BeforeProcess(ctx context.Context, cmd redis.Cmder) (context.Context, error) { 20 | span, ctx := opentracing.StartSpanFromContext(ctx, cmd.FullName()) 21 | 22 | ext.Component.Set(span, "memdb") 23 | ext.DBInstance.Set(span, o.name) 24 | ext.DBStatement.Set(span, rediscmd.CmdString(cmd)) 25 | 26 | return ctx, nil 27 | } 28 | 29 | func (o observer) AfterProcess(ctx context.Context, cmd redis.Cmder) error { 30 | span := opentracing.SpanFromContext(ctx) 31 | if err := cmd.Err(); err != nil && err != redis.Nil { 32 | ext.Error.Set(span, true) 33 | ext.LogError(span, err) 34 | } 35 | span.Finish() 36 | 37 | d := trace.GetDuration(span) 38 | log.Get(ctx).Debugf("[memdb] %s, cost:%v", rediscmd.CmdString(cmd), d) 39 | 40 | redisDurations.WithLabelValues( 41 | o.name, 42 | cmd.FullName(), 43 | ).Observe(d.Seconds()) 44 | 45 | return nil 46 | } 47 | 48 | func (o observer) BeforeProcessPipeline(ctx context.Context, cmds []redis.Cmder) (context.Context, error) { 49 | return ctx, nil 50 | } 51 | 52 | func (o observer) AfterProcessPipeline(ctx context.Context, cmds []redis.Cmder) error { 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/memdb/utils.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "reflect" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/go-redis/redis/v8" 12 | ) 13 | 14 | func setOptions(opts *redis.UniversalOptions, dsn string) { 15 | url, err := url.Parse(dsn) 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | args := url.Query() 21 | 22 | rv := reflect.ValueOf(opts).Elem() 23 | rt := rv.Type() 24 | 25 | for i := 0; i < rv.NumField(); i++ { 26 | f := rv.Field(i) 27 | if !f.CanInterface() { 28 | continue 29 | } 30 | name := rt.Field(i).Name 31 | arg := args.Get(name) 32 | if arg == "" { 33 | continue 34 | } 35 | switch f.Interface().(type) { 36 | case time.Duration: 37 | v, err := time.ParseDuration(arg) 38 | if err != nil { 39 | panic(fmt.Sprintf("%s=%s, err:%v", name, arg, err)) 40 | } 41 | f.Set(reflect.ValueOf(v)) 42 | case int: 43 | v, err := strconv.Atoi(arg) 44 | if err != nil { 45 | panic(fmt.Sprintf("%s=%s, err:%v", name, arg, err)) 46 | } 47 | f.SetInt(int64(v)) 48 | case bool: 49 | v, err := strconv.ParseBool(arg) 50 | if err != nil { 51 | panic(fmt.Sprintf("%s=%s, err:%v", name, arg, err)) 52 | } 53 | f.SetBool(v) 54 | case string: 55 | f.SetString(arg) 56 | } 57 | } 58 | 59 | opts.Addrs = []string{url.Host} 60 | opts.Username = url.User.Username() 61 | if p, ok := url.User.Password(); ok { 62 | opts.Password = p 63 | } 64 | } 65 | 66 | func name(ctx context.Context) string { 67 | v, _ := ctx.Value(nameKey{}).(string) 68 | if v == "" { 69 | v = "unknown" 70 | } 71 | return v 72 | } 73 | -------------------------------------------------------------------------------- /pkg/memdb/utils_test.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/go-redis/redis/v8" 10 | ) 11 | 12 | func TestSetOptions(t *testing.T) { 13 | opts := redis.UniversalOptions{} 14 | dsn := "redis://foo:bar@localhost:6379/?DB=1&" + 15 | "&Network=tcp" + 16 | "&MaxRetries=1" + 17 | "&MinRetryBackoff=1s" + 18 | "&MaxRetryBackoff=1s" + 19 | "&DialTimeout=1s" + 20 | "&ReadTimeout=1s" + 21 | "&WriteTimeout=1s" + 22 | "&PoolFIFO=true" + 23 | "&PoolSize=1" + 24 | "&MinIdleConns=1" + 25 | "&MaxConnAge=1s" + 26 | "&PoolTimeout=1s" + 27 | "&IdleTimeout=1s" + 28 | "&IdleCheckFrequency=1s" 29 | 30 | setOptions(&opts, dsn) 31 | v := redis.Options{ 32 | Network: "tcp", 33 | Addr: "localhost:6379", 34 | Username: "foo", 35 | Password: "bar", 36 | DB: 1, 37 | MaxRetries: 1, 38 | MinRetryBackoff: 1 * time.Second, 39 | MaxRetryBackoff: 1 * time.Second, 40 | DialTimeout: 1 * time.Second, 41 | ReadTimeout: 1 * time.Second, 42 | WriteTimeout: 1 * time.Second, 43 | PoolFIFO: true, 44 | PoolSize: 1, 45 | MinIdleConns: 1, 46 | MaxConnAge: 1 * time.Second, 47 | PoolTimeout: 1 * time.Second, 48 | IdleTimeout: 1 * time.Second, 49 | IdleCheckFrequency: 1 * time.Second, 50 | } 51 | 52 | if !reflect.DeepEqual(opts, v) { 53 | fmt.Println(opts) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/pkg.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | import ( 4 | _ "github.com/go-kiss/sniper/pkg/conf" // init conf 5 | _ "github.com/go-kiss/sniper/pkg/http" // init http 6 | 7 | "github.com/go-kiss/sniper/pkg/log" 8 | ) 9 | 10 | // Reset all utils 11 | func Reset() { 12 | log.Reset() 13 | } 14 | 15 | // Stop all utils 16 | func Stop() { 17 | } 18 | -------------------------------------------------------------------------------- /pkg/sqldb/README.md: -------------------------------------------------------------------------------- 1 | # sqldb 2 | 3 | sqldb 主要解决以下问题： 4 | 5 | - 加载数据库配置 6 | - 记录 sql 执行日志 7 | - 上报 opentracing 追踪数据 8 | - 汇总 prometheus 监控指标 9 | 10 | 核心思想是用`github.com/ngrok/sqlmw`把现有的`database/sql`驱动包起来， 11 | 拦截所有数据库操作进行观察。 12 | 13 | ## 配置 14 | 15 | 框架默认支持 sqlite 和 mysql。 16 | 17 | 每个数据库的配置需要指定一个名字，并添加`SQLDB_DSN_`前缀。 18 | 19 | ```yaml 20 | # sqlite 配置示例 21 | SQLDB_DSN_lite1 = "file:///tmp/foo.db" 22 | # mysql 配置示例 23 | SQLDB_DSN_mysql1 = "username:password@protocol(address)/dbname?param=value" 24 | ``` 25 | 26 | 不同的驱动需要不同的配置内容： 27 | 28 | - sqlite 请参考 29 | - mysql 请参考 30 | 31 | ## 使用 32 | 33 | 框架通过`sqldb.Get(name)`函数获取数据库实例，入参是配置名（去掉前缀）， 34 | 返回的是`*sqlx.DB`对象。 35 | 36 | 框架会根据配置内容自动识别数据库驱动。 37 | 38 | ```go 39 | import "github.com/go-kiss/sniper/pkg/sqldb" 40 | 41 | db := sqldb.Get(ctx, "name") 42 | db.ExecContext(ctx, "delete from ...") 43 | ``` 44 | 45 | ## ORM 46 | 47 | sqldb 提供简单的 Insert/Update/StructScan 方法，替换常用的 ORM 使用场景。 48 | 49 | 所有的模型对象都必须实现 `Modler` 接口，支持查询所属的表名和主键字段名。 50 | 51 | 比如我们定义一个 user 对象： 52 | 53 | ```go 54 | type user struct { 55 | ID int 56 | Name string 57 | Age int 58 | Created time.Time 59 | } 60 | func (u *user) TableName() string { return "users" } 61 | func (u *user) KeyName() string { return "id" } 62 | ``` 63 | 64 | 保存对象： 65 | 66 | ```go 67 | u := {Name:"foo", Age:18, Created:time.Now()} 68 | result, err := db.Insert(&u) 69 | ``` 70 | 71 | 更新对象： 72 | 73 | ```go 74 | u.Name = "bar" 75 | result, err := db.Update(&u) 76 | ``` 77 | 78 | 查询对象： 79 | 80 | ```go 81 | var u user 82 | err := db.Get(&u, "select * from users where id = ?", id) 83 | ``` 84 | 85 | ## 现有问题 86 | 87 | 受限于 database/sql 驱动的设计，我们无法在提交或者回滚事务的时候确定总耗时。 88 | 89 | 目前只能监控 begin/commit/rollback 单个查询耗时，而非事务总耗时。 90 | 91 | database/sql 不支持添加 hooks，而使用 sql.Regster 只能进行全局注册。为了实现 92 | 不同数据库实例注册不同 hooks 的效果（主要用于保存数据库配置名字），我们为每个 93 | 数据库配置分别注册 sql.Driver 实例。 94 | 95 | 96 | ## 添加新驱动 97 | 98 | 如果想添加 sqlite 和 mysql 之外的数据库驱动（比如 postgres），需要初始化 99 | driverName 和 driver 两个变量。driverName 中应该包含数据库配置名（实例名）。 100 | 101 | ```go 102 | driverName = "db-sqlite:" + name 103 | driver = sqlmw.Driver(&sqlite.Driver{}, observer{name: name}) 104 | ``` 105 | -------------------------------------------------------------------------------- /pkg/sqldb/metrics.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | var defBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1} 8 | 9 | var sqlDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 10 | Namespace: "sniper", 11 | Subsystem: "sqldb", 12 | Name: "sql_durations_seconds", 13 | Help: "sql latency distributions", 14 | Buckets: defBuckets, 15 | }, []string{"db_name", "table", "cmd"}) 16 | 17 | func init() { 18 | prometheus.MustRegister(sqlDurations) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/sqldb/model.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "reflect" 8 | "sort" 9 | "strings" 10 | 11 | "github.com/jmoiron/sqlx/reflectx" 12 | ) 13 | 14 | // Modeler 接口提供查询模型的表结构信息 15 | // 所有模型都需要实现本接口 16 | type Modeler interface { 17 | // TableName 返回表名 18 | TableName() string 19 | // TableName 返回主键字段名 20 | KeyName() string 21 | } 22 | 23 | // 统一 DB 和 Tx 对象 24 | type mapExecer interface { 25 | DriverName() string 26 | GetMapper() *reflectx.Mapper 27 | Rebind(string) string 28 | ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) 29 | } 30 | 31 | // MustBegin 封装 sqlx.DB.MustBegin，返回自定义的 *Tx 32 | func (db *DB) MustBegin() *Tx { 33 | tx := db.DB.MustBegin() 34 | return &Tx{tx} 35 | } 36 | 37 | // Beginx 封装 sqlx.DB.Beginx，返回自定义的 *Tx 38 | func (db *DB) Beginx() (*Tx, error) { 39 | tx, err := db.DB.Beginx() 40 | if err != nil { 41 | return nil, err 42 | } 43 | return &Tx{tx}, nil 44 | } 45 | 46 | // BeginTxx 封装 sqlx.DB.BeginTxx，返回自定义的 *Tx 47 | func (db *DB) BeginTxx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) { 48 | tx, err := db.DB.BeginTxx(ctx, opts) 49 | if err != nil { 50 | return nil, err 51 | } 52 | return &Tx{tx}, nil 53 | } 54 | 55 | // InsertContext 生成并执行 insert 语句 56 | func (db *DB) InsertContext(ctx context.Context, m Modeler) (sql.Result, error) { 57 | return insert(ctx, db, m) 58 | } 59 | 60 | func (db *DB) Insert(m Modeler) (sql.Result, error) { 61 | return db.InsertContext(context.Background(), m) 62 | } 63 | 64 | // UpdateContext 生成并执行 update 语句 65 | func (db *DB) UpdateContext(ctx context.Context, m Modeler) (sql.Result, error) { 66 | return update(ctx, db, m) 67 | } 68 | 69 | func (db *DB) Update(m Modeler) (sql.Result, error) { 70 | return db.UpdateContext(context.Background(), m) 71 | } 72 | 73 | // InsertContext 生成并执行 insert 语句 74 | func (tx *Tx) InsertContext(ctx context.Context, m Modeler) (sql.Result, error) { 75 | return insert(ctx, tx, m) 76 | } 77 | 78 | func (tx *Tx) Insert(m Modeler) (sql.Result, error) { 79 | return tx.InsertContext(context.Background(), m) 80 | } 81 | 82 | // UpdateContext 生成并执行 update 语句 83 | func (tx *Tx) UpdateContext(ctx context.Context, m Modeler) (sql.Result, error) { 84 | return update(ctx, tx, m) 85 | } 86 | 87 | func (tx *Tx) Update(m Modeler) (sql.Result, error) { 88 | return tx.UpdateContext(context.Background(), m) 89 | } 90 | 91 | // 添加 GetMapper 方法，方便与 Tx 统一 92 | func (db *DB) GetMapper() *reflectx.Mapper { 93 | return db.Mapper 94 | } 95 | 96 | // 添加 GetMapper 方法，方便与 DB 统一 97 | func (tx *Tx) GetMapper() *reflectx.Mapper { 98 | return tx.Mapper 99 | } 100 | 101 | func insert(ctx context.Context, db mapExecer, m Modeler) (sql.Result, error) { 102 | names, args, err := bindModeler(m, db.GetMapper()) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | marks := "" 108 | k := -1 109 | for i := 0; i < len(names); i++ { 110 | if names[i] == m.KeyName() { 111 | v := reflect.ValueOf(args[i]) 112 | if v.IsZero() { 113 | k = i 114 | args = append(args[:i], args[i+1:]...) 115 | continue 116 | } 117 | } 118 | marks += "?," 119 | } 120 | if k >= 0 { 121 | names = append(names[:k], names[k+1:]...) 122 | } 123 | marks = marks[:len(marks)-1] 124 | query := "INSERT INTO " + m.TableName() + "(" + strings.Join(names, ",") + ") VALUES (" + marks + ")" 125 | query = db.Rebind(query) 126 | return db.ExecContext(ctx, query, args...) 127 | } 128 | 129 | func update(ctx context.Context, db mapExecer, m Modeler) (sql.Result, error) { 130 | names, args, err := bindModeler(m, db.GetMapper()) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | query := "UPDATE " + m.TableName() + " set " 136 | var id interface{} 137 | for i := 0; i < len(names); i++ { 138 | name := names[i] 139 | if name == m.KeyName() { 140 | id = args[i] 141 | args = append(args[:i], args[i+1:]...) 142 | continue 143 | } 144 | query += name + "=?," 145 | } 146 | query = query[:len(query)-1] + " WHERE " + m.KeyName() + " = ?" 147 | query = db.Rebind(query) 148 | args = append(args, id) 149 | return db.ExecContext(ctx, query, args...) 150 | } 151 | 152 | func bindModeler(arg interface{}, m *reflectx.Mapper) ([]string, []interface{}, error) { 153 | t := reflect.TypeOf(arg) 154 | names := []string{} 155 | for k := range m.TypeMap(t).Names { 156 | names = append(names, k) 157 | } 158 | sort.Stable(sort.StringSlice(names)) 159 | args, err := bindArgs(names, arg, m) 160 | if err != nil { 161 | return nil, nil, err 162 | } 163 | 164 | return names, args, nil 165 | } 166 | 167 | func bindArgs(names []string, arg interface{}, m *reflectx.Mapper) ([]interface{}, error) { 168 | arglist := make([]interface{}, 0, len(names)) 169 | 170 | // grab the indirected value of arg 171 | v := reflect.ValueOf(arg) 172 | for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; { 173 | v = v.Elem() 174 | } 175 | 176 | err := m.TraversalsByNameFunc(v.Type(), names, func(i int, t []int) error { 177 | if len(t) == 0 { 178 | return fmt.Errorf("could not find name %s in %#v", names[i], arg) 179 | } 180 | 181 | val := reflectx.FieldByIndexesReadOnly(v, t) 182 | arglist = append(arglist, val.Interface()) 183 | 184 | return nil 185 | }) 186 | 187 | return arglist, err 188 | } 189 | -------------------------------------------------------------------------------- /pkg/sqldb/observer.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import ( 4 | "context" 5 | "database/sql/driver" 6 | "time" 7 | 8 | "github.com/go-kiss/sniper/pkg/log" 9 | "github.com/ngrok/sqlmw" 10 | "github.com/opentracing/opentracing-go" 11 | "github.com/opentracing/opentracing-go/ext" 12 | ) 13 | 14 | // 观察所有 sql 执行情况 15 | type observer struct { 16 | sqlmw.NullInterceptor 17 | name string 18 | } 19 | 20 | func (o observer) ConnExecContext(ctx context.Context, 21 | conn driver.ExecerContext, 22 | query string, args []driver.NamedValue) (driver.Result, error) { 23 | 24 | span, ctx := opentracing.StartSpanFromContext(ctx, "Exec") 25 | defer span.Finish() 26 | 27 | ext.Component.Set(span, "sqldb") 28 | ext.DBInstance.Set(span, o.name) 29 | ext.DBStatement.Set(span, query) 30 | 31 | s := time.Now() 32 | result, err := conn.ExecContext(ctx, query, args) 33 | d := time.Since(s) 34 | 35 | log.Get(ctx).Debugf("[sqldb] name:%s, exec: %s, args: %v, cost: %v", 36 | o.name, query, values(args), d) 37 | 38 | table, cmd := parseSQL(query) 39 | sqlDurations.WithLabelValues( 40 | o.name, 41 | table, 42 | cmd, 43 | ).Observe(d.Seconds()) 44 | 45 | return result, err 46 | } 47 | 48 | func (o observer) ConnQueryContext(ctx context.Context, 49 | conn driver.QueryerContext, 50 | query string, args []driver.NamedValue) (driver.Rows, error) { 51 | 52 | span, ctx := opentracing.StartSpanFromContext(ctx, "Query") 53 | defer span.Finish() 54 | 55 | ext.Component.Set(span, "sqldb") 56 | ext.DBInstance.Set(span, o.name) 57 | ext.DBStatement.Set(span, query) 58 | 59 | s := time.Now() 60 | rows, err := conn.QueryContext(ctx, query, args) 61 | d := time.Since(s) 62 | 63 | log.Get(ctx).Debugf("[sqldb] name:%s, query: %s, args: %v, cost: %v", 64 | o.name, query, values(args), d) 65 | 66 | table, cmd := parseSQL(query) 67 | sqlDurations.WithLabelValues( 68 | o.name, 69 | table, 70 | cmd, 71 | ).Observe(d.Seconds()) 72 | 73 | return rows, err 74 | } 75 | 76 | func (o observer) ConnPrepareContext(ctx context.Context, 77 | conn driver.ConnPrepareContext, 78 | query string) (driver.Stmt, error) { 79 | 80 | span, ctx := opentracing.StartSpanFromContext(ctx, "Prepare") 81 | defer span.Finish() 82 | 83 | ext.Component.Set(span, "sqldb") 84 | ext.DBInstance.Set(span, o.name) 85 | ext.DBStatement.Set(span, query) 86 | 87 | s := time.Now() 88 | stmt, err := conn.PrepareContext(ctx, query) 89 | d := time.Since(s) 90 | 91 | log.Get(ctx).Debugf("[sqldb] name:%s, prepare: %s, args: %v, cost: %v", 92 | o.name, query, nil, d) 93 | 94 | table, _ := parseSQL(query) 95 | sqlDurations.WithLabelValues( 96 | o.name, 97 | table, 98 | "prepare", 99 | ).Observe(d.Seconds()) 100 | 101 | return stmt, err 102 | } 103 | 104 | func (o observer) StmtExecContext(ctx context.Context, 105 | stmt driver.StmtExecContext, 106 | query string, args []driver.NamedValue) (driver.Result, error) { 107 | 108 | span, ctx := opentracing.StartSpanFromContext(ctx, "PreparedExec") 109 | defer span.Finish() 110 | 111 | ext.Component.Set(span, "sqldb") 112 | ext.DBInstance.Set(span, o.name) 113 | ext.DBStatement.Set(span, query) 114 | 115 | s := time.Now() 116 | result, err := stmt.ExecContext(ctx, args) 117 | d := time.Since(s) 118 | 119 | log.Get(ctx).Debugf("[sqldb] name:%s, prepared exec: %s, args: %v, cost: %v", 120 | o.name, query, values(args), d) 121 | 122 | table, cmd := parseSQL(query) 123 | sqlDurations.WithLabelValues( 124 | o.name, 125 | table, 126 | cmd+"-prepared", 127 | ).Observe(d.Seconds()) 128 | 129 | return result, err 130 | } 131 | 132 | func (o observer) StmtQueryContext(ctx context.Context, 133 | stmt driver.StmtQueryContext, 134 | query string, args []driver.NamedValue) (driver.Rows, error) { 135 | 136 | span, ctx := opentracing.StartSpanFromContext(ctx, "PreparedQuery") 137 | defer span.Finish() 138 | 139 | ext.Component.Set(span, "sqldb") 140 | ext.DBInstance.Set(span, o.name) 141 | ext.DBStatement.Set(span, query) 142 | 143 | s := time.Now() 144 | rows, err := stmt.QueryContext(ctx, args) 145 | d := time.Since(s) 146 | 147 | log.Get(ctx).Debugf("[sqldb] name:%s, prepared query: %s, args: %v, cost: %v", 148 | o.name, query, values(args), d) 149 | 150 | table, cmd := parseSQL(query) 151 | sqlDurations.WithLabelValues( 152 | o.name, 153 | table, 154 | cmd+"-prepared", 155 | ).Observe(d.Seconds()) 156 | 157 | return rows, err 158 | } 159 | 160 | func (o observer) ConnBeginTx(ctx context.Context, conn driver.ConnBeginTx, 161 | txOpts driver.TxOptions) (driver.Tx, error) { 162 | 163 | span, ctx := opentracing.StartSpanFromContext(ctx, "Begin") 164 | defer span.Finish() 165 | 166 | ext.Component.Set(span, "sqldb") 167 | ext.DBInstance.Set(span, o.name) 168 | 169 | s := time.Now() 170 | tx, err := conn.BeginTx(ctx, txOpts) 171 | d := time.Since(s) 172 | 173 | log.Get(ctx).Debugf("[sqldb] name:%s, begin, cost: %v", o.name, d) 174 | 175 | sqlDurations.WithLabelValues( 176 | o.name, 177 | "", 178 | "begin", 179 | ).Observe(d.Seconds()) 180 | 181 | return tx, err 182 | } 183 | 184 | func (o observer) TxCommit(ctx context.Context, tx driver.Tx) error { 185 | span, ctx := opentracing.StartSpanFromContext(ctx, "Commit") 186 | defer span.Finish() 187 | 188 | ext.Component.Set(span, "sqldb") 189 | ext.DBInstance.Set(span, o.name) 190 | 191 | s := time.Now() 192 | err := tx.Commit() 193 | d := time.Since(s) 194 | 195 | log.Get(ctx).Debugf("[sqldb] name:%s, commit, cost: %v", o.name, d) 196 | 197 | sqlDurations.WithLabelValues( 198 | o.name, 199 | "", 200 | "commit", 201 | ).Observe(d.Seconds()) 202 | 203 | return err 204 | } 205 | 206 | func (o observer) TxRollback(ctx context.Context, tx driver.Tx) error { 207 | span, ctx := opentracing.StartSpanFromContext(ctx, "Rollback") 208 | defer span.Finish() 209 | 210 | ext.Component.Set(span, "sqldb") 211 | ext.DBInstance.Set(span, o.name) 212 | 213 | s := time.Now() 214 | err := tx.Rollback() 215 | d := time.Since(s) 216 | 217 | log.Get(ctx).Debugf("[sqldb] name:%s, rollback, cost: %v", o.name, d) 218 | 219 | sqlDurations.WithLabelValues( 220 | o.name, 221 | "", 222 | "rollback", 223 | ).Observe(d.Seconds()) 224 | 225 | return err 226 | } 227 | -------------------------------------------------------------------------------- /pkg/sqldb/sqldb.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "database/sql/driver" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/dlmiddlecote/sqlstats" 11 | "github.com/go-kiss/sniper/pkg/conf" 12 | "github.com/go-sql-driver/mysql" 13 | "github.com/jmoiron/sqlx" 14 | "github.com/ngrok/sqlmw" 15 | "github.com/prometheus/client_golang/prometheus" 16 | "golang.org/x/sync/singleflight" 17 | "modernc.org/sqlite" 18 | ) 19 | 20 | var ( 21 | sfg singleflight.Group 22 | rwl sync.RWMutex 23 | 24 | dbs = map[string]*DB{} 25 | ) 26 | 27 | type nameKey struct{} 28 | 29 | // DB 扩展 sqlx.DB 30 | type DB struct { 31 | *sqlx.DB 32 | } 33 | 34 | // Tx 扩展 sqlx.Tx 35 | type Tx struct { 36 | *sqlx.Tx 37 | } 38 | 39 | // Get 获取数据库实例 40 | // 41 | // db := sqldb.Get(ctx, "foo") 42 | // db.ExecContext(ctx, "select ...") 43 | func Get(ctx context.Context, name string) *DB { 44 | rwl.RLock() 45 | if db, ok := dbs[name]; ok { 46 | rwl.RUnlock() 47 | return db 48 | } 49 | rwl.RUnlock() 50 | 51 | v, _, _ := sfg.Do(name, func() (interface{}, error) { 52 | dsn := conf.Get("SQLDB_DSN_" + name) 53 | isSqlite := strings.HasPrefix(dsn, "file:") || dsn == ":memory:" 54 | var driverName string 55 | var driver driver.Driver 56 | if isSqlite { 57 | driverName = "db-sqlite:" + name 58 | driver = sqlmw.Driver(&sqlite.Driver{}, observer{name: name}) 59 | } else { 60 | driverName = "db-mysql:" + name 61 | driver = sqlmw.Driver(mysql.MySQLDriver{}, observer{name: name}) 62 | } 63 | 64 | sql.Register(driverName, driver) 65 | sdb := sqlx.MustOpen(driverName, dsn) 66 | 67 | db := &DB{sdb} 68 | 69 | rwl.Lock() 70 | defer rwl.Unlock() 71 | dbs[name] = db 72 | 73 | collector := sqlstats.NewStatsCollector(name, db) 74 | prometheus.MustRegister(collector) 75 | 76 | return db, nil 77 | }) 78 | 79 | return v.(*DB) 80 | } 81 | -------------------------------------------------------------------------------- /pkg/sqldb/sqldb_test.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/go-kiss/sniper/pkg/conf" 9 | ) 10 | 11 | var schema = ` 12 | CREATE TABLE IF NOT EXISTS users ( 13 | id integer primary key, 14 | age integer, 15 | name varchar(30), 16 | created datetime default CURRENT_TIMESTAMP 17 | ) 18 | ` 19 | 20 | type user struct { 21 | ID int 22 | Name string 23 | Age int 24 | Created time.Time 25 | } 26 | 27 | func (u *user) TableName() string { return "users" } 28 | func (u *user) KeyName() string { return "id" } 29 | 30 | func TestSqlDb(t *testing.T) { 31 | conf.Set("SQLDB_DSN_foo", ":memory:") 32 | ctx := context.Background() 33 | 34 | db := Get(ctx, "foo") 35 | db.MustExecContext(ctx, schema) 36 | 37 | result, err := db.ExecContext(ctx, 38 | "insert into users(name,age) values (?,?)", "a", 1) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | 43 | id, _ := result.LastInsertId() 44 | row := db.QueryRowxContext(ctx, "select * from users where id = ?", id) 45 | var u1 user 46 | if err := row.StructScan(&u1); err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | if u1.ID != 1 || u1.Name != "a" || u1.Age != 1 || u1.Created.IsZero() { 51 | t.Fatal("invalid user", u1) 52 | } 53 | 54 | tx := db.MustBegin() 55 | tx.Exec("delete from users") 56 | tx.Rollback() 57 | 58 | stmt, err := db.PreparexContext(ctx, "select * from users where id = ?") 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | row = stmt.QueryRowxContext(ctx, id) 64 | var u2 user 65 | if err := row.StructScan(&u2); err != nil { 66 | t.Fatal(err) 67 | } 68 | 69 | if u2.ID != 1 || u2.Name != "a" || u2.Age != 1 || u2.Created.IsZero() { 70 | t.Fatal("invalid user", u2) 71 | } 72 | } 73 | 74 | func TestModel(t *testing.T) { 75 | conf.Set("SQLDB_DSN_foo", ":memory:") 76 | ctx := context.Background() 77 | 78 | db := Get(ctx, "foo") 79 | db.MustExecContext(ctx, schema) 80 | 81 | now := time.Now() 82 | u1 := &user{Name: "foo", Age: 18, Created: now} 83 | result, err := db.Insert(u1) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | 88 | id, _ := result.LastInsertId() 89 | 90 | u1.Name = "bar" 91 | u1.ID = int(id) 92 | 93 | _, err = db.Update(u1) 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | 98 | var u2 user 99 | err = db.Get(&u2, "select * from users where id = ?", id) 100 | if err != nil { 101 | t.Fatal(err) 102 | } 103 | 104 | if u2.Name != "bar" || u2.Age != 18 || !u2.Created.Equal(now) { 105 | t.Fatal("invalid user", u2) 106 | } 107 | u3 := *u1 108 | u3.ID = 10 109 | _, err = db.Insert(&u3) 110 | if err != nil { 111 | t.Fatal(err) 112 | } 113 | 114 | var u4 user 115 | err = db.Get(&u4, "select * from users where id = ?", u3.ID) 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | if u4.ID != u3.ID { 120 | t.Fatal("invalid user", u4) 121 | } 122 | } 123 | 124 | func TestName(t *testing.T) { 125 | conf.Set("SQLDB_DSN_bar", ":memory:") 126 | conf.Set("SQLDB_DSN_baz", ":memory:") 127 | ctx := context.Background() 128 | 129 | db1 := Get(ctx, "bar") 130 | db1.MustExecContext(ctx, schema) 131 | db2 := Get(ctx, "baz") 132 | db2.MustExecContext(ctx, schema) 133 | 134 | now := time.Now() 135 | u1 := &user{Name: "foo", Age: 18, Created: now} 136 | 137 | result1, err := db1.Insert(u1) 138 | if err != nil { 139 | t.Fatal(err) 140 | } 141 | 142 | id1, _ := result1.LastInsertId() 143 | 144 | result2, err := db2.Insert(u1) 145 | if err != nil { 146 | t.Fatal(err) 147 | } 148 | 149 | id2, _ := result2.LastInsertId() 150 | 151 | if id1 != id2 { 152 | t.Fatal("invalid id", id1, id2) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /pkg/sqldb/utils.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import ( 4 | "database/sql/driver" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | func values(args []driver.NamedValue) []driver.Value { 10 | values := make([]driver.Value, 0, len(args)) 11 | for _, a := range args { 12 | values = append(values, a.Value) 13 | } 14 | return values 15 | } 16 | 17 | var sqlreg = regexp.MustCompile(`(?i)` + 18 | `(?Pselect)\s+.+?from\s+(?P\w+)\s*|` + 19 | `(?Pupdate)\s+(?P
\w+)\s+|` + 20 | `(?Pdelete)\s+from\s+(?P
\w+)\s*|` + 21 | `(?Pinsert)\s+into\s+(?P
\w+)`) 22 | 23 | // 提取 sql 的表名和指令 24 | // 25 | // "select * from foo ..." => foo,select 26 | func parseSQL(sql string) (table, cmd string) { 27 | matches := sqlreg.FindStringSubmatch(sql) 28 | 29 | results := map[string]string{} 30 | names := sqlreg.SubexpNames() 31 | for i, match := range matches { 32 | if match != "" { 33 | results[names[i]] = match 34 | } 35 | } 36 | 37 | table = strings.ToLower(results["table"]) 38 | cmd = strings.ToLower(results["cmd"]) 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /pkg/sqldb/utils_test.go: -------------------------------------------------------------------------------- 1 | package sqldb 2 | 3 | import "testing" 4 | 5 | func TestParseSQL(t *testing.T) { 6 | cases := [][]string{ 7 | {"select * from foo where", "foo", "select"}, 8 | {"update foo set", "foo", "update"}, 9 | {"insert into foo value", "foo", "insert"}, 10 | {"DELETE from foo where", "foo", "delete"}, 11 | {"select * from foo", "foo", "select"}, 12 | {"DELETE from foo", "foo", "delete"}, 13 | } 14 | 15 | for _, c := range cases { 16 | table, cmd := parseSQL(c[0]) 17 | if table != c[1] || cmd != c[2] { 18 | t.Fatal("invalid sql", table, cmd) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pkg/trace/README.md: -------------------------------------------------------------------------------- 1 | # trace 2 | 3 | 框架支持 [opentracing](https://opentracing.io/)，默认集成 [jaeger](https://github.com/jaegertracing/jaeger-client-go)。 4 | 5 | 如果想开启 jaeger 收集 opentracing 数据，需要以下配置： 6 | 7 | - `JAEGER_AGENT_HOST` jaeger 服务器IP或域名，默认为 127.0.0.1 8 | - `JAEGER_AGENT_PORT` jaeger 服务器端口，默认为 6831 9 | - `JAEGER_SAMPLER_PARAM` 采样率，0-1 之间的浮点数，默认为0，也就是不采集 10 | 11 | 个人开发环境可以使用 docker 体验： 12 | 13 | ```bash 14 | docker run -d --name jaeger \ 15 | -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \ 16 | -p 5775:5775/udp \ 17 | -p 6831:6831/udp \ 18 | -p 6832:6832/udp \ 19 | -p 5778:5778 \ 20 | -p 16686:16686 \ 21 | -p 14268:14268 \ 22 | -p 14250:14250 \ 23 | -p 9411:9411 \ 24 | jaegertracing/all-in-one:1.25 25 | ``` 26 | 27 | 启动后访问 即可打开查询界面。 28 | -------------------------------------------------------------------------------- /pkg/trace/trace.go: -------------------------------------------------------------------------------- 1 | package trace 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "time" 7 | 8 | "github.com/go-kiss/sniper/pkg/conf" 9 | "github.com/opentracing/opentracing-go" 10 | "github.com/uber/jaeger-client-go" 11 | "github.com/uber/jaeger-client-go/config" 12 | "github.com/uber/jaeger-client-go/log" 13 | "github.com/uber/jaeger-lib/metrics" 14 | ) 15 | 16 | var closer io.Closer 17 | 18 | func init() { 19 | host := conf.Get("JAEGER_AGENT_HOST") 20 | if host == "" { 21 | host = "127.0.0.1" 22 | } 23 | 24 | port := conf.Get("JAEGER_AGENT_PORT") 25 | if port == "" { 26 | port = "6831" 27 | } 28 | 29 | cfg := config.Configuration{ 30 | ServiceName: conf.App, 31 | Sampler: &config.SamplerConfig{ 32 | Type: jaeger.SamplerTypeProbabilistic, 33 | Param: conf.GetFloat64("JAEGER_SAMPLER_PARAM"), 34 | }, 35 | Reporter: &config.ReporterConfig{ 36 | LocalAgentHostPort: host + ":" + port, 37 | }, 38 | } 39 | 40 | tracer, c, err := cfg.NewTracer( 41 | config.Logger(log.NullLogger), 42 | config.Metrics(metrics.NullFactory), 43 | ) 44 | if err != nil { 45 | panic(err) 46 | } 47 | 48 | closer = c 49 | opentracing.SetGlobalTracer(tracer) 50 | } 51 | 52 | // GetTraceID 查询 trace_id 53 | func GetTraceID(ctx context.Context) (traceID string) { 54 | traceID = "no-trace-id" 55 | 56 | span := opentracing.SpanFromContext(ctx) 57 | if span == nil { 58 | return 59 | } 60 | 61 | jctx, ok := (span.Context()).(jaeger.SpanContext) 62 | if !ok { 63 | return 64 | } 65 | 66 | traceID = jctx.TraceID().String() 67 | 68 | return 69 | } 70 | 71 | // GetDuration 查询当前 span 耗时 72 | func GetDuration(span opentracing.Span) time.Duration { 73 | jspan, ok := span.(*jaeger.Span) 74 | if !ok { 75 | return 0 76 | } 77 | 78 | return jspan.Duration() 79 | } 80 | 81 | // StartFollowSpanFromContext 开起一个 follow 类型 span 82 | // follow 类型用于异步任务，可能在 root span 结束之后才完成。 83 | func StartFollowSpanFromContext(ctx context.Context, operation string) (opentracing.Span, context.Context) { 84 | span := opentracing.SpanFromContext(ctx) 85 | if span == nil { 86 | return opentracing.StartSpanFromContext(ctx, operation) 87 | } 88 | 89 | return opentracing.StartSpanFromContext(ctx, operation, opentracing.FollowsFrom(span.Context())) 90 | } 91 | 92 | // Stop 停止 trace 协程 93 | func Stop() { 94 | closer.Close() 95 | } 96 | -------------------------------------------------------------------------------- /pkg/twirp/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing # 2 | 3 | Thanks for helping make Twirp better! This is great! 4 | 5 | First, if you have run into a bug, please file an issue. We try to get back to 6 | issue reporters within a day or two. We might be able to help you right away. 7 | 8 | If you'd rather not publicly discuss the issue, please email spencer@twitch.tv 9 | and/or security@twitch.tv. 10 | 11 | Issues are also a good place to present experience reports or requests for new 12 | features. 13 | 14 | If you'd like to make changes to Twirp, read on: 15 | 16 | ## Setup Requirements ## 17 | 18 | You will need git, Go 1.9+, and Python 2.7 installed and on your system's path. 19 | Install them however you feel. 20 | 21 | ## Developer Loop ## 22 | 23 | Generally you want to make changes and run `make`, which will install all 24 | dependencies we know about, build the core, and run all of the tests that we 25 | have against all of the languages we support. 26 | 27 | Most tests of the Go server are in `internal/twirptest/service_test.go`. Tests 28 | of cross-language clients are in the [clientcompat](./clientcompat) directory. 29 | 30 | ## Contributing Code ## 31 | 32 | Twirp uses github pull requests. Fork, hack away at your changes, run the test 33 | suite with `make`, and submit a PR. 34 | 35 | ## Contributing Documentation ## 36 | 37 | Twirp's docs are generated with [Docusaurus](https://docusaurus.io/). You can 38 | safely edit anything inside the [docs](./docs) directory, adding new pages or 39 | editing them. You can edit the sidebar by editing 40 | [website/sidebars.json](./website/sidebars.json). 41 | 42 | Then, to render your changes, run docusaurus's local server. To do this: 43 | 44 | 1. [Install docusaurus on your machine](https://docusaurus.io/docs/en/installation.html). 45 | 2. `cd website` 46 | 3. `npm start` 47 | 4. Navigate to http://localhost:3000/. 48 | 49 | ## Releasing Versions ## 50 | 51 | Releasing versions is the responsibility of the core maintainers. Most people 52 | don't need to know this stuff. 53 | 54 | Twirp uses [Semantic versioning](http://semver.org/): `v..`. 55 | 56 | * Increment major if you're making a backwards-incompatible change. 57 | * Increment minor if you're adding a feature that's backwards-compatible. 58 | * Increment patch if you're making a bugfix. 59 | 60 | To make a release, remember to update the version number in 61 | [internal/gen/version.go](./internal/gen/version.go). 62 | 63 | Twirp uses Github releases. To make a new release: 64 | 1. Merge all changes that should be included in the release into the master 65 | branch. 66 | 2. Update the version constant in `internal/gen/version.go`. 67 | 3. Add a new commit to master with a message like "Version vX.X.X release". 68 | 4. Tag the commit you just made: `git tag ` and `git push 69 | origin --tags` 70 | 5. Go to Github https://github.com/bilibili/twirp/releases and 71 | "Draft a new release". 72 | 6. Make sure to document changes, specially when upgrade instructions are 73 | needed. 74 | 75 | 76 | ## Code of Conduct 77 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 78 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 79 | opensource-codeofconduct@amazon.com with any additional questions or comments. 80 | 81 | 82 | ## Licensing 83 | 84 | See the [LICENSE](https://github.com/bilibili/twirp/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 85 | 86 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 87 | -------------------------------------------------------------------------------- /pkg/twirp/NOTICE: -------------------------------------------------------------------------------- 1 | Twirp 2 | Copyright 2018 Twitch Interactive, Inc. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /pkg/twirp/PROTOCOL.md: -------------------------------------------------------------------------------- 1 | # Twirp Wire Protocol 2 | 3 | This document defines the Twirp wire protocol over HTTP. The 4 | current protocol version is v5. 5 | 6 | ## Overview 7 | 8 | The Twirp wire protocol is a simple RPC protocol based on HTTP and 9 | Protocol Buffers (proto). The protocol uses HTTP URLs to specify the 10 | RPC endpoints, and sends/receives proto messages as HTTP 11 | request/response bodies. 12 | 13 | To use Twirp, developers first define their APIs using proto files, 14 | then use Twirp tools to generate the client and the server libraries. 15 | The generated libraries implement the Twirp wire protocol, using the 16 | standard HTTP library provided by the programming language runtime or 17 | the operating system. Once the client and the server are implemented, 18 | the client can communicate with the server by making RPC calls. 19 | 20 | The Twirp wire protocol supports both binary and JSON encodings of 21 | proto messages, and works with any HTTP client and any HTTP version. 22 | 23 | ### URLs 24 | 25 | In [ABNF syntax](https://tools.ietf.org/html/rfc5234), Twirp's URLs 26 | have the following format: 27 | 28 | **URL ::= Base-URL "/twirp/" [ Package "." ] Service "/" Method** 29 | 30 | The Twirp wire protocol uses HTTP URLs to specify the RPC 31 | endpoints on the server for sending the requests. Such direct mapping 32 | makes the request routing simple and efficient. The Twirp URLs have 33 | the following components. 34 | 35 | * **Base-URL** is the virtual location of a Twirp API server, which is 36 | typically published via API documentation or service discovery. 37 | Currently, it should only contain URL `scheme` and `authority`. For 38 | example, "https://example.com". 39 | 40 | * **Package** is the proto `package` name for an API, which is often 41 | considered as an API version. For example, 42 | `example.calendar.v1`. This component is omitted if the API 43 | definition doesn't have a package name. 44 | 45 | * **Service** is the proto `service` name for an API. For example, 46 | `CalendarService`. 47 | 48 | * **Method** is the proto `rpc` name for an API method. For example, 49 | `CreateEvent`. 50 | 51 | ### Requests 52 | 53 | Twirp always uses HTTP POST method to send requests, because it 54 | closely matches the semantics of RPC methods. 55 | 56 | The **Request-Headers** are normal HTTP headers. The Twirp wire 57 | protocol uses the following headers. 58 | 59 | * **Content-Type** header indicates the proto message encoding, which 60 | should be one of "application/protobuf", "application/json". The 61 | server uses this value to decide how to parse the request body, 62 | and encode the response body. 63 | 64 | The **Request-Body** is the encoded request message, contained in the 65 | HTTP request body. The encoding is specified by the `Content-Type` 66 | header. 67 | 68 | ### Responses 69 | 70 | The **Response-Headers** are just normal HTTP response headers. The 71 | Twirp wire protocol uses the following headers. 72 | 73 | * **Content-Type** The value should be either "application/protobuf" 74 | or "application/json" to indicate the encoding of the response 75 | message. It must match the "Content-Type" header in the request. 76 | 77 | The **Request-Body** is the encoded response message contained in the 78 | HTTP response body. The encoding is specified by the `Content-Type` 79 | header. 80 | 81 | ### Example 82 | 83 | The following example shows a simple Echo API definition and its 84 | corresponding wire payloads. 85 | 86 | The example assumes the server base URL is "https://example.com". 87 | 88 | ```proto 89 | syntax = "proto3"; 90 | 91 | package example.echoer; 92 | 93 | service Echo { 94 | rpc Hello(HelloRequest) returns (HelloResponse); 95 | } 96 | 97 | message HelloRequest { 98 | string message; 99 | } 100 | 101 | message HelloResponse { 102 | string message; 103 | } 104 | ``` 105 | 106 | **Proto Request** 107 | 108 | ``` 109 | POST /twirp/example.echoer.Echo/Hello HTTP/1.1 110 | Host: example.com 111 | Content-Type: application/protobuf 112 | Content-Length: 15 113 | 114 | 115 | ``` 116 | 117 | **JSON Request** 118 | 119 | ``` 120 | POST /twirp/example.echoer.Echo/Hello HTTP/1.1 121 | Host: example.com 122 | Content-Type: application/json 123 | Content-Length: 27 124 | 125 | {"message":"Hello, World!"} 126 | ``` 127 | 128 | **Proto Response** 129 | 130 | ``` 131 | HTTP/1.1 200 OK 132 | Content-Type: application/protobuf 133 | Content-Length: 15 134 | 135 | 136 | ``` 137 | 138 | **JSON Response** 139 | 140 | ``` 141 | HTTP/1.1 200 OK 142 | Content-Type: application/json 143 | Content-Length: 27 144 | 145 | {"message":"Hello, World!"} 146 | ``` 147 | 148 | ## Errors 149 | 150 | Twirp error responses are always JSON-encoded, regardless of 151 | the request's Content-Type, with a corresponding 152 | `Content-Type: application/json` header. This ensures that 153 | the errors are human-readable in any setting. 154 | 155 | Twirp errors are a JSON object with three keys: 156 | 157 | * **code**: One of the Twirp error codes as a string. 158 | * **msg**: A human-readable message describing the error 159 | as a string. 160 | * **meta**: An object with string keys and values holding 161 | arbitrary additional metadata describing the error. 162 | 163 | Example: 164 | ``` 165 | { 166 | "code": "permission_denied", 167 | "msg": "thou shall not pass", 168 | "meta": { 169 | "target": "Balrog" 170 | } 171 | } 172 | ``` 173 | 174 | For more information, see https://github.com/bilibili/twirp/wiki/Errors. 175 | -------------------------------------------------------------------------------- /pkg/twirp/THIRD_PARTY: -------------------------------------------------------------------------------- 1 | ** Protobuf -- https://github.com/golang/protobuf 2 | Copyright 2010 The Go Authors. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | -------------------------------------------------------------------------------- /pkg/twirp/client.go: -------------------------------------------------------------------------------- 1 | package twirp 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net/http" 11 | "strconv" 12 | 13 | "google.golang.org/protobuf/encoding/protojson" 14 | "google.golang.org/protobuf/proto" 15 | ) 16 | 17 | // HTTPClient is the interface used by generated clients to send HTTP requests. 18 | // It is fulfilled by *(net/http).Client, which is sufficient for most users. 19 | // Users can provide their own implementation for special retry policies. 20 | // 21 | // HTTPClient implementations should not follow redirects. Redirects are 22 | // automatically disabled if *(net/http).Client is passed to client 23 | // constructors. See the withoutRedirects function in this file for more 24 | // details. 25 | type HTTPClient interface { 26 | Do(req *http.Request) (*http.Response, error) 27 | } 28 | 29 | // DoProtobufRequest is common code to make a request to the remote twirp service. 30 | func DoProtobufRequest(ctx context.Context, client HTTPClient, url string, in, out proto.Message) (err error) { 31 | reqBodyBytes, err := proto.Marshal(in) 32 | if err != nil { 33 | return clientError("failed to marshal proto request", err) 34 | } 35 | reqBody := bytes.NewBuffer(reqBodyBytes) 36 | if err = ctx.Err(); err != nil { 37 | return clientError("aborted because context was done", err) 38 | } 39 | 40 | req, err := newRequest(ctx, url, reqBody, "application/protobuf") 41 | if err != nil { 42 | return clientError("could not build request", err) 43 | } 44 | resp, err := client.Do(req) 45 | if err != nil { 46 | return clientError("failed to do request", err) 47 | } 48 | 49 | defer func() { 50 | cerr := resp.Body.Close() 51 | if err == nil && cerr != nil { 52 | err = clientError("failed to close response body", cerr) 53 | } 54 | }() 55 | 56 | if err = ctx.Err(); err != nil { 57 | return clientError("aborted because context was done", err) 58 | } 59 | 60 | if resp.StatusCode != 200 { 61 | return errorFromResponse(resp) 62 | } 63 | 64 | respBodyBytes, err := ioutil.ReadAll(resp.Body) 65 | if err != nil { 66 | return clientError("failed to read response body", err) 67 | } 68 | if err = ctx.Err(); err != nil { 69 | return clientError("aborted because context was done", err) 70 | } 71 | 72 | if err = proto.Unmarshal(respBodyBytes, out); err != nil { 73 | return clientError("failed to unmarshal proto response", err) 74 | } 75 | return nil 76 | } 77 | 78 | // DoJSONRequest is common code to make a request to the remote twirp service. 79 | func DoJSONRequest(ctx context.Context, client HTTPClient, url string, in, out proto.Message) (err error) { 80 | marshaler := protojson.MarshalOptions{UseProtoNames: true} 81 | var buf []byte 82 | if buf, err = marshaler.Marshal(in); err != nil { 83 | return clientError("failed to marshal json request", err) 84 | } 85 | if err = ctx.Err(); err != nil { 86 | return clientError("aborted because context was done", err) 87 | } 88 | 89 | reqBody := bytes.NewReader(buf) 90 | 91 | req, err := newRequest(ctx, url, reqBody, "application/json") 92 | if err != nil { 93 | return clientError("could not build request", err) 94 | } 95 | resp, err := client.Do(req) 96 | if err != nil { 97 | return clientError("failed to do request", err) 98 | } 99 | 100 | defer func() { 101 | cerr := resp.Body.Close() 102 | if err == nil && cerr != nil { 103 | err = clientError("failed to close response body", cerr) 104 | } 105 | }() 106 | 107 | if err = ctx.Err(); err != nil { 108 | return clientError("aborted because context was done", err) 109 | } 110 | 111 | if resp.StatusCode != 200 { 112 | return errorFromResponse(resp) 113 | } 114 | 115 | unmarshaler := protojson.UnmarshalOptions{} 116 | body, err := ioutil.ReadAll(resp.Body) 117 | if err != nil { 118 | return clientError("failed to read response body", err) 119 | } 120 | if err = unmarshaler.Unmarshal(body, out); err != nil { 121 | return clientError("failed to unmarshal json response", err) 122 | } 123 | if err = ctx.Err(); err != nil { 124 | return clientError("aborted because context was done", err) 125 | } 126 | return nil 127 | } 128 | 129 | // newRequest makes an http.Request from a client, adding common headers. 130 | func newRequest(ctx context.Context, url string, reqBody io.Reader, contentType string) (*http.Request, error) { 131 | req, err := http.NewRequest("POST", url, reqBody) 132 | if err != nil { 133 | return nil, err 134 | } 135 | req = req.WithContext(ctx) 136 | if customHeader := getCustomHTTPReqHeaders(ctx); customHeader != nil { 137 | req.Header = customHeader 138 | } 139 | req.Header.Set("Accept", contentType) 140 | req.Header.Set("Content-Type", contentType) 141 | req.Header.Set("Twirp-Version", "v5.5.0") 142 | return req, nil 143 | } 144 | 145 | // getCustomHTTPReqHeaders retrieves a copy of any headers that are set in 146 | // a context through the WithHTTPRequestHeaders function. 147 | // If there are no headers set, or if they have the wrong type, nil is returned. 148 | func getCustomHTTPReqHeaders(ctx context.Context) http.Header { 149 | header, ok := HTTPRequestHeaders(ctx) 150 | if !ok || header == nil { 151 | return nil 152 | } 153 | copied := make(http.Header) 154 | for k, vv := range header { 155 | if vv == nil { 156 | copied[k] = nil 157 | continue 158 | } 159 | copied[k] = make([]string, len(vv)) 160 | copy(copied[k], vv) 161 | } 162 | return copied 163 | } 164 | 165 | // clientError adds consistency to errors generated in the client 166 | func clientError(desc string, err error) Error { 167 | return InternalErrorWith(wrapErr(err, desc)) 168 | } 169 | 170 | // wrappedError implements the github.com/pkg/errors.Causer interface, allowing errors to be 171 | // examined for their root cause. 172 | type wrappedError struct { 173 | msg string 174 | cause error 175 | } 176 | 177 | func wrapErr(err error, msg string) error { return &wrappedError{msg: msg, cause: err} } 178 | func (e *wrappedError) Cause() error { return e.cause } 179 | func (e *wrappedError) Error() string { return e.msg + ": " + e.cause.Error() } 180 | 181 | // errorFromResponse builds a Error from a non-200 HTTP response. 182 | // If the response has a valid serialized Twirp error, then it's returned. 183 | // If not, the response status code is used to generate a similar twirp 184 | // error. See twirpErrorFromIntermediary for more info on intermediary errors. 185 | func errorFromResponse(resp *http.Response) Error { 186 | statusCode := resp.StatusCode 187 | statusText := http.StatusText(statusCode) 188 | 189 | if isHTTPRedirect(statusCode) { 190 | // Unexpected redirect: it must be an error from an intermediary. 191 | // Twirp clients don't follow redirects automatically, Twirp only handles 192 | // POST requests, redirects should only happen on GET and HEAD requests. 193 | location := resp.Header.Get("Location") 194 | msg := fmt.Sprintf("unexpected HTTP status code %d %q received, Location=%q", statusCode, statusText, location) 195 | return twirpErrorFromIntermediary(statusCode, msg, location) 196 | } 197 | 198 | respBodyBytes, err := ioutil.ReadAll(resp.Body) 199 | if err != nil { 200 | return clientError("failed to read server error response body", err) 201 | } 202 | var tj twerr 203 | if err := json.Unmarshal(respBodyBytes, &tj); err != nil { 204 | // Invalid JSON response; it must be an error from an intermediary. 205 | msg := fmt.Sprintf("Error from intermediary with HTTP status code %d %q", statusCode, statusText) 206 | return twirpErrorFromIntermediary(statusCode, msg, string(respBodyBytes)) 207 | } 208 | 209 | if !IsValidErrorCode(tj.Code()) { 210 | msg := "invalid type returned from server error response: " + string(tj.Code()) 211 | return InternalError(msg) 212 | } 213 | 214 | return &tj 215 | } 216 | 217 | // twirpErrorFromIntermediary maps HTTP errors from non-twirp sources to twirp errors. 218 | // The mapping is similar to gRPC: https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md. 219 | // Returned twirp Errors have some additional metadata for inspection. 220 | func twirpErrorFromIntermediary(status int, msg string, bodyOrLocation string) Error { 221 | var code ErrorCode 222 | if isHTTPRedirect(status) { // 3xx 223 | code = Internal 224 | } else { 225 | switch status { 226 | case 400: // Bad Request 227 | code = Internal 228 | case 401: // Unauthorized 229 | code = Unauthenticated 230 | case 403: // Forbidden 231 | code = PermissionDenied 232 | case 404: // Not Found 233 | code = BadRoute 234 | case 429, 502, 503, 504: // Too Many Requests, Bad Gateway, Service Unavailable, Gateway Timeout 235 | code = Unavailable 236 | default: // All other codes 237 | code = Unknown 238 | } 239 | } 240 | 241 | twerr := NewError(code, msg) 242 | twerr = twerr.WithMeta("http_error_from_intermediary", "true") // to easily know if this error was from intermediary 243 | twerr = twerr.WithMeta("status_code", strconv.Itoa(status)) 244 | if isHTTPRedirect(status) { 245 | twerr = twerr.WithMeta("location", bodyOrLocation) 246 | } else { 247 | twerr = twerr.WithMeta("body", bodyOrLocation) 248 | } 249 | return twerr 250 | } 251 | 252 | func isHTTPRedirect(status int) bool { 253 | return status >= 300 && status <= 399 254 | } 255 | -------------------------------------------------------------------------------- /pkg/twirp/context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Twitch Interactive, Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may not 4 | // use this file except in compliance with the License. A copy of the License is 5 | // located at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed on 10 | // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package twirp 15 | 16 | import ( 17 | "context" 18 | "errors" 19 | "net/http" 20 | 21 | "google.golang.org/protobuf/proto" 22 | ) 23 | 24 | type contextKey int 25 | 26 | const ( 27 | MethodNameKey contextKey = 1 + iota 28 | ServiceNameKey 29 | PackageNameKey 30 | StatusCodeKey 31 | RequestHeaderKey 32 | HttpRequestKey 33 | RequestKey 34 | ResponseWriterKey 35 | ResponseKey 36 | AllowGETKey 37 | MethodOptionKey 38 | ) 39 | 40 | // MethodName extracts the name of the method being handled in the given 41 | // context. If it is not known, it returns ("", false). 42 | func MethodName(ctx context.Context) (string, bool) { 43 | name, ok := ctx.Value(MethodNameKey).(string) 44 | return name, ok 45 | } 46 | 47 | // ServiceName extracts the name of the service handling the given context. If 48 | // it is not known, it returns ("", false). 49 | func ServiceName(ctx context.Context) (string, bool) { 50 | name, ok := ctx.Value(ServiceNameKey).(string) 51 | return name, ok 52 | } 53 | 54 | // PackageName extracts the fully-qualified protobuf package name of the service 55 | // handling the given context. If it is not known, it returns ("", false). If 56 | // the service comes from a proto file that does not declare a package name, it 57 | // returns ("", true). 58 | // 59 | // Note that the protobuf package name can be very different than the go package 60 | // name; the two are unrelated. 61 | func PackageName(ctx context.Context) (string, bool) { 62 | name, ok := ctx.Value(PackageNameKey).(string) 63 | return name, ok 64 | } 65 | 66 | // StatusCode retrieves the status code of the response (as string like "200"). 67 | // If it is known returns (status, true). 68 | // If it is not known, it returns ("", false). 69 | func StatusCode(ctx context.Context) (string, bool) { 70 | code, ok := ctx.Value(StatusCodeKey).(string) 71 | return code, ok 72 | } 73 | 74 | // HttpRequest retrieves the request. 75 | // If it is known returns (req, true). 76 | // If it is not known, it returns (nil, false). 77 | func HttpRequest(ctx context.Context) (*http.Request, bool) { 78 | req, ok := ctx.Value(HttpRequestKey).(*http.Request) 79 | return req, ok 80 | } 81 | 82 | // Request 返回解析后的请求对象 83 | func Request(ctx context.Context) (proto.Message, bool) { 84 | req, ok := ctx.Value(RequestKey).(proto.Message) 85 | return req, ok 86 | } 87 | 88 | // MethodOption retrieves the option of service method. 89 | // If it is known returns (string, true). 90 | // If it is not known, it returns ("", false). 91 | func MethodOption(ctx context.Context) (string, bool) { 92 | option, ok := ctx.Value(MethodOptionKey).(string) 93 | return option, ok 94 | } 95 | 96 | // Response retrieves the response. 97 | // If it is known returns (resp, true). 98 | // If it is not known, it returns (nil, false). 99 | func Response(ctx context.Context) (proto.Message, bool) { 100 | req, ok := ctx.Value(ResponseKey).(proto.Message) 101 | return req, ok 102 | } 103 | 104 | // AllowGET check whether allow the current request using post method. 105 | func AllowGET(ctx context.Context) bool { 106 | allow, ok := ctx.Value(AllowGETKey).(bool) 107 | 108 | return ok && allow 109 | } 110 | 111 | // WithAllowGET allow get method. 112 | func WithAllowGET(ctx context.Context, f bool) context.Context { 113 | return context.WithValue(ctx, AllowGETKey, f) 114 | } 115 | 116 | // WithHTTPRequestHeaders stores an http.Header in a context.Context. When 117 | // using a Twirp-generated client, you can pass the returned context 118 | // into any of the request methods, and the stored header will be 119 | // included in outbound HTTP requests. 120 | // 121 | // This can be used to set custom HTTP headers like authorization tokens or 122 | // client IDs. But note that HTTP headers are a Twirp implementation detail, 123 | // only visible by middleware, not by the server implementation. 124 | // 125 | // WithHTTPRequestHeaders returns an error if the provided http.Header 126 | // would overwrite a header that is needed by Twirp, like "Content-Type". 127 | func WithHTTPRequestHeaders(ctx context.Context, h http.Header) (context.Context, error) { 128 | if _, ok := h["Accept"]; ok { 129 | return nil, errors.New("provided header cannot set Accept") 130 | } 131 | if _, ok := h["Content-Type"]; ok { 132 | return nil, errors.New("provided header cannot set Content-Type") 133 | } 134 | if _, ok := h["Twirp-Version"]; ok { 135 | return nil, errors.New("provided header cannot set Twirp-Version") 136 | } 137 | 138 | copied := make(http.Header, len(h)) 139 | for k, vv := range h { 140 | if vv == nil { 141 | copied[k] = nil 142 | continue 143 | } 144 | copied[k] = make([]string, len(vv)) 145 | copy(copied[k], vv) 146 | } 147 | 148 | return context.WithValue(ctx, RequestHeaderKey, copied), nil 149 | } 150 | 151 | func HTTPRequestHeaders(ctx context.Context) (http.Header, bool) { 152 | h, ok := ctx.Value(RequestHeaderKey).(http.Header) 153 | return h, ok 154 | } 155 | 156 | // SetHTTPResponseHeader sets an HTTP header key-value pair using a context 157 | // provided by a twirp-generated server, or a child of that context. 158 | // The server will include the header in its response for that request context. 159 | // 160 | // This can be used to respond with custom HTTP headers like "Cache-Control". 161 | // But note that HTTP headers are a Twirp implementation detail, 162 | // only visible by middleware, not by the clients or their responses. 163 | // 164 | // The header will be ignored (noop) if the context is invalid (i.e. using a new 165 | // context.Background() instead of passing the context from the handler). 166 | // 167 | // If called multiple times with the same key, it replaces any existing values 168 | // associated with that key. 169 | // 170 | // SetHTTPResponseHeader returns an error if the provided header key 171 | // would overwrite a header that is needed by Twirp, like "Content-Type". 172 | func SetHTTPResponseHeader(ctx context.Context, key, value string) error { 173 | if key == "Content-Type" { 174 | return errors.New("header key can not be Content-Type") 175 | } 176 | 177 | responseWriter, ok := ctx.Value(ResponseWriterKey).(http.ResponseWriter) 178 | if ok { 179 | responseWriter.Header().Set(key, value) 180 | } // invalid context is ignored, not an error, this is to allow easy unit testing with mock servers 181 | 182 | return nil 183 | } 184 | 185 | // AddHTTPResponseHeader adds an HTTP header key-value pair using a context 186 | // provided by a twirp-generated server, or a child of that context. 187 | // The server will include the header in its response for that request context. 188 | func AddHTTPResponseHeader(ctx context.Context, key, value string) error { 189 | if key == "Content-Type" { 190 | return errors.New("header key can not be Content-Type") 191 | } 192 | 193 | responseWriter, ok := ctx.Value(ResponseWriterKey).(http.ResponseWriter) 194 | if ok { 195 | responseWriter.Header().Add(key, value) 196 | } // invalid context is ignored, not an error, this is to allow easy unit testing with mock servers 197 | 198 | return nil 199 | } 200 | -------------------------------------------------------------------------------- /pkg/twirp/ctxsetters.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Twitch Interactive, Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may not 4 | // use this file except in compliance with the License. A copy of the License is 5 | // located at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed on 10 | // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | // ctxsetters is an implementation detail for twirp generated code, used 15 | // by the generated servers to set values in contexts for later access with the 16 | // twirp package's accessors. 17 | // 18 | // DO NOT USE CTXSETTERS OUTSIDE OF TWIRP'S GENERATED CODE. 19 | package twirp 20 | 21 | import ( 22 | "context" 23 | "net/http" 24 | "strconv" 25 | 26 | "google.golang.org/protobuf/proto" 27 | ) 28 | 29 | func WithMethodName(ctx context.Context, name string) context.Context { 30 | return context.WithValue(ctx, MethodNameKey, name) 31 | } 32 | 33 | func WithServiceName(ctx context.Context, name string) context.Context { 34 | return context.WithValue(ctx, ServiceNameKey, name) 35 | } 36 | 37 | func WithPackageName(ctx context.Context, name string) context.Context { 38 | return context.WithValue(ctx, PackageNameKey, name) 39 | } 40 | 41 | func WithStatusCode(ctx context.Context, code int) context.Context { 42 | return context.WithValue(ctx, StatusCodeKey, strconv.Itoa(code)) 43 | } 44 | 45 | func WithResponseWriter(ctx context.Context, w http.ResponseWriter) context.Context { 46 | return context.WithValue(ctx, ResponseWriterKey, w) 47 | } 48 | 49 | func WithHttpRequest(ctx context.Context, r *http.Request) context.Context { 50 | return context.WithValue(ctx, HttpRequestKey, r) 51 | } 52 | 53 | func WithRequest(ctx context.Context, r proto.Message) context.Context { 54 | return context.WithValue(ctx, RequestKey, r) 55 | } 56 | 57 | func WithResponse(ctx context.Context, resp proto.Message) context.Context { 58 | return context.WithValue(ctx, ResponseKey, resp) 59 | } 60 | 61 | func WithMethodOption(ctx context.Context, option string) context.Context { 62 | return context.WithValue(ctx, MethodOptionKey, option) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/twirp/docs/best_practices.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "best_practices" 3 | title: "Best Practices" 4 | sidebar_label: "Best Practices" 5 | --- 6 | 7 | Twirp simplifies service design when compared with a REST endpoint: method 8 | definitions, message types and parsing is handled by the framework (i.e. you 9 | don’t have to worry about JSON fields or types). However, there are still some 10 | things to consider when making a new service in Twirp, mainly to keep 11 | consistency. 12 | 13 | ## Folder/Package Structure 14 | 15 | The recommended folder/package structure for your twirp `` is: 16 | ``` 17 | /cmd 18 | / 19 | main.go 20 | /rpc 21 | / 22 | service.proto 23 | // and auto-generated files 24 | /internal 25 | /server 26 | server.go 27 | // and usually one other file per method 28 | ``` 29 | 30 | For example, for the Haberdasher service it would be: 31 | ``` 32 | /cmd 33 | /haberdasherserver 34 | main.go 35 | /rpc 36 | /haberdasher 37 | service.proto 38 | service.pb.go 39 | service.twirp.go 40 | /internal 41 | /haberdasherserver 42 | server_test.go 43 | server.go 44 | make_hat_test.go 45 | make_hat.go 46 | ``` 47 | 48 | Notes: 49 | * Keep the `.proto` and generated files in their own package. 50 | * Do not implement the server or other files in the same package. This allows 51 | other services to do a "clean import" of the autogenerated client. 52 | * Do not name the package something generic like `api`, `client` or `service`; 53 | name it after your service. Remember that the package is going to be imported 54 | by other projects that will likely import other clients from other services 55 | as well. 56 | 57 | ## `.proto` File 58 | 59 | The `.proto` file is the source of truth for your service design. 60 | 61 | * The first step to design your service is to write a `.proto` file. Use that 62 | to discuss design with your coworkers before starting the implementation. 63 | * Use proto3 (first line should be `syntax="proto3"`). Do not use proto2. 64 | * Use `option go_package = "";` for the Go package name. 65 | * Add comments on message fields; they translate to the generated Go 66 | interfaces. 67 | * Don’t worry about fields like `user_id` being auto-converted into `UserId` in 68 | Go. I know, the right case should be `UserID`, but it's not your fault how 69 | the protoc-gen-go compiler decides to translate it. Avoid doing hacks like 70 | naming it `user_i_d` so it looks "good" in Go (`UserID`). In the future, we 71 | may use a better Go code generator, or generate clients for other languages 72 | like Python or JavaScript. 73 | * rpc methods should clearly be named with (i.e.: ListBooks, 74 | GetBook, CreateBook, UpdateBook, RenameBook, DeleteBook). See more in "Naming 75 | Conventions" below. 76 | 77 | The header of the `.proto` file should look like this (change and 78 | with your values): 79 | ```go 80 | syntax = "proto3" 81 | 82 | package ..; 83 | 84 | option go_package = ""; 85 | ``` 86 | 87 | ## Specifying protoc version and using retool for protoc-gen-go and protoc-gen-twirp 88 | 89 | Code generation depends on `protoc` and its plugins `protoc-gen-go` and 90 | `protoc-gen-twirp`. Having different versions may cause problems. 91 | 92 | Make sure to specify the required `protoc` version in your README or 93 | CONTRIBUTING file. 94 | 95 | For the plugins, you can use [retool](https://github.com/bilibili/retool). Like 96 | with most Go commands used to manage your source code, retool makes it easy to 97 | lock versions for all team members. 98 | 99 | ```sh 100 | $ retool add github.com/golang/protobuf/protoc-gen-go master 101 | $ retool add github.com/bilibili/twirp/protoc-gen-twirp master 102 | ``` 103 | 104 | Using a Makefile is a good way to simplify code generation: 105 | 106 | ```Makefile 107 | gen: 108 | # Auto-generate code 109 | retool do protoc --proto_path=. --twirp_out=. --go_out=. rpc//service.proto 110 | 111 | upgrade: 112 | # Upgrade glide dependencies 113 | retool do glide update --strip-vendor 114 | retool do glide-vc --only-code --no-tests --keep '**/*.proto' 115 | ``` 116 | 117 | ## Naming Conventions 118 | 119 | Like in any other API or interface, it is very important to have names that are 120 | simple, intuitive and consistent. 121 | 122 | Respect the [Protocol Buffers Style Guide](https://developers.google.com/protocol-buffers/docs/style): 123 | * Use `CamelCase` for `Service`, `Message` and `Type` names. 124 | * Use `underscore_separated_names` for field names. 125 | * Use `CAPITALS_WITH_UNDERSCORES` for enum value names. 126 | 127 | For naming conventions, the 128 | [Google Cloud Platform design guides](https://cloud.google.com/apis/design/naming_convention) 129 | are a good reference: 130 | * Use the same name for the same concept, even across APIs. 131 | * Avoid name overloading. Use different names for different concepts. 132 | * Include units on field names for durations and quantities (e.g. 133 | `delay_seconds` is better than just `delay`). 134 | 135 | For times, we have a few Twitch-specific conventions that have worked for us: 136 | * Timestamp names should end with `_at` whenever possible (i.e. `created_at`, 137 | `updated_at`). 138 | * Timestamps should be [RFC3339](https://tools.ietf.org/html/rfc3339) strings 139 | (in Go it's very easy to generate these with `t.Format(time.RFC3339)` and 140 | parse them with `time.Parse(time.RFC3339Nano, t)`). 141 | * Timestamps can also be a `google.protobuf.Timestamp`, in which case their 142 | names should end with `_time` for clarity. 143 | 144 | ## Default Values and Required Fields 145 | 146 | In proto3 all fields have zero-value defaults (string is `""`, int32 is `0`), so 147 | all fields are optional. 148 | 149 | If you want to make a required field (i.e. "name is required"), it needs to be 150 | handled by the service implementation. But to make this clear in the `.proto` 151 | file: 152 | * Add a "required" comment on the field. For example `string name = 1; // 153 | required` implies that the server implementation will return an 154 | `twirp.RequiredArgumentError("name")` if the name is empty. 155 | 156 | If you need a different default (e.g. limit default 20 for paginated 157 | collections), it needs to be handled by the service implementation. But to make 158 | this clear in the `.proto` file: 159 | * Add a "(default X)" comment on the field. For example `int32 limit = 1; // 160 | (default 20)` implies that the server implementation will convert the 161 | zero-value 0 to 20 (0 == 20). 162 | * For enums, the first item is the default. 163 | 164 | Your service implementation cannot tell the difference between empty and missing 165 | fields (this is by design). If you really need to tell them apart, you need to 166 | use an extra bool field, or use `google/protobuf.wrappers.proto` messages (which 167 | can be nil in go). 168 | 169 | ## Twirp Errors 170 | 171 | Protocol Buffers do not specify errors. You can always add an extra field on the 172 | returned message for the error, but Twirp has an excellent system that you 173 | should use instead: 174 | 175 | * Familiarize yourself with the possible [Twirp error codes](errors.md) and use 176 | the ones that make sense for each situation (i.e. `InvalidArgument`, 177 | `NotFound`, `Internal`). The codes are very straightforward and are almost 178 | the same as in gRPC. 179 | * Always return a `twirp.Error`. Twirp allows you to return a regular `error`, 180 | that will get wrapped with `twirp.InternalErrorWith(err)`, but it is better 181 | if you explicitly wrap it yourself. Being explicit makes the server and the 182 | client to always return the same twirp errors, which is more predictable and 183 | easier for unit tests. 184 | * Include possible errors on the `.proto` file (add comments to RPC methods). 185 | * But there's no need to document all the obvious `Internal` errors, which can 186 | always happen for diverse reasons (e.g. backend service is down, or there was 187 | a problem on the client). 188 | * Make sure to document (with comments) possible validation errors on the 189 | specific message fields. For example `int32 amount = 1; // must be positive` 190 | implies that the server implementation will return a 191 | `twirp.InvalidArgument("amount", "must be positive")` error if the condition 192 | is not met. 193 | * Required fields are also validation errors. For example, if a given string 194 | field cannot be empty, you should add a "required" comment in the proto file, 195 | which implies that a `twirp.RequiredArgumentError(field)` will be returned if 196 | the field is empty (or missing, which is the same thing in proto3). If you 197 | are using proto2 (I hope not), the "required" comment is still preferred over 198 | the required field type. 199 | -------------------------------------------------------------------------------- /pkg/twirp/docs/command_line.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "command_line" 3 | title: "Command line parameters" 4 | sidebar_label: "Command line parameters" 5 | --- 6 | 7 | In general, Twirp's Go generator shouldn't need command line parameters. There 8 | are some complex cases, though, where they are the only way to get things done, 9 | particularly when setting the import path to be used in generated code. 10 | 11 | # How to pass command line parameters 12 | 13 | Command line parameters are passed to the Twirp generator, `protoc-gen-twirp`, 14 | by specifying them in the `--twirp_out` argument. The parameters are key-values, 15 | separated by `,` characters, and you the parameter list is terminated with a `:` character. 16 | 17 | So, for example, `--twirp_out=k1=v1,k2=v2,k3=v3:.` would pass `k1=v1`, `k2=v2`, 18 | and `k3=v3` to twirp. 19 | 20 | # Modifying imports 21 | 22 | When working with multiple proto files that use import statements, 23 | `protoc-gen-twirp` uses the `option go_package` field in the `.proto` files to 24 | determine the import paths for imported message types. Usually, this is 25 | sufficient, but in some complex setups, you need to be able to directly override 26 | import lines. 27 | 28 | You should usually set import paths by using `option go_package` in your .proto 29 | files. A line like this: 30 | 31 | ```protobuf 32 | option go_package = "github.com/bilibili/thisisanexample"; 33 | ``` 34 | 35 | will set things up properly. But if a file needs to be imported at different 36 | paths for different users, you might need to resort to command-line parameters. 37 | 38 | 39 | This behavior can be customized by using two different command line parameters: 40 | 41 | * `import_prefix`, which prefixes all generated import paths with something. 42 | * `go_import_mapping`, which lets you set an explicit mapping of import paths to 43 | use for particular .proto files. 44 | 45 | ## Import prefix parameter 46 | 47 | The `import_prefix` parameter can be passed to `--twirp_out` in order to prefix 48 | the generated import path with something. 49 | 50 | ```sh 51 | $ PROTO_SRC_PATH=./ 52 | $ IMPORT_PREFIX="github.com/example/rpc/haberdasher" 53 | $ protoc \ 54 | --proto_path=$PROTO_SRC_PATH \ 55 | --twirp_out=import_prefix=$IMPORT_PREFIX:$PROTO_SRC_PATH \ 56 | --go_out=import_prefix=$IMPORT_PREFIX:$PROTO_SRC_PATH \ 57 | $PROTO_SRC_PATH/rpc/haberdasher/service.proto 58 | ``` 59 | 60 | ## Import mapping parameter 61 | 62 | The import mapping parameter can be passed multiple times to `--twirp_out` in 63 | order to substitute the import path for a given proto file with something else. 64 | By passing the parameter multiple times you can build up a map of proto file to 65 | import path inside the generator. 66 | 67 | This parameter should be used when one of your proto files `import`s a proto 68 | file from another package and you're not generating your code at the 69 | `$GOPATH/src` root. 70 | 71 | There are two ways to provide this parameter to `--twirp_out`: 72 | 73 | ### As provided to `protoc-gen-go` 74 | 75 | Just like `proto-gen-go`, you can use a shorthand, formatted as: `M=`. For example, you could tell `protoc-gen-twirp` that 77 | `rpcutil/empty.proto` can be found at `github.com/example/rpcutil` by using 78 | `Mrpcutil/empty.proto=github.com/example/rpcutil`. Here's a full example: 79 | 80 | ```sh 81 | $ PROTO_SRC_PATH=./ 82 | $ IMPORT_MAPPING="rpcutil/empty.proto=github.com/example/rpcutil" 83 | $ protoc \ 84 | --proto_path=$PROTO_SRC_PATH \ 85 | --twirp_out=M$IMPORT_MAPPING:$PROTO_SRC_PATH \ 86 | --go_out=M$IMPORT_MAPPING:$PROTO_SRC_PATH \ 87 | $PROTO_SRC_PATH/rpc/haberdasher/service.proto 88 | ``` 89 | 90 | ### Using the `go_import_mapping@` prefix 91 | 92 | This is exactly the same as the previous method; it's just a little more verbose 93 | and a little clearer. The format is `go_import_mapping@=`. For example, you could tell `protoc-gen-twirp` that 95 | `rpcutil/empty.proto` can be found at `github.com/example/rpcutil` by using 96 | `go_import_mapping@rpcutil/empty.proto=github.com/example/rpcutil`. Here's a 97 | full example: 98 | 99 | ```sh 100 | $ IMPORT_MAPPING="rpcutil/empty.proto=github.com/example/rpcutil" 101 | $ protoc \ 102 | --proto_path=$PROTO_SRC_PATH \ 103 | --twirp_out=go_import_mapping@$IMPORT_MAPPING:$PROTO_SRC_PATH \ 104 | --go_out=M$IMPORT_MAPPING:$PROTO_SRC_PATH \ 105 | $PROTO_SRC_PATH/rpc/haberdasher/service.proto 106 | ``` 107 | 108 | Note: this is a `protoc-gen-twirp` flavor of the parameter. `protoc-gen-go` does 109 | not support this prefix. 110 | -------------------------------------------------------------------------------- /pkg/twirp/docs/curl.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "curl" 3 | title: "cURL" 4 | sidebar_label: "cURL" 5 | --- 6 | 7 | Twirp allows you to cURL your service with either Protobuf or JSON. 8 | 9 | ## Example 10 | 11 | A cURL request to the Haberdasher example `MakeHat` RPC with the following request and reply could be executed as Protobuf or JSON with the snippets further below.: 12 | 13 | Request proto: 14 | ``` 15 | message Size { 16 | int32 inches = 1; 17 | } 18 | ``` 19 | 20 | Reply proto: 21 | 22 | ``` 23 | message Hat { 24 | int32 inches = 1; 25 | string color = 2; 26 | string name = 3; 27 | } 28 | ``` 29 | 30 | ### Protobuf 31 | 32 | ```sh 33 | echo "inches:10" \ 34 | | protoc --proto_path=$GOPATH/src --encode twirp.example.haberdasher.Size ./rpc/haberdasher/service.proto \ 35 | | curl -s --request POST \ 36 | --header "Content-Type: application/protobuf" \ 37 | --data-binary @- 38 | http://localhost:8080/twirp/twirp.example.haberdasher.Haberdasher/MakeHat \ 39 | | protoc --proto_path=$GOPATH/src --decode twirp.example.haberdasher.Hat ./rpc/haberdasher/service.proto 40 | ``` 41 | 42 | We signal Twirp that we're sending Protobuf data by setting the `Content-Type` as `application/protobuf`. 43 | 44 | The request is: 45 | 46 | ``` 47 | inches:10 48 | ``` 49 | 50 | The reply from Twirp would look something like this: 51 | 52 | ``` 53 | inches:1 54 | color:"black" 55 | name:"bowler" 56 | ``` 57 | 58 | ### JSON 59 | 60 | ```sh 61 | curl --request "POST" \ 62 | --location "http://localhost:8080/twirp/twirp.example.haberdasher.Haberdasher/MakeHat" \ 63 | --header "Content-Type:application/json" \ 64 | --data '{"inches": 10}' \ 65 | --verbose 66 | ``` 67 | 68 | We signal Twirp that we're sending JSON data by setting the `Content-Type` as `application/json`. 69 | 70 | The request is: 71 | 72 | ```json 73 | {"inches": 10} 74 | ``` 75 | 76 | The JSON response from Twirp would look something like this: 77 | 78 | ```json 79 | {"inches":1, "color":"black", "name":"bowler"} 80 | ``` 81 | -------------------------------------------------------------------------------- /pkg/twirp/docs/errors.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "errors" 3 | title: "Errors" 4 | sidebar_label: "Errors" 5 | --- 6 | 7 | You probably noticed that all methods on a Twirp-made interface return `(..., 8 | error)`. 9 | 10 | Twirp clients always return errors that can be cast to `twirp.Error`. Even 11 | transport-level errors will be `twirp.Error`s. 12 | 13 | Twirp server implementations can return regular errors too, but those 14 | will be wrapped with `twirp.InternalErrorWith(err)`, so they are also 15 | `twirp.Error` values when received by the clients. 16 | 17 | Check the [Errors Spec](spec_v5.md) for more information on error 18 | codes and the wire protocol. 19 | 20 | Also don't be afraid to open the [source code](https://github.com/bilibili/twirp/blob/master/errors.go) 21 | for details, it is pretty straightforward. 22 | 23 | ### twirp.Error interface 24 | 25 | Twirp Errors have this interface: 26 | ```go 27 | type Error interface { 28 | Code() ErrorCode // identifies a valid error type 29 | Msg() string // free-form human-readable message 30 | 31 | WithMeta(key string, val string) Error // set metadata 32 | Meta(key string) string // get metadata value 33 | MetaMap() map[string]string // see all metadata 34 | 35 | Error() string // as an error returns "twirp error : " 36 | } 37 | ``` 38 | 39 | ### Error Codes 40 | 41 | Each error code is defined by a constant in the `twirp` package: 42 | 43 | | twirp.ErrorCode | JSON/String | HTTP status code 44 | | ------------------ | ------------------- | ------------------ 45 | | Canceled | canceled | 408 RequestTimeout 46 | | Unknown | unknown | 500 Internal Server Error 47 | | InvalidArgument | invalid_argument | 400 BadRequest 48 | | DeadlineExceeded | deadline_exceeded | 408 RequestTimeout 49 | | NotFound | not_found | 404 Not Found 50 | | BadRoute | bad_route | 404 Not Found 51 | | AlreadyExists | already_exists | 409 Conflict 52 | | PermissionDenied | permission_denied | 403 Forbidden 53 | | Unauthenticated | unauthenticated | 401 Unauthorized 54 | | ResourceExhausted | resource_exhausted | 403 Forbidden 55 | | FailedPrecondition | failed_precondition | 412 Precondition Failed 56 | | Aborted | aborted | 409 Conflict 57 | | OutOfRange | out_of_range | 400 Bad Request 58 | | Unimplemented | unimplemented | 501 Not Implemented 59 | | Internal | internal | 500 Internal Server Error 60 | | Unavailable | unavailable | 503 Service Unavailable 61 | | DataLoss | dataloss | 500 Internal Server Error 62 | 63 | For more information on each code, see the [Errors Spec](spec_v5.md). 64 | 65 | ### HTTP Errors from Intermediary Proxies 66 | 67 | It is also possible for Twirp Clients to receive HTTP responses with non-200 status 68 | codes but without an expected error message. For example, proxies or load balancers 69 | might return a "503 Service Temporarily Unavailable" body, which cannot be 70 | deserialized into a Twirp error. 71 | 72 | In these cases, generated Go clients will return twirp.Errors with a code which 73 | depends upon the HTTP status of the invalid response: 74 | 75 | | HTTP status code | Twirp Error Code 76 | | ------------------------ | ------------------ 77 | | 3xx (redirects) | Internal 78 | | 400 Bad Request | Internal 79 | | 401 Unauthorized | Unauthenticated 80 | | 403 Forbidden | PermissionDenied 81 | | 404 Not Found | BadRoute 82 | | 429 Too Many Requests | Unavailable 83 | | 502 Bad Gateway | Unavailable 84 | | 503 Service Unavailable | Unavailable 85 | | 504 Gateway Timeout | Unavailable 86 | | ... other | Unknown 87 | 88 | Additional metadata is added to make it easy to identify intermediary errors: 89 | 90 | * `"http_error_from_intermediary": "true"` 91 | * `"status_code": string` (original status code on the HTTP response, e.g. `"500"`). 92 | * `"body": string` (original non-Twirp error response as string). 93 | * `"location": url-string` (only on 3xx responses, matching the `Location` header). 94 | 95 | ### Metadata 96 | 97 | You can add arbitrary string metadata to any error. For example, the service may return an error like this: 98 | 99 | ```go 100 | if unavailable { 101 | twerr := twirp.NewError(twirp.Unavailable, "taking a nap ...") 102 | twerr = twerr.WithMeta("retryable", "true") 103 | twerr = twerr.WithMeta("retry_after", "15s") 104 | return nil, twerr 105 | } 106 | ``` 107 | 108 | And the metadata is available on the client: 109 | 110 | ```go 111 | if twerr.Code() == twirp.Unavailable { 112 | if twerr.Meta("retryable") != "" { 113 | // do stuff... maybe retry after twerr.Meta("retry_after") 114 | } 115 | } 116 | ``` 117 | 118 | Error metadata can only have string values. This is to simplify error parsing by clients. 119 | If your service requires errors with complex metadata, you should consider adding client 120 | wrappers on top of the auto-generated clients, or just include business-logic errors as 121 | part of the Protobuf messages (add an error field to proto messages). 122 | 123 | -------------------------------------------------------------------------------- /pkg/twirp/docs/example.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "example" 3 | title: "Usage Example: Haberdasher" 4 | sidebar_label: "Usage Example" 5 | --- 6 | 7 | Let's make the canonical Twirp example service: a `Haberdasher`. 8 | 9 | The `Haberdasher` service makes hats. It has only one RPC method, `MakeHat`, 10 | which makes a new hat of a particular size. 11 | 12 | By the end of this, we'll run a Haberdasher server and have a client that can 13 | send it requests. 14 | 15 | There are 5 steps here: 16 | 17 | 1. [Write a Protobuf service definition](#write-a-protobuf-service-definition) 18 | 2. [Generate code](#generate-code) 19 | 3. [Implement the server](#implement-the-server) 20 | 4. [Mount and run the server](#mount-and-run-the-server) 21 | 5. [Use the client](#use-the-client) 22 | 23 | ## Write a Protobuf Service Definition 24 | 25 | Start with the [Protobuf](https://developers.google.com/protocol-buffers/) 26 | definition file, placed in `rpc/haberdasher/service.proto`: 27 | 28 | ```protobuf 29 | syntax = "proto3"; 30 | 31 | package twirp.example.haberdasher; 32 | option go_package = "haberdasher"; 33 | 34 | // Haberdasher service makes hats for clients. 35 | service Haberdasher { 36 | // MakeHat produces a hat of mysterious, randomly-selected color! 37 | rpc MakeHat(Size) returns (Hat); 38 | } 39 | 40 | // Size of a Hat, in inches. 41 | message Size { 42 | int32 inches = 1; // must be > 0 43 | } 44 | 45 | // A Hat is a piece of headwear made by a Haberdasher. 46 | message Hat { 47 | int32 inches = 1; 48 | string color = 2; // anything but "invisible" 49 | string name = 3; // i.e. "bowler" 50 | } 51 | ``` 52 | 53 | It's a good idea to add comments on your Protobuf file. These files can work as 54 | the primary documentation of your API. They'll also show up in the generated Go 55 | code, so they'll show up in Godoc. 56 | 57 | ## Generate code 58 | 59 | To generate code run the `protoc` compiler pointed at your service's `.proto` 60 | files: 61 | 62 | ```sh 63 | $ protoc --proto_path=$GOPATH/src:. --twirp_out=. --go_out=. ./rpc/haberdasher/service.proto 64 | ``` 65 | 66 | The code is generated in the same directory as the `.proto` files. 67 | 68 | You should see the generated files next to the `service.proto` file: 69 | 70 | ```text 71 | /rpc 72 | /haberdasher 73 | service.pb.go # auto-generated by protoc-gen-go (for protobuf serialization) 74 | service.proto # original protobuf definition 75 | service.twirp.go # auto-generated by protoc-gen-twirp (servers, clients and interfaces) 76 | ``` 77 | 78 | If you open the generated `.twirp.go` file, you will see a Go interface like 79 | this: 80 | 81 | ```go 82 | // A Haberdasher makes hats for clients. 83 | type Haberdasher interface { 84 | // MakeHat produces a hat of mysterious, randomly-selected color! 85 | MakeHat(context.Context, *Size) (*Hat, error) 86 | } 87 | ``` 88 | 89 | along with code to instantiate clients and servers. 90 | 91 | 92 | ## Implement the Server 93 | 94 | Now, our job is to write code that fulfills the `Haberdasher` interface. This 95 | will be the "backend" full of logic that we'll be serving. 96 | 97 | For example, the implementation could go in 98 | `internal/haberdasherserver/server.go`: 99 | 100 | ```go 101 | package haberdasherserver 102 | 103 | import ( 104 | "context" 105 | "math/rand" 106 | 107 | "github.com/bilibili/twirp" 108 | pb "github.com/bilibili/twirpexample/rpc/haberdasher" 109 | ) 110 | 111 | // Server implements the Haberdasher service 112 | type Server struct {} 113 | 114 | func (s *Server) MakeHat(ctx context.Context, size *pb.Size) (hat *pb.Hat, err error) { 115 | if size.Inches <= 0 { 116 | return nil, twirp.InvalidArgumentError("inches", "I can't make a hat that small!") 117 | } 118 | return &pb.Hat{ 119 | Inches: size.Inches, 120 | Color: []string{"white", "black", "brown", "red", "blue"}[rand.Intn(4)], 121 | Name: []string{"bowler", "baseball cap", "top hat", "derby"}[rand.Intn(3)], 122 | }, nil 123 | } 124 | ``` 125 | 126 | This meets the `Haberdasher` interface because it implements the `MakeHat` method, so we're ready to serve this over Twirp! 127 | 128 | ## Mount and run the server 129 | 130 | To serve our Haberdasher over HTTP, use the auto-generated server constructor 131 | `New{{Service}}Server`. For Haberdasher, this is: 132 | 133 | ```go 134 | func NewHaberdasherServer(svc Haberdasher, hooks *twirp.ServerHooks) TwirpServer 135 | ``` 136 | 137 | This constructor wraps your interface implementation as an `TwirpServer`, which 138 | is a `http.Handler` with a few extra bells and whistles. 139 | 140 | It's easy to serve a `http.Handler` straight away with the standard library. 141 | Just call `http.ListenAndServe`. For example, you might write the following in 142 | `cmd/server/main.go`: 143 | 144 | ```go 145 | package main 146 | 147 | import ( 148 | "net/http" 149 | 150 | "github.com/bilibili/twirpexample/internal/haberdasherserver" 151 | "github.com/bilibili/twirpexample/rpc/haberdasher" 152 | ) 153 | 154 | func main() { 155 | server := &haberdasherserver.Server{} // implements Haberdasher interface 156 | twirpHandler := haberdasher.NewHaberdasherServer(server, nil) 157 | 158 | http.ListenAndServe(":8080", twirpHandler) 159 | } 160 | ``` 161 | 162 | If you `go run ./cmd/server/main.go`, you'll be running your server at 163 | `localhost:8080`. All that's left is to create a client! 164 | 165 | ## Use the Client 166 | 167 | Client stubs are automatically generated, hooray! 168 | 169 | For each service, there are 2 client constructors: 170 | * `New{{Service}}ProtobufClient` for Protobuf requests. 171 | * `New{{Service}}JSONClient` for JSON requests. 172 | 173 | The JSONClient is included as reference, to show that a Twirp server can handle 174 | JSON requests, but you should really use ProtobufClient (see 175 | [Protobuf vs JSON](protobuf_and_json.md)). 176 | 177 | To use the `Haberdasher` service from another Go project, just import the 178 | auto-generated client and then use it. You might do this, in `cmd/client/main.go`: 179 | 180 | ```go 181 | package main 182 | 183 | import ( 184 | "context" 185 | "net/http" 186 | "os" 187 | "fmt" 188 | "github.com/bilibili/twirpexample/rpc/haberdasher" 189 | ) 190 | 191 | func main() { 192 | client := haberdasher.NewHaberdasherProtobufClient("http://localhost:8080", &http.Client{}) 193 | 194 | hat, err := client.MakeHat(context.Background(), &haberdasher.Size{Inches: 12}) 195 | if err != nil { 196 | fmt.Printf("oh no: %v", err) 197 | os.Exit(1) 198 | } 199 | fmt.Printf("I have a nice new hat: %+v", hat) 200 | } 201 | ``` 202 | 203 | If you have the server running in another terminal, try running this client with 204 | `go run ./cmd/client/main.go`. Enjoy the new hat! 205 | -------------------------------------------------------------------------------- /pkg/twirp/docs/headers.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "headers" 3 | title: "Using custom HTTP Headers" 4 | sidebar_label: "Custom HTTP Headers" 5 | 6 | --- 7 | Sometimes, you need to send custom HTTP headers. 8 | 9 | For Twirp, HTTP headers are a transport implementation detail. You should not 10 | have to worry about them, but maybe your HTTP middleware requires them. 11 | 12 | If so, there's nothing the Twirp spec that _forbids_ extra headers, so go ahead. 13 | The rest of this doc is a guide on how to do this. 14 | 15 | ## Client side 16 | 17 | ### Send HTTP Headers with client requests 18 | 19 | Use `twirp.WithHTTPRequestHeaders` to attach the `http.Header` to a particular 20 | `context.Context`, then use that context in the client request: 21 | 22 | ```go 23 | // Given a client ... 24 | client := haberdasher.NewHaberdasherProtobufClient(addr, &http.Client{}) 25 | 26 | // Given some headers ... 27 | header := make(http.Header) 28 | header.Set("Twitch-Authorization", "uDRlDxQYbFVXarBvmTncBoWKcZKqrZTY") 29 | header.Set("Twitch-Client-ID", "FrankerZ") 30 | 31 | // Attach the headers to a context 32 | ctx := context.Background() 33 | ctx, err := twirp.WithHTTPRequestHeaders(ctx, header) 34 | if err != nil { 35 | log.Printf("twirp error setting headers: %s", err) 36 | return 37 | } 38 | 39 | // And use the context in the request. Headers will be included in the request! 40 | resp, err := client.MakeHat(ctx, &haberdasher.Size{Inches: 7}) 41 | ``` 42 | 43 | ### Read HTTP Headers from responses 44 | 45 | Twirp client responses are structs that depend only on the Protobuf response. 46 | HTTP headers can not be used by the Twirp client in any way. 47 | 48 | However, remember that the Twirp client is instantiated with an `http.Client`, 49 | which can be configured with any `http.RoundTripper` transport. You could make a 50 | RoundTripper that reads some response headers and does something with them. 51 | 52 | ## Server side 53 | 54 | ### Send HTTP Headers on server responses 55 | 56 | In your server implementation code, set response headers one by one with the 57 | helper `twirp.SetHTTPResponseHeader`, using the same context provided by the 58 | handler. For example: 59 | 60 | ```go 61 | func (h *myServer) MyRPC(ctx context.Context, req *pb.Req) (*pb.Resp, error) { 62 | 63 | // Add Cache-Control custom header to HTTP response 64 | err := twirp.SetHTTPResponseHeader(ctx, "Cache-Control", "public, max-age=60") 65 | if err != nil { 66 | return nil, twirp.InternalErrorWith(err) 67 | } 68 | 69 | return &pb.Resp{}, nil 70 | } 71 | ``` 72 | 73 | ### Read HTTP Headers from requests 74 | 75 | Twirp server methods are abstracted away from HTTP, therefore they don't have 76 | direct access to HTTP Headers. 77 | 78 | However, they receive the `http.Request`'s `context.Context` as parameter that 79 | can be modified by HTTP middleware before being used by the Twirp method. 80 | 81 | In more detail, you could do the following: 82 | 83 | * Write some middleware (a `func(http.Handler) http.Handler)` that reads the 84 | header's value and stores it in the request context. 85 | * Wrap your Twirp server with the middleware you wrote. 86 | * Inside your service, pull the header value out through the context. 87 | 88 | For example, lets say you want to read the 'User-Agent' HTTP header inside a 89 | twirp server method. You might write this middleware: 90 | 91 | ```go 92 | func WithUserAgent(base http.Handler) http.Handler { 93 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 94 | ctx := r.Context() 95 | ua := r.Header.Get("User-Agent") 96 | ctx = context.WithValue(ctx, "user-agent", ua) 97 | r = r.WithContext(ctx) 98 | 99 | base.ServeHTTP(w, r) 100 | }) 101 | } 102 | ``` 103 | 104 | Then, you could wrap your generated Twirp server with this middleware: 105 | 106 | ```go 107 | h := haberdasher.NewHaberdasherServer(...) 108 | wrapped := WithUserAgent(h) 109 | http.ListenAndServe(":8080", wrapped) 110 | ``` 111 | 112 | Now, in your application code, you would have access to the header through the 113 | context, so you can do whatever you like with it: 114 | 115 | ```go 116 | func (h *haberdasherImpl) MakeHat(ctx context.Context, req *pb.MakeHatRequest) (*pb.Hat, error) { 117 | ua := ctx.Value("user-agent").(string) 118 | log.Printf("user agent: %v", ua) 119 | } 120 | ``` 121 | -------------------------------------------------------------------------------- /pkg/twirp/docs/hooks.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "hooks" 3 | title: "Server Hooks" 4 | sidebar_label: "Server Hooks" 5 | --- 6 | 7 | The service constructor can use `hooks *twirp.ServerHooks` to plug in additional 8 | functionality: 9 | 10 | ```go 11 | func NewHaberdasherServer(svc Haberdasher, hooks *twirp.ServerHooks) http.Handler 12 | ``` 13 | 14 | These _hooks_ provide a framework for side-effects at important points while a 15 | request gets handled. You can do things like log requests, record response times 16 | in statsd, authenticate requests, and so on. 17 | 18 | To enable hooks for your service, you pass a `*twirp.ServerHooks` in to the 19 | server constructor. The `ServerHooks` struct holds a pile of callbacks, each of 20 | which receives the current context.Context, and can provide a new 21 | context.Context (possibly including a new value through the 22 | [`context.WithValue`](https://godoc.org/golang.org/x/net/context#WithValue) 23 | function). 24 | 25 | Check out 26 | [the godoc for `ServerHooks`](http://godoc.org/github.com/bilibili/twirp#ServerHooks) 27 | for information on the specific callbacks. For an example hooks implementation, 28 | [`github.com/bilibili/twirp/hooks/statsd`](https://github.com/bilibili/twirp/blob/master/hooks/statsd/) 29 | is a good tutorial. 30 | -------------------------------------------------------------------------------- /pkg/twirp/docs/install.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: install 3 | title: Installing Twirp 4 | sidebar_label: Installation 5 | --- 6 | 7 | You'll need a few things to install Twirp: 8 | * Go1.7+ 9 | * The protobuf compiler `protoc` 10 | * Go and Twirp protoc plugins `protoc-gen-go` and `protoc-gen-twirp` 11 | 12 | ## Install protoc 13 | 14 | [Install Protocol Buffers v3](https://developers.google.com/protocol-buffers/docs/gotutorial), 15 | the `protoc` compiler that is used to auto-generate code. The simplest way to do 16 | this is to download pre-compiled binaries for your platform from here: 17 | https://github.com/google/protobuf/releases 18 | 19 | It is also available in MacOS through Homebrew: 20 | 21 | ```sh 22 | $ brew install protobuf 23 | ``` 24 | 25 | ## Get protoc-gen-go and protoc-gen-twirp plugins 26 | 27 | ### With retool 28 | 29 | We recommend using [retool](https://github.com/bilibili/retool) to manage go 30 | tools like commands and linters: 31 | 32 | ```sh 33 | $ go get github.com/bilibili/retool 34 | ``` 35 | 36 | Install the plugins into your project's `_tools` folder: 37 | ```sh 38 | $ retool add github.com/golang/protobuf/protoc-gen-go master 39 | $ retool add github.com/bilibili/twirp/protoc-gen-twirp master 40 | ``` 41 | 42 | This will make it easier to manage and update versions without causing problems 43 | to other project collaborators. 44 | 45 | If the plugins were installed with retool, when run the `protoc` command make 46 | sure to prefix with `retool do`, for example: 47 | 48 | ```sh 49 | $ retool do protoc --proto_path=$GOPATH/src:. --twirp_out=. --go_out=. ./rpc/haberdasher/service.proto 50 | ``` 51 | 52 | ### With go get 53 | 54 | Download and install `protoc-gen-go` using the normal Go tools: 55 | 56 | ```sh 57 | $ go get -u github.com/golang/protobuf/protoc-gen-go 58 | $ go get -u github.com/bilibili/twirp/protoc-gen-twirp 59 | ``` 60 | 61 | The normal Go tools will install `protoc-gen-go` in `$GOBIN`, defaulting to 62 | `$GOPATH/bin`. It must be in your `$PATH` for the protocol compiler, `protoc`, 63 | to find it, so you might need to explicitly add it to your path: 64 | 65 | ```sh 66 | $ export PATH=$PATH:$GOPATH/bin 67 | ``` 68 | 69 | ## Updating Twirp ## 70 | 71 | Twirp releases are tagged with semantic versioning and releases are managed by 72 | Github. See the [releases](https://github.com/bilibili/twirp/releases) page. 73 | 74 | To stay up to date, you update `protoc-gen-twirp` and regenerate your code. If 75 | you are using [retool](https://github.com/bilibili/retool), that's done with 76 | 77 | ```sh 78 | $ retool upgrade github.com/bilibili/twirp/protoc-gen-twirp v5.2.0 79 | ``` 80 | 81 | If you're not using retool, you can also do a system-wide install with checking 82 | out the package new version and using `go install`: 83 | 84 | ```sh 85 | $ cd $GOPATH/src/github.com/bilibili/twirp 86 | $ git checkout v5.2.0 87 | $ go install ./protoc-gen-twirp 88 | ``` 89 | 90 | With the new version of `protoc-gen-twirp`, you can re-generate code to update 91 | your servers. Then, any of the clients of your service can update their vendored 92 | copy of your service to get the latest version. 93 | -------------------------------------------------------------------------------- /pkg/twirp/docs/intro.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: intro 3 | title: Meet Twirp! 4 | sidebar_label: Overview 5 | --- 6 | 7 | Twirp is a simple RPC framework built on 8 | [protobuf](https://developers.google.com/protocol-buffers/). You define a 9 | service in a `.proto` specification file, then Twirp will _generate_ servers and 10 | clients for that service. It's your job to fill in the "business logic" that 11 | powers the server, and then generated clients can consume your service straight 12 | away. 13 | 14 | This doc is an overview of how you use Twirp - how you interact with it, what 15 | you write, and what it generates. 16 | 17 | ## Making a Twirp Service 18 | 19 | To make a Twirp service: 20 | 21 | 1. Define your service in a **Proto** file. 22 | 2. Use the `protoc` command to generate go code from the **Proto** file, it 23 | will generate an **interface**, a **client** and some **server utils** (to 24 | easily start an http listener). 25 | 3. Implement the generated **interface** to implement the service. 26 | 27 | For example, a HelloWorld **Proto** file: 28 | 29 | ```protobuf 30 | syntax = "proto3"; 31 | package twitch.twirp.example.helloworld; 32 | option go_package = "helloworld"; 33 | 34 | service HelloWorld { 35 | rpc Hello(HelloReq) returns (HelloResp); 36 | } 37 | 38 | message HelloReq { 39 | string subject = 1; 40 | } 41 | 42 | message HelloResp { 43 | string text = 1; 44 | } 45 | ``` 46 | 47 | From which Twirp can auto-generate this **interface** (running the `protoc` command): 48 | 49 | ```go 50 | type HelloWorld interface { 51 | Hello(context.Context, *HelloReq) (*HelloResp, error) 52 | } 53 | ``` 54 | 55 | You provide the **implementation**: 56 | 57 | ```go 58 | package main 59 | 60 | import ( 61 | "context" 62 | "net/http" 63 | 64 | pb "github.com/bilibili/twirp-example/rpc/helloworld" 65 | ) 66 | 67 | type HelloWorldServer struct{} 68 | 69 | func (s *HelloWorldServer) Hello(ctx context.Context, req *pb.HelloReq) (*pb.HelloResp, error) { 70 | return &pb.HelloResp{Text: "Hello " + req.Subject}, nil 71 | } 72 | 73 | // Run the implementation in a local server 74 | func main() { 75 | twirpHandler := pb.NewHelloWorldServer(&HelloWorldServer{}, nil) 76 | // You can use any mux you like - NewHelloWorldServer gives you an http.Handler. 77 | mux := http.NewServeMux() 78 | // The generated code includes a const, PathPrefix, which 79 | // can be used to mount your service on a mux. 80 | mux.Handle(pb.HelloWorldPathPrefix, twirpHandler) 81 | http.ListenAndServe(":8080", mux) 82 | } 83 | ``` 84 | 85 | And voila! Now you can just use the auto-generated **Client** to make remote calls to your new service: 86 | 87 | ```go 88 | package main 89 | 90 | import ( 91 | "context" 92 | "fmt" 93 | "net/http" 94 | 95 | pb "github.com/bilibili/twirp-example/rpc/helloworld" 96 | ) 97 | 98 | func main() { 99 | client := pb.NewHelloWorldProtobufClient("http://localhost:8080", &http.Client{}) 100 | 101 | resp, err := client.Hello(context.Background(), &pb.HelloReq{Subject: "World"}) 102 | if err == nil { 103 | fmt.Println(resp.Text) // prints "Hello World" 104 | } 105 | } 106 | ``` 107 | 108 | ## Why this is good 109 | 110 | There's no need to worry about JSON serialization or HTTP verbs/routes! Twirp 111 | [routing and serialization](routing.md) handles that for you, reducing the risk 112 | of introducing bugs. Both JSON and Protobuf are supported. The 113 | [Protobuf protocol](https://developers.google.com/protocol-buffers/docs/proto3) 114 | is designed to allow backwards compatible changes (unlike JSON, it is trivial to 115 | rename fields), it is faster than JSON and also works well as documentation for 116 | your service, because it is easy to tell what a service does by just looking at 117 | the Proto file. 118 | 119 | ## What's next? 120 | 121 | * [Install Twirp](install.md): instructions to install or upgrade Twirp tools 122 | for code auto-generation (protoc, protoc-gen-go and protoc-gen-twirp). 123 | * [Usage Example](example.md): step by step guide to build an awesome 124 | Haberdasher service. 125 | * [How Twirp routes requests](routing.md): learn more about how Twirp works 126 | under the covers. 127 | -------------------------------------------------------------------------------- /pkg/twirp/docs/mux.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "mux" 3 | title: "Muxing Twirp services" 4 | sidebar_label: "Muxing Twirp with other HTTP services" 5 | --- 6 | 7 | If you want run your server next to other `http.Handler`s, you'll need to use a 8 | mux. The generated code includes a path prefix that you can use for routing 9 | Twirp requests correctly. It's an exported string const, always as 10 | `PathPrefix`, and it is the prefix for all Twirp requests. 11 | 12 | For example, you could use it with [`http.ServeMux`](https://golang.org/pkg/net/http/#ServeMux) like this: 13 | 14 | ```go 15 | server := &haberdasherserver.Server{} 16 | twirpHandler := haberdasher.NewHaberdasherServer(server, nil) 17 | 18 | mux := http.NewServeMux() 19 | mux.Handle(haberdasher.HaberdasherPathPrefix, twirpHandler) 20 | mux.Handle("/some/other/path", someOtherHandler) 21 | 22 | http.ListenAndServe(":8080", mux) 23 | ``` 24 | 25 | You can also serve your Handler on many third-party muxes which accept 26 | `http.Handler`s. For example, on a `goji.Mux`: 27 | 28 | ```go 29 | server := &haberdasherserver.Server{} // implements Haberdasher interface 30 | twirpHandler := haberdasher.NewHaberdasherServer(server, nil) 31 | 32 | mux := goji.NewMux() 33 | mux.Handle(pat.Post(haberdasher.NewHaberdasherPathPrefix+"*"), twirpHandler) 34 | // mux.Handle other things like health checks ... 35 | http.ListenAndServe("localhost:8000", mux) 36 | ``` 37 | -------------------------------------------------------------------------------- /pkg/twirp/docs/protobuf_and_json.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "proto_and_json" 3 | title: "Twirp's Serialization Schemes" 4 | sidebar_label: "Protobuf and JSON" 5 | --- 6 | 7 | Twirp can handle both Protobuf and JSON encoding for requests and responses. 8 | 9 | This is transparent to your service implementation. Twirp parses the HTTP 10 | request (returning an Internal error if the `Content-Type` or the body are 11 | invalid) and converts it into the request struct defined in the interface. Your 12 | implementation returns a response struct, which Twirp serializes back to a 13 | Protobuf or JSON response (depending on the request `Content-Type`). 14 | 15 | See [the spec](spec.md) for more details on routing and serialization. 16 | 17 | Twirp can generates two types of clients for your service: 18 | 19 | * `New{{Service}}ProtobufClient`: makes Protobuf requests to your service. 20 | * `New{{Service}}JSONClient`: makes JSON requests to your service. 21 | 22 | ### Which one should I use, ProtobufClient or JSONClient? 23 | 24 | You should use the **ProtobufClient**. 25 | 26 | Protobuf uses fewer bytes to encode than JSON (it is more compact), and it 27 | serializes faster. 28 | 29 | In addition, Protobuf is designed to gracefully handle schema 30 | updates. Did you notice the numbers added after each field? They allow you to 31 | change a field and it still works if the client and the server have a different 32 | versions (that doesn't work with JSON clients). 33 | 34 | ### If Protobuf is better, why does Twirp support JSON? 35 | 36 | You will probably never need to use a Twirp **JSONClient** in Go, but having 37 | your servers automatically handle JSON requests is still very convenient. It 38 | makes it easier to debug (see [cURL requests](curl.md)), allows to easily write 39 | clients in other languages like Python, or make REST mappings to Twirp 40 | services. 41 | 42 | The JSON client is generated to provide a reference for implementations in other 43 | languages, and because in some rare circumstances, binary encoding of request 44 | bodies is unacceptable, and you just need to use JSON. 45 | -------------------------------------------------------------------------------- /pkg/twirp/docs/routing.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "routing" 3 | title: "HTTP Routing and Serialization" 4 | sidebar_label: "How Twirp routes requests" 5 | --- 6 | 7 | Routing and Serialization is handled by Twirp. All you really need to know is 8 | that "it just works". However you may find this interesting and useful for 9 | debugging. 10 | 11 | ### HTTP Routes 12 | 13 | Twirp works over HTTP 1.1; all RPC methods map to routes that follow the format: 14 | 15 | ``` 16 | POST /twirp/./ 17 | ``` 18 | 19 | The `` name is whatever value is used for `package` in the `.proto` 20 | file where the service was defined. The `` and `` names are 21 | CamelCased just as they would be in Go. 22 | 23 | For example, to call the `MakeHat` RPC method on the example 24 | [Haberdasher service](https://github.com/bilibili/twirp/wiki/Usage-Example:-Haberdasher) 25 | the route would be: 26 | 27 | ``` 28 | POST /twirp/twirp.example.haberdasher.Haberdasher/MakeHat 29 | ``` 30 | 31 | ### Content-Type Header (json or protobuf) 32 | 33 | The `Content-Type` header is required and must be either `application/json` or 34 | `application/protobuf`. JSON is easier for debugging (particularly when making 35 | requests with cURL), but Protobuf is better in almost every other way. Please 36 | use Protobuf in real code. See 37 | [Protobuf and JSON](https://github.com/bilibili/twirp/wiki/Protobuf-and-JSON) 38 | for more details. 39 | 40 | ### JSON serialization 41 | 42 | The JSON format should match the 43 | [official spec](https://developers.google.com/protocol-buffers/docs/proto3#json)'s 44 | rules for JSON serialization. In a nutshell: names are `camelCased`, _all_ 45 | fields must be set, _no_ extra fields may be set, and `null` means "I want to 46 | leave this field blank". 47 | 48 | ### Error responses 49 | 50 | Errors returned by Twirp servers use non-200 HTTP status codes and always have 51 | JSON-encoded bodies (even if the request was Protobuf-encoded). The body JSON 52 | has three fields `{type, msg, meta}`. For example: 53 | 54 | ``` 55 | POST /twirp/twirp.example.haberdasher.Haberdasher/INVALIDROUTE 56 | 57 | 404 Not Found 58 | { 59 | "type": "bad_route", 60 | "msg": "no handler for path /twirp/twirp.example.haberdasher.Haberdasher/INVALIDROUTE", 61 | "meta": {"twirp_invalid_route": "POST /twirp/twirp.example.haberdasher.Haberdasher/INVALIDROUTE"} 62 | } 63 | ``` 64 | 65 | ## Making requests on the command line with cURL 66 | 67 | It's easy to hand-write a Twirp request on the command line. 68 | 69 | For example, a cURL request to the Haberdasher example `MakeHat` RPC would look 70 | like this: 71 | 72 | ```sh 73 | curl --request "POST" \ 74 | --location "http://localhost:8080/twirp/twirp.example.haberdasher.Haberdasher/MakeHat" \ 75 | --header "Content-Type:application/json" \ 76 | --data '{"inches": 10}' \ 77 | --verbose 78 | ``` 79 | 80 | We need to signal Twirp that we're sending JSON data (instead of protobuf), so 81 | it can use the right deserializer. If we were using protobuf, the `--header` 82 | would be `Content-Type:application/protobuf` (and `--data` a protobuf-encoded 83 | message). 84 | 85 | The `Size` request in JSON is `{"inches": 10}`, matching the Protobuf message 86 | type: 87 | 88 | ```protobuf 89 | message Size { 90 | int32 inches = 1; 91 | } 92 | ``` 93 | 94 | The JSON response from Twirp would look something like this (`--verbose` stuff 95 | omitted): 96 | 97 | ```json 98 | {"inches":1, "color":"black", "name":"bowler"} 99 | ``` 100 | 101 | Matching the Protobuf message type: 102 | 103 | ```protobuf 104 | message Hat { 105 | int32 inches = 1; 106 | string color = 2; 107 | string name = 3; 108 | } 109 | ``` 110 | -------------------------------------------------------------------------------- /pkg/twirp/docs/spec_changelog.md: -------------------------------------------------------------------------------- 1 | --- 2 | id: "spec_changelog" 3 | title: "Twirp Wire Protocol Changelog" 4 | sidebar_label: "Changelog" 5 | --- 6 | 7 | This document lists changes to the Twirp specification. 8 | 9 | ## Changed in v6 10 | 11 | ### URL scheme 12 | 13 | In [v5](./PROTOCOL.md), URLs followed this format: 14 | 15 | ```bnf 16 | **URL ::= Base-URL "/twirp/" [ Package "." ] Service "/" Method** 17 | ``` 18 | 19 | Version 6 changes this format to remove the mandatory `"/twirp"` prefix: 20 | 21 | ```bnf 22 | **URL ::= Base-URL "/" [ Package "." ] Service "/" Method** 23 | ``` 24 | 25 | Also, `Base-URL` can now contain a path component - in other words, it's legal 26 | to set any prefix you like. 27 | 28 | If you loved the old `/twirp` prefix, you can still use it by using a base URL 29 | that ends with `/twirp`. You're no longer forced into use it, however. 30 | 31 | The "/twirp/" prefix is no longer required for three reasons: 32 | 33 | - Trademark concerns: some very large organizations don't want to 34 | take any legal risks and are concerned that "twirp" could become 35 | trademarked. 36 | - Feels like advertising: To some users, putting "twirp" in all your 37 | routes feels like it's just supposed to pump Twirp's brand, and 38 | provides no value back to users. 39 | - Homophonous with "twerp": In some Very Serious settings (like 40 | government websites), it's not okay that "twirp" sounds like 41 | "twerp", which means something like "insignificant pest." 42 | -------------------------------------------------------------------------------- /pkg/twirp/errors_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Twitch Interactive, Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may not 4 | // use this file except in compliance with the License. A copy of the License is 5 | // located at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed on 10 | // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package twirp 15 | 16 | import ( 17 | "fmt" 18 | "sync" 19 | "testing" 20 | ) 21 | 22 | func TestWithMetaRaces(t *testing.T) { 23 | err := NewError(Internal, "msg") 24 | err = err.WithMeta("k1", "v1") 25 | 26 | var wg sync.WaitGroup 27 | for i := 0; i < 1000; i++ { 28 | wg.Add(1) 29 | go func(i int) { 30 | _ = err.WithMeta(fmt.Sprintf("k-%d", i), "v") 31 | wg.Done() 32 | }(i) 33 | } 34 | 35 | wg.Wait() 36 | 37 | if len(err.MetaMap()) != 1 { 38 | t.Errorf("err was mutated") 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pkg/twirp/hooks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Twitch Interactive, Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may not 4 | // use this file except in compliance with the License. A copy of the License is 5 | // located at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed on 10 | // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package twirp 15 | 16 | import ( 17 | "context" 18 | "encoding/json" 19 | "net/http" 20 | ) 21 | 22 | // ServerHooks is a container for callbacks that can instrument a 23 | // Twirp-generated server. These callbacks all accept a context and return a 24 | // context. They can use this to add to the request context as it threads 25 | // through the system, appending values or deadlines to it. 26 | // 27 | // The RequestReceived and RequestRouted hooks are special: they can return 28 | // errors. If they return a non-nil error, handling for that request will be 29 | // stopped at that point. The Error hook will be triggered, and the error will 30 | // be sent to the client. This can be used for stuff like auth checks before 31 | // deserializing a request. 32 | // 33 | // The RequestReceived hook is always called first, and it is called for every 34 | // request that the Twirp server handles. The last hook to be called in a 35 | // request's lifecycle is always ResponseSent, even in the case of an error. 36 | // 37 | // Details on the timing of each hook are documented as comments on the fields 38 | // of the ServerHooks type. 39 | type ServerHooks struct { 40 | // RequestReceived is called as soon as a request enters the Twirp 41 | // server at the earliest available moment. 42 | RequestReceived func(context.Context) (context.Context, error) 43 | 44 | // RequestRouted is called when a request has been routed to a 45 | // particular method of the Twirp server. 46 | RequestRouted func(context.Context) (context.Context, error) 47 | 48 | // ResponsePrepared is called when a request has been handled and a 49 | // response is ready to be sent to the client. 50 | ResponsePrepared func(context.Context) context.Context 51 | 52 | // ResponseSent is called when all bytes of a response (including an error 53 | // response) have been written. Because the ResponseSent hook is terminal, it 54 | // does not return a context. 55 | ResponseSent func(context.Context) 56 | 57 | // Error hook is called when an error occurs while handling a request. The 58 | // Error is passed as argument to the hook. 59 | Error func(context.Context, Error) context.Context 60 | } 61 | 62 | // CallRequestReceived call twirp.ServerHooks.RequestReceived if the hook is available 63 | func (h *ServerHooks) CallRequestReceived(ctx context.Context) (context.Context, error) { 64 | if h == nil || h.RequestReceived == nil { 65 | return ctx, nil 66 | } 67 | return h.RequestReceived(ctx) 68 | } 69 | 70 | // CallRequestRouted call twirp.ServerHooks.RequestRouted if the hook is available 71 | func (h *ServerHooks) CallRequestRouted(ctx context.Context) (context.Context, error) { 72 | if h == nil || h.RequestRouted == nil { 73 | return ctx, nil 74 | } 75 | return h.RequestRouted(ctx) 76 | } 77 | 78 | // CallResponsePrepared call twirp.ServerHooks.ResponsePrepared if the hook is available 79 | func (h *ServerHooks) CallResponsePrepared(ctx context.Context) context.Context { 80 | if h == nil || h.ResponsePrepared == nil { 81 | return ctx 82 | } 83 | return h.ResponsePrepared(ctx) 84 | } 85 | 86 | // CallResponseSent call twirp.ServerHooks.ResponseSent if the hook is available 87 | func (h *ServerHooks) CallResponseSent(ctx context.Context) { 88 | if h == nil || h.ResponseSent == nil { 89 | return 90 | } 91 | h.ResponseSent(ctx) 92 | } 93 | 94 | // CallError call twirp.ServerHooks.Error if the hook is available 95 | func (h *ServerHooks) CallError(ctx context.Context, err Error) context.Context { 96 | if h == nil || h.Error == nil { 97 | return ctx 98 | } 99 | return h.Error(ctx, err) 100 | } 101 | 102 | // WriteError writes Twirp errors in the response and triggers hooks. 103 | func (h *ServerHooks) WriteError(ctx context.Context, resp http.ResponseWriter, err error) { 104 | // Non-twirp errors are wrapped as Internal (default) 105 | twerr, ok := err.(Error) 106 | if !ok { 107 | twerr = InternalErrorWith(err) 108 | } 109 | 110 | statusCode := ServerHTTPStatusFromErrorCode(twerr.Code()) 111 | ctx = WithStatusCode(ctx, statusCode) 112 | ctx = h.CallError(ctx, twerr) 113 | 114 | resp.Header().Set("Content-Type", "application/json") // Error responses are always JSON (instead of protobuf) 115 | resp.WriteHeader(statusCode) // HTTP response status code 116 | 117 | respBody := marshalErrorToJSON(twerr) 118 | _, writeErr := resp.Write(respBody) 119 | if writeErr != nil { 120 | // We have three options here. We could log the error, call the Error 121 | // hook, or just silently ignore the error. 122 | // 123 | // Logging is unacceptable because we don't have a user-controlled 124 | // logger; writing out to stderr without permission is too rude. 125 | // 126 | // Calling the Error hook would confuse users: it would mean the Error 127 | // hook got called twice for one request, which is likely to lead to 128 | // duplicated log messages and metrics, no matter how well we document 129 | // the behavior. 130 | // 131 | // Silently ignoring the error is our least-bad option. It's highly 132 | // likely that the connection is broken and the original 'err' says 133 | // so anyway. 134 | _ = writeErr 135 | } 136 | 137 | h.CallResponseSent(ctx) 138 | } 139 | 140 | // marshalErrorToJSON returns JSON from a twirp.Error, that can be used as HTTP error response body. 141 | // If serialization fails, it will use a descriptive Internal error instead. 142 | func marshalErrorToJSON(twerr Error) []byte { 143 | // make sure that msg is not too large 144 | msg := twerr.Msg() 145 | if len(msg) > 1000000 { 146 | msg = msg[:1000000] 147 | } 148 | 149 | type twerrJSON struct { 150 | Code string `json:"code"` 151 | Msg string `json:"msg"` 152 | Meta map[string]string `json:"meta,omitempty"` 153 | } 154 | 155 | tj := twerrJSON{ 156 | Code: string(twerr.Code()), 157 | Msg: msg, 158 | Meta: twerr.MetaMap(), 159 | } 160 | 161 | buf, err := json.Marshal(&tj) 162 | if err != nil { 163 | buf = []byte("{\"type\": \"" + Internal + "\", \"msg\": \"There was an error but it could not be serialized into JSON\"}") // fallback 164 | } 165 | 166 | return buf 167 | } 168 | 169 | // ChainHooks creates a new *ServerHooks which chains the callbacks in 170 | // each of the constituent hooks passed in. Each hook function will be 171 | // called in the order of the ServerHooks values passed in. 172 | // 173 | // For the erroring hooks, RequestReceived and RequestRouted, any returned 174 | // errors prevent processing by later hooks. 175 | func ChainHooks(hooks ...*ServerHooks) *ServerHooks { 176 | if len(hooks) == 0 { 177 | return nil 178 | } 179 | if len(hooks) == 1 { 180 | return hooks[0] 181 | } 182 | return &ServerHooks{ 183 | RequestReceived: func(ctx context.Context) (context.Context, error) { 184 | var err error 185 | for _, h := range hooks { 186 | if h != nil && h.RequestReceived != nil { 187 | ctx, err = h.RequestReceived(ctx) 188 | if err != nil { 189 | return ctx, err 190 | } 191 | } 192 | } 193 | return ctx, nil 194 | }, 195 | RequestRouted: func(ctx context.Context) (context.Context, error) { 196 | var err error 197 | for _, h := range hooks { 198 | if h != nil && h.RequestRouted != nil { 199 | ctx, err = h.RequestRouted(ctx) 200 | if err != nil { 201 | return ctx, err 202 | } 203 | } 204 | } 205 | return ctx, nil 206 | }, 207 | ResponsePrepared: func(ctx context.Context) context.Context { 208 | for _, h := range hooks { 209 | if h != nil && h.ResponsePrepared != nil { 210 | ctx = h.ResponsePrepared(ctx) 211 | } 212 | } 213 | return ctx 214 | }, 215 | ResponseSent: func(ctx context.Context) { 216 | for _, h := range hooks { 217 | if h != nil && h.ResponseSent != nil { 218 | h.ResponseSent(ctx) 219 | } 220 | } 221 | }, 222 | Error: func(ctx context.Context, twerr Error) context.Context { 223 | for _, h := range hooks { 224 | if h != nil && h.Error != nil { 225 | ctx = h.Error(ctx, twerr) 226 | } 227 | } 228 | return ctx 229 | }, 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /pkg/twirp/hooks_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Twitch Interactive, Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"). You may not 4 | // use this file except in compliance with the License. A copy of the License is 5 | // located at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // or in the "license" file accompanying this file. This file is distributed on 10 | // an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | // express or implied. See the License for the specific language governing 12 | // permissions and limitations under the License. 13 | 14 | package twirp 15 | 16 | import ( 17 | "context" 18 | "reflect" 19 | "testing" 20 | ) 21 | 22 | func TestChainHooks(t *testing.T) { 23 | var ( 24 | hook1 = new(ServerHooks) 25 | hook2 = new(ServerHooks) 26 | hook3 = new(ServerHooks) 27 | ) 28 | 29 | const key = "key" 30 | 31 | hook1.RequestReceived = func(ctx context.Context) (context.Context, error) { 32 | return context.WithValue(ctx, key, []string{"hook1"}), nil 33 | } 34 | hook2.RequestReceived = func(ctx context.Context) (context.Context, error) { 35 | v := ctx.Value(key).([]string) 36 | return context.WithValue(ctx, key, append(v, "hook2")), nil 37 | } 38 | hook3.RequestReceived = func(ctx context.Context) (context.Context, error) { 39 | v := ctx.Value(key).([]string) 40 | return context.WithValue(ctx, key, append(v, "hook3")), nil 41 | } 42 | 43 | hook1.RequestRouted = func(ctx context.Context) (context.Context, error) { 44 | return context.WithValue(ctx, key, []string{"hook1"}), nil 45 | } 46 | 47 | hook2.ResponsePrepared = func(ctx context.Context) context.Context { 48 | return context.WithValue(ctx, key, []string{"hook2"}) 49 | } 50 | 51 | chain := ChainHooks(hook1, hook2, hook3) 52 | 53 | ctx := context.Background() 54 | 55 | // When all three chained hooks have a handler, all should be called in order. 56 | want := []string{"hook1", "hook2", "hook3"} 57 | haveCtx, err := chain.RequestReceived(ctx) 58 | if err != nil { 59 | t.Fatalf("RequestReceived chain has unexpected err %v", err) 60 | } 61 | have := haveCtx.Value(key) 62 | if !reflect.DeepEqual(want, have) { 63 | t.Errorf("RequestReceived chain has unexpected ctx, have=%v, want=%v", have, want) 64 | } 65 | 66 | // When only the first chained hook has a handler, it should be called, and 67 | // there should be no panic. 68 | want = []string{"hook1"} 69 | haveCtx, err = chain.RequestRouted(ctx) 70 | if err != nil { 71 | t.Fatalf("RequestRouted chain has unexpected err %v", err) 72 | } 73 | have = haveCtx.Value(key) 74 | if !reflect.DeepEqual(want, have) { 75 | t.Errorf("RequestRouted chain has unexpected ctx, have=%v, want=%v", have, want) 76 | } 77 | 78 | // When only the second chained hook has a handler, it should be called, and 79 | // there should be no panic. 80 | want = []string{"hook2"} 81 | have = chain.ResponsePrepared(ctx).Value(key) 82 | if !reflect.DeepEqual(want, have) { 83 | t.Errorf("RequestRouted chain has unexpected ctx, have=%v, want=%v", have, want) 84 | } 85 | 86 | // When none of the chained hooks has a handler there should be no panic. 87 | chain.ResponseSent(ctx) 88 | } 89 | -------------------------------------------------------------------------------- /pkg/twirp/server.go: -------------------------------------------------------------------------------- 1 | package twirp 2 | 3 | import "net/http" 4 | 5 | // Server is the interface generated server structs will support: they're 6 | // HTTP handlers with additional methods for accessing metadata about the 7 | // service. Those accessors are a low-level API for building reflection tools. 8 | // Most people can think of TwirpServers as just http.Handlers. 9 | type Server interface { 10 | http.Handler 11 | // ServiceDescriptor returns gzipped bytes describing the .proto file that 12 | // this service was generated from. Once unzipped, the bytes can be 13 | // unmarshalled as a 14 | // github.com/golang/protobuf/protoc-gen-go/descriptor.FileDescriptorProto. 15 | // 16 | // The returned integer is the index of this particular service within that 17 | // FileDescriptorProto's 'Service' slice of ServiceDescriptorProtos. This is a 18 | // low-level field, expected to be used for reflection. 19 | ServiceDescriptor() ([]byte, int) 20 | // ProtocGenTwirpVersion is the semantic version string of the version of 21 | // twirp used to generate this file. 22 | ProtocGenTwirpVersion() string 23 | } 24 | -------------------------------------------------------------------------------- /rpc/README.md: -------------------------------------------------------------------------------- 1 | # rpc 2 | 3 | 接口定义层，基于 protobuf 严格定义 RPC 接口路由、参数和文档。 4 | 5 | ## 目录结构 6 | 7 | 通常一个服务一个文件夹。服务下有版本，一个版本一个文件夹。内部服务一般使用 `v0` 作为版本。 8 | 9 | 一个版本可以定义多个 service，每个 service 一个 proto 文件。 10 | 11 | 典型的目录结构如下： 12 | ``` 13 | rpc/user # 业务服务 14 | └── v0 # 服务版本 15 | ├── echo.go # rpc 方法实现，方法签名由脚手架自动生成 16 | ├── echo.pb.go # protobuf message 定义代码[自动生成] 17 | ├── echo.proto # protobuf 描述文件[业务方定义] 18 | └── echo.twirp.go # rpc 接口和路由代码[自动生成] 19 | ``` 20 | 21 | ## 定义接口 22 | 23 | 服务接口使用 [protobuf](https://developers.google.com/protocol-buffers/docs/proto3#services) 描述。 24 | ```proto 25 | syntax = "proto3"; 26 | 27 | package user.v0; // 包名，与目录保持一致 28 | 29 | // 服务名，只要能定义一个 service 30 | service Echo { 31 | // 服务方法，按需定义 32 | rpc Hello(HelloRequest) returns (HelloResponse); 33 | } 34 | 35 | // 入参定义 36 | message HelloRequest { 37 | // 字段定义，如果使用 form 表单传输，则只支持 38 | // int32, int64, uint32, unint64, double, float, bool, string, 39 | // 以及对应的 repeated 类型，不支持 map 和 message 类型！ 40 | // 框架会自动解析并转换参数类型 41 | // 如果用 json 或 protobuf 传输则没有限制 42 | string message = 1; // 这是行尾注释，业务方一般不要使用 43 | int32 age = 2; 44 | // form 表单格式只能部分支持 repeated 语义 45 | // 但客户端需要发送英文逗号分割的字符串 46 | // 如 ids=1,2,3 将会解析为 []int32{1,2,3} 47 | repeated int32 ids = 3; 48 | } 49 | 50 | message HelloMessage { 51 | string message = 1; 52 | } 53 | 54 | // 出参定义, 55 | // 理论上可以输出任意消息 56 | // 但我们的业务要求只能包含 code, msg, data 三个字段， 57 | // 其中 data 需要定义成 message 58 | // 开源版本可以怱略这一约定 59 | message HelloResponse { 60 | // 业务错误码[机读]，必须大于零 61 | // 小于零的主站框架在用，注意避让。 62 | int32 code = 1; 63 | // 业务错误信息[人读] 64 | string msg = 2; 65 | // 业务数据对象 66 | HelloMessage data = 3; 67 | } 68 | ``` 69 | 70 | ### GET 请求 71 | 72 | 有些业务场景需提供 GET 接口，原生的 twirp 框架并不支持。但 sniper 框架是支持的。 73 | 74 | 只需要在 `hook.RequestReceived` 阶段调用 `ctx = twirp.WithAllowGET(ctx, true)` 将 GET 开关注入 ctx 即可。 75 | 76 | 但原则上不建议使用 GET 请求。 77 | 78 | ### 文件下载 79 | 80 | 有些业务场景需提供 json/protobuf 之外的数据，如 xml、txt 甚至是 xlsx。 81 | 82 | sniper 为这类情况留有「后门」。只需要定义并返回一个特殊的 response 消息： 83 | ```proto 84 | // 消息名可以随便取 85 | message DownloadMsg { 86 | // content_type 内容用于设置 http 的 content-type 字段 87 | string content_type = 1; 88 | // data 内容会直接以 http body 的形式发送给调用方 89 | bytes data = 2; 90 | } 91 | ``` 92 | 93 | ## 接口映射 94 | 95 | - 请求方法 **POST** 96 | - 请求路径 **/twirp**/package.Service/Method 97 | - 请求协议 http/1.1、http/2 98 | - Content-Type 99 | - application/x-www-form-urlencoded 100 | - application/json 101 | - application/protobuf 102 | - 请求内容 103 | - urlencoded 字符串 104 | - json 105 | - protobuf 106 | 107 | 最新版的[protobuf-gen-twirp](./cmd/protoc-gen-twirp)生成的 `*.twirp.go` 文件已经 108 | 不再硬编码 `/twirp` 前缀。接口前缀可以通过 `RPC_PREFIX` 配置项控制，默认前缀为 `/api`。 109 | 110 | 表单请求 111 | ``` 112 | POST /user.v0.Echo/Hello HTTP/1.1 113 | Host: example.com 114 | Content-Type: application/x-www-form-urlencoded 115 | Content-Length: 19 116 | 117 | message=hello&age=1 118 | 119 | HTTP/1.1 200 OK 120 | Content-Type: application/json 121 | Content-Length: 27 122 | 123 | {"message":"Hello, World!"} 124 | ``` 125 | json 请求 126 | ``` 127 | POST /user.v0.Echo/Hello HTTP/1.1 128 | Host: example.com 129 | Content-Type: application/json 130 | Content-Length: 19 131 | 132 | {"message":"hello","age":1} 133 | 134 | HTTP/1.1 200 OK 135 | Content-Type: application/json 136 | Content-Length: 27 137 | 138 | {"message":"Hello, World!"} 139 | ``` 140 | 141 | 原始英文协议在[这里](../util/twirp/PROTOCOL.md) 142 | 143 | ## 生成代码 144 | 145 | ```bash 146 | # 首次使用需要安装 protoc-gen-twirp 工具 147 | make cmd 148 | # 针对指定服务 149 | protoc --go_out=. --twirp_out=. echo.proto 150 | 151 | # 针对所有服务 152 | find rpc -name '*.proto' -exec protoc --twirp_out=. --go_out=. {} \; 153 | 154 | # 建议直接使用框架提供的 make 规则 155 | make rpc 156 | ``` 157 | 158 | 生成的文件中 `*.pb.go` 是由 protobuf 消息的定义代码，同时支持 protobuf 和 json。`*.twirp.go` 则是 rpc 路由相关代码。 159 | 160 | ## 自动注册 161 | 162 | sniper 提供的脚手架可以自动生成 proto 模版、server 模版，并注册路由。 163 | 运行以下命令： 164 | ```bash 165 | go run cmd/sniper/main.go rpc --server=foo --service=echo 166 | ``` 167 | 会自动生成： 168 | ``` 169 | rpc 170 | └── foo 171 | └── v1 172 | ├── echo.go 173 | ├── echo.pb.go 174 | ├── echo.proto 175 | └── echo.twirp.go 176 | ``` 177 | 178 | ## 实现接口 179 | 180 | 服务接口定义在 rpc 目录对应的 echo.twirp.go 中，是自动生成的。 181 | 182 | 接口实现代码则会自动生成并保存到 echo.go 中。 183 | 184 | ```go 185 | package foo_v0 186 | 187 | import ( 188 | // 标准库单列一组 189 | "context" 190 | 191 | // 框架库单列一组 192 | "sniper/dao/login" 193 | "sniper/pkg/conf" 194 | ) 195 | 196 | // 服务对象，约定为 Server 197 | type EchoServer struct{} 198 | 199 | // 接口实现，三步走：处理入参、调用服务、返回出参 200 | func (s *EchoServer) ClearLoginCache(ctx context.Context, req *ClearRequest) (*EmptyReply, error) { 201 | // 处理入参 202 | mid := req.GetMid() 203 | 204 | // 调用 service 层或者 dao 层完成业务逻辑 205 | login.ClearUID(ctx, mid) 206 | 207 | // 返回出参 208 | reply := &EmptyReply{} 209 | 210 | return reply, nil 211 | } 212 | ``` 213 | 214 | ## 注册服务 215 | 216 | 请参考 [cmd/server/README.md](../cmd/server/README.md)。 217 | 218 | ## 错误处理 219 | 220 | ### 异常/错误 221 | 222 | **错误** 是 __计划内__ 的情形，例如用户输入密码不匹配、用户余额不足等等。 223 | **异常** 是 __计划外__ 的情形，例如用户提交的参数类型跟接口定义不匹配、DB 连接超时等等。 224 | 225 | **错误** 可以认为是一种特殊的“正常情况”, **异常** 则是真正的“不正常情况”。 226 | 227 | ### 处理错误 228 | 229 | 客户端需要根据不同业务需求处理 **错误**, 例如用户未登录则需要跳转到登录页面。所以，我需要使用错误码来返回错误信息。 230 | 231 | 处理代码示例如下： 232 | ```go 233 | resp := &pb.Resp{} 234 | 235 | resp.Code = 100 236 | resp.Msg = "Need Login" 237 | 238 | return nil, resp 239 | ``` 240 | 以上代码会返回如下 HTTP 信息： 241 | ``` 242 | HTTP/1.1 200 OK 243 | Content-Length: 355 244 | Content-Type: application/json 245 | Date: Tue, 14 Aug 2018 03:05:41 GMT 246 | X-Trace-Id: 3kclnknyzmamo 247 | 248 | { 249 | "code": 100, 250 | "msg": "Need Login", 251 | "data": {} 252 | } 253 | ``` 254 | 255 | ### 处理异常 256 | 257 | 正常的客户端会严格按照接口定义调用接口，只有客户端有 bug 或者服务端有问题的时候才会遇到 **异常**。 258 | 在这种情况下，首先，我们无法从错误中恢复；其次，这类错误的处理方式跟具体的业务没有关系的；最后，我们需要 **及时发现** 这类问题并修复。 259 | 所以，我们需要使用 HTTP 的 4xx 和 5xx 状态码来返回错误信息。 260 | 261 | 处理代码示例如下： 262 | ```go 263 | import "sniper/pkg/errors" 264 | // ... 265 | 266 | // 这是客户端问题，返回 HTTP 4xx 状态码 267 | if req.ID <= 0 { 268 | return nil, errors.InvalidArgumentError("id", "must > 0") 269 | } 270 | 271 | // HTTP/1.1 400 Bad Request 272 | // Content-Length: 104 273 | // Content-Type: application/json 274 | // Date: Tue, 14 Aug 2018 03:09:30 GMT 275 | // X-Trace-Id: kg1od386gjto 276 | // 277 | // { 278 | // "code": "invalid_argument", 279 | // "meta": { 280 | // "argument": "page_size" 281 | // }, 282 | // "msg": "page_size page_size must be > 0" 283 | // } 284 | 285 | // 这是服务端问题，返回 HTTP 5xx 状态码 286 | if err := bookshelf.AddFavorite(ctx, id); err != nil { 287 | return nil, err 288 | } 289 | 290 | // HTTP/1.1 500 Internal Server Error 291 | // Content-Length: 112 292 | // Content-Type: application/json 293 | // Date: Wed, 15 Aug 2018 08:50:47 GMT 294 | // X-Trace-Id: 3njq5120j3c1n 295 | // 296 | // { 297 | // "code": "internal", 298 | // "meta": { 299 | // "cause": "*net.OpError" 300 | // }, 301 | // "msg": "dial tcp :0: connect: can't assign requested address" 302 | // } 303 | ``` 304 | 305 | 我们可以通过 SLB 报警及时发现此类错误并减少业务损失。 306 | -------------------------------------------------------------------------------- /sniper.toml: -------------------------------------------------------------------------------- 1 | # 全局日志级别 2 | LOG_LEVEL = "debug" 3 | # rpc 接口路径前缀 4 | RPC_PREFIX = "/api" 5 | # sqlite 配置 6 | SQLDB_DSN_FOO = "file:memory:" 7 | # redis 配置 8 | MEMDB_DSN_BAR = "redis://localhost:6379" 9 | -------------------------------------------------------------------------------- /svc/README.md: -------------------------------------------------------------------------------- 1 | # service 2 | 3 | 业务逻辑层，处于 rpc 层和 dao 层之间。service 只能通过 dao 层获取数据。 4 | 5 | 业务接口必须接受 `ctx context.Context` 对象，并向下传递。 6 | 7 | ## 错误日志 8 | --------------------------------------------------------------------------------