├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── .gitignore ├── config │ ├── errors.zh.yaml │ ├── gateway.yaml │ └── greeter.yaml ├── main.go └── srv │ ├── all.go │ ├── gateway.go │ └── greeter.go ├── docs ├── application-auth.md ├── application-cache.md ├── application-db.md ├── application-log.md ├── application-opentracing.md ├── application.md ├── data │ └── filebeat.yml ├── error-handle.md ├── graphql.md ├── handle-request.md ├── img │ └── goland-build.jpg ├── index.md ├── service-layer.md ├── subject-jenkins.md ├── subject-log.md ├── subject-micro.md ├── subject-opentracing.md ├── subject-profile.md ├── tools.md └── use-protobuf.md ├── gateway ├── app │ ├── app.go │ └── middleware.go ├── errors │ └── errors.go ├── handle │ ├── graphql.go │ └── parse.go ├── loader │ ├── loader.go │ ├── loader_keys.go │ └── user.go ├── public │ └── html │ │ └── graphiql.html ├── resolver │ ├── greeter.go │ ├── node.go │ ├── page.go │ ├── query.go │ └── resolver.go ├── router │ └── router.go └── schema │ ├── bindata.go │ ├── schema.go │ ├── schema.graphql │ └── type │ ├── base.graphql │ └── greeter.graphql ├── go.mod ├── go.sum ├── helper └── relay │ └── relay.go ├── schemas └── greeter │ ├── greeter.pb.go │ └── greeter.proto └── services └── greetersrv └── greeter.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | vendor -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # golang starter kit Change Log 2 | 3 | 本更新涉及的包更新记录包括go-common,gin-contrib 4 | 5 | ### 2018-09-30 6 | 7 | * enh: http,RPC通讯增加gzip支持 8 | 9 | ### 2018-09-28 10 | * enh: 增强graphql的错误定制输出 11 | * enh: jwt验证完善,与go-common组件统一 12 | * fix: auth验证链的传递 13 | * fix: 用户ID 由 userId统一至小写 userid 14 | 15 | ### 2018-09-23 16 | * enh: go-common为grpx服务定义增加grpc原生ServerOption支持 17 | * enh: 客户端rpc配置,及服务端grpc服务配置定义 18 | * enh: 日志记录整合grpc日志,请区分gateway及grpc服务端的调用方式. 19 | * enh: go-common logger优化,增加上下文信息记录支持,区分类型化日志及糖方式 v -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | ENV TIMEZONE=Asia/Shanghai HOSTIP=0.0.0.0 4 | RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \ 5 | apk update && \ 6 | apk --no-cache add ca-certificates tzdata && \ 7 | echo "$TIMEZONE" > /etc/timezone && \ 8 | ln -sf "/usr/share/zoneinfo/$TIME_ZONE" /etc/localtime 9 | # in alpine,cross compile binany file maynot appear 'sh: file not found ' error 10 | RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 11 | 12 | VOLUME /app/runtime 13 | 14 | WORKDIR /app 15 | 16 | COPY cmd/serve . 17 | COPY cmd/config ./config 18 | COPY cmd/public ./public 19 | CMD ./serve all -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 qeelyn.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Binary names 2 | BUILD_NAME=serve 3 | GO=$(shell which go) 4 | 5 | .PHONY: dep 6 | dep: 7 | vgo mod vendor 8 | 9 | .PHONY: test 10 | test: 11 | $(GO) test -json ./... 12 | 13 | .PHONY: build 14 | build: pre-build build-linux 15 | 16 | pre-build: 17 | cp -rf gateway/public cmd/ 18 | 19 | build-default: 20 | $(GO) build -o cmd/$(BUILD_NAME) cmd/main.go 21 | build-linux: 22 | GOOS=linux GOARCH=amd64 $(GO) build -o cmd/$(BUILD_NAME) cmd/main.go 23 | 24 | build-window: 25 | GOOS=window GOARCH=amd64 $(GO) build -o cmd/$(BUILD_NAME) cmd/main.go 26 | 27 | .PHONY: clean 28 | clean: 29 | rm -f cmd/$(BUILD_NAME) 30 | rm -rf cmd/public 31 | rm -rf cmd/runtime/*.* 32 | 33 | .PHONY: docker 34 | docker: clean build 35 | docker build -t sms-std-api . 36 | 37 | .PHONY: run 38 | run: pre-build build-default 39 | cd cmd && ./serve all 40 | 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 基于Go的应用开发入门套件 2 | ======================== 3 | 4 | 本工具包旨于让您快速构建起项目结构,以便通过Go来开发WebApi或RPC服务,遵循SOLID的最佳实践来编写GO代码. 5 | 6 | > 现阶段本项目例子还相对简单,但框架及组件的使用都是在实际项目使用的,未来再提供近于实战的例子. 7 | 8 | 本工具包提供下列功能: 9 | 10 | * 应用与组件的可配置性,并支持配置中心方式 11 | * 基于Gin的Web服务支持 12 | * GraqhQl服务支持 13 | * 基于Gorm的数据库操作及事务控制 14 | * JWT-based 验证 15 | * 异常处理及可控的错误响应 16 | * 应用日志及访问日志支持 17 | * 围绕protobuf为模型中心,生成通用性代码 18 | * 采用Service层,并可扩展为RPC服务或微服务 19 | * 测试环境可配置 20 | 21 | 本工具包使用了常见的GoPKG,你可以很容易的替换为自己喜欢的包.因为这些流行的PKG进行了良好的抽象 22 | . 23 | 24 | * 路由框架: [gin](http://github.com/gin-gonic/gin) 25 | * 数据库及ORM: [gorm](http://github.com/jinzhu/gorm) 26 | * 数据验证: 目前通过Gin在路由层处理,还有很式工作 [want help] 27 | * 配置文件: [viper](http://github.com/spf13/viper) 28 | * 日志: [Uber Zap](http://go.uber.org/zap) 29 | * graphql: [gopher-graphql](github.com/graph-gophers/graphql-go) 30 | * 依赖管理: [DEP](https://golang.github.io/dep/docs/introduction.html) 将被[vgo](https://github.com/golang/vgo)取代 31 | * 基础套件:[qeelyn-common](http://github.com/qeelyn/go-common) 32 | - 缓存 cache 内置支持local,redis,memcached 33 | - protobuf工具包 34 | - grpc 一些的微服务工具包 35 | * 中间件与组件: [qeelyn-contrib](http://github.com/qeelyn/gin-contrib) 36 | * protoc生成工具扩展: [protoc-gen-goql](http://github.com/tsingsun/protoc-gen-goql) 37 | 38 | 微服务部分 39 | 40 | * 服务注册与发现: 实现了[etcd](https://github.com/coreos/etcd),留有其他组件扩展的能力 41 | * GRPC组件: 主要采用了[grpc-ecosystem](https://github.com/grpc-ecosystem)提供的组件 42 | * 系统监控: [prometheus](https://prometheus.io),可配合[grafana](https://grafana.com)搭建监控平台 43 | * 通过Docker构建部署.可通过[基于jenkins的持续构建](./docs/subject-jenkins.md)进一部了解 44 | 45 | 本套件可以做什么 46 | ---------------- 47 | 本套件面向是的企业级应用开发,做为通用的API编程框架.包括常见的RESTapi,微服务架构支持. 48 | 49 | 本套件的目标不是为了实现像beego这样的全栈框架,通常认为每个项目特性不同,除了提供一些基础包,应该由项目自行装配. 50 | 51 | 开发环境 52 | --------- 53 | 54 | - go环境安装 55 | - IDE vscode or goland 56 | - 以前go的开发离开不了翻墙,现在可以不翻了,具体可看[工具篇](./docs/tools.md) 57 | 58 | 快速入门 59 | --------- 60 | ### 新建项目 61 | ``` 62 | git clone https://github.com/qeelyn/golang-starter-kit.git $GOPATH/src/xxx.com/your-vendor/project-name 63 | ``` 64 | 65 | ### 运行 66 | ``` 67 | make run 68 | ``` 69 | 默认采用微服务结构,通过浏览器访问: `http://localhost:18000/graphiql`,graphiql正常显示时在graphiql中输入: 70 | ``` 71 | query test { 72 | hello(name:"qsli") { 73 | id 74 | } 75 | } 76 | ``` 77 | 执行,没有看到异常时表示项目正常运行 78 | 79 | ### 资源 80 | 81 | 开发手册: [戳我](./docs/index.md) 82 | 83 | QQ: 21997272 -------------------------------------------------------------------------------- /cmd/.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | /config/*-local.yaml 3 | /config/*.pem 4 | /runtime 5 | /public 6 | /serve -------------------------------------------------------------------------------- /cmd/config/errors.zh.yaml: -------------------------------------------------------------------------------- 1 | INTERNAL_SERVER_ERROR: 2 | code: 500 3 | message: "内部服务器错误!" 4 | debug: "Internal server error: {error}" 5 | 6 | PERMISSION_DENIED: 7 | code: 403 8 | message: "无请求权限!" 9 | debug: "无请求权限: {error}" 10 | 11 | Unauthorized: 12 | code: 401 13 | message: "用户未验证!" 14 | debug: "用户未验证:{error}" 15 | 16 | LOAD_WRONG_TYPE: 17 | code: 500 18 | message: "加载的数据类型不符合" 19 | 20 | DeadlineExceeded: 21 | code: 408 22 | message: "请求超时,请刷新重试" 23 | 24 | GRPCUnavailable: 25 | code: 503 26 | message: "请求超时,请刷新重试" -------------------------------------------------------------------------------- /cmd/config/gateway.yaml: -------------------------------------------------------------------------------- 1 | appname: gst-gateway 2 | listen: ":18000" 3 | appmode: release 4 | gzip: 0 5 | log: 6 | file: 7 | filename: runtime/gateway.log 8 | maxsize: 100 9 | level: 1 #debug:-1;info:0;warning:1;error:2;DPanic:3;Panic:4;Fatal:5 10 | 11 | error-template: config/errors.zh.yaml 12 | 13 | cache: 14 | dataloader: 15 | type: local 16 | 17 | web: 18 | staticdir: "public" 19 | 20 | jwt: 21 | enable: false 22 | public-key: "" # pem file if use rs algorithm 23 | encryption-key: "abcderf" 24 | 25 | auth: 26 | auth-server: "" 27 | check-access: "/access/can" 28 | check-access-timeout: 1000 #microsecond 29 | router-prefix: "" 30 | 31 | rpc: 32 | greeter: 33 | name: :18001 # if use etcd ,the rpc connection will be: qeelyn://author/srv-greeter 34 | #compressor: gzip # turn on if client and server communicate through internet 35 | -------------------------------------------------------------------------------- /cmd/config/greeter.yaml: -------------------------------------------------------------------------------- 1 | appname: srv-greeter 2 | listen: ":18001" 3 | registryListen: ":18001" 4 | appmode: dev 5 | log: 6 | file: 7 | filename: runtime/greeter.log 8 | maxsize: 100 9 | level: 1 #debug:-1;info:0;warning:1;error:2;DPanic:3;Panic:4;Fatal:5 10 | 11 | error-template: config/errors.zh.yaml 12 | 13 | #db: 14 | # default: 15 | # dialect: mysql 16 | # dsn: root:@tcp(localhost:3306)/yak 17 | 18 | jwt: 19 | enable: false 20 | public-key: "" # pem file if use rs algorithm 21 | encryption-key: "abcderf" 22 | 23 | auth: 24 | auth-server: "" 25 | check-access: "" 26 | check-access-timeout: 1000 #microsecond 27 | router-prefix: "" 28 | 29 | #opentracing: 30 | # sampler: 31 | # type: const 32 | # param: 1 33 | # reporter: 34 | # logSpans: false 35 | # localAgentHostPort: "127.0.0.1:6831" 36 | # headers: 37 | # TraceContextHeaderName: trace.traceid 38 | 39 | 40 | metrics: 41 | enable: false 42 | backend: prometheus -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/prometheus/client_golang/prometheus/promhttp" 7 | "github.com/qeelyn/go-common/config" 8 | "github.com/qeelyn/go-common/config/etcdv3" 9 | "github.com/qeelyn/go-common/config/options" 10 | "github.com/qeelyn/go-common/grpcx/registry" 11 | _ "github.com/qeelyn/go-common/grpcx/registry/etcdv3" 12 | "github.com/qeelyn/golang-starter-kit/cmd/srv" 13 | _ "google.golang.org/grpc/encoding/gzip" 14 | "google.golang.org/grpc/resolver" 15 | "log" 16 | "net/http" 17 | "os" 18 | "strings" 19 | ) 20 | 21 | func usage() { 22 | fmt.Fprintf(os.Stderr, "golang start kit services\n") 23 | fmt.Fprintf(os.Stderr, "USAGE\n") 24 | fmt.Fprintf(os.Stderr, " serve command [flags]\n") 25 | fmt.Fprintf(os.Stderr, "\n") 26 | fmt.Fprintf(os.Stderr, "The commands are\n") 27 | fmt.Fprintf(os.Stderr, " all Boots all services\n") 28 | fmt.Fprintf(os.Stderr, " gateway Api gateway\n") 29 | fmt.Fprintf(os.Stderr, " gteeter Greeter service\n") 30 | fmt.Fprintf(os.Stderr, "Flags\n") 31 | fmt.Fprintf(os.Stderr, " -c Config file path,default is using local path at './config'\n") 32 | fmt.Fprintf(os.Stderr, " -n Service discovery address\n") 33 | fmt.Fprintf(os.Stderr, " -m Http listen for prometheus monitor\n") 34 | fmt.Fprintf(os.Stderr, "\n") 35 | } 36 | 37 | const LocalConfigPath = "./config" 38 | 39 | func main() { 40 | var ( 41 | configPath = flag.String("c", "config", "app config path") 42 | namingAddr = flag.String("n", "", "service register and discovery server address") 43 | monitorListen = flag.String("m", "", "http listen for prometheus monitor") 44 | useLocalConfigPath = false 45 | ) 46 | if len(os.Args) < 2 { 47 | usage() 48 | os.Exit(1) 49 | } 50 | flag.CommandLine.Parse(os.Args[2:]) 51 | 52 | configOptions := []options.Option{config.Path(*configPath)} 53 | if *configPath == LocalConfigPath { 54 | useLocalConfigPath = true 55 | } 56 | 57 | var run func(options.Options, registry.Registry) error 58 | 59 | switch cmd := strings.ToLower(os.Args[1]); cmd { 60 | case "all": 61 | run = srv.RunAll 62 | case "gateway": 63 | run = srv.RunGateway 64 | case "greeter": 65 | run = srv.RunGreeter 66 | default: 67 | usage() 68 | os.Exit(1) 69 | } 70 | 71 | //discover service 72 | //registry.DefaultRegistry 73 | //rrBalancer := balancer.Get("round_robin") 74 | var ( 75 | register registry.Registry 76 | err error 77 | ) 78 | 79 | if *namingAddr != "" { 80 | register, err = registry.DefaultRegistry( 81 | registry.Dsn(*namingAddr), 82 | ) 83 | 84 | if err != nil { 85 | panic(err) 86 | } 87 | resolver.Register(register.(resolver.Builder)) 88 | if !useLocalConfigPath { 89 | configOptions = append(configOptions, config.Registry(register)) 90 | } 91 | } 92 | if *monitorListen != "" { 93 | go func() { 94 | log.Printf("starting prometheus http server at:%s", *monitorListen) 95 | http.Handle("/metrics", promhttp.Handler()) 96 | httpServer := &http.Server{ 97 | Addr: *monitorListen, 98 | } 99 | httpServer.ListenAndServe() 100 | }() 101 | } 102 | 103 | cnfOpts := config.ParseOptions(configOptions...) 104 | if register != nil && !useLocalConfigPath { 105 | // TODO 只支持了etcd3 106 | etcdv3.Build(cnfOpts) 107 | } 108 | 109 | if err := run(*cnfOpts, register); err != nil { 110 | fmt.Fprintf(os.Stderr, "%v\n", err) 111 | os.Exit(1) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /cmd/srv/all.go: -------------------------------------------------------------------------------- 1 | package srv 2 | 3 | import ( 4 | "errors" 5 | "github.com/grpc-ecosystem/go-grpc-prometheus" 6 | "github.com/opentracing/opentracing-go" 7 | "github.com/qeelyn/go-common/cache" 8 | qconfig "github.com/qeelyn/go-common/config" 9 | "github.com/qeelyn/go-common/config/options" 10 | "github.com/qeelyn/go-common/grpcx" 11 | "github.com/qeelyn/go-common/grpcx/authfg" 12 | "github.com/qeelyn/go-common/grpcx/dialer" 13 | "github.com/qeelyn/go-common/grpcx/registry" 14 | "github.com/qeelyn/go-common/grpcx/tracing" 15 | "github.com/qeelyn/go-common/logger" 16 | "github.com/qeelyn/go-common/tracer" 17 | "github.com/spf13/viper" 18 | "github.com/uber/jaeger-client-go/config" 19 | "google.golang.org/grpc" 20 | "google.golang.org/grpc/keepalive" 21 | "log" 22 | "strings" 23 | "time" 24 | ) 25 | 26 | func RunAll(cnfOpts options.Options, register registry.Registry) error { 27 | go RunGreeter(cnfOpts, register) 28 | return RunGateway(cnfOpts, register) 29 | } 30 | 31 | // viper is the section for rpc 32 | func newDialer(isGateway bool, viper *viper.Viper, tracer opentracing.Tracer, gopts ...grpc.DialOption) *grpc.ClientConn { 33 | dcopts := make([]grpc.CallOption, 0) 34 | if viper.IsSet("compressor") { 35 | dcopts = append(dcopts, grpc.UseCompressor(viper.GetString("compressor"))) 36 | grpc.WithDefaultCallOptions(grpc.UseCompressor("gzip")) 37 | } 38 | dialOptions := append(gopts, 39 | grpc.WithWaitForHandshake(), 40 | grpc.WithInsecure(), 41 | grpc.WithBalancerName("round_robin"), 42 | grpc.WithDefaultCallOptions(dcopts...), 43 | ) 44 | if viper.GetInt("keepalive") != 0 { 45 | cp := keepalive.ClientParameters{Time: time.Duration(viper.GetInt("keepalive")) * time.Second} 46 | dialOptions = append(dialOptions, grpc.WithKeepaliveParams(cp)) 47 | } 48 | cc, err := dialer.Dial(viper.GetString("name"), 49 | dialer.WithDialOption(dialOptions...), 50 | dialer.WithUnaryClientInterceptor( 51 | grpc_prometheus.UnaryClientInterceptor, 52 | authfg.WithAuthClient(isGateway), 53 | ), 54 | dialer.WithTracer(tracer), 55 | dialer.WithTraceIdFunc(tracing.DefaultClientTraceIdFunc(isGateway)), 56 | ) 57 | if err != nil { 58 | log.Panicf("dialer error: %v", err) 59 | } 60 | return cc 61 | } 62 | 63 | func newTracing(viper *viper.Viper, serviceName string) opentracing.Tracer { 64 | var ter opentracing.Tracer 65 | if viper.IsSet("opentracing") { 66 | cfg := &config.Configuration{} 67 | viper.Sub("opentracing").Unmarshal(cfg) 68 | ter = tracer.NewTracer(cfg, serviceName) 69 | if _, ok := opentracing.GlobalTracer().(opentracing.NoopTracer); ok { 70 | opentracing.InitGlobalTracer(ter) 71 | } else if strings.Contains(cfg.ServiceName, "gateway") { 72 | //set gateway's ter to global ter 73 | opentracing.InitGlobalTracer(ter) 74 | } 75 | } 76 | return ter 77 | } 78 | 79 | func newLogger(viper *viper.Viper) *logger.Logger { 80 | fl := logger.NewFileLogger(viper.GetStringMap("log.file")) 81 | if viper.GetBool("debug") { 82 | return logger.NewLogger(fl, logger.NewStdLogger()) 83 | } else { 84 | return logger.NewLogger(fl) 85 | } 86 | } 87 | 88 | func tryAppendAuthInterceptor(viper *viper.Viper, opts []grpcx.Option) []grpcx.Option { 89 | if viper.GetBool("jwt.enable") { 90 | if viper.IsSet("jwt.public-key") { 91 | if err := qconfig.ResetFromSource(viper, "jwt.public-key"); err != nil { 92 | panic(err) 93 | } 94 | } 95 | opts = append(opts, grpcx.WithAuthFunc(authfg.ServerJwtAuthFunc(viper.GetStringMap("jwt")))) 96 | } 97 | return opts 98 | } 99 | 100 | func tryAppendKeepAlive(viper *viper.Viper, opts []grpcx.Option) []grpcx.Option { 101 | if viper.IsSet("keepalive") { 102 | ksp := keepalive.ServerParameters{ 103 | Time: viper.GetDuration("keepalive") * time.Second, 104 | } 105 | return append(opts, grpcx.WithGrpcOption(grpc.KeepaliveParams(ksp))) 106 | } 107 | return opts 108 | } 109 | 110 | func tryAppendMetrics(viper *viper.Viper, opts []grpcx.Option) []grpcx.Option { 111 | if viper.GetBool("metrics.enable") { 112 | return append(opts, grpcx.WithPrometheus(viper.GetString("metrics.listen"))) 113 | } 114 | return opts 115 | } 116 | 117 | // mgo logger adapter 118 | type mgoLogger struct { 119 | *logger.Logger 120 | } 121 | 122 | func NewMgoLogger(log *logger.Logger) *mgoLogger { 123 | return &mgoLogger{ 124 | Logger: log, 125 | } 126 | } 127 | 128 | func (t mgoLogger) Output(calldepth int, s string) error { 129 | t.Logger.Sugared().Info(s) 130 | return nil 131 | } 132 | 133 | func newBatchCache(viper *viper.Viper) (caches map[string]cache.Cache, err error) { 134 | if viper.IsSet("cache") { 135 | return nil, nil 136 | } 137 | batchCnf := viper.GetStringMap("cache") 138 | caches = make(map[string]cache.Cache) 139 | for key, value := range batchCnf { 140 | cnf := value.(map[string]interface{}) 141 | if ins, err := cache.NewCache(cnf["type"].(string), cnf); err != nil { 142 | return nil, err 143 | } else { 144 | caches[key] = ins 145 | } 146 | } 147 | if len(caches) == 0 { 148 | return nil, errors.New("initial cache failure,please check the config") 149 | } 150 | return 151 | } 152 | -------------------------------------------------------------------------------- /cmd/srv/gateway.go: -------------------------------------------------------------------------------- 1 | package srv 2 | 3 | import ( 4 | "github.com/gin-contrib/gzip" 5 | "github.com/gin-gonic/gin" 6 | "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" 7 | "github.com/opentracing/opentracing-go" 8 | "github.com/qeelyn/gin-contrib/errorhandle" 9 | ginTracing "github.com/qeelyn/gin-contrib/tracing" 10 | "github.com/qeelyn/go-common/config" 11 | "github.com/qeelyn/go-common/config/options" 12 | "github.com/qeelyn/go-common/grpcx/registry" 13 | "github.com/qeelyn/go-common/logger" 14 | "github.com/qeelyn/golang-starter-kit/gateway/app" 15 | "github.com/qeelyn/golang-starter-kit/gateway/router" 16 | "github.com/qeelyn/golang-starter-kit/schemas/greeter" 17 | "net/http" 18 | "time" 19 | ) 20 | 21 | func RunGateway(cnfOpts options.Options, register registry.Registry) error { 22 | var ( 23 | err error 24 | tracer opentracing.Tracer 25 | ) 26 | cnfOpts.FileName = "gateway.yaml" 27 | // load application configurations 28 | if app.Config, err = config.LoadConfig(&cnfOpts); err != nil { 29 | return err 30 | } 31 | 32 | appName, listen := app.Config.GetString("appname"), app.Config.GetString("listen") 33 | app.IsDebug = app.Config.GetBool("debug") 34 | // create the logger 35 | app.Logger = newLogger(app.Config) 36 | defer app.Logger.Strict().Sync() 37 | //use grpc log for rpc client 38 | grpc_zap.ReplaceGrpcLogger(app.Logger.Strict()) 39 | 40 | if app.Caches, err = newBatchCache(app.Config); err != nil { 41 | panic(err) 42 | } 43 | //tracing 44 | tracer = newTracing(app.Config, appName) 45 | 46 | app.TracerFunc = ginTracing.HandleFunc(map[string]interface{}{"useOpentracing": tracer != nil}) 47 | 48 | //rpc client 49 | cc := newDialer(true, app.Config.Sub("rpc.greeter"), tracer) 50 | app.GreeterClient = greeter.NewGreeterClient(cc) 51 | defer cc.Close() 52 | 53 | router := routers.DefaultRouter() 54 | initRouter(router) 55 | 56 | server := &http.Server{ 57 | Addr: listen, 58 | Handler: router, 59 | } 60 | 61 | return server.ListenAndServe() 62 | } 63 | 64 | func initRouter(g *gin.Engine) { 65 | g.Use(app.TracerFunc) 66 | if glevel := app.Config.GetInt("gzip"); glevel != 0 { 67 | g.Use(gzip.Gzip(glevel)) 68 | } 69 | if app.Config.IsSet("log.access") { 70 | c := logger.NewFileLogger(app.Config.GetStringMap("log.access")) 71 | accessLogger := logger.NewLogger(c) 72 | g.Use(app.AccessLogHandleFunc(accessLogger.Strict(), time.RFC3339, false)) 73 | } 74 | // load error messages 75 | ef := app.Config.GetString("error-template") 76 | if ef != "" { 77 | g.Use(errorhandle.ErrorHandle(map[string]interface{}{ 78 | "error-template": ef, 79 | }, app.Logger)) 80 | } 81 | 82 | if app.Config.GetBool("jwt.enable") { 83 | pubKeyKey := "jwt.public-key" 84 | if app.Config.IsSet(pubKeyKey) { 85 | if err := config.ResetFromSource(app.Config, pubKeyKey); err != nil { 86 | panic(err) 87 | } 88 | } 89 | authConfig := app.Config.GetStringMap("jwt") 90 | //init middleware 91 | app.AuthHanlerFunc = app.NewAuthMiddleware(authConfig).Handle() 92 | } 93 | // check access 94 | if app.Config.IsSet("auth") { 95 | app.CheckAccessMiddleware = app.NewCheckAccessMiddleware(app.Config.GetStringMap("auth")) 96 | } 97 | 98 | routers.SetupRouterGroup(g) 99 | routers.SetGraphQlRouterGroup(g) 100 | } 101 | -------------------------------------------------------------------------------- /cmd/srv/greeter.go: -------------------------------------------------------------------------------- 1 | package srv 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" 7 | "github.com/qeelyn/go-common/config/options" 8 | "github.com/qeelyn/go-common/logger" 9 | "github.com/spf13/viper" 10 | "go.uber.org/zap/zapcore" 11 | 12 | "github.com/jinzhu/gorm" 13 | _ "github.com/jinzhu/gorm/dialects/postgres" 14 | "github.com/opentracing/opentracing-go" 15 | "github.com/qeelyn/go-common/config" 16 | "github.com/qeelyn/go-common/gormx" 17 | "github.com/qeelyn/go-common/grpcx" 18 | "github.com/qeelyn/go-common/grpcx/registry" 19 | "github.com/qeelyn/golang-starter-kit/schemas/greeter" 20 | "github.com/qeelyn/golang-starter-kit/services/greetersrv" 21 | ) 22 | 23 | const greeterSrvName = "srv-greeter" 24 | 25 | func RunGreeter(cnfOpts options.Options, register registry.Registry) error { 26 | var ( 27 | err error 28 | cnf *viper.Viper 29 | tracer opentracing.Tracer 30 | db *gorm.DB 31 | dlog *logger.Logger 32 | ) 33 | cnfOpts.FileName = "greeter.yaml" 34 | 35 | if cnf, err = config.LoadConfig(&cnfOpts); err != nil { 36 | panic(fmt.Errorf("Invalid application configuration: %s", err)) 37 | } 38 | 39 | appName, listen, isDebug := cnf.GetString("appname"), cnf.GetString("listen"), cnf.GetBool("debug") 40 | // create the logger 41 | dlog = newLogger(cnf) 42 | defer dlog.Strict().Sync() 43 | 44 | dlog.ToZapField = func(values []interface{}) []zapcore.Field { 45 | return gormx.CreateGormLog(values).ToZapFields() 46 | } 47 | //db 48 | if cnf.IsSet("db") { 49 | db, err = gormx.NewDb(cnf.GetStringMap("db.default")) 50 | if err != nil { 51 | panic(err) 52 | } 53 | db.LogMode(isDebug) 54 | defer db.Close() 55 | 56 | if !isDebug { 57 | db.SetLogger(dlog) 58 | } 59 | } 60 | //opentracing 61 | tracer = newTracing(cnf, appName) 62 | 63 | service := greetersrv.NewGreeterService() 64 | service.Db = db 65 | 66 | var opts = []grpcx.Option{ 67 | grpcx.WithTracer(tracer), 68 | grpcx.WithLogger(dlog.Strict()), 69 | grpcx.WithRegistry(register, greeterSrvName, cnf.GetString("registryListen")), 70 | } 71 | // Payload log the request and response,it usually use in debug 72 | if cnf.IsSet("log.access") { 73 | c := logger.NewFileLogger(cnf.GetStringMap("log.access")) 74 | accessLogger := logger.NewLogger(c) 75 | opts = append(opts, grpcx.WithUnaryServerInterceptor(grpc_zap.PayloadUnaryServerInterceptor(accessLogger.Strict(), 76 | func(ctx context.Context, fullMethodName string, servingObject interface{}) bool { 77 | return true 78 | }))) 79 | } 80 | 81 | opts = tryAppendMetrics(cnf, opts) 82 | opts = tryAppendKeepAlive(cnf, opts) 83 | opts = tryAppendAuthInterceptor(cnf, opts) 84 | server, err := grpcx.Micro(appName, opts...) 85 | 86 | if err != nil { 87 | panic(fmt.Errorf("%s server start error:%s", greeterSrvName, err)) 88 | } 89 | 90 | rpc := server.BuildGrpcServer() 91 | greeter.RegisterGreeterServer(rpc, service) 92 | if err = server.Run(rpc, listen); err != nil { 93 | return fmt.Errorf("%s server run error:%s", greeterSrvName, err) 94 | } 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /docs/application-auth.md: -------------------------------------------------------------------------------- 1 | # 认证与授权 2 | 3 | ## 认证 4 | 5 | 本starter kit采用Json Web Token来支持常见系统的认证及授权.涉及到认证服务系统在此不多解释. 6 | 7 | 未来有机会也提供认证服务系统 8 | 9 | 经由用户认证服务获取的JWT在系统中通过各种方式传递. 10 | * gateway 11 | 12 | gateway接受http请求,接收Http Header的Authorization头信息传递.在Log Middleware中进行将该值记录到上下文向后传递. 13 | ``` 14 | func AccessLogHandleFunc(logger *zap.Logger, timeFormat string, utc bool) gin.HandlerFunc { 15 | if orgId := c.GetHeader("Qeelyn-Org-Id"); orgId != "" { 16 | c.Set("orgid", orgId) 17 | } 18 | // pass to context 19 | if authHeader := c.GetHeader("Authorization"); authHeader != "" { 20 | c.Set("authorization", authHeader) 21 | } 22 | c.Next() 23 | } 24 | ``` 25 | * Gateway向GRPC服务传递 26 | 27 | 利用grpc client加载Auth拦截器以获取上下文中的Header信息 28 | ``` 29 | cc, err := dialer.Dial(viper.GetString("name"), 30 | dialer.WithUnaryClientInterceptor( 31 | authfg.WithAuthClient(isGateway), 32 | ), 33 | ) 34 | ``` 35 | * GRPC服务接收 36 | 37 | 利用grpc_auth中间件接收MD对象中的Header信息 38 | ``` 39 | var opts = []grpcx.Option{} 40 | opts = append(opts, grpcx.WithAuthFunc( 41 | authfg.ServerJwtAuthFunc(viper.GetStringMap("jwt")) 42 | ) 43 | ) 44 | ``` 45 | ## 授权 46 | 47 | * mvc 48 | * graphql 49 | 50 | 由于GraphQl不像传统的MVC那样具有规则,经常做法就是独立设置授权编码,并检验. 51 | 可见gateway的CheckAccess组件实现. 52 | (待细化) -------------------------------------------------------------------------------- /docs/application-cache.md: -------------------------------------------------------------------------------- 1 | 缓存 2 | =========== 3 | 通过对比现在go的一些应用缓存组件,比如beego的cache组件,让人意外的是这些组件的能力太弱,如Get返回Interface,支持的数据类型有限.只好自行实现. 4 | 提供如下功能. 5 | * 提取自动根据入参识别并转换类型. 6 | * 全类型的支持 保存至缓存. 7 | * 采用msgpack进行序列化. 8 | * 内置支持local,redis,memcache 9 | * 多缓存组件并存,支持组件命名 10 | 11 | 配置 12 | -------- 13 | ``` 14 | cache: 15 | default: 16 | type: memcache 17 | addr: :11211 18 | auth: 19 | type: redis 20 | addr: :6379 21 | password: 22 | db: 3 23 | connectionTimeout: 3 24 | local: 25 | type: local 26 | duration: 10 #默认时间 分钟 27 | gc: 30 # 分钟 28 | ``` 29 | 30 | 使用 31 | -------- 32 | ```go 33 | app.Cache.Get //使用默认default设置 34 | app.Caches["auth"].Get 使用auth设置 35 | ``` 36 | -------------------------------------------------------------------------------- /docs/application-db.md: -------------------------------------------------------------------------------- 1 | 配合数据库工作 2 | ================= 3 | 本套件的数据库服务组件采用的是GROM,我们即可以使用ORM方式,也可以使用RAWSQL方式. 4 | 实际在业务开发中,大部分使用的是ORM方式.请参考[gorm](http://gorm.io/docs/) 5 | 6 | 数据库访问对象 7 | --------------- 8 | GORM默认支持的数据库如下: 9 | * mysql/mariadb 10 | * progreSQL 11 | 12 | 只列出实践过的,其他的Github DAO层应该都有. 13 | 14 | ### 配置 15 | 支持多数据库访问支持,具体的数据库配置请参考GORM 16 | ``` 17 | db: 18 | default: 19 | dsn: root:@tcp(localhost:3306)/test 20 | maxidleconns: 10 21 | maxopenconns: 100 22 | connmaxlifetime: 7200 23 | test: 24 | dsn: root:@tcp(localhost:3306)/test2 25 | ``` 26 | 27 | ### 使用 28 | 29 | * 使用默认DAO 30 | ``` 31 | db := app.db 32 | ``` 33 | * 使用指定的DAO 34 | ``` 35 | db := app.GetORMByName("test") 36 | ``` 37 | ### 配合protobuf使用 38 | 默认生成的protobuf文件是不包括GORM的规则的.一般手动在pd.go同级目录加入.gorm.go文件,如以下文件目录 39 | ``` 40 | schemas 41 | - meta 42 | - calendar.gorm.go 43 | - calendar.pb.go 44 | - calendar.proto 45 | ``` 46 | calendar.gorm.go的内容 47 | ```go 48 | package meta 49 | 50 | func (Calendar) TableName() string { 51 | return "meta_date" 52 | } 53 | ``` 54 | 根据grom的需要,定义相关的处理方法 55 | 56 | ### 注意事项 57 | * 链接池设置,需要根据实际环境设置,如并发要求,线上数据库支持 58 | * 连接的生命周期,注意应该设置比数据库的小,否则会出现部分请求失败 59 | 60 | [下一节 Graphql](graphql.md) -------------------------------------------------------------------------------- /docs/application-log.md: -------------------------------------------------------------------------------- 1 | # 日志 2 | 3 | 本套件的日志以uber.zap为核心,具有一定的定制性及可扩展性,可以很容易的记录各种类型的消息并分类处理,并将他们收集至特定的存储位置. 4 | 5 | 默认提供access.log及app.log,来记录访问日志与应用日志. 6 | 7 | ## 日志消息 8 | 9 | 跟系统日志一样,提供以下方法, 10 | - Strict 该方法提供类型化日志 11 | - Sugared 简单方式,与原生log方法原型最接近. 12 | - WithContext 会把上下文相关信息增加到日志字段中. 13 | 14 | 配置 15 | ``` 16 | log: 17 | file: 18 | filename: "runtime/app.log" 19 | maxsize: 500 20 | maxbackups: 3 21 | maxage: 28 22 | level: -1 23 | access: 24 | filename: "runtime/access.log" 25 | maxsize: 200 26 | maxbackups: 3 27 | maxage: 28 28 | level: 0 29 | ``` 30 | 目前的日志主要还是采用文件日志,后续再考虑结合其他方式,如Logstash 31 | 32 | 在api的访问日志中采用了zap记录,可根据需求自行更改显示的格式 33 | ``` 34 | logger.Info(path, 35 | zap.Int("status", c.Writer.Status()), 36 | zap.String("method", c.Request.Method), 37 | zap.String("path", path), 38 | zap.String("query", query), 39 | zap.ByteString("body", bodyCopy.Bytes()), 40 | zap.String("ip", c.ClientIP()), 41 | zap.String("auth", c.GetString("userid")), 42 | zap.String("user-agent", c.Request.UserAgent()), 43 | zap.String("time", end.Format(timeFormat)), 44 | zap.Duration("latency", latency), 45 | ) 46 | ``` 47 | ## 使用方式 48 | 49 | gateway: 50 | ``` 51 | app.Config.Strict().Error(...) //zap方式 52 | app.Config.WithContext().Error(....) //附带上下文的zap方式 53 | app.Config.Sugared().Error(....) // 54 | ``` 55 | micro: 56 | 57 | 在微服务下,采用拦截器方式提供上下文信息的记录支持,大部分情况你应该使用如下方式进行记录 58 | ``` 59 | ctxzap.Extract(ctx).Error("query holding error:" + err.Error()) 60 | ``` 61 | > 取消了之前在微服务构建增加Log组件的方法.统一使用ctxzap 62 | 63 | 相关专题: [日志采集](subject-log.md) 64 | 65 | [下一节 Service](service-layer.md) -------------------------------------------------------------------------------- /docs/application-opentracing.md: -------------------------------------------------------------------------------- 1 | OpenTracing 与 Jaeger 2 | ====================== 3 | 4 | 可通过[OpenTracing](./subject-opentracing.md)了解基本的知识 5 | 6 | 配置 7 | ----------- 8 | ``` 9 | opentracing: 10 | serviceName:"" // 该配置不需要的,自动采用appname 11 | sampler: 12 | type: const 13 | param: 1 14 | reporter: 15 | logSpans: false 16 | localAgentHostPort: "127.0.0.1:6831" 17 | ``` 18 | 主要配置组件为 19 | * sampler: 取样 20 | * reporter: 提交 21 | 22 | 详细文档https://github.com/jaegertracing/jaeger-client-go 23 | 24 | 只要提供配置后,就可以在jaeger服务端查看到相应的查询信息 25 | 26 | 部署 27 | ------------ 28 | 测试环境可直接用 jaegertracing/all-in-one:latest 进行部署 29 | ``` 30 | docker pull jaegertracing/all-in-one:latest 31 | ``` 32 | 33 | 实际生产中.agent需要独立部署,以处理应用产生的span数据 -------------------------------------------------------------------------------- /docs/application.md: -------------------------------------------------------------------------------- 1 | 应用结构 2 | ============== 3 | 4 | 应用 5 | -------- 6 | 应用可定义为单一服务程序的管理主体,每个服务程序只能包含一个应用主体,应用主体是通过app包进行访问. 7 | 8 | 应用配置以文件形式展现,并支持远程配置. 9 | 10 | 为了便于在开发环境的个性化,支持本地配置文件(以-local为后缀的形式),并避免敏感信息的泄露. 11 | 12 | ### 应用主体配置 13 | 如下所示,应用主体的配置目前还比较简单 14 | ``` 15 | // 应用程序名 16 | appname: "myApp" 17 | // 监听地址 host:port 18 | listen: ":9097" 19 | // 应用模式,生产或测试环境切换 20 | appmode: debug 21 | ``` 22 | 组件级别在应用配置的一级节点展开 23 | 24 | 应用主体是个很灵活的,应该而由项目管理者自行决定引入的组件,这边是有一些代码成本,但实际上带来的自由度让人感觉更好. 25 | 26 | ### 应用组件 27 | 28 | 以下列出一些顶层组件, 29 | 30 | * [缓存](application-cache.md) 31 | * [数据库](application-db.md) 32 | * [认证与授权](application-auth.md) 33 | * [OpenTracing](application-opentracing.md) 分布式跟踪 34 | * [Metrics](application-metrics.md) 系统运行指标监控 35 | 36 | ### 本地配置 37 | 38 | 默认采用./config路径为应用配置路径,应用对应的配置文件已经在程序中指定完成. 39 | 当应用的配置文件为`gateway.yaml`时,将默认加载后缀为`-local`的配置文件`gateway-local.yaml` 40 | 41 | 需要注意当通过`-c`参数变更路径时,将与配置中心进行组合获取配置文件 42 | 43 | ### 远程配置 44 | Kit默认以本地文件加载配置文件,如果启用远程配置时,注册中心与配置中心依赖服务是一致的,这时将同时启动服务注册与发现. 45 | 46 | * 代码 47 | ``` 48 | cnfOpts := config.ParseOptions(configOptions...) 49 | //采用etcd 客户端v3版本 50 | etcdv3.Build(cnfOpts) 51 | 52 | cnfOpts.FileName = "gateway.yaml" 53 | if app.Config, err = config.LoadConfig(&cnfOpts); err != nil { 54 | return err 55 | } 56 | ``` 57 | * 启动参数 58 | ``` 59 | {cmd} -c {配置路径} -n {etcd connection string} 60 | 配置路径为uri : golang-start-kit/config 格式的基路径, 61 | etcd connection string: 需要用host:port?key=value形式,如127.0.0.1:2379?username=qeelyn 62 | 例如启动gateway时,最终对应到etcd的配置路径为 : 127.0.0.1:2379/golang-start-kit/config/gateway.yaml, 63 | 所以往配置中心发布时,需要注意key取值 64 | ``` 65 | > etcd的连接串是根据etcd config的配置,当启用TLS时,需要定制初始化 66 | * 发布配置 67 | 68 | 命令行方式可以用如下命令,自己可以结合一些开源的etcd管理工具或者自动开发,实现配置的管理 69 | ``` 70 | cat cmd/config/gateway.yaml | etcdctl put golang-start-kit/config/gateway.yaml 71 | ``` 72 | 73 | [下一节 模型定义-protobuf](use-protobuf.md) -------------------------------------------------------------------------------- /docs/data/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.inputs: 2 | - type: log 3 | paths: 4 | - /var/log/system.log 5 | - /var/log/wifi.log 6 | 7 | output.elasticsearch: 8 | hosts: ["myEShost:9200"] -------------------------------------------------------------------------------- /docs/error-handle.md: -------------------------------------------------------------------------------- 1 | 异常处理 2 | ========== 3 | 针对在响应过程,针对异常的响应是必不可少的环节. 4 | golang自身的错误处理方式相对于其他语言可以说比较独特.异常做为方法的返回值处理. 5 | 6 | Error只提供了如何转化为String方法,转化为文件输出.完全没有其他语言的序列化问题.这种单一性对于错误的提醒不够完备,因此需要自定义错误处理. 7 | 8 | 错误消息 9 | ----------- 10 | 11 | 套件中对错误消息的定义在config/errors.{语言}.yaml文件中.格式如下 12 | ```yaml 13 | INTERNAL_SERVER_ERROR: 14 | code: 500 15 | message: "内部服务器错误!" 16 | debug: "Internal server error: {error}" 17 | 18 | PERMISSION_DENIED: 19 | code: 403 20 | message: "无请求权限!" 21 | debug: "无请求权限: {error}" 22 | ``` 23 | INTERNAL_SERVER_ERROR 对应error的消息文本,通过消息文本的全等匹配来定位code与message,输出符合code,message这样的输出. 24 | 25 | 统一异常 26 | ---------- 27 | 28 | ### gin框架 29 | 30 | 以gin为路由的错误处理是通过套件提供的ErrorHandle中间件实现的.异常的注入方式 31 | ```go 32 | gin.Context.Error(error) 33 | ``` 34 | 最终ErrorHandle将error转化为code,message响应输出 35 | > 涉及到gin的作用范围 36 | 37 | ### RPC 38 | 39 | 延用GRPC框架的异常处理. 40 | 41 | ### 异常跟踪 42 | 43 | 如果采用了OperatingTracer中间件时,会在Context中保存TracerID,来做为整个请求链的全局ID 44 | ``` 45 | // gin 46 | g *gin.Engine 47 | g.Use(app.NewJeagerTracer()) 48 | 49 | // grpc client 50 | cc, err := dialer.Dial(serviceName, 51 | dialer.WithTracer(tracer), 52 | ) 53 | 54 | // grpc server 初始化option时加入 55 | var opts = []grpcx.Option{ 56 | grpcx.WithTracer(tracer), 57 | } 58 | server, err := grpcx.Micro(appName, opts...) 59 | ``` 60 | 访问日志及GRPC日志将会产生相应的key记录: "trace.traceid":"1fa3ff926212922" 61 | 同时tracerid可用于JeagerUI查询对应的operationtracer记录. 62 | 63 | [下一节 日志](application-log.md) 64 | -------------------------------------------------------------------------------- /docs/graphql.md: -------------------------------------------------------------------------------- 1 | Graphql 2 | ============== 3 | 除了采用传统的WEB API方式外,还可以采用graphql的方式提供,实际上,这种方式已经越来越被大家接受,相比web api的优势明显. 4 | * 字段级别的解释,变更可控性强.减少很多烦人的版本控制 5 | * 统一的编程模型,方便跟踪与性能提升,比api这样的黑洞更友好. 6 | * 基于模型的定义访问,能应对各种需求变化,原型上,基于同一套模型的前端设计,接口是不需要更改的. 7 | 8 | [graphql-go](github.com/graph-gophers/graphql-go)的参考资料较少,本质上需要对js版本的graphql有一定的了解,再参考示例代码,学习一下基础知识. 9 | 10 | graphql属于Web层,与gin是相配置的. 11 | 12 | 模型定义 13 | ----------- 14 | 采用JS同样的定义支持,采用golang的资源加载方式,目录结构如下 15 | ``` 16 | api 17 | - schema //在该文件夹存放graphql定义相关文件 18 | - type //类型文件夹 在该文件夹定义类型 19 | - schema.graphql 20 | - schema.go //资源加载,graphql只需要将内容进行拼接 21 | ``` 22 | 23 | go代码文件还依赖go-bindata 24 | ``` 25 | go get -u github.com/jteeuwen/go-bindata/... 26 | ``` 27 | 28 | Resolver 29 | ------------- 30 | 对定义文件的解释器,为golang实现代码 31 | 32 | Tracer 33 | ------------- 34 | 由于graphql的灵活性,可能一个类型中,就会并发访问不同的remote service,这样在跟踪及性能分析上就存在一定的困难,需要借助一些分析工具. 35 | 目前采用 opentracing + jaeger 来提供跟踪服务 36 | 37 | 后续再来专门讲. -------------------------------------------------------------------------------- /docs/handle-request.md: -------------------------------------------------------------------------------- 1 | 处理请求 2 | ============== 3 | 4 | 运行机制 5 | -------------- 6 | 7 | 经过应用主体的路由机制后,采用的是洋葱模型,由一层层中间件来完成整个请求的处理,具体请参考Gin的处理方式, 8 | 9 | 路由 10 | -------------- 11 | 路由的初始化文件在router文件夹中.一般在本router.go文件中完成路由分组与中间件的集成 12 | 13 | 14 | 可直接参考Gin的方式来编写web服务,如果你钟情于MVC,就自定义controller文件夹.本套件建议就用GO的方式定义API,即直接定义Handle方法 15 | 16 | ``` 17 | v1 := g.Group("oauth2/v1") 18 | { 19 | v1.POST("token", tokenHandle) 20 | v1.POST("authorize", authorizeHandle) 21 | } 22 | 23 | authMd := app.BearerAuth(app.Config.GetStringMap("auth")) 24 | 25 | g.POST("login", api.Login) 26 | g.POST("logout", authMd, api.Logout) 27 | g.GET("userinfo", authMd, api.UserInfo) 28 | 29 | ``` 30 | 31 | 请求参数 32 | ------------- 33 | Gin已经提供了方便的处理方法,我们可以直接使用它 34 | ``` 35 | func Get(c *gin.Context){ 36 | c.Qeury("key") 37 | c.Param("key") 38 | } 39 | ``` 40 | ### 参数绑定 41 | ``` 42 | gin.Context.Bind(struct) 43 | ``` 44 | 45 | 响应 46 | ---------- 47 | 48 | 除了Gin提供的方法外.你完全可以定义一些helper方法来辅助格式化响应 49 | ``` 50 | gin.Json 51 | gin.Html 52 | 等等 53 | ``` 54 | 55 | ### gin的作用范围 56 | 57 | gin框架的作用范围在路由及handle方法,gin.context的显式调用就结束,此时上下文会转化为context.Context接口或其封装往下传递.如service层或者graphql服务中传递,不推荐再将context转化为gin.context. 58 | > 实际上将context转化为gin.context这也是很困难的,封装的层级是未知的. 59 | 60 | ### 请求ID 61 | 62 | 为了方便跟踪及日志,默认启用了traceid记录.支持opentracing与自定义ID,当启用opentracing时.则采用TraceID. 63 | 这时,你会在日志中看到trace.traceid的字段. 64 | 65 | [下一节 异常处理](error-handle.md) -------------------------------------------------------------------------------- /docs/img/goland-build.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qeelyn/golang-starter-kit/f2353e5be646341fe2518ec0feab0da26965a35c/docs/img/goland-build.jpg -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # 开发手册 2 | 3 | ## 项目目录 4 | ### 项目结构 5 | 6 | ``` 7 | - gateway api gateway接口项目,基于Web http server 8 | - app 应用程序域相关组件,包含配置,日志及启动有关的处理 9 | - config 配置文件目录 10 | - controllers 控制器文件夹 11 | - public 站点静态文件目录 12 | - routers 路由配置目录 13 | - schema graphql定义文件夹 14 | - resolver graphql golang解释器目录 15 | - loader graphql 的dataloader目录 16 | server.go 应用服务文件 17 | - cmd 程序运行及配置 18 | - config 19 | - gateway.yaml 应用配置文件 20 | - gateway-local.yaml 个人的环境配置文件 21 | - srv 服务目录,主要针对Server的初始化 22 | - gateway 23 | main.go 入口,具体由决定 24 | - schemas protobuf协议定义目录 25 | - services 逻辑服务代码文件夹 26 | - greeter.go RPC服务的Service文件,包括应用程序域组件 27 | ``` 28 | 配置 29 | --------- 30 | 31 | 在config目录中新建本地配置文件,有命名要求: app-local.yaml 32 | 33 | 默认所有配置应该集中于app.yaml与app-local.yaml中 34 | 35 | * 应用配置 36 | ``` 37 | appname: myApp 38 | listen: ":9097" 39 | appmode: debug 40 | web: 41 | staticdir: public 42 | 43 | ``` 44 | 45 | * 日志配置 46 | ``` 47 | log: 48 | file: 49 | filename: runtime/app.log" 50 | maxsize: 500 51 | maxbackups: 3 52 | maxage: 28 53 | level: -1 54 | access: 55 | filename: runtime/access.log" 56 | maxsize: 200 57 | maxbackups: 3 58 | maxage: 28 59 | level: 0 60 | 61 | ``` 62 | * DB配置 - 具体数据库类型请查看grom的配置 63 | ``` 64 | db: 65 | default: 66 | dialect: mysql 67 | dsn: root:@tcp(localhost:3306)/yak 68 | ``` 69 | 70 | 基于数据库的应用在配置完后,就可以进行服务响应测试了.更多的配置内容可查看[应用程序配置](./application.md) 71 | 72 | [下一节 应用结构](./application.md) -------------------------------------------------------------------------------- /docs/service-layer.md: -------------------------------------------------------------------------------- 1 | Service 2 | ============== 3 | 为了兼容单体服务与微服务架构,业务逻辑采用Repository模式,但为直接提供Service. 4 | 5 | 可以认为由protobuf生成的文件包含了IRepostiory定义,Service只是对这些契约的实现. 6 | 7 | Service开发时,应该注意,这是可扩展为独立服务程序的,在组件应用时应该特别注意,如不要引用WebSite的包.特别是Gin的作用范围内的包. 8 | 在注意这边细节后,开发一个service主要就是纯业务逻辑的实现 9 | 10 | 为了做为可分离共用的服务层,应用程序组件App是独立的,具有独立的初始化步骤. 11 | ```go 12 | type HelloService struct{ 13 | //可以定义一些依赖组件,目前的组件直接使用App包,看个人喜欢进行调整 14 | } 15 | 16 | func NewHelloService(){ 17 | return &HelloService{} 18 | } 19 | 20 | func World(ctx context.Context,req *WorldRequest) (*WorldResponse,error){ 21 | db = app.Db 22 | // ...业务逻辑实现 23 | return &WorldResponse{Data:"hello world"},nil 24 | } 25 | ``` 26 | 调用 27 | -------------- 28 | 如果是单体项目,可在handle中这样定义,直接引入service层.如果需要单例管理,可以再多定义如ClientManager这样的管理类来维持. 29 | 而做为微服务提供的服务层是个独立的应用程序,采用GRPC服务. 30 | 31 | ### 调用 - 单体程序 32 | ```go 33 | type HelloController struct { 34 | client *hello.HelloService 35 | } 36 | 37 | func ServeHelloResource(group *gin.RouterGroup) { 38 | helloCtr := HelloController{ 39 | client: NewHelloService(), 40 | } 41 | group.GET("/hello", helloCtr.world) 42 | } 43 | func (t *HelloController)world(c *gin.Context){ 44 | //todo 45 | } 46 | ``` 47 | 48 | GRPC 49 | --------- 50 | 做为微服务支持,将Service变成服务代码时,如果遵循Service的开发约定,只需要增加服务应用初始化即时 51 | 52 | 请直接转[微服务专题](./subject-micro.md) 53 | 54 | [下一节 配合数据库工作](application-db.md) -------------------------------------------------------------------------------- /docs/subject-jenkins.md: -------------------------------------------------------------------------------- 1 | # 基于jenkins的持续集成 2 | 3 | ## 写在前面 4 | 5 | 本文内容把个性化的环境隐藏或替换,如xxx等.请按照自己情况调整 6 | 7 | * docker源,选国内,如果是阿里云,请进入[加速器](https://cr.console.aliyun.com/#/accelerator),注意,本文写时,阿里上的配置是错误的,请下看 8 | * docker的配置文件 9 | 10 | 配置文件位于/etc/docker/daemon.json 没有的话自己建一个. 11 | ``` 12 | { 13 | "registry-mirrors": ["http://harbor.test.com"], #镜像加速地址 14 | "insecure-registries": ["xx.xx.xxx.231","registry.cn-shenzhen.aliyuncs.com"], # Docker如果需要从非SSL源管理镜像,这里加上。 15 | "max-concurrent-downloads": 10 16 | } 17 | ``` 18 | >对于上传方也需要设置insecure-registries才可push至私有仓库,不过jenkins内置docker插件不用 19 | ## 安装 20 | 21 | 为了适用于各种开发环境,我们需要安装的以基础镜像打包的.注意不要采用基于alpine的,库支持不足,会遇到不少问题. 22 | ``` 23 | docker run \ 24 | -u root \ 25 | -d \ 26 | -p 58080:8080 \ 27 | -p 50000:50000 \ 28 | -v jenkins-data:/var/jenkins_home \ 29 | -v /usr/bin/docker:/usr/bin/docker \ 30 | -v /var/run/docker.sock:/var/run/docker.sock \ 31 | --restart=always \ 32 | jenkinsci/jenkins 33 | ``` 34 | ### docker out docker 35 | 36 | 官方的安装方式只是将docker.sock映射内容器,但如果需要docker的话,会出现`docker not found`.因此你看到在run 命令中加入docker的映射. 37 | 但还是会出现一个libltdl.so.7文件不存在,这个可以进容器,执行apt-get安装 38 | ``` 39 | apt-get update && apt-get install -y libltdl7 40 | ``` 41 | > 可以自定义Jenkins的Dockfile,将so安装好. 42 | 43 | ### SSH KEY 44 | 45 | 进入容器,创建SSH KEY,这个KEY与GitLab会用到 46 | ``` 47 | ssh-keygen -t rsa -C "jenkins" 48 | # 一路回车, 默认路径和文件名, 不要密码 49 | cd ~/.ssh 50 | //私钥和密钥可另保存起来, 51 | mv id_rsa id_rsa_jk 52 | mv id_rsa.pub id_rsa_jk.pub 53 | ``` 54 | 55 | 在gitlab的项目下, 点击右侧配置菜单 -> Deploy Keys, 用刚才创建的 id_rsa_tho.pub 的内容, 创建一个key, 名称为 Readonly Key for Jenkins, 如果有多个项目都需要这个私钥, 则在每个项目的deploy keys下enable这个key即可. 可以用下面的方式验证是否生效 56 | 57 | 检验: 在Jenkins中, 新建一个freestyle的项目, 点击项目 -> Source Code Management, 选择 Git, 填入gitlab中给的项目地址 git@192.168.1.109:cc/tho.git 在下面add new credential, Username:git, Private Key Enter Directly, 输入刚才创建的 id_rsa_tho 的内容, 注意这个是私钥. 58 | 59 | 如果切换鼠标焦点后, 项目地址栏下方没有错误提示, 就说明Jenkins检查没错 60 | 61 | 62 | ## gitlab 63 | 64 | 重启 65 | ``` 66 | cd /data/gitlab-8.9.6 67 | ./ctlscript.sh restart 68 | ``` 69 | 70 | ## registry 71 | 72 | registry做为Docker镜像仓库,目前阿里云的容器镜像是免费的,可以优先使用. 73 | 74 | [Docker官方文档](https://docs.docker.com/registry/deploying/#support-for-lets-encrypt) 75 | 76 | registry官方的简易安装是不包含安全性的,我们直接采用auth方式 77 | 78 | ### auth 79 | ``` 80 | docker run \ 81 | --entrypoint htpasswd \ 82 | registry:2 -Bbn jenkins {pwd} > /opt/data/registry/auth/htpasswd 83 | 84 | docker run -d \ 85 | -p 5000:5000 \ 86 | --restart=always \ 87 | --name registry \ 88 | -v /nas/registry:/var/lib/registry \ 89 | -v /opt/data/registry/auth:/auth \ 90 | -e "REGISTRY_AUTH=htpasswd" \ 91 | -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ 92 | -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ 93 | registry:2 94 | 95 | ``` 96 | 检查: 97 | * netstat -an | grep 5000 98 | * curl -x http://localhost:5000 99 | 100 | ### 客户端设置 101 | insecure registries: 配置上私有的HOST:PORT,见/etc/docker/deamon.json 102 | 103 | ### 测试 104 | 105 | 找个包,tag 需要按registry的格式: HOST:PORT/{包包} 106 | ``` 107 | //如果配置了auth验证 108 | docker login myregistrydomain.com:5000 109 | 110 | docker tag busybox xx.xx.xxx.231:5000/busybox 111 | docker push xx.xx.xxx.231:5000/busybox 112 | ``` 113 | 114 | ## 基于pipeline的golang项目接入 115 | 116 | ### 设置GO基本环境 117 | 118 | 现在go为1.10版本,之前dep很流行,但很不幸,它无法解决翻墙问题.目前vgo是最适合国内的版本工具,而且将是go的标准工具. 119 | 120 | 121 | * 默认的Jenkins镜像是不带有Go的编译工具的,安装GO Plugin插件.位置在/var/jenkins_home/tools下对应到插件位置. 122 | * 进入镜像docker exec -it {containerid} bash 123 | * ln -s /var/jenkins_home/tools/{}/bin/go /usr/bin/go 124 | * 设置GOPATH为{$workspace}/go,建好{bin,pkg,src}三个目录 125 | * 安装vgo: 126 | ``` 127 | // 自已将代码上传到GOPATH目录 128 | go get golang.org/golang/x/vgo 129 | ln -s /var/jenkins_home/workspace/go/bin/vgo /usr/bin/vgo 130 | ``` 131 | ### 项目要求 132 | 133 | 采用vgo来做版本管理,并将一些无法获取的项目做replace 134 | 135 | * 将包地址都改成国内码云的镜像地址,我使用githubmirror组同步github项目,但每天只能同像20个项目,珍惜吧. 136 | * 所有以golang.org为结束的包 137 | 138 | replace例子 139 | ``` 140 | replace( 141 | github.com/prometheus/client_golang => gitee.com/githubmirror/prometheus_client_golang v1 142 | ) 143 | ``` 144 | 145 | 146 | 该包由于依赖特殊,还依赖github的一些项目,放入GOPATH会提示找不到相关包. 147 | 也是需要override的. 148 | 149 | ### pipeline例子 150 | ``` 151 | node { 152 | // Install the desired Go version 153 | def root = tool name: '1.10', type: 'go' 154 | def BUILD_NAME = "serve" 155 | def IMAGE_NAME = "xxx.xx.xx.244:5000/fof-api" 156 | def APP_ROOT = "src/github.com/tsingsun/fof-api" 157 | // Export environment variables pointing to the directory where Go was installed 158 | withEnv(["GOROOT=${root}", "PATH+GO=${root}/bin","GOPATH=${WORKSPACE}/../go",]) { 159 | stage("prepare") { 160 | dir("src") { 161 | git branch: 'master',credentialsId: 'befbfc02-d877-4f6f-b3db-7ad4cb65119f', url: 'git@xx.xx.xxx.xx:advisorq/fof-api.git' 162 | sh "mkdir -p ${GOPATH}/src/github.com/tsingsun && rm -rf ${GOPATH}/${APP_ROOT} && ln -s ${WORKSPACE}/src ${GOPATH}/${APP_ROOT}" 163 | } 164 | } 165 | stage("build") { 166 | dir("src") { 167 | sh "vgo get && vgo mod vendor" 168 | sh "cp -rf gateway/public cmd/" 169 | } 170 | sh "cd ${GOPATH}/${APP_ROOT} && GOOS=linux GOARCH=amd64 go build -o cmd/${BUILD_NAME} cmd/main.go" 171 | } 172 | stage("deploy") { 173 | dir("src") { 174 | docker.withRegistry('http://172.16.61.244:5000','pushToRegistry') { 175 | def image = docker.build("${IMAGE_NAME}:${env.BUILD_ID}") 176 | image.push("latest") 177 | } 178 | } 179 | } 180 | } 181 | } 182 | ``` 183 | 184 | ## 部署 185 | 186 | 目前由于没有k8s与swarm环境,可采用dock pull的方式手工部署. 187 | 由于registry没有配套UI,但对于企业应用,它的restapi足够了. 188 | ``` 189 | //以下为安全验证下的请求 190 | //列出image repo 191 | http://jenkins:pwd@4xx.xx.xx.xxx:5000/v2/_catolog 192 | //列出image tggs 193 | http://jenkins:pwd@4xx.xx.xx.xxx:5000/v2/nginx/tags/list 194 | ``` 195 | 196 | ## 常见问题 197 | 198 | * 忘记Jenkins管理员密码的解决办法 199 | 按[此文可解](https://blog.csdn.net/jlminghui/article/details/54952148) 200 | * centos构建的无法在alpine运行.glibc库问题,但musl库是兼容的,可以用,在你的Dockfile加 201 | ``` 202 | RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 203 | ``` 204 | 据说CGO编译也可以,但我没试 205 | 206 | * jenkins出现非登陆用户,这是根据git历史自动生成问题,这与git的用户配置有关.需要针对项目进行配置,在项目目录下 207 | ``` 208 | git config user.name “gitlab’s Name” 209 | git config user.email “gitlab’s Name” 210 | ``` 211 | 212 | ## 待完善 213 | * docker build的镜像是在宿主机中,因为已经往registry发送了,保留有些多余,需要有删除机制. 214 | * jenkins没有回滚机制,部署不在jenkins这端做了,需要再找方案. 215 | -------------------------------------------------------------------------------- /docs/subject-log.md: -------------------------------------------------------------------------------- 1 | # 日志 2 | 3 | 这些我们先介绍elastic全家桶 4 | 5 | ## 采集 6 | 7 | ## filebeat 8 | 9 | ### docker使用 10 | 11 | * 获取镜像 12 | 13 | 使用elastic自家的镜像仓库 14 | ``` 15 | docker pull docker.elastic.co/beats/filebeat:6.4.2 16 | ``` 17 | 18 | * 运行 19 | ``` 20 | docker run \ 21 | --mount type=bind,source="$(pwd)"/filebeat.yml,target=/usr/share/filebeat/filebeat.yml \ 22 | docker.elastic.co/beats/filebeat:6.4.2 23 | ``` 24 | 25 | -------------------------------------------------------------------------------- /docs/subject-micro.md: -------------------------------------------------------------------------------- 1 | # 微服务-microservices 2 | 本开发套件提供了很简单的方式来支持微服务架构.具备以下能力 3 | 4 | * 服务发现与注册: 采用基于etcd的方式,支持`HOSTIP`环境变量,简化配置 5 | * 负载均衡: GRPC提供,默认采用round_robin 6 | * 消息格式: 目前为默认的GRPC的protobuf格式 7 | * 消息流: 内置的拦截器支持unary和stream方式 8 | * 跟踪服务: uber jeager 9 | * 日志收集: uber zap,计划采用ELK技术来完善日志收集流程 10 | * 监控与预警: prometheus + grafana 11 | 12 | 这些能力可通过grpcx.Option来初始化开启 13 | 14 | ## 服务构建 15 | 16 | 可通过`go-common/grpx`包提供的默认定义初始化.用户定制要求高的,可参考mirco方法实现 17 | ``` 18 | tracer := tracing.NewTracer(app.Config.Sub("opentracing"), appName) 19 | serverPayloadLoggingDecider := func(ctx context.Context, fullMethodName string, servingObject interface{}) bool { 20 | if fullMethodName == "healthcheck" { 21 | return false 22 | } 23 | return true 24 | } 25 | // option 初始化 26 | var opts = []grpcx.Option{ 27 | grpcx.WithTracer(tracer), 28 | grpcx.WithLogger(app.Logger.GetZap()), 29 | grpcx.WithUnaryServerInterceptor( 30 | grpc_zap.PayloadUnaryServerInterceptor( 31 | app.Logger.GetZap(), 32 | serverPayloadLoggingDecider)), 33 | grpcx.WithAuthFunc(grpcx.AuthFunc(config.GetString("auth.public-key"))), 34 | grpcx.WithPrometheus(config.GetString("metrics.listen")), 35 | grpcx.WithRegistry(register, fofSrvName), 36 | } 37 | //采用默认的微服务组件初始化 38 | server, err := grpcx.Micro(appName, opts...) 39 | 40 | if err != nil { 41 | panic(fmt.Errorf("fof server start error:%s", err)) 42 | } 43 | 44 | rpc := server.BuildGrpcServer() 45 | // grpc的服务注册 46 | fof.RegisterFofServiceServer(rpc, fofsrv.NewFofService()) 47 | // 启动 48 | if err = server.Run(rpc, listen); err != nil { 49 | return fmt.Errorf("Server run error:", err) 50 | } 51 | return nil 52 | ``` 53 | 54 | ### 注册配置 55 | 56 | 以qeelyn://author为基地址,相关参数如下 57 | ``` 58 | appname: srv-notice //注册中心路径为: qeelyn://author/srv-notice 59 | registryListen: ":8033" 服务地址 60 | ``` 61 | 62 | 如果设置了HOSTIP环境变理,配置文件为`:8033`格式的变根据环境变量修改成`${HOSTIP}:8033` 63 | 该支持主要为让不同机器保持相同的配置. 64 | 65 | ### go客户端 66 | 67 | 基于grpc的注册与发现可通过配置的方式 68 | * 服务名: 采用GRPC的naming方式,格式如: qeelyn://author/srv-pool 69 | * IP: 传统方式,如"127.0.0.1:8000" 70 | ``` 71 | func newDialer(serviceName string, tracer opentracing.Tracer) *grpc.ClientConn { 72 | cc, err := dialer.Dial(serviceName, 73 | dialer.WithDialOption( 74 | grpc.WithInsecure(), 75 | grpc.WithBalancerName("round_robin"), 76 | ), 77 | dialer.WithUnaryClientInterceptor( 78 | grpc_prometheus.UnaryClientInterceptor, 79 | dialer.WithAuth(), 80 | ), 81 | dialer.WithTracer(tracer), 82 | ) 83 | if err != nil { 84 | log.Panicf("dialer error: %v", err) 85 | } 86 | return cc 87 | } 88 | //服务名 89 | poolcc := newDialer(app.Config.GetString("rpc.pool"), tracer) 90 | //grpc client 91 | app.PoolClient = pool.NewPoolServiceClient(poolcc) 92 | 93 | ``` 94 | 95 | ### 复杂网络下的rpc通信 96 | 97 | * 心跳 98 | 采用gRPC长连接方式构建微服务通信,已经考虑到心跳设计,对于简单环境下,默认即可.但在复杂网络环境下,必须考虑无活动连接被防火墙释放问题.如 99 | 客户IDC网络与云环境的联通情况下,需要自定义心跳,特别需要采用双向心跳,一般只需要定义发送间隔. 100 | 101 | 客户端: 在rpc下定义keepalive 102 | 服务端: 在根配置下定义keepalive 103 | 104 | * 断线重连 105 | 由gRPC定义实现 106 | -------------------------------------------------------------------------------- /docs/subject-opentracing.md: -------------------------------------------------------------------------------- 1 | OpenTracing 2 | ====================== 3 | 4 | [开放分布式追踪(OpenTracing)入门与 Jaeger 实现](https://yq.aliyun.com/articles/514488) 5 | 6 | 分布式系统的运维挑战 7 | ------------------ 8 | 容器、Serverless 编程方式的诞生极大提升了软件交付与部署的效率。在架构的演化过程中,可以看到两个变化: 9 | 10 |  11 | * 应用架构开始从单体系统逐步转变为微服务,其中的业务逻辑随之而来就会变成微服务之间的调用与请求 12 | * 资源角度来看,传统服务器这个物理单位也逐渐淡化,变成了看不见摸不到的虚拟资源模式 13 | 14 | 从以上两个变化可以看到这种弹性、标准化的架构背后,原先运维与诊断的需求也变得越来越复杂。为了应对这种变化趋势,诞生一系列面向 DevOps 的诊断与分析系统,包括集中式日志系统(Logging),集中式度量系统(Metrics)和分布式追踪系统(Tracing) 15 | 16 | ### Logging,Metrics 和 Tracing 17 | 18 | Logging,Metrics 和 Tracing 有各自专注的部分。 19 | 20 | Logging - 用于记录离散的事件。例如,应用程序的调试信息或错误信息。它是我们诊断问题的依据。 21 | Metrics - 用于记录可聚合的数据。例如,队列的当前深度可被定义为一个度量值,在元素入队或出队时被更新;HTTP 请求个数可被定义为一个计数器,新请求到来时进行累加。 22 | Tracing - 用于记录请求范围内的信息。例如,一次远程方法调用的执行过程和耗时。它是我们排查系统性能问题的利器。 23 | 这三者也有相互重叠的部分,如下图所示: 24 |  25 | 26 | 通过上述信息,我们可以对已有系统进行分类。例如,Zipkin 专注于 tracing 领域;Prometheus 开始专注于 metrics,随着时间推移可能会集成更多的 tracing 功能,但不太可能深入 logging 领域; ELK,阿里云日志服务这样的系统开始专注于 logging 领域,但同时也不断地集成其他领域的特性到系统中来,正向上图中的圆心靠近。 27 | 28 | 关于三者关系的更详细信息可参考 [Metrics, tracing, and logging](http://peter.bourgon.org/blog/2017/02/21/metrics-tracing-and-logging.html?spm=a2c4e.11153940.blogcont514488.18.11b730c2dYH4KD)下面我们重点介绍下 tracing 29 | 30 | ### Tracing 的诞生 31 | Tracing 是在90年代就已出现的技术。但真正让该领域流行起来的还是源于 Google 的一篇论文"Dapper, a Large-Scale Distributed Systems Tracing Infrastructure",而另一篇论文"Uncertainty in Aggregate Estimates from Sampled Distributed Traces"中则包含关于采样的更详细分析。论文发表后一批优秀的 Tracing 软件孕育而生,比较流行的有: 32 | 33 | * Dapper(Google) : 各 tracer 的基础 34 | * StackDriver Trace (Google) 35 | * Zipkin(twitter) 36 | * Appdash(golang) 37 | * 鹰眼(taobao) 38 | * 谛听(盘古,阿里云云产品使用的Trace系统) 39 | * 云图(蚂蚁Trace系统) 40 | * sTrace(神马) 41 | * X-ray(aws) 42 | 分布式追踪系统发展很快,种类繁多,但核心步骤一般有三个:代码埋点,数据存储、查询展示。 43 | 44 | 下图是一个分布式调用的例子,客户端发起请求,请求首先到达负载均衡器,接着经过认证服务,计费服务,然后请求资源,最后返回结果。 45 | 46 |  47 | 48 | 数据被采集存储后,分布式追踪系统一般会选择使用包含时间轴的时序图来呈现这个 Trace。 49 | 50 |  51 | 52 | 但在数据采集过程中,由于需要侵入用户代码,并且不同系统的 API 并不兼容,这就导致了如果您希望切换追踪系统,往往会带来较大改动。 53 | 54 | 其他内容请参考[原文](https://yq.aliyun.com/articles/514488) 55 | 56 | ### Jaeger 57 | 58 | 中文名应该是-耶格,由uber开发的实现OpenTracing API的系统.可通过Docker部署测试 59 | 60 | 阿里云提供了Jaeger的支持,其他云厂商还未查阅资料. -------------------------------------------------------------------------------- /docs/subject-profile.md: -------------------------------------------------------------------------------- 1 | # 性能分析 2 | 3 | go提供了基础的分析工具pprof,下面介绍一些常用的 4 | 5 | * qcachegrind: 通过GUI查看pprof导出的报告 6 | * wrk: 测试请求工具 7 | 8 | ## 开始 9 | 10 | ### 嵌入pprof代码 11 | ``` 12 | import _ "net/http/pprof" 13 | ``` 14 | ### 运行HTTP服务器 15 | ``` 16 | go func() { 17 | http.ListenAndServe("localhost:8081", nil) 18 | }() 19 | ``` 20 | 21 | 目标服务启动后 22 | 23 | * 请求端启动 24 | ``` 25 | wrk -c 200 -t 4 -d 3m -s pprof.lua http://localhost:8040/v2/query 26 | ``` 27 | 28 | 29 | ### 使用pprof 30 | 31 | 服务端用以下命令进入跟踪 32 | ``` 33 | go tool pprof -seconds 200 ./all_go http://localhost:8081//debug/pprof/profile 34 | ``` 35 | 在指定时间结束后,将进入pprof命令行: 36 | * 导出qcachegrind可识别文件 37 | ``` 38 | callgrind 39 | ``` 40 | 41 | -------------------------------------------------------------------------------- /docs/tools.md: -------------------------------------------------------------------------------- 1 | ## 代理设置 2 | 3 | 在Go 1.13中,我们可以通过GOPROXY来控制代理,以及通过GOPRIVATE控制私有库不走代理。 4 | 5 | 设置GOPROXY代理: 6 | 7 | go env -w GOPROXY=https://goproxy.cn,direct 8 | 设置GOPRIVATE来跳过私有库,比如常用的Gitlab或Gitee,中间使用逗号分隔: 9 | go env -w GOPRIVATE=*.gitlab.com,*.gitee.com 10 | 11 | 如果在运行go mod vendor时,提示Get https://sum.golang.org/lookup/xxxxxx: dial tcp 216.58.200.49:443: i/o timeout,则是因为Go 1.13设置了默认的GOSUMDB=sum.golang.org,这个网站是被墙了的,用于验证包的有效性,可以通过如下命令关闭: 12 | 13 | go env -w GOSUMDB=off 14 | 15 | 可以设置 GOSUMDB="sum.golang.google.cn", 这个是专门为国内提供的sum 验证服务。 16 | 17 | go env -w GOSUMDB="sum.golang.google.cn" 18 | 19 | > 有些IDE(如goland)会集成包管理工具,需要注意一下,是否存在环境变量被覆盖的情况 20 | 21 | ## go < 1.13 22 | ### 翻墙 23 | 24 | 目前套件使用翻墙的主要原因为依赖golang.org的包.而golang.org是被墙的. 25 | 26 | 但发现在使用GOPATH的包时编译速度下降,建议有条件翻墙就上. 27 | 28 | * 使用带有http proxy,如shadowsocks(mac),因为如果使用全局模式时,在socks模式下会被站点墙 29 | 30 | 1. 设置好terminal: 31 | ``` 32 | // linux 33 | export https_proxy=http://127.0.0.1:1087 34 | 35 | // window(在win10、shadowsocks PAC模式下测试过,goland中配置proxy无效) 36 | set https_proxy=http://localhost:1080 37 | ``` 38 | ### DEP(deprecated) 39 | 40 | ``` 41 | go get -u github.com/golang/dep/cmd/dep 42 | 43 | dep ensure 同步包 44 | ``` 45 | 46 | 第一次初始化时,可用以下命令 47 | ``` 48 | dep init -gopath -v -no-examples 49 | ``` 50 | 51 | ### VGO 52 | 53 | 在Go 1.11后,vgo正式成为go工具链的一部分,也意味着go官方正式推出版本管理工具.在笔者的使用过程来看,确实是最优秀的. 54 | 55 | 配合Gitee,完全可以不用翻墙,由于1.11还未发布,这边就不写了,请根据官方文档来 56 | 57 | 可以将被墙的包通过go.mod文件的replace方式替换,竟味着可以直接指向github. 58 | 如果github也被墙,可以通过gitee导入github的项目,然后将包指向gitee,这样就可以达到不用翻墙开发了 59 | 替换获取包常见的命令为 60 | ``` 61 | vgo get package@[commit|version] 62 | ``` 63 | > 通过gitee访问github,非常之快.目前为止,github又不抽疯了..请忽略gitee的包 64 | ``` 65 | //以下列出常见需要替换的包 66 | replace ( 67 | //本项目需要 68 | github.com/graph-gophers/graphql-go => github.com/qeelyn/graphql-go v0.0.0-20181012014650-03df3acf1181 69 | // github 70 | golang.org/x/net => github.com/golang/net v0.0.0-20180811021610-c39426892332 71 | golang.org/x/sys => github.com/golang/sys v0.0.0-20180810173357-98c5dad5d1a0 72 | golang.org/x/text => github.com/golang/text v0.3.0 73 | google.golang.org/appengine => github.com/golang/appengine v1.1.0 74 | google.golang.org/genproto => github.com/google/go-genproto v0.0.0-20180808183934-383e8b2c3b9e 75 | google.golang.org/grpc => github.com/grpc/grpc-go v1.14.0 76 | 77 | // gitee.com/githubmirror. 78 | golang.org/x/net => gitee.com/githubmirror/golang-net v0.0.0-20180811021610-c39426892332 79 | google.golang.org/appengine => gitee.com/githubmirror/appengine v1.1.0 80 | google.golang.org/genproto => gitee.com/githubmirror/go-genproto v0.0.0-20180808183934-383e8b2c3b9e 81 | google.golang.org/grpc => gitee.com/githubmirror/grpc-go v1.14.0 82 | ) 83 | ``` 84 | 85 | > 现发现vgo的问题为,加载不出非go文件目录.对于protobuf需要外部protobuf文件编译会产生问题,请自行COPY 86 | 87 | ### Build 88 | 89 | 本套件默认的编译路径为cmd,所有的配置也是针对该路径的.请在开发时调整一下IDE配置 90 | * goland 91 |  -------------------------------------------------------------------------------- /docs/use-protobuf.md: -------------------------------------------------------------------------------- 1 | 使用protobuf 2 | =============== 3 | 4 | protobuf是gRPC的协议,扩展名为.proto,利用一系列代码生成工具可以生成各层级代码. 5 | [protobuf协议说明-要翻墙](https://developers.google.com/protocol-buffers/docs/reference/go-generated) 6 | 7 | 代码生成工具 8 | ------------ 9 | 10 | ``` 11 | brew install protobuf 12 | go get -u github.com/golang/protobuf/protoc-gen-go 13 | go get -u github.com/tsingsun/protoc-gen-goql 14 | ``` 15 | 16 | 这里利用到了自定义的生成工具,主要作用是替换协议中默认的协议处理类,以使生成的代码可以在各逻辑层中传递. 17 | 18 | * 处理像timestamp的非标量定义,控制序列化与反序列化 19 | * 与GORM的配套使用,数据库命名推荐采用下划线方式,来匹配protobuf规范 20 | * 支持采用inject注释为属性增加TAG定义,以及修改JSON定义 21 | * nullable支持,但需要其他平台的客户端代码协同支持 22 | * proto本身具备了json定义,可直接当成DTO 23 | * request对象可直接做为gin的入参对象 24 | 25 | 通过proto定义输出的模型文件可用于数据处理层,传输层,省掉了我们重复定义模型及值传递这类无意义的过程 26 | 27 | 编译protobuf文件 28 | ------------------ 29 | ``` 30 | protoc schemas/fund/fund.proto -I vendor/github.com/qeelyn/go-common -I . --goql_out=paths=source_relative:./ 31 | //需要rpc的话,加plugins=grpc 32 | ``` 33 | 34 | 注意确保protoc-gen-go和protoc在PATH环境变量中,protoc-gen-go一般在$GOPATH/bin目录下。 35 | > 注意paths参数,针对我们项目请加上该参数 36 | 37 | * protbuf生成的文件没有的修改必要,如果需要修改,由生成器来调整. 38 | * 请在protobuf文件头部注释出生成命令,执行命令的路径应该为当前项目目录. 39 | * 推荐直接将RPC Service一起定义,后期可直接转RPC 40 | 41 | ### IDE支持 42 | 43 | * goland 可以配置外部工具,直接右键菜单就可以生成代码. 44 | 45 | [上一节 首页](README.md) 46 | 47 | [下一节 处理请求](handle-request.md) 48 | -------------------------------------------------------------------------------- /gateway/app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | _ "github.com/patrickmn/go-cache" 5 | "github.com/qeelyn/go-common/cache" 6 | _ "github.com/qeelyn/go-common/cache/local" 7 | _ "github.com/qeelyn/go-common/cache/memcache" 8 | "github.com/spf13/viper" 9 | 10 | "context" 11 | "github.com/jinzhu/gorm" 12 | "github.com/qeelyn/go-common/logger" 13 | "github.com/qeelyn/golang-starter-kit/schemas/greeter" 14 | ) 15 | 16 | var ( 17 | Config *viper.Viper 18 | IsDebug bool 19 | Cache cache.Cache 20 | Caches map[string]cache.Cache 21 | Logger *logger.Logger 22 | Db *gorm.DB 23 | GreeterClient greeter.GreeterClient 24 | ) 25 | 26 | func init() { 27 | Caches = make(map[string]cache.Cache) 28 | } 29 | 30 | func GetUserId(ctx context.Context) string { 31 | v := ctx.Value("userid") 32 | if v == nil { 33 | return "" 34 | } 35 | return v.(string) 36 | } 37 | 38 | func GetOrgId(ctx context.Context) string { 39 | v := ctx.Value("orgid") 40 | if v == nil { 41 | return "0" 42 | } 43 | return v.(string) 44 | } 45 | -------------------------------------------------------------------------------- /gateway/app/middleware.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "github.com/dgrijalva/jwt-go" 10 | "github.com/gin-gonic/gin" 11 | "github.com/gin-gonic/gin/binding" 12 | "github.com/qeelyn/gin-contrib/auth" 13 | auth2 "github.com/qeelyn/go-common/auth" 14 | commonLogger "github.com/qeelyn/go-common/logger" 15 | errors2 "github.com/qeelyn/golang-starter-kit/gateway/errors" 16 | "go.uber.org/zap" 17 | "io" 18 | "io/ioutil" 19 | "net/http" 20 | "strings" 21 | "time" 22 | ) 23 | 24 | var ( 25 | AuthHanlerFunc gin.HandlerFunc 26 | CheckAccessMiddleware *auth.CheckAccess 27 | TracerFunc gin.HandlerFunc 28 | ) 29 | 30 | // Ginzap returns a gin.HandlerFunc (middleware) that logs requests using uber-go/zap. 31 | // 32 | // Requests with errors are logged using zap.Error(). 33 | // Requests without errors are logged using zap.Info(). 34 | // 35 | func AccessLogHandleFunc(logger *zap.Logger, timeFormat string, utc bool) gin.HandlerFunc { 36 | return func(c *gin.Context) { 37 | start := time.Now() 38 | // some evil middlewares modify this values 39 | reqPath := c.Request.URL.Path 40 | query := c.Request.URL.RawQuery 41 | bodyCopy := &bytes.Buffer{} 42 | if c.Request.Method == "POST" { 43 | switch c.ContentType() { 44 | case binding.MIMEJSON, binding.MIMEPOSTForm, binding.MIMEXML: 45 | io.Copy(bodyCopy, c.Request.Body) 46 | c.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyCopy.Bytes())) 47 | } 48 | } 49 | if orgId := c.GetHeader("Qeelyn-Org-Id"); orgId != "" { 50 | c.Set("orgid", orgId) 51 | } 52 | // pass to context 53 | if authHeader := c.GetHeader("Authorization"); authHeader != "" { 54 | c.Set("authorization", authHeader) 55 | } 56 | c.Next() 57 | 58 | end := time.Now() 59 | latency := end.Sub(start) 60 | if utc { 61 | end = end.UTC() 62 | } 63 | 64 | logger.Info(reqPath, 65 | zap.Int("status", c.Writer.Status()), 66 | zap.String("method", c.Request.Method), 67 | zap.String("path", reqPath), 68 | zap.String("query", query), 69 | zap.ByteString("body", bodyCopy.Bytes()), 70 | zap.String("ip", c.ClientIP()), 71 | zap.String("auth", c.GetString("userid")), 72 | zap.String("user-agent", c.Request.UserAgent()), 73 | zap.String("time", end.Format(timeFormat)), 74 | zap.Duration("latency", latency), 75 | commonLogger.TraceIdField(c), 76 | ) 77 | } 78 | } 79 | 80 | // auth will check the jwt token basically 81 | func NewAuthMiddleware(config map[string]interface{}) *auth.GinJWTMiddleware { 82 | // the jwt middleware 83 | pKey, _ := config["public-key"].([]byte) 84 | eKey, _ := config["encryption-key"].(string) 85 | algo, _ := config["algorithm"].(string) 86 | if strings.HasPrefix(algo, "RS") && (pKey == nil) { 87 | panic("miss pubKeyFile or priKeyFile setting when in RS signing algorithm") 88 | } 89 | if strings.HasPrefix(algo, "HS") && eKey == "" { 90 | panic("miss encryption-key setting when in HS signing algorithm") 91 | } 92 | middle := &auth.GinJWTMiddleware{ 93 | BearerTokenValidator: &auth2.BearerTokenValidator{ 94 | Realm: "auth server", 95 | PubKeyFile: pKey, 96 | Key: []byte(eKey), 97 | }, 98 | SigningAlgorithm: algo, //RS256 99 | UnauthorizedHandle: func(c *gin.Context, code int, message string) bool { 100 | if IsDebug && c.GetHeader("Authorization") == "" { 101 | if tid, ok := config["testuserid"]; ok { 102 | c.Set("userid", tid.(string)) 103 | } 104 | c.Next() 105 | return false 106 | } 107 | c.JSON(code, gin.H{ 108 | "errors": []map[string]interface{}{ 109 | { 110 | "code": code, 111 | "message": message, 112 | }, 113 | }, 114 | }) 115 | return true 116 | }, 117 | TokenValidator: func(token *jwt.Token, c *gin.Context) bool { 118 | c.Set("authorization", c.GetHeader("Authorization")) 119 | return true 120 | }, 121 | TokenLookup: "header:Authorization", 122 | TokenHeadName: "Bearer", 123 | } 124 | return middle 125 | 126 | } 127 | 128 | // userId will be exist after bearer auth middleware execute 129 | func NewCheckAccessMiddleware(config map[string]interface{}) *auth.CheckAccess { 130 | checkAccessUrl := config["auth-server"].(string) + config["check-access"].(string) 131 | routerPrefix := config["router-prefix"].(string) 132 | checkAccessTimeout := config["check-access-timeout"].(int) 133 | instance := &auth.CheckAccess{ 134 | GetPermissionFunc: func(context *gin.Context) string { 135 | reqPath := context.Request.URL.Path 136 | if strings.HasPrefix(reqPath, routerPrefix) { 137 | return reqPath[len(routerPrefix):] 138 | } else { 139 | return reqPath 140 | } 141 | }, 142 | CheckFunc: func(context *http.Request, userId string, permission string, params map[string]interface{}) int { 143 | if IsDebug && context.Header.Get("Authorization") == "" { 144 | return http.StatusOK 145 | } 146 | body, err := json.Marshal(map[string]interface{}{ 147 | "permission": permission, 148 | "params": params, 149 | }) 150 | if err != nil { 151 | Logger.Strict().Error(fmt.Sprintf("error on CheckFunc : %s", err)) 152 | return http.StatusBadRequest 153 | } 154 | client := http.Client{ 155 | Timeout: time.Duration(checkAccessTimeout) * time.Millisecond, 156 | } 157 | req, _ := http.NewRequest("POST", checkAccessUrl, bytes.NewReader(body)) 158 | req.Header.Set("Content-Type", "application/json") 159 | req.Header.Set("Authorization", context.Header.Get("Authorization")) 160 | 161 | if authRes, err := client.Do(req); err == nil { 162 | return authRes.StatusCode 163 | } else { 164 | Logger.Strict().Error(fmt.Sprintf("error on auth client request : %s", err)) 165 | return http.StatusInternalServerError 166 | } 167 | }, 168 | } 169 | return instance 170 | } 171 | 172 | func CheckAccess(ctx context.Context, permission string, params map[string]interface{}) (bool, error) { 173 | var userId, orgId string 174 | var ok bool 175 | var err error 176 | if userId, ok = ctx.Value("userid").(string); !ok { 177 | err = errors2.ErrUnauthorized 178 | } 179 | if orgId, ok = ctx.Value("orgid").(string); ok { 180 | if params == nil { 181 | params = map[string]interface{}{} 182 | } 183 | params["org_id"] = orgId 184 | } 185 | req := ctx.Value(0).(*http.Request) 186 | 187 | if code := CheckAccessMiddleware.CheckFunc(req, userId, permission, params); code != http.StatusOK { 188 | if code == http.StatusForbidden { 189 | err = errors2.ErrPermissionDenied 190 | Logger.Strict().Warn(fmt.Sprintf("userId %s has no permission at %s", userId, permission)) 191 | } 192 | err = errors.New(http.StatusText(code)) 193 | } 194 | return err == nil, err 195 | } 196 | -------------------------------------------------------------------------------- /gateway/errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "errors" 5 | graphql "github.com/graph-gophers/graphql-go/errors" 6 | "github.com/qeelyn/gin-contrib/errorhandle" 7 | "google.golang.org/grpc/codes" 8 | "google.golang.org/grpc/status" 9 | ) 10 | 11 | var ( 12 | //check data loader key,and service returns type 13 | ErrLoaderWrongType = errors.New("LOADER_WRONG_TYPE") 14 | ErrPermissionDenied = errors.New("PERMISSION_DENIED") 15 | ErrUnauthorized = errors.New("UNAUTHORIZED") 16 | ErrGRPCUnavailable = errors.New("GRPCUnavailable") 17 | ErrDeadlineExceeded = errors.New("DeadlineExceeded") 18 | ) 19 | 20 | func Expand(errs []*graphql.QueryError) []*graphql.QueryError { 21 | expanded := make([]*graphql.QueryError, 0, len(errs)) 22 | 23 | for _, err := range errs { 24 | switch t := err.ResolverError.(type) { 25 | case interface{ GRPCStatus() *status.Status }: //for grpc 26 | switch t.GRPCStatus().Code() { 27 | case codes.DeadlineExceeded: //timeout 28 | err.Message = errorhandle.ErrMessage.GetErrorDescription(ErrDeadlineExceeded).Message 29 | case codes.Unavailable: 30 | err.Message = errorhandle.ErrMessage.GetErrorDescription(ErrGRPCUnavailable).Message 31 | default: 32 | } 33 | expanded = append(expanded, err) 34 | default: 35 | if errorhandle.ErrMessage != nil && err.ResolverError != nil { 36 | err.Message = errorhandle.ErrMessage.GetErrorDescription(err.ResolverError).Message 37 | } 38 | expanded = append(expanded, err) 39 | } 40 | } 41 | 42 | return expanded 43 | } 44 | -------------------------------------------------------------------------------- /gateway/handle/graphql.go: -------------------------------------------------------------------------------- 1 | package handle 2 | 3 | import ( 4 | "context" 5 | navErr "errors" 6 | "github.com/gin-gonic/gin" 7 | "github.com/graph-gophers/graphql-go" 8 | "github.com/opentracing/opentracing-go" 9 | "github.com/qeelyn/gin-contrib/auth" 10 | "github.com/qeelyn/gin-contrib/tracing" 11 | "github.com/qeelyn/go-common/logger" 12 | "github.com/qeelyn/golang-starter-kit/gateway/app" 13 | "github.com/qeelyn/golang-starter-kit/gateway/errors" 14 | "github.com/qeelyn/golang-starter-kit/gateway/loader" 15 | "github.com/qeelyn/golang-starter-kit/gateway/resolver" 16 | "github.com/qeelyn/golang-starter-kit/gateway/schema" 17 | "net/http" 18 | "sync" 19 | ) 20 | 21 | type query struct { 22 | Query string `json:"query"` 23 | OperationName string `json:"operationName"` 24 | Variables map[string]interface{} `json:"variables"` 25 | } 26 | 27 | type request struct { 28 | queries []query 29 | isBatch bool 30 | } 31 | 32 | type GraphQL struct { 33 | Schema *graphql.Schema 34 | Loaders loader.Collection 35 | CheckAccess *auth.CheckAccess 36 | } 37 | 38 | func ServeGraphqlResource(r *gin.RouterGroup) { 39 | //tracer := graphql.Tracer(opentracing.GlobalTracer()) 40 | graphqlSchema := graphql.MustParseSchema(schema.GetRootSchema(), &resolver.Resolver{}) 41 | graphql.Logger(NewGraphqlLogger(app.Logger)) 42 | h := &GraphQL{ 43 | Schema: graphqlSchema, 44 | Loaders: loader.NewLoaderCollection(), 45 | CheckAccess: app.CheckAccessMiddleware, 46 | } 47 | r.POST("query", h.Query) 48 | } 49 | 50 | type graphqlLoggerAdapter struct { 51 | logger *logger.Logger 52 | } 53 | 54 | func NewGraphqlLogger(l *logger.Logger) *graphqlLoggerAdapter { 55 | return &graphqlLoggerAdapter{ 56 | logger: l, 57 | } 58 | } 59 | 60 | func (t *graphqlLoggerAdapter) LogPanic(ctx context.Context, value interface{}) { 61 | app.Logger.WithContext(ctx).Error(value.(string)) 62 | } 63 | 64 | func (t *GraphQL) Query(c *gin.Context) { 65 | req, err := parse(c.Request) 66 | if err != nil { 67 | c.AbortWithError(http.StatusBadRequest, err) 68 | return 69 | } 70 | n := len(req.queries) 71 | if n == 0 { 72 | c.AbortWithError(http.StatusBadRequest, navErr.New("err-request")) 73 | return 74 | } 75 | var ( 76 | ctx = t.Loaders.Attach(c) 77 | responses = make([]*graphql.Response, n) 78 | wg sync.WaitGroup 79 | ) 80 | spanCtx, _ := tracing.SpanFromContext(c) 81 | if spanCtx != nil { 82 | span := opentracing.StartSpan(c.Request.RequestURI, opentracing.ChildOf(spanCtx)) 83 | ctx = opentracing.ContextWithSpan(ctx, span) 84 | defer span.Finish() 85 | } 86 | 87 | wg.Add(n) 88 | for i, q := range req.queries { 89 | go func(i int, q query) { 90 | res := t.Schema.Exec(ctx, q.Query, q.OperationName, q.Variables) 91 | res.Errors = errors.Expand(res.Errors) 92 | 93 | responses[i] = res 94 | wg.Done() 95 | }(i, q) 96 | } 97 | 98 | wg.Wait() 99 | if req.isBatch { 100 | c.JSON(200, responses) 101 | } else if len(responses) > 0 { 102 | c.JSON(200, responses[0]) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /gateway/handle/parse.go: -------------------------------------------------------------------------------- 1 | package handle 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | ) 10 | 11 | func parse(r *http.Request) (request, error) { 12 | // We always need to read and close the request body. 13 | body, err := ioutil.ReadAll(r.Body) 14 | if err != nil { 15 | return request{}, errors.New("unable to read request body") 16 | } 17 | _ = r.Body.Close() 18 | 19 | var req request 20 | 21 | switch r.Method { 22 | case "POST": 23 | req = parsePost(body) 24 | case "GET": 25 | req = parseGet(r.URL.Query()) 26 | default: 27 | err = errors.New("only POST and GET requests are supported") 28 | } 29 | 30 | return req, err 31 | } 32 | 33 | func parseGet(v url.Values) request { 34 | var ( 35 | queries = v["query"] 36 | names = v["operationName"] 37 | variables = v["variables"] 38 | qLen = len(queries) 39 | nLen = len(names) 40 | vLen = len(variables) 41 | ) 42 | 43 | if qLen == 0 { 44 | return request{} 45 | } 46 | 47 | var requests = make([]query, 0, qLen) 48 | var isBatch bool 49 | 50 | // This loop assumes there will be a corresponding element at each index 51 | // for query, operation name, and variable fields. 52 | // 53 | // NOTE: This could be a bad assumption. Maybe we want to do some validation? 54 | for i, q := range queries { 55 | var n string 56 | if i < nLen { 57 | n = names[i] 58 | } 59 | 60 | var m = map[string]interface{}{} 61 | if i < vLen { 62 | str := variables[i] 63 | if err := json.Unmarshal([]byte(str), &m); err != nil { 64 | m = nil // TODO: Improve error handling here. 65 | } 66 | } 67 | 68 | requests = append(requests, query{Query: q, OperationName: n, Variables: m}) 69 | } 70 | 71 | if qLen > 1 { 72 | isBatch = true 73 | } 74 | 75 | return request{queries: requests, isBatch: isBatch} 76 | } 77 | 78 | func parsePost(b []byte) request { 79 | if len(b) == 0 { 80 | return request{} 81 | } 82 | 83 | var queries []query 84 | var isBatch bool 85 | 86 | // Inspect the first character to inform how the body is parsed. 87 | switch b[0] { 88 | case '{': 89 | q := query{} 90 | err := json.Unmarshal(b, &q) 91 | if err == nil { 92 | queries = append(queries, q) 93 | } 94 | case '[': 95 | isBatch = true 96 | _ = json.Unmarshal(b, &queries) 97 | } 98 | 99 | return request{queries: queries, isBatch: isBatch} 100 | } 101 | -------------------------------------------------------------------------------- /gateway/loader/loader.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/gin-gonic/gin" 7 | "github.com/graph-gophers/dataloader" 8 | "github.com/qeelyn/go-common/cache" 9 | "github.com/qeelyn/golang-starter-kit/gateway/app" 10 | "time" 11 | ) 12 | 13 | var ( 14 | dataloaderCache dataloader.Cache 15 | ) 16 | 17 | const loaderInContextKey string = "loader_cls" 18 | 19 | type Loader struct { 20 | } 21 | 22 | type Collection struct { 23 | lookup map[loaderKey]*dataloader.Loader 24 | } 25 | 26 | func NewLoaderCollection() Collection { 27 | var cacheOpt dataloader.Option 28 | if c, ok := app.Caches["dataloader"]; ok { 29 | //use cache's default duration 30 | dataloaderCache = NewLoaderCache(c, time.Duration(app.Config.GetInt("cache.dataloader.duration"))*time.Second) 31 | cacheOpt = dataloader.WithCache(dataloaderCache) 32 | } else { 33 | cacheOpt = dataloader.WithCache(&dataloader.NoCache{}) 34 | } 35 | 36 | return Collection{ 37 | lookup: map[loaderKey]*dataloader.Loader{ 38 | UserLoaderKey: NewUserLoader(cacheOpt), 39 | }, 40 | } 41 | } 42 | 43 | func (c Collection) Attach(ctx *gin.Context) context.Context { 44 | ctx.Set(loaderInContextKey, &c) 45 | //for k, batchFn := range c.lookup { 46 | // ctx.Set(string(k), dataloader.NewBatchedLoader(batchFn, dataloader.WithCache(dataloaderCache))) 47 | // //ctx = context.WithValue(ctx, k, dataloader.NewBatchedLoader(batchFn)) 48 | //} 49 | return ctx 50 | } 51 | 52 | func (c Collection) GetLoader(k loaderKey) *dataloader.Loader { 53 | ldr, ok := c.lookup[k] 54 | if ok { 55 | return ldr 56 | } 57 | switch k { 58 | } 59 | return nil 60 | } 61 | 62 | func extract(ctx context.Context, k loaderKey) (*dataloader.Loader, error) { 63 | // k need same type as attach ctx.Set type 64 | coll, ok := ctx.Value(loaderInContextKey).(*Collection) 65 | if !ok { 66 | return nil, fmt.Errorf("unable to find %s loader on the request context", k) 67 | } 68 | // find the loader 69 | ldr := coll.GetLoader(k) 70 | if ldr == nil { 71 | return nil, fmt.Errorf("unable to find %s loader on loader collection", k) 72 | } 73 | 74 | return ldr, nil 75 | } 76 | 77 | func Load(tk loaderKey, ctx context.Context, key dataloader.Key) (interface{}, error) { 78 | ldr, err := extract(ctx, tk) 79 | if err != nil { 80 | return nil, err 81 | } 82 | thunk := ldr.Load(ctx, key) 83 | data, err := thunk() 84 | if err != nil { 85 | ldr.Clear(ctx, key) 86 | return nil, err 87 | } 88 | return data, nil 89 | } 90 | 91 | type loaderCache struct { 92 | cache cache.Cache 93 | duration time.Duration 94 | } 95 | 96 | func NewLoaderCache(c cache.Cache, duration time.Duration) *loaderCache { 97 | return &loaderCache{cache: c, duration: duration} 98 | } 99 | 100 | func (l *loaderCache) Get(ctx context.Context, key dataloader.Key) (dataloader.Thunk, bool) { 101 | var thunk dataloader.Thunk 102 | err := l.cache.Get(key.String(), &thunk) 103 | if err != nil { 104 | return nil, false 105 | } 106 | 107 | return thunk, true 108 | } 109 | 110 | func (l *loaderCache) Set(ctx context.Context, key dataloader.Key, thunk dataloader.Thunk) { 111 | l.cache.Set(key.String(), thunk, l.duration) 112 | } 113 | 114 | func (l *loaderCache) Delete(ctx context.Context, key dataloader.Key) bool { 115 | return l.cache.Delete(key.String()) == nil 116 | } 117 | 118 | func (l *loaderCache) Clear() { 119 | l.cache.FlushAll() 120 | } 121 | -------------------------------------------------------------------------------- /gateway/loader/loader_keys.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | type loaderKey string 4 | 5 | func (k loaderKey) String() string { 6 | return string(k) 7 | } 8 | 9 | const ( 10 | UserLoaderKey loaderKey = "user" 11 | ) 12 | 13 | // 需要对应proto文件中的node type 14 | type GlobalType string 15 | 16 | const ( 17 | User GlobalType = "user" 18 | ) 19 | 20 | type DataKey struct { 21 | string string 22 | raw interface{} 23 | } 24 | 25 | func NewDataKey(key string, raw interface{}) *DataKey { 26 | return &DataKey{key, raw} 27 | } 28 | 29 | func (d *DataKey) String() string { 30 | return d.string 31 | } 32 | 33 | func (d *DataKey) Raw() interface{} { 34 | if d != nil { 35 | return d.raw 36 | } 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /gateway/loader/user.go: -------------------------------------------------------------------------------- 1 | package loader 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/graph-gophers/dataloader" 8 | "github.com/graph-gophers/graphql-go" 9 | "github.com/pkg/errors" 10 | "github.com/qeelyn/golang-starter-kit/gateway/app" 11 | "github.com/qeelyn/golang-starter-kit/helper/relay" 12 | "io/ioutil" 13 | "net/http" 14 | "strconv" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | type UserLoader struct { 20 | userInfoUrl string 21 | } 22 | 23 | func NewUserLoader(opts ...dataloader.Option) *dataloader.Loader { 24 | val := UserLoader{ 25 | userInfoUrl: app.Config.GetString("auth.auth-server") + "/userinfo", 26 | }.loadBatch 27 | return dataloader.NewBatchedLoader(val, opts...) 28 | } 29 | 30 | func (t UserLoader) loadBatch(ctx context.Context, keys dataloader.Keys) []*dataloader.Result { 31 | var ( 32 | n = len(keys) 33 | results = make([]*dataloader.Result, n) 34 | wg sync.WaitGroup 35 | ) 36 | wg.Add(n) 37 | for i, key := range keys { 38 | go func(i int, key dataloader.Key) { 39 | defer wg.Done() 40 | var oid int 41 | if err := relay.UnmarshalSpec(graphql.ID(key.String()), &oid); err != nil { 42 | results[i] = &dataloader.Result{Data: nil, Error: err} 43 | return 44 | } 45 | gin := ctx.Value(0).(*http.Request) 46 | if userInfo, err := t.getUserInfo(gin, strconv.Itoa(oid)); err != nil { 47 | results[i] = &dataloader.Result{Data: nil, Error: err} 48 | } else { 49 | results[i] = &dataloader.Result{Data: userInfo, Error: err} 50 | } 51 | }(i, key) 52 | 53 | } 54 | wg.Wait() 55 | return results 56 | } 57 | 58 | func (t UserLoader) getUserInfo(context *http.Request, openId string) (map[string]interface{}, error) { 59 | client := http.Client{ 60 | Timeout: 1 * time.Second, 61 | } 62 | req, _ := http.NewRequest("GET", t.userInfoUrl+"?openid="+openId, nil) 63 | req.Header.Set("Content-Type", "application/json") 64 | req.Header.Set("Authorization", context.Header.Get("Authorization")) 65 | authRes, err := client.Do(req) 66 | if err == nil { 67 | var body []byte 68 | body, err = ioutil.ReadAll(authRes.Body) 69 | if err != nil { 70 | return nil, err 71 | } 72 | var res map[string]interface{} 73 | json.Unmarshal(body, &res) 74 | if authRes.StatusCode != http.StatusOK { 75 | msg := res["errors"].([]interface{})[0].(map[string]interface{})["message"] 76 | err = errors.New(msg.(string)) 77 | } else { 78 | return res, nil 79 | } 80 | } 81 | err = fmt.Errorf("error on auth client request : %s", err) 82 | return nil, err 83 | } 84 | 85 | func LoadUserNickName(ctx context.Context, key dataloader.Key) (string, error) { 86 | if val, err := Load(UserLoaderKey, ctx, key); err != nil { 87 | return "", err 88 | } else { 89 | ui := val.(map[string]interface{}) 90 | return ui["data"].(map[string]interface{})["nickname"].(string), nil 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /gateway/public/html/graphiql.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |