├── .gitignore ├── etcdv3 ├── globals.go ├── resolver.go ├── watcher.go └── register.go ├── helloword ├── helloword.proto └── helloword.pb.go ├── service ├── service.crt ├── service.key └── service.go └── client └── client.go /.gitignore: -------------------------------------------------------------------------------- 1 | service/debug 2 | service/service 3 | client/client 4 | .idea 5 | .vscode 6 | -------------------------------------------------------------------------------- /etcdv3/globals.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import "github.com/coreos/etcd/clientv3" 4 | 5 | var Prefix = "etcd3_naming" 6 | var client *clientv3.Client 7 | 8 | var service_key string 9 | 10 | var stop_signal chan bool = make(chan bool, 1) 11 | -------------------------------------------------------------------------------- /etcdv3/resolver.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/coreos/etcd/clientv3" 8 | "github.com/pkg/errors" 9 | "google.golang.org/grpc/naming" 10 | ) 11 | 12 | type Resolver struct { 13 | ServiceName string 14 | } 15 | 16 | func NewResolver(name string) *Resolver { 17 | return &Resolver{ 18 | ServiceName: name, 19 | } 20 | } 21 | 22 | func (re *Resolver) Resolve(target string) (naming.Watcher, error) { 23 | if re.ServiceName == "" { 24 | return nil, fmt.Errorf("grpclib: no service name provided") 25 | } 26 | client, err := clientv3.New(clientv3.Config{ 27 | Endpoints: strings.Split(target, ","), 28 | }) 29 | 30 | if err != nil { 31 | return nil, errors.Wrap(err, "grpclib: create etcd3 client faield") 32 | } 33 | 34 | return &Watcher{re: re, client: *client}, nil 35 | } 36 | -------------------------------------------------------------------------------- /helloword/helloword.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package helloword; 4 | 5 | import "github.com/golang/protobuf/ptypes/timestamp/timestamp.proto"; 6 | 7 | service Greeter { 8 | rpc Login(LoginRequest) returns (LoginReply) {}; 9 | rpc SayHello(stream HelloRequest) returns (stream HelloReply) {}; 10 | } 11 | 12 | message LoginRequest { 13 | string username = 1; 14 | } 15 | message LoginReply { 16 | string message = 1; 17 | bool success = 2; 18 | string token = 3; 19 | } 20 | 21 | message HelloRequest { 22 | string message = 1; 23 | } 24 | 25 | message HelloReply { 26 | string message = 1; 27 | google.protobuf.Timestamp TS = 2; 28 | MessageType message_type = 3; 29 | 30 | enum MessageType{ 31 | CONNECT_SUCCESS = 0; 32 | CONNECT_FAILED = 1; 33 | NORMAL_MESSAGE = 2; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /service/service.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID/DCCAuSgAwIBAgIJAIrnXLKGduTtMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV 3 | BAYTAmNuMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX 4 | aWRnaXRzIFB0eSBMdGQxFDASBgNVBAMTC2R1b3Nob3ViYW5nMCAXDTE4MDMyMTA4 5 | MzcwMFoYDzIxMTgwMjI1MDgzNzAwWjBbMQswCQYDVQQGEwJjbjETMBEGA1UECBMK 6 | U29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRQw 7 | EgYDVQQDEwtkdW9zaG91YmFuZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC 8 | ggEBAL3+dZ4Q1170DaTp4bB4XQjWvNCv8zM+4Nzibad+kiGkZNLP2xIm89LWlJxs 9 | kvXwMdoSIA548K02oe2lfou/wvmTsJUp+CIGUNlgyKWV6mjH6YGeLxlRWugemW5b 10 | j+t6OuzCF2dROnC6DgdiLkI8tx7+DRT901EExgvGhxkPoWWPAS3+WqpR3rDixXJa 11 | hxRhgDNht+zNhLXb0ISgbBv/BdfGilW5/3OSN55a6IVfcyhLprVGTRRbJtPmB3Ja 12 | e566qmdtC1jK2tjUTazIB/Re0rdtt54j1z8/ovcMBtS1Jw+KTqBV9VlBnK3K0SjM 13 | hURvLRLdZN4/Qqmy6TPQ4Qt/8L8CAwEAAaOBwDCBvTAdBgNVHQ4EFgQUQcw1sRwF 14 | 0kPsbhLX8LZ4FTkFfokwgY0GA1UdIwSBhTCBgoAUQcw1sRwF0kPsbhLX8LZ4FTkF 15 | fomhX6RdMFsxCzAJBgNVBAYTAmNuMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYD 16 | VQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxFDASBgNVBAMTC2R1b3Nob3Vi 17 | YW5nggkAiudcsoZ25O0wDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEA 18 | CTcT0EQlqbOpVtENPSJphc0+G52/9uDHYXoH0RRF6ivFzKzlUFqINQbtEhv9RwQG 19 | 29wx/ZpT5qoxVuI4049LgQJmxvtT20lbZXVkvmdyeAx0vn1f1yNxT6+o94yfXskz 20 | xFYgCWRKAYN9hMXmRw69vKMvAbm+FeYDKeliz3G3mB8NzcKEHwv5rFTgJJWgZwEp 21 | 4FFxpHjXG/UHSL/rghMJXqFF0CuueYrBCTt596Ii8EX7xXLU4cz3h5ZVyghiunNF 22 | GXXs87phjY/6+x1sqIyX4fi8CGLGCekPVjUlz+GE6wacnDyMGJVm9Hkh69L3xHw/ 23 | 21t2MsY1enPmoJttvZJBGw== 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /service/service.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEAvf51nhDXXvQNpOnhsHhdCNa80K/zMz7g3OJtp36SIaRk0s/b 3 | Eibz0taUnGyS9fAx2hIgDnjwrTah7aV+i7/C+ZOwlSn4IgZQ2WDIpZXqaMfpgZ4v 4 | GVFa6B6ZbluP63o67MIXZ1E6cLoOB2IuQjy3Hv4NFP3TUQTGC8aHGQ+hZY8BLf5a 5 | qlHesOLFclqHFGGAM2G37M2EtdvQhKBsG/8F18aKVbn/c5I3nlrohV9zKEumtUZN 6 | FFsm0+YHclp7nrqqZ20LWMra2NRNrMgH9F7St223niPXPz+i9wwG1LUnD4pOoFX1 7 | WUGcrcrRKMyFRG8tEt1k3j9CqbLpM9DhC3/wvwIDAQABAoIBAGgYedK/wXCJ4Mvh 8 | RMFOQRLtrIfeTy5dnhAHkzK421HJY2BJd+rLIr36yXOm3SIYlwe29I9ZXexZSYEO 9 | MnZY+3eInrArTYM+2J8xMt3edI8yMNLOm1mQxHD3CvN3ATRwRMahVPdLfuxyU+th 10 | tJyf7WjyorJEm7oBFo0vGaf2c2RTch13Qtmt8i8ETncewkxdBd/AUiGGW2YM8I68 11 | 2K8uyAJPzIa81RxPl9LkNtVB3ZOf7GZspvntBg7sN2TBAYIydEf3NQZan6AeMKBG 12 | 8hNDduAwxuRjSF+XavNbll3bGPqFcqT7DFsmeK9KYUmyVDWWv6Db9gHTCJuEv81c 13 | ZqRp2zECgYEA8SEEHGB3qTh70io7D9yoSC3b2h1fLlRlPJFt3QXVkikRROY1KTCW 14 | IqqLvebx6YnN9DrctYxZXGIcalNijRURrge555I4NNMYgpDLqZXoEm7H0vzPbmGo 15 | L2+x+wtOeQQ9Biiceo00IgEIgvjcjW49I2VwmySR2i3c4xblSm8EdxcCgYEAybYd 16 | o6DG7KnxFrDpP/EKAHhK7W6jFQzIirK6utyWcIt3287NcCMjorDrc4d5Ewe4I0PM 17 | xmsGhI3Y8KLwuqXHe4h6KR5rFGzMoppnDWllj5H2pDwwYPIRsLka2hOUndk8PDV+ 18 | Ty8wotZRCqYP6LkB6XRrkAq0RVL3jxPbcJ6L3JkCgYBnjadHTIJ3MOO0KSC+OiQn 19 | A1LEhBKpQeNMNKR65BOJNovNDBROgSgo8RccUf8YLW8+cLzyVrjVvOi5HOBY+HBk 20 | Tbsf4SK0ROfkwqxEQzWsHNlCGgI3REdHP++ugXeM7y4J5Az3cIawB8ORA7EsJ+in 21 | t9u5NOZSTZnsCJwl8DF5twKBgBCPBrfcBvh+qu/17e/11aBcP6pmHrUnq37XVUTp 22 | vt/WUyyUOYEvIUnFxE5jnKjXRWJ/ulu/hXV0P47hkWmBGnMlrQGbIac3l0Tx7+vF 23 | 7zRuAxVQa5hJxvH/ABlO6jBbGPrIWWoesZtshJKfN0cfiOylRf85IbtfKDtyod0I 24 | 5uphAoGAAXAjdB/rybwroPGSCVPAzE//FRHOFPV6p8wWU3/vCSQiY23jJOUYYtSr 25 | 4/ARgmJavSxITTWeeCzfkTn5TtrY60QOrJjKGiqgAZhV5NQouBV8cOQAs61rJ4Ck 26 | QhXjE0mMzJHtaRT7X4rIqMdgLuTw2MdDnPCxw6L03/NjOLrg4pA= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /etcdv3/watcher.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/coreos/etcd/clientv3" 8 | "github.com/coreos/etcd/mvcc/mvccpb" 9 | "google.golang.org/grpc/naming" 10 | ) 11 | 12 | type Watcher struct { 13 | re *Resolver 14 | client clientv3.Client 15 | is_initialized bool 16 | } 17 | 18 | func extract_addrs(resp *clientv3.GetResponse) []string { 19 | addrs := []string{} 20 | 21 | if resp == nil || resp.Kvs == nil { 22 | return addrs 23 | } 24 | 25 | for _, kv := range resp.Kvs { 26 | if v := kv.Value; v != nil { 27 | addrs = append(addrs, string(v)) 28 | } 29 | } 30 | return addrs 31 | } 32 | 33 | func (w *Watcher) Close() { 34 | w.client.Close() 35 | } 36 | 37 | func (w *Watcher) Next() ([]*naming.Update, error) { 38 | prefix := fmt.Sprintf("%s/%s/nodes/", Prefix, w.re.ServiceName) 39 | 40 | if !w.is_initialized { 41 | resp, err := w.client.Get(context.Background(), prefix, clientv3.WithPrefix()) 42 | w.is_initialized = true 43 | if err == nil { 44 | addrs := extract_addrs(resp) 45 | if l := len(addrs); l != 0 { 46 | updates := make([]*naming.Update, l) 47 | for i := range addrs { 48 | updates[i] = &naming.Update{ 49 | Op: naming.Add, Addr: addrs[i], 50 | } 51 | } 52 | 53 | return updates, nil 54 | } 55 | } 56 | } 57 | 58 | rch := w.client.Watch(context.Background(), prefix, clientv3.WithPrefix()) 59 | for wresp := range rch { 60 | for _, ev := range wresp.Events { 61 | switch ev.Type { 62 | case mvccpb.PUT: 63 | return []*naming.Update{{ 64 | Op: naming.Add, 65 | Addr: string(ev.Kv.Value), 66 | }}, nil 67 | 68 | case mvccpb.DELETE: 69 | return []*naming.Update{{ 70 | Op: naming.Delete, 71 | Addr: string(ev.Kv.Value), 72 | }}, nil 73 | } 74 | } 75 | } 76 | return nil, nil 77 | } 78 | -------------------------------------------------------------------------------- /etcdv3/register.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "strings" 8 | "time" 9 | 10 | "github.com/coreos/etcd/clientv3" 11 | ) 12 | 13 | func Register(name string, host string, port int, target string, interval time.Duration, ttl int) error { 14 | service_value := fmt.Sprintf("%s:%d", host, port) 15 | service_key = fmt.Sprintf("%s/%s/nodes/%s", Prefix, name, service_value) 16 | 17 | var err error 18 | client, err = clientv3.New(clientv3.Config{ 19 | Endpoints: strings.Split(target, ","), 20 | }) 21 | 22 | if err != nil { 23 | return fmt.Errorf("grpclb: create etcd3 client failed: %v", err) 24 | } 25 | 26 | go func() { 27 | var err error 28 | var get_resp *clientv3.GetResponse 29 | ticker := time.NewTicker(interval) 30 | grant_resp, _ := client.Grant(context.Background(), int64(ttl)) 31 | for { 32 | get_resp, err = client.Get(context.Background(), service_key) 33 | if err != nil { 34 | log.Printf("grpclb: set service '%s' with ttl to etcd3 failed: %s", name, err.Error()) 35 | 36 | } else if get_resp.Count == 0 { 37 | if _, err = client.Put(context.Background(), service_key, service_value, clientv3.WithLease(grant_resp.ID)); err != nil { 38 | log.Printf("grpclb: set service '%s' with ttl to etcd3 failed: %s", name, err.Error()) 39 | } 40 | } else { 41 | if _, err = client.KeepAliveOnce(context.Background(), grant_resp.ID); err != nil { 42 | log.Printf("grpclb: refresh service '%s' with ttl to etcd3 failed: %s", name, err.Error()) 43 | } 44 | } 45 | 46 | select { 47 | case <-stop_signal: 48 | return 49 | 50 | case <-ticker.C: 51 | } 52 | } 53 | }() 54 | return nil 55 | } 56 | 57 | func UnRegister() error { 58 | stop_signal <- true 59 | stop_signal = make(chan bool, 1) 60 | 61 | var err error 62 | if _, err = client.Delete(context.Background(), service_key); err != nil { 63 | 64 | log.Printf("grpclb: deregister '%s' failed: %s", service_key, err.Error()) 65 | } else { 66 | log.Printf("grpclb: deregister '%s' success", service_key) 67 | } 68 | return err 69 | } 70 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "flag" 7 | "fmt" 8 | grpclb "grpclb/etcdv3" 9 | "io" 10 | "log" 11 | "os" 12 | "sync" 13 | "time" 14 | "utils" 15 | 16 | "github.com/pkg/errors" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/codes" 19 | "google.golang.org/grpc/credentials" 20 | "google.golang.org/grpc/metadata" 21 | "google.golang.org/grpc/status" 22 | 23 | pb "grpclb/helloword" 24 | ) 25 | 26 | var name *string = flag.String("name", "guess", "what's your name?") 27 | var reg *string = flag.String("reg", "http://127.0.0.1:2479", "register etcd address") 28 | var serv *string = flag.String("service", "chat_service", "service name") 29 | var cert_file *string = flag.String("cert", "", "cert file") 30 | var mutex sync.Mutex 31 | 32 | func ConsoleLog(message string) { 33 | mutex.Lock() 34 | defer mutex.Unlock() 35 | fmt.Printf("\n------ %s -----\n%s\n> ", time.Now(), message) 36 | } 37 | 38 | func Input(prompt string) string { 39 | fmt.Print(prompt) 40 | reader := bufio.NewReader(os.Stdin) 41 | line, _, err := reader.ReadLine() 42 | if err != nil { 43 | if err == io.EOF { 44 | return "" 45 | } else { 46 | panic(errors.Wrap(err, "Input")) 47 | } 48 | } 49 | return string(line) 50 | } 51 | 52 | type Robot struct { 53 | sync.Mutex 54 | conn *grpc.ClientConn 55 | client pb.GreeterClient 56 | chat_stream pb.Greeter_SayHelloClient 57 | ctx context.Context 58 | cancel context.CancelFunc 59 | token string 60 | } 61 | 62 | func (robot *Robot) Cancel() { 63 | robot.cancel() 64 | } 65 | 66 | func (robot *Robot) Done() <-chan struct{} { 67 | return robot.ctx.Done() 68 | } 69 | 70 | func (robot *Robot) Connect() error { 71 | robot.Lock() 72 | defer robot.Unlock() 73 | 74 | if robot.conn != nil { 75 | robot.conn.Close() 76 | } 77 | r := grpclb.NewResolver(*serv) 78 | lb := grpc.RoundRobin(r) 79 | 80 | ctx, cancel := context.WithCancel(context.Background()) 81 | robot.ctx = ctx 82 | robot.cancel = cancel 83 | 84 | options := []grpc.DialOption{ 85 | grpc.WithDecompressor(grpc.NewGZIPDecompressor()), 86 | grpc.WithCompressor(grpc.NewGZIPCompressor()), 87 | grpc.WithBalancer(lb), 88 | grpc.WithBlock(), 89 | grpc.WithTimeout(10 * time.Second), 90 | } 91 | if *cert_file != "" { 92 | creds, err := credentials.NewClientTLSFromFile(*cert_file, "duoshoubang") 93 | utils.CheckErrorPanic(err) 94 | options = append(options, grpc.WithTransportCredentials(creds)) 95 | } else { 96 | options = append(options, grpc.WithInsecure()) 97 | } 98 | conn, err := grpc.DialContext(ctx, *reg, options...) 99 | utils.CheckErrorPanic(err) 100 | 101 | if err != nil { 102 | return errors.Wrap(err, "Client Connect") 103 | } 104 | 105 | client := pb.NewGreeterClient(conn) 106 | 107 | robot.conn = conn 108 | robot.client = client 109 | robot.chat_stream = nil 110 | return nil 111 | } 112 | 113 | func (robot *Robot) GetChatStream() pb.Greeter_SayHelloClient { 114 | robot.Lock() 115 | defer robot.Unlock() 116 | if robot.chat_stream != nil { 117 | return robot.chat_stream 118 | } 119 | ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs("token", robot.token)) 120 | for { 121 | stream, err := robot.client.SayHello(ctx) 122 | if err != nil { 123 | fmt.Printf("get chat stream failed. %s", err.Error()) 124 | time.Sleep(1 * time.Second) 125 | } else { 126 | robot.chat_stream = stream 127 | break 128 | } 129 | } 130 | 131 | return robot.chat_stream 132 | } 133 | 134 | func (robot *Robot) Login(username string) error { 135 | robot.Lock() 136 | defer robot.Unlock() 137 | reply, err := robot.client.Login(context.Background(), &pb.LoginRequest{ 138 | Username: username, 139 | }) 140 | if err != nil { 141 | return errors.Wrap(err, "Login") 142 | } 143 | robot.token = reply.GetToken() 144 | return nil 145 | } 146 | 147 | func NewRobot() *Robot { 148 | robot := &Robot{} 149 | utils.CheckErrorPanic(robot.Connect()) 150 | 151 | return robot 152 | } 153 | 154 | func main() { 155 | flag.Parse() 156 | 157 | robot := NewRobot() 158 | utils.CheckErrorPanic(robot.Login(*name)) 159 | ConsoleLog("登录成功") 160 | 161 | // 监听服务端通知 162 | go func() { 163 | var ( 164 | reply *pb.HelloReply 165 | err error 166 | ) 167 | for { 168 | reply, err = robot.GetChatStream().Recv() 169 | reply_status, _ := status.FromError(err) 170 | if err != nil && reply_status.Code() == codes.Unavailable { 171 | ConsoleLog("与服务器的连接被断开, 进行重试") 172 | robot.Connect() 173 | ConsoleLog("重连成功") 174 | time.Sleep(time.Second) 175 | continue 176 | } 177 | utils.CheckErrorPanic(err) 178 | ConsoleLog(reply.Message) 179 | if reply.MessageType == pb.HelloReply_CONNECT_FAILED { 180 | log.Println("Connect failed.") 181 | robot.Cancel() 182 | break 183 | } 184 | } 185 | }() 186 | 187 | // 接受聊天信息并发送聊天内容 188 | go func() { 189 | var ( 190 | line string 191 | err error 192 | ) 193 | for { 194 | line = Input("") 195 | if line == "exit" { 196 | robot.Cancel() 197 | break 198 | } 199 | err = robot.GetChatStream().Send(&pb.HelloRequest{ 200 | Message: line, 201 | }) 202 | fmt.Print("> ") 203 | if err != nil { 204 | ConsoleLog(fmt.Sprintf("there was error sending data. %s", err.Error())) 205 | continue 206 | } 207 | } 208 | }() 209 | <-robot.Done() 210 | 211 | fmt.Println("Bye") 212 | } 213 | -------------------------------------------------------------------------------- /service/service.go: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Author : tuxpy 4 | * Email : q8886888@qq.com.com 5 | * Create time : 3/7/18 9:18 AM 6 | * Filename : service.go 7 | * Description : 8 | * 9 | * 10 | */ 11 | 12 | package main 13 | 14 | import ( 15 | "bytes" 16 | "context" 17 | "crypto/rand" 18 | "encoding/gob" 19 | "encoding/hex" 20 | "encoding/json" 21 | "flag" 22 | "fmt" 23 | grpclb "grpclb/etcdv3" 24 | pb "grpclb/helloword" 25 | "io" 26 | "log" 27 | "net" 28 | "os" 29 | "os/signal" 30 | "strings" 31 | "sync" 32 | "syscall" 33 | "time" 34 | "utils" 35 | 36 | "github.com/coreos/etcd/clientv3" 37 | "github.com/coreos/etcd/mvcc/mvccpb" 38 | "github.com/golang/protobuf/ptypes/timestamp" 39 | "github.com/pkg/errors" 40 | "google.golang.org/grpc" 41 | "google.golang.org/grpc/codes" 42 | "google.golang.org/grpc/credentials" 43 | "google.golang.org/grpc/metadata" 44 | "google.golang.org/grpc/peer" 45 | "google.golang.org/grpc/status" 46 | ) 47 | 48 | // Service implements grpclib/helloword.GreeterServer 49 | type Service struct{} 50 | 51 | // ConnectPool 管理本地的stream池. 该结构下的操作是线程安全的 52 | type ConnectPool struct { 53 | sync.Map 54 | } 55 | 56 | // RemoteCommand 一个remote channel传输的数据 57 | type RemoteCommand struct { 58 | Command string 59 | Args map[string]string 60 | } 61 | 62 | // RemoteChannel 使用etcd3的PUT和watch实现远程channel. 63 | type RemoteChannel struct { 64 | In chan RemoteCommand 65 | Out chan RemoteCommand 66 | cli *clientv3.Client 67 | } 68 | 69 | // SessionManager 利用etcd3实现的session集群化管理器 70 | type SessionManager struct { 71 | cli *clientv3.Client 72 | } 73 | 74 | // Session 将在Marshal成bytes存储为etcd3中 75 | type Session struct { 76 | Name string 77 | Token string 78 | } 79 | 80 | var connect_pool *ConnectPool 81 | var remote_channel *RemoteChannel 82 | 83 | var session_manger *SessionManager 84 | 85 | // ReadyBroadCast 将消息广播给其他service. 因为做了负载均衡，一个client stream有可能落在不同节点，需要将行为广播给所有的节点 86 | func ReadyBroadCast(from, message string) { 87 | remote_channel.Out <- RemoteCommand{ 88 | Command: "broadcast", 89 | Args: map[string]string{ 90 | "from": from, 91 | "message": message, 92 | }, 93 | } 94 | } 95 | 96 | // Get 根据token获取出一个Session 97 | // 每次获取一个Session, 就将它的ttl刷新一遍 98 | func (sm *SessionManager) Get(token string) (*Session, error) { 99 | key := fmt.Sprintf("%s/%s/session/%s", grpclb.Prefix, *srv, token) 100 | resp, err := sm.cli.Get(context.Background(), key) 101 | if err != nil { 102 | return nil, err 103 | } 104 | kv := resp.Kvs[0] 105 | session := &Session{} 106 | err = json.Unmarshal(kv.Value, session) 107 | if err != nil { 108 | return nil, errors.Wrap(err, "failed to unmarshal session data") 109 | } 110 | 111 | _, err = sm.cli.KeepAliveOnce(context.Background(), clientv3.LeaseID(kv.Lease)) 112 | utils.CheckErrorPanic(err) 113 | 114 | return session, nil 115 | } 116 | 117 | // GetFromContext 根据一个grpc的context获取出Session. 118 | // 本质上是根据ctx创建一个md, 然后获取token, 再调用SessionManager.Get 119 | func (sm *SessionManager) GetFromContext(ctx context.Context) (*Session, error) { 120 | md, _ := metadata.FromIncomingContext(ctx) 121 | tokens := md["token"] 122 | if len(tokens) == 0 { 123 | return nil, errors.New("Miss token") 124 | } 125 | 126 | return sm.Get(tokens[0]) 127 | } 128 | 129 | // New 根据用户名新建一个Session 130 | func (sm *SessionManager) New(name string) (*Session, error) { 131 | buf := make([]byte, 16) 132 | io.ReadFull(rand.Reader, buf) 133 | token := hex.EncodeToString(buf) 134 | 135 | key := fmt.Sprintf("%s/%s/session/%s", grpclb.Prefix, *srv, token) 136 | grant, err := sm.cli.Grant(context.Background(), 60*5) // token有效期5分钟 137 | if err != nil { 138 | return nil, errors.Wrap(err, "grant etcd lease ") 139 | } 140 | 141 | session := &Session{ 142 | Name: name, 143 | Token: token, 144 | } 145 | buf, err = json.Marshal(session) 146 | 147 | _, err = sm.cli.Put(context.Background(), key, string(buf), clientv3.WithLease(grant.ID)) 148 | if err != nil { 149 | return nil, errors.Wrap(err, "etcd3 put") 150 | } 151 | 152 | return session, nil 153 | } 154 | 155 | // Get 根据用户名获取出该用户的stream 156 | func (p *ConnectPool) Get(name string) pb.Greeter_SayHelloServer { 157 | if stream, ok := p.Load(name); ok { 158 | 159 | return stream.(pb.Greeter_SayHelloServer) 160 | } else { 161 | return nil 162 | } 163 | } 164 | 165 | // Add 添加一个stream 166 | func (p *ConnectPool) Add(name string, stream pb.Greeter_SayHelloServer) { 167 | p.Store(name, stream) 168 | } 169 | 170 | // Del 删除一个stream 171 | func (p *ConnectPool) Del(name string) { 172 | p.Delete(name) 173 | } 174 | 175 | // Broadcast 向连接上自己的所有客户端发送信息(除了发送者) 176 | func (p *ConnectPool) BroadCast(from, message string) { 177 | log.Printf("BroadCast from: %s, message: %s\n", from, message) 178 | p.Range(func(username_i, stream_i interface{}) bool { 179 | username := username_i.(string) 180 | stream := stream_i.(pb.Greeter_SayHelloServer) 181 | if username == from { 182 | return true 183 | 184 | } else { 185 | log.Printf("From %s to %s\n", from, username) 186 | utils.CheckErrorPanic(stream.Send(&pb.HelloReply{ 187 | Message: message, 188 | MessageType: pb.HelloReply_NORMAL_MESSAGE, 189 | TS: ×tamp.Timestamp{Seconds: time.Now().Unix()}, 190 | })) 191 | } 192 | return true 193 | }) 194 | } 195 | 196 | // Login 登录用户, 返回token给客户端, 并广播给所有已连接的客户一条欢迎的消息 197 | func (s *Service) Login(ctx context.Context, in *pb.LoginRequest) (*pb.LoginReply, error) { 198 | // 当同一个用户名在多台节点上登录了, 就会出来一个聊天室中有多个同名的用户. 199 | // 可以在etcd中以用户名为主来创建一个key, 退出时删除该key来防止用户名大多节点中重复 200 | if connect_pool.Get(in.GetUsername()) != nil { 201 | return nil, errors.Errorf("username %s already exists", in.GetUsername()) 202 | } 203 | session, err := session_manger.New(in.GetUsername()) 204 | if err != nil { 205 | return nil, err 206 | } 207 | ReadyBroadCast(in.GetUsername(), fmt.Sprintf("Welcome %s!", in.GetUsername())) 208 | 209 | return &pb.LoginReply{ 210 | Token: session.Token, 211 | Success: true, 212 | Message: "success", 213 | }, nil 214 | } 215 | 216 | // SayHello 和客户端的stream进行通信 217 | func (s *Service) SayHello(stream pb.Greeter_SayHelloServer) error { 218 | var ( 219 | session *Session 220 | err error 221 | ) 222 | 223 | stream.Context().Value("session") 224 | peer, _ := peer.FromContext(stream.Context()) 225 | log.Printf("Received new connection. %s", peer.Addr.String()) 226 | session, err = session_manger.GetFromContext(stream.Context()) // context中带有token, 取出session 227 | 228 | if err != nil { 229 | stream.Send(&pb.HelloReply{ 230 | Message: err.Error(), 231 | MessageType: pb.HelloReply_CONNECT_FAILED, 232 | }) 233 | return nil 234 | } 235 | username := session.Name 236 | 237 | connect_pool.Add(username, stream) 238 | stream.Send(&pb.HelloReply{ 239 | Message: fmt.Sprintf("Connect success!"), 240 | MessageType: pb.HelloReply_CONNECT_SUCCESS, 241 | }) // 发送连接成功的提醒 242 | go func() { 243 | <-stream.Context().Done() 244 | connect_pool.Del(username) // 用户离开聊天室时, 从连接池中删除它 245 | ReadyBroadCast(username, fmt.Sprintf("%s leval room", username)) 246 | }() 247 | for { 248 | req, err := stream.Recv() 249 | if err != nil { 250 | break 251 | } 252 | ReadyBroadCast(username, fmt.Sprintf("%s: %s", username, req.Message)) 253 | } 254 | return nil 255 | } 256 | 257 | var ( 258 | srv = flag.String("service", "chat_service", "service name") 259 | port = flag.Int("port", 8880, "listening port") 260 | reg = flag.String("reg", "http://127.0.0.1:2479", "register etcd address") 261 | cert_file = flag.String("cert", "", "cert file") 262 | key_file = flag.String("key", "", "key file") 263 | ) 264 | 265 | // GetListen 获取服务端监听的地址 266 | func GetListen() string { 267 | return fmt.Sprintf("0.0.0.0:%d", *port) 268 | } 269 | 270 | // NewSessionManager 根据etcd创建SessionManager 271 | func NewSessionManager(cli *clientv3.Client) *SessionManager { 272 | return &SessionManager{ 273 | cli: cli, 274 | } 275 | } 276 | 277 | // NewRemoteChannel 根据etcd创建remote channel 278 | func NewRemoteChannel(cli *clientv3.Client) *RemoteChannel { 279 | qc := &RemoteChannel{ 280 | cli: cli, 281 | In: make(chan RemoteCommand, 1), 282 | Out: make(chan RemoteCommand, 1), 283 | } 284 | 285 | go func() { 286 | var command RemoteCommand 287 | var channel string = fmt.Sprintf("%s/%s/channel/remote", grpclb.Prefix, *srv) 288 | var buf bytes.Buffer 289 | var err error 290 | 291 | var dec *gob.Decoder 292 | 293 | rch := qc.cli.Watch(context.Background(), channel) 294 | for wresp := range rch { 295 | for _, ev := range wresp.Events { 296 | buf.Reset() 297 | dec = gob.NewDecoder(&buf) 298 | switch ev.Type { 299 | case mvccpb.PUT: 300 | buf.Write(ev.Kv.Value) 301 | err = dec.Decode(&command) 302 | if err != nil { 303 | log.Printf("recv an error message. %s\n", err.Error()) 304 | } else { 305 | qc.In <- command 306 | } 307 | } 308 | } 309 | } 310 | }() 311 | 312 | go func() { 313 | var command RemoteCommand 314 | var channel string = fmt.Sprintf("%s/%s/channel", grpclb.Prefix, *srv) 315 | var buf bytes.Buffer 316 | var enc *gob.Encoder 317 | 318 | for { 319 | buf.Reset() 320 | enc = gob.NewEncoder(&buf) 321 | command = <-qc.Out 322 | utils.CheckErrorPanic(enc.Encode(command)) 323 | qc.cli.Put(context.Background(), 324 | channel, 325 | buf.String()) 326 | } 327 | }() 328 | 329 | return qc 330 | } 331 | 332 | // NewEtcd3Client 根据命令行参数--reg, 创建etcd3客户端 333 | func NewEtcd3Client() (*clientv3.Client, error) { 334 | cli, err := clientv3.New(clientv3.Config{ 335 | Endpoints: strings.Split(*reg, ","), 336 | }) 337 | if err != nil { 338 | return nil, errors.Wrap(err, fmt.Sprintf("Create etcd3 client failed: %s", err.Error())) 339 | } 340 | 341 | return cli, nil 342 | } 343 | 344 | func StreamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { 345 | if _, err := session_manger.GetFromContext(ss.Context()); err != nil { 346 | return status.Error(codes.Unauthenticated, 347 | fmt.Sprintf("Authenticated failed(%s)", err.Error())) 348 | } 349 | return handler(srv, ss) 350 | } 351 | 352 | func main() { 353 | var err error 354 | flag.Parse() 355 | connect_pool = &ConnectPool{} 356 | etcd_cli, err := NewEtcd3Client() 357 | 358 | utils.CheckErrorPanic(err) 359 | 360 | remote_channel = NewRemoteChannel(etcd_cli) 361 | session_manger = NewSessionManager(etcd_cli) 362 | 363 | go func() { 364 | var command RemoteCommand 365 | for command = range remote_channel.In { 366 | switch command.Command { 367 | case "broadcast": 368 | connect_pool.BroadCast(command.Args["from"], command.Args["message"]) 369 | } 370 | } 371 | }() 372 | 373 | lis, err := net.Listen("tcp", GetListen()) 374 | utils.CheckErrorPanic(err) 375 | fmt.Println("Listen on", GetListen()) 376 | 377 | err = grpclb.Register(*srv, "127.0.0.1", *port, *reg, time.Second*3, 15) // 注册当前节点到etcd 378 | utils.CheckErrorPanic(err) 379 | ch := make(chan os.Signal) 380 | signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT) 381 | go func() { 382 | s := <-ch 383 | log.Printf("receive signal '%v'\n", s) 384 | 385 | grpclb.UnRegister() // 程序被退出后，主动去unregister 386 | signal.Stop(ch) 387 | 388 | switch s := s.(type) { 389 | case syscall.Signal: 390 | syscall.Kill(os.Getpid(), s) 391 | 392 | default: 393 | os.Exit(1) 394 | } 395 | }() 396 | options := []grpc.ServerOption{ 397 | grpc.StreamInterceptor(StreamInterceptor), 398 | grpc.RPCCompressor(grpc.NewGZIPCompressor()), 399 | grpc.RPCDecompressor(grpc.NewGZIPDecompressor()), 400 | } 401 | if *cert_file != "" { 402 | creds, err := credentials.NewServerTLSFromFile(*cert_file, *key_file) 403 | if err != nil { 404 | log.Fatalf("NewServerTLSFromFile has error. %s\n", err.Error()) 405 | } 406 | options = append(options, grpc.Creds(creds)) 407 | } 408 | 409 | s := grpc.NewServer(options...) 410 | pb.RegisterGreeterServer(s, &Service{}) 411 | fmt.Println("Resiger server success") 412 | 413 | utils.CheckErrorPanic(s.Serve(lis)) 414 | } 415 | -------------------------------------------------------------------------------- /helloword/helloword.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: helloword.proto 3 | 4 | /* 5 | Package helloword is a generated protocol buffer package. 6 | 7 | It is generated from these files: 8 | helloword.proto 9 | 10 | It has these top-level messages: 11 | LoginRequest 12 | LoginReply 13 | HelloRequest 14 | HelloReply 15 | */ 16 | package helloword 17 | 18 | import proto "github.com/golang/protobuf/proto" 19 | import fmt "fmt" 20 | import math "math" 21 | import google_protobuf "github.com/golang/protobuf/ptypes/timestamp" 22 | 23 | import ( 24 | context "golang.org/x/net/context" 25 | grpc "google.golang.org/grpc" 26 | ) 27 | 28 | // Reference imports to suppress errors if they are not otherwise used. 29 | var _ = proto.Marshal 30 | var _ = fmt.Errorf 31 | var _ = math.Inf 32 | 33 | // This is a compile-time assertion to ensure that this generated file 34 | // is compatible with the proto package it is being compiled against. 35 | // A compilation error at this line likely means your copy of the 36 | // proto package needs to be updated. 37 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 38 | 39 | type HelloReply_MessageType int32 40 | 41 | const ( 42 | HelloReply_CONNECT_SUCCESS HelloReply_MessageType = 0 43 | HelloReply_CONNECT_FAILED HelloReply_MessageType = 1 44 | HelloReply_NORMAL_MESSAGE HelloReply_MessageType = 2 45 | ) 46 | 47 | var HelloReply_MessageType_name = map[int32]string{ 48 | 0: "CONNECT_SUCCESS", 49 | 1: "CONNECT_FAILED", 50 | 2: "NORMAL_MESSAGE", 51 | } 52 | var HelloReply_MessageType_value = map[string]int32{ 53 | "CONNECT_SUCCESS": 0, 54 | "CONNECT_FAILED": 1, 55 | "NORMAL_MESSAGE": 2, 56 | } 57 | 58 | func (x HelloReply_MessageType) String() string { 59 | return proto.EnumName(HelloReply_MessageType_name, int32(x)) 60 | } 61 | func (HelloReply_MessageType) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 0} } 62 | 63 | type LoginRequest struct { 64 | Username string `protobuf:"bytes,1,opt,name=username" json:"username,omitempty"` 65 | } 66 | 67 | func (m *LoginRequest) Reset() { *m = LoginRequest{} } 68 | func (m *LoginRequest) String() string { return proto.CompactTextString(m) } 69 | func (*LoginRequest) ProtoMessage() {} 70 | func (*LoginRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } 71 | 72 | func (m *LoginRequest) GetUsername() string { 73 | if m != nil { 74 | return m.Username 75 | } 76 | return "" 77 | } 78 | 79 | type LoginReply struct { 80 | Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` 81 | Success bool `protobuf:"varint,2,opt,name=success" json:"success,omitempty"` 82 | Token string `protobuf:"bytes,3,opt,name=token" json:"token,omitempty"` 83 | } 84 | 85 | func (m *LoginReply) Reset() { *m = LoginReply{} } 86 | func (m *LoginReply) String() string { return proto.CompactTextString(m) } 87 | func (*LoginReply) ProtoMessage() {} 88 | func (*LoginReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } 89 | 90 | func (m *LoginReply) GetMessage() string { 91 | if m != nil { 92 | return m.Message 93 | } 94 | return "" 95 | } 96 | 97 | func (m *LoginReply) GetSuccess() bool { 98 | if m != nil { 99 | return m.Success 100 | } 101 | return false 102 | } 103 | 104 | func (m *LoginReply) GetToken() string { 105 | if m != nil { 106 | return m.Token 107 | } 108 | return "" 109 | } 110 | 111 | type HelloRequest struct { 112 | Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` 113 | } 114 | 115 | func (m *HelloRequest) Reset() { *m = HelloRequest{} } 116 | func (m *HelloRequest) String() string { return proto.CompactTextString(m) } 117 | func (*HelloRequest) ProtoMessage() {} 118 | func (*HelloRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } 119 | 120 | func (m *HelloRequest) GetMessage() string { 121 | if m != nil { 122 | return m.Message 123 | } 124 | return "" 125 | } 126 | 127 | type HelloReply struct { 128 | Message string `protobuf:"bytes,1,opt,name=message" json:"message,omitempty"` 129 | TS *google_protobuf.Timestamp `protobuf:"bytes,2,opt,name=TS" json:"TS,omitempty"` 130 | MessageType HelloReply_MessageType `protobuf:"varint,3,opt,name=message_type,json=messageType,enum=helloword.HelloReply_MessageType" json:"message_type,omitempty"` 131 | } 132 | 133 | func (m *HelloReply) Reset() { *m = HelloReply{} } 134 | func (m *HelloReply) String() string { return proto.CompactTextString(m) } 135 | func (*HelloReply) ProtoMessage() {} 136 | func (*HelloReply) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } 137 | 138 | func (m *HelloReply) GetMessage() string { 139 | if m != nil { 140 | return m.Message 141 | } 142 | return "" 143 | } 144 | 145 | func (m *HelloReply) GetTS() *google_protobuf.Timestamp { 146 | if m != nil { 147 | return m.TS 148 | } 149 | return nil 150 | } 151 | 152 | func (m *HelloReply) GetMessageType() HelloReply_MessageType { 153 | if m != nil { 154 | return m.MessageType 155 | } 156 | return HelloReply_CONNECT_SUCCESS 157 | } 158 | 159 | func init() { 160 | proto.RegisterType((*LoginRequest)(nil), "helloword.LoginRequest") 161 | proto.RegisterType((*LoginReply)(nil), "helloword.LoginReply") 162 | proto.RegisterType((*HelloRequest)(nil), "helloword.HelloRequest") 163 | proto.RegisterType((*HelloReply)(nil), "helloword.HelloReply") 164 | proto.RegisterEnum("helloword.HelloReply_MessageType", HelloReply_MessageType_name, HelloReply_MessageType_value) 165 | } 166 | 167 | // Reference imports to suppress errors if they are not otherwise used. 168 | var _ context.Context 169 | var _ grpc.ClientConn 170 | 171 | // This is a compile-time assertion to ensure that this generated file 172 | // is compatible with the grpc package it is being compiled against. 173 | const _ = grpc.SupportPackageIsVersion4 174 | 175 | // Client API for Greeter service 176 | 177 | type GreeterClient interface { 178 | Login(ctx context.Context, in *LoginRequest, opts ...grpc.CallOption) (*LoginReply, error) 179 | SayHello(ctx context.Context, opts ...grpc.CallOption) (Greeter_SayHelloClient, error) 180 | } 181 | 182 | type greeterClient struct { 183 | cc *grpc.ClientConn 184 | } 185 | 186 | func NewGreeterClient(cc *grpc.ClientConn) GreeterClient { 187 | return &greeterClient{cc} 188 | } 189 | 190 | func (c *greeterClient) Login(ctx context.Context, in *LoginRequest, opts ...grpc.CallOption) (*LoginReply, error) { 191 | out := new(LoginReply) 192 | err := grpc.Invoke(ctx, "/helloword.Greeter/Login", in, out, c.cc, opts...) 193 | if err != nil { 194 | return nil, err 195 | } 196 | return out, nil 197 | } 198 | 199 | func (c *greeterClient) SayHello(ctx context.Context, opts ...grpc.CallOption) (Greeter_SayHelloClient, error) { 200 | stream, err := grpc.NewClientStream(ctx, &_Greeter_serviceDesc.Streams[0], c.cc, "/helloword.Greeter/SayHello", opts...) 201 | if err != nil { 202 | return nil, err 203 | } 204 | x := &greeterSayHelloClient{stream} 205 | return x, nil 206 | } 207 | 208 | type Greeter_SayHelloClient interface { 209 | Send(*HelloRequest) error 210 | Recv() (*HelloReply, error) 211 | grpc.ClientStream 212 | } 213 | 214 | type greeterSayHelloClient struct { 215 | grpc.ClientStream 216 | } 217 | 218 | func (x *greeterSayHelloClient) Send(m *HelloRequest) error { 219 | return x.ClientStream.SendMsg(m) 220 | } 221 | 222 | func (x *greeterSayHelloClient) Recv() (*HelloReply, error) { 223 | m := new(HelloReply) 224 | if err := x.ClientStream.RecvMsg(m); err != nil { 225 | return nil, err 226 | } 227 | return m, nil 228 | } 229 | 230 | // Server API for Greeter service 231 | 232 | type GreeterServer interface { 233 | Login(context.Context, *LoginRequest) (*LoginReply, error) 234 | SayHello(Greeter_SayHelloServer) error 235 | } 236 | 237 | func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) { 238 | s.RegisterService(&_Greeter_serviceDesc, srv) 239 | } 240 | 241 | func _Greeter_Login_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 242 | in := new(LoginRequest) 243 | if err := dec(in); err != nil { 244 | return nil, err 245 | } 246 | if interceptor == nil { 247 | return srv.(GreeterServer).Login(ctx, in) 248 | } 249 | info := &grpc.UnaryServerInfo{ 250 | Server: srv, 251 | FullMethod: "/helloword.Greeter/Login", 252 | } 253 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 254 | return srv.(GreeterServer).Login(ctx, req.(*LoginRequest)) 255 | } 256 | return interceptor(ctx, in, info, handler) 257 | } 258 | 259 | func _Greeter_SayHello_Handler(srv interface{}, stream grpc.ServerStream) error { 260 | return srv.(GreeterServer).SayHello(&greeterSayHelloServer{stream}) 261 | } 262 | 263 | type Greeter_SayHelloServer interface { 264 | Send(*HelloReply) error 265 | Recv() (*HelloRequest, error) 266 | grpc.ServerStream 267 | } 268 | 269 | type greeterSayHelloServer struct { 270 | grpc.ServerStream 271 | } 272 | 273 | func (x *greeterSayHelloServer) Send(m *HelloReply) error { 274 | return x.ServerStream.SendMsg(m) 275 | } 276 | 277 | func (x *greeterSayHelloServer) Recv() (*HelloRequest, error) { 278 | m := new(HelloRequest) 279 | if err := x.ServerStream.RecvMsg(m); err != nil { 280 | return nil, err 281 | } 282 | return m, nil 283 | } 284 | 285 | var _Greeter_serviceDesc = grpc.ServiceDesc{ 286 | ServiceName: "helloword.Greeter", 287 | HandlerType: (*GreeterServer)(nil), 288 | Methods: []grpc.MethodDesc{ 289 | { 290 | MethodName: "Login", 291 | Handler: _Greeter_Login_Handler, 292 | }, 293 | }, 294 | Streams: []grpc.StreamDesc{ 295 | { 296 | StreamName: "SayHello", 297 | Handler: _Greeter_SayHello_Handler, 298 | ServerStreams: true, 299 | ClientStreams: true, 300 | }, 301 | }, 302 | Metadata: "helloword.proto", 303 | } 304 | 305 | func init() { proto.RegisterFile("helloword.proto", fileDescriptor0) } 306 | 307 | var fileDescriptor0 = []byte{ 308 | // 372 bytes of a gzipped FileDescriptorProto 309 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x51, 0x4d, 0x6f, 0xaa, 0x40, 310 | 0x14, 0x75, 0x78, 0xf1, 0xa9, 0x57, 0xa3, 0x66, 0xde, 0x7b, 0x79, 0x84, 0x95, 0x65, 0x45, 0x5c, 311 | 0x40, 0x63, 0x57, 0x4d, 0x37, 0x35, 0x48, 0x6d, 0x1b, 0x3f, 0x12, 0x86, 0x76, 0x6b, 0xd0, 0x4e, 312 | 0xd1, 0x14, 0x18, 0xca, 0x0c, 0x69, 0xf8, 0x03, 0xfd, 0xa5, 0xfd, 0x21, 0x0d, 0x28, 0x82, 0x69, 313 | 0xda, 0xdd, 0x9c, 0x7b, 0xcf, 0xb9, 0xf7, 0xdc, 0x33, 0xd0, 0xdb, 0x52, 0xdf, 0x67, 0x6f, 0x2c, 314 | 0x7e, 0xd2, 0xa3, 0x98, 0x09, 0x86, 0x5b, 0xc7, 0x82, 0x72, 0xe5, 0xed, 0xc4, 0x36, 0x59, 0xeb, 315 | 0x1b, 0x16, 0x18, 0x1e, 0xf3, 0xdd, 0xd0, 0x33, 0x72, 0xce, 0x3a, 0x79, 0x36, 0x22, 0x91, 0x46, 316 | 0x94, 0x1b, 0x62, 0x17, 0x50, 0x2e, 0xdc, 0x20, 0x2a, 0x5f, 0xfb, 0x39, 0xea, 0x10, 0x3a, 0x33, 317 | 0xe6, 0xed, 0x42, 0x9b, 0xbe, 0x26, 0x94, 0x0b, 0xac, 0x40, 0x33, 0xe1, 0x34, 0x0e, 0xdd, 0x80, 318 | 0xca, 0x68, 0x80, 0xb4, 0x96, 0x7d, 0xc4, 0xea, 0x23, 0xc0, 0x81, 0x1b, 0xf9, 0x29, 0x96, 0xa1, 319 | 0x11, 0x50, 0xce, 0x5d, 0xaf, 0x20, 0x16, 0x30, 0xeb, 0xf0, 0x64, 0xb3, 0xa1, 0x9c, 0xcb, 0xd2, 320 | 0x00, 0x69, 0x4d, 0xbb, 0x80, 0xf8, 0x2f, 0xd4, 0x05, 0x7b, 0xa1, 0xa1, 0xfc, 0x2b, 0x57, 0xec, 321 | 0x81, 0xaa, 0x41, 0xe7, 0x36, 0xbb, 0xa6, 0xf0, 0xf0, 0xed, 0x64, 0xf5, 0x03, 0x01, 0x1c, 0xa8, 322 | 0x3f, 0x5b, 0x18, 0x82, 0xe4, 0x90, 0x7c, 0x7b, 0x7b, 0xa4, 0xe8, 0x1e, 0x63, 0x9e, 0x4f, 0xf5, 323 | 0x22, 0x15, 0xdd, 0x29, 0x42, 0xb0, 0x25, 0x87, 0xe0, 0x09, 0x74, 0x0e, 0xb2, 0x55, 0x96, 0x56, 324 | 0xee, 0xad, 0x3b, 0x3a, 0xd3, 0xcb, 0xc8, 0xcb, 0x95, 0xfa, 0x7c, 0xcf, 0x74, 0xd2, 0x88, 0xda, 325 | 0xed, 0xa0, 0x04, 0xea, 0x3d, 0xb4, 0x2b, 0x3d, 0xfc, 0x07, 0x7a, 0xe6, 0x72, 0xb1, 0xb0, 0x4c, 326 | 0x67, 0x45, 0x1e, 0x4c, 0xd3, 0x22, 0xa4, 0x5f, 0xc3, 0x18, 0xba, 0x45, 0xf1, 0x66, 0x7c, 0x37, 327 | 0xb3, 0x26, 0x7d, 0x94, 0xd5, 0x16, 0x4b, 0x7b, 0x3e, 0x9e, 0xad, 0xe6, 0x16, 0x21, 0xe3, 0xa9, 328 | 0xd5, 0x97, 0x46, 0xef, 0x08, 0x1a, 0xd3, 0x98, 0x52, 0x41, 0x63, 0x7c, 0x09, 0xf5, 0x3c, 0x74, 329 | 0xfc, 0xbf, 0x62, 0xa8, 0xfa, 0x65, 0xca, 0xbf, 0xaf, 0x8d, 0xc8, 0x4f, 0xd5, 0x1a, 0xbe, 0x86, 330 | 0x26, 0x71, 0xd3, 0xdc, 0xfc, 0x89, 0xba, 0x1a, 0xf6, 0x89, 0xba, 0xbc, 0x53, 0xad, 0x69, 0xe8, 331 | 0x1c, 0xad, 0x7f, 0xe7, 0x91, 0x5d, 0x7c, 0x06, 0x00, 0x00, 0xff, 0xff, 0x51, 0x16, 0x3b, 0x49, 332 | 0x7f, 0x02, 0x00, 0x00, 333 | } 334 | --------------------------------------------------------------------------------