├── .gitignore ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── tools ├── README.md ├── cache │ ├── README.md │ ├── images │ │ └── class-tuopu.png │ ├── informer_sample │ │ └── informer.go │ └── sample │ │ └── workqueue.go ├── event-record │ ├── README.md │ ├── event.go │ ├── images │ │ └── archtect.png │ └── record.go ├── leader-election │ ├── README.md │ └── leader-election.go ├── port-forward │ ├── README.md │ └── port-forward.go └── rest-client │ ├── README.md │ └── client-pod.go ├── transport └── README.md └── util ├── README.md ├── certs ├── README.md ├── cert_test.go ├── images │ ├── create-cert.png │ └── create-csr.png ├── key_test.go ├── server │ └── server.go └── testdata │ ├── ca-key.pem │ └── ca.pem ├── flowcontrol ├── README.md ├── backoff_test.go ├── retry_test.go └── throttle_test.go ├── jsonpath ├── README.md ├── jsonpath_test.go └── testdata │ └── pod.yaml └── workqueue ├── README.md ├── clock.md ├── cond.md ├── default-rate-limiter.md ├── delaying-queue.md ├── parallelizer.md ├── queue.md ├── rate-limiting-queue.md ├── rate.md └── rate_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # client-go 2 | Kubernetes client go 源码阅读 3 | 4 | ## Utils 5 | - [util](/util/README.md) 6 | 7 | ## transport 8 | - [transport](/transport/README.md) 9 | 10 | 11 | ## Tools 12 | - [cache](/tools/cache/README.md) ListWatcher, SharedIndexInformer,Controller 13 | - [rest-client](/tools/rest-client/README.md) rest-config rest-client resouce create/updated/delete 14 | - [event-record](/tools/event-record/README.md) event broadcaster, sinker ,recorder 15 | - [leader-election](/tools/leader-election/README.md) leader election 16 | - [port-forward](/tools/port-forward/README.md) 17 | 18 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/weenxin/client-go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/gin-gonic/gin v1.7.7 7 | github.com/go-logr/logr v1.2.0 // indirect 8 | github.com/gogo/protobuf v1.3.2 // indirect 9 | github.com/google/gofuzz v1.1.0 // indirect 10 | github.com/json-iterator/go v1.1.12 // indirect 11 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 12 | github.com/modern-go/reflect2 v1.0.2 // indirect 13 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect 14 | golang.org/x/text v0.3.7 // indirect 15 | golang.org/x/time v0.0.0-20220411224347-583f2d630306 16 | gopkg.in/inf.v0 v0.9.1 // indirect 17 | gopkg.in/yaml.v2 v2.4.0 // indirect 18 | k8s.io/apimachinery v0.21.0 19 | k8s.io/client-go v0.21.0 20 | k8s.io/klog/v2 v2.60.1 21 | k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 22 | sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect 23 | ) 24 | 25 | require ( 26 | go.uber.org/zap v1.21.0 27 | k8s.io/api v0.21.0 28 | k8s.io/klog v1.0.0 29 | ) 30 | 31 | require ( 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/gin-contrib/sse v0.1.0 // indirect 34 | github.com/go-playground/locales v0.13.0 // indirect 35 | github.com/go-playground/universal-translator v0.17.0 // indirect 36 | github.com/go-playground/validator/v10 v10.4.1 // indirect 37 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 38 | github.com/golang/protobuf v1.5.2 // indirect 39 | github.com/google/gnostic v0.5.7-v3refs // indirect 40 | github.com/google/go-cmp v0.5.5 // indirect 41 | github.com/googleapis/gnostic v0.4.1 // indirect 42 | github.com/hashicorp/golang-lru v0.5.1 // indirect 43 | github.com/imdario/mergo v0.3.5 // indirect 44 | github.com/leodido/go-urn v1.2.0 // indirect 45 | github.com/mattn/go-isatty v0.0.12 // indirect 46 | github.com/moby/spdystream v0.2.0 // indirect 47 | github.com/spf13/pflag v1.0.5 // indirect 48 | github.com/ugorji/go/codec v1.1.7 // indirect 49 | go.uber.org/atomic v1.7.0 // indirect 50 | go.uber.org/multierr v1.6.0 // indirect 51 | golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect 52 | golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect 53 | golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect 54 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect 55 | google.golang.org/appengine v1.6.7 // indirect 56 | google.golang.org/protobuf v1.27.1 // indirect 57 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 58 | k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect 59 | sigs.k8s.io/yaml v1.2.0 // indirect 60 | ) 61 | -------------------------------------------------------------------------------- /tools/README.md: -------------------------------------------------------------------------------- 1 | # Tools 2 | 3 | - [Cache](/tools/cache/README.md) 4 | - [rest-client](/tools/rest-client/README.md) 5 | - [event-record](/tools/event-record/README.md) 6 | - [leader-election](/tools/leader-election/README.md) 7 | - [port-ward](/tools/port-forward/README.md) 8 | -------------------------------------------------------------------------------- /tools/cache/images/class-tuopu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weenxin/client-go/c7db1f7d9a182fd5b9b3d303e10705ffa5834c3f/tools/cache/images/class-tuopu.png -------------------------------------------------------------------------------- /tools/cache/informer_sample/informer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | v1 "k8s.io/api/core/v1" 6 | "k8s.io/apimachinery/pkg/api/meta" 7 | "k8s.io/apimachinery/pkg/fields" 8 | "k8s.io/client-go/kubernetes" 9 | "k8s.io/client-go/tools/cache" 10 | "k8s.io/client-go/tools/clientcmd" 11 | "k8s.io/klog/v2" 12 | "os" 13 | "os/signal" 14 | "syscall" 15 | "time" 16 | ) 17 | 18 | type ActionHandler struct{ 19 | } 20 | 21 | func (handler *ActionHandler) OnAdd(obj interface{}) { 22 | accessor, err := meta.Accessor(obj) 23 | if err != nil { 24 | panic(err.Error()) 25 | } 26 | fmt.Printf("object added : %s, %s \n ", accessor.GetName() , informer.LastSyncResourceVersion()) 27 | } 28 | func (handler *ActionHandler) OnUpdate(oldObj, newObj interface{}) { 29 | accessor, err := meta.Accessor(newObj) 30 | if err != nil { 31 | panic(err.Error()) 32 | } 33 | fmt.Printf("object updated : %s, %s \n ", accessor.GetName() , informer.LastSyncResourceVersion()) 34 | } 35 | 36 | func (handler *ActionHandler) OnDelete(obj interface{}) { 37 | accessor, err := meta.Accessor(obj) 38 | if err != nil { 39 | panic(err.Error()) 40 | } 41 | fmt.Printf("object deleted : %s, %s \n ", accessor.GetName() , informer.LastSyncResourceVersion()) 42 | } 43 | 44 | 45 | var informer cache.Controller 46 | 47 | 48 | func main() { 49 | // creates the connection 50 | config, err := clientcmd.BuildConfigFromFlags("", "/Users/weenxin/.kube/config") 51 | if err != nil { 52 | klog.Fatal(err) 53 | } 54 | 55 | // creates the clientset 56 | clientset, err := kubernetes.NewForConfig(config) 57 | if err != nil { 58 | klog.Fatal(err) 59 | } 60 | 61 | // create the pod watcher 62 | lw := cache.NewListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", v1.NamespaceDefault, fields.Everything()) 63 | 64 | _, informer = cache.NewInformer(lw,&v1.Pod{},time.Minute,&ActionHandler{}) 65 | stopInformer := make(chan struct{} ) 66 | go informer.Run(stopInformer) 67 | 68 | 69 | fmt.Printf("informer begined") 70 | 71 | stopMain := make(chan os.Signal,1) 72 | 73 | signal.Notify(stopMain,syscall.SIGHUP,syscall.SIGINT,syscall.SIGQUIT) 74 | 75 | <- stopMain 76 | 77 | } 78 | -------------------------------------------------------------------------------- /tools/cache/sample/workqueue.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "time" 23 | 24 | "k8s.io/klog/v2" 25 | 26 | v1 "k8s.io/api/core/v1" 27 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/fields" 29 | "k8s.io/apimachinery/pkg/util/runtime" 30 | "k8s.io/apimachinery/pkg/util/wait" 31 | "k8s.io/client-go/kubernetes" 32 | "k8s.io/client-go/tools/cache" 33 | "k8s.io/client-go/tools/clientcmd" 34 | "k8s.io/client-go/util/workqueue" 35 | ) 36 | 37 | // Controller demonstrates how to implement a controller with client-go. 38 | type Controller struct { 39 | indexer cache.Indexer 40 | queue workqueue.RateLimitingInterface 41 | informer cache.Controller 42 | } 43 | 44 | // NewController creates a new Controller. 45 | func NewController(queue workqueue.RateLimitingInterface, indexer cache.Indexer, informer cache.Controller) *Controller { 46 | return &Controller{ 47 | informer: informer, 48 | indexer: indexer, 49 | queue: queue, 50 | } 51 | } 52 | 53 | func (c *Controller) processNextItem() bool { 54 | // Wait until there is a new item in the working queue 55 | key, quit := c.queue.Get() 56 | if quit { 57 | return false 58 | } 59 | // Tell the queue that we are done with processing this key. This unblocks the key for other workers 60 | // This allows safe parallel processing because two pods with the same key are never processed in 61 | // parallel. 62 | defer c.queue.Done(key) 63 | 64 | // Invoke the method containing the business logic 65 | err := c.syncToStdout(key.(string)) 66 | // Handle the error if something went wrong during the execution of the business logic 67 | c.handleErr(err, key) 68 | return true 69 | } 70 | 71 | // syncToStdout is the business logic of the controller. In this controller it simply prints 72 | // information about the pod to stdout. In case an error happened, it has to simply return the error. 73 | // The retry logic should not be part of the business logic. 74 | func (c *Controller) syncToStdout(key string) error { 75 | obj, exists, err := c.indexer.GetByKey(key) 76 | if err != nil { 77 | klog.Errorf("Fetching object with key %s from store failed with %v", key, err) 78 | return err 79 | } 80 | 81 | if !exists { 82 | // Below we will warm up our cache with a Pod, so that we will see a delete for one pod 83 | fmt.Printf("Pod %s does not exist anymore\n", key) 84 | } else { 85 | // Note that you also have to check the uid if you have a local controlled resource, which 86 | // is dependent on the actual instance, to detect that a Pod was recreated with the same name 87 | fmt.Printf("Sync/Add/Update for Pod %s\n", obj.(*v1.Pod).GetName()) 88 | } 89 | return nil 90 | } 91 | 92 | // handleErr checks if an error happened and makes sure we will retry later. 93 | func (c *Controller) handleErr(err error, key interface{}) { 94 | if err == nil { 95 | // Forget about the #AddRateLimited history of the key on every successful synchronization. 96 | // This ensures that future processing of updates for this key is not delayed because of 97 | // an outdated error history. 98 | c.queue.Forget(key) 99 | return 100 | } 101 | 102 | // This controller retries 5 times if something goes wrong. After that, it stops trying. 103 | if c.queue.NumRequeues(key) < 5 { 104 | klog.Infof("Error syncing pod %v: %v", key, err) 105 | 106 | // Re-enqueue the key rate limited. Based on the rate limiter on the 107 | // queue and the re-enqueue history, the key will be processed later again. 108 | c.queue.AddRateLimited(key) 109 | return 110 | } 111 | 112 | c.queue.Forget(key) 113 | // Report to an external entity that, even after several retries, we could not successfully process this key 114 | runtime.HandleError(err) 115 | klog.Infof("Dropping pod %q out of the queue: %v", key, err) 116 | } 117 | 118 | // Run begins watching and syncing. 119 | func (c *Controller) Run(workers int, stopCh chan struct{}) { 120 | defer runtime.HandleCrash() 121 | 122 | // Let the workers stop when we are done 123 | defer c.queue.ShutDown() 124 | klog.Info("Starting Pod controller") 125 | 126 | go c.informer.Run(stopCh) 127 | 128 | // Wait for all involved caches to be synced, before processing items from the queue is started 129 | if !cache.WaitForCacheSync(stopCh, c.informer.HasSynced) { 130 | runtime.HandleError(fmt.Errorf("Timed out waiting for caches to sync")) 131 | return 132 | } 133 | 134 | for i := 0; i < workers; i++ { 135 | go wait.Until(c.runWorker, time.Second, stopCh) 136 | } 137 | 138 | <-stopCh 139 | klog.Info("Stopping Pod controller") 140 | } 141 | 142 | func (c *Controller) runWorker() { 143 | for c.processNextItem() { 144 | } 145 | } 146 | 147 | func main() { 148 | var kubeconfig string 149 | var master string 150 | 151 | flag.StringVar(&kubeconfig, "kubeconfig", "", "absolute path to the kubeconfig file") 152 | flag.StringVar(&master, "master", "", "master url") 153 | flag.Parse() 154 | 155 | // creates the connection 156 | config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig) 157 | if err != nil { 158 | klog.Fatal(err) 159 | } 160 | 161 | // creates the clientset 162 | clientset, err := kubernetes.NewForConfig(config) 163 | if err != nil { 164 | klog.Fatal(err) 165 | } 166 | 167 | // create the pod watcher 168 | podListWatcher := cache.NewListWatchFromClient(clientset.CoreV1().RESTClient(), "pods", v1.NamespaceDefault, fields.Everything()) 169 | 170 | // create the workqueue 171 | queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) 172 | 173 | // Bind the workqueue to a cache with the help of an informer. This way we make sure that 174 | // whenever the cache is updated, the pod key is added to the workqueue. 175 | // Note that when we finally process the item from the workqueue, we might see a newer version 176 | // of the Pod than the version which was responsible for triggering the update. 177 | indexer, informer := cache.NewIndexerInformer(podListWatcher, &v1.Pod{}, 0, cache.ResourceEventHandlerFuncs{ 178 | AddFunc: func(obj interface{}) { 179 | key, err := cache.MetaNamespaceKeyFunc(obj) 180 | if err == nil { 181 | queue.Add(key) 182 | } 183 | }, 184 | UpdateFunc: func(old interface{}, new interface{}) { 185 | key, err := cache.MetaNamespaceKeyFunc(new) 186 | if err == nil { 187 | queue.Add(key) 188 | } 189 | }, 190 | DeleteFunc: func(obj interface{}) { 191 | // IndexerInformer uses a delta queue, therefore for deletes we have to use this 192 | // key function. 193 | key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) 194 | if err == nil { 195 | queue.Add(key) 196 | } 197 | }, 198 | }, cache.Indexers{}) 199 | 200 | controller := NewController(queue, indexer, informer) 201 | 202 | // We can now warm up the cache for initial synchronization. 203 | // Let's suppose that we knew about a pod "mypod" on our last run, therefore add it to the cache. 204 | // If this pod is not there anymore, the controller will be notified about the removal after the 205 | // cache has synchronized. 206 | indexer.Add(&v1.Pod{ 207 | ObjectMeta: meta_v1.ObjectMeta{ 208 | Name: "mypod", 209 | Namespace: v1.NamespaceDefault, 210 | }, 211 | }) 212 | 213 | // Now let's start the controller 214 | stop := make(chan struct{}) 215 | defer close(stop) 216 | go controller.Run(1, stop) 217 | 218 | // Wait forever 219 | select {} 220 | } -------------------------------------------------------------------------------- /tools/event-record/README.md: -------------------------------------------------------------------------------- 1 | # Event-Recorder 2 | 3 | 记录对象发生的各类事件。 4 | 5 | [blog](https://www.cncf.io/blog/2021/12/21/extracting-value-from-the-kubernetes-events-feed/) 6 | 7 | ## 架构图 8 | 9 | ![架构图](/tools/event-record/images/archtect.png) 10 | 11 | ## 使用方式 12 | 13 | [record.go](/tools/event-record/record.go) 14 | 15 | 16 | - 创建rest-client 17 | - 基于rest-client，建立一个`EventBroadCaster` 18 | - `events.NewEventBroadcasterAdapter(client)` 19 | - 开启EventBroadcaster消息处理循环； 20 | - `go eventBroadcast.StartRecordingToSink(closeCh)` 21 | - 基于 `EventBroadCaster`创建一个`Record` 22 | - `eventBroadcast.NewRecorder("test")` 23 | - 基于`Record`开始记录事件 24 | `- `r.Eventf(pod, pod, v1.EventTypeNormal, "test event record", "add", "")` 25 | - 关闭Broadcaster 26 | - `eventBroadcast.Shutdown()` 27 | 28 | ## tools/record 包代码分解 29 | 30 | ### EventBroadcaster 31 | EventBroadcaster 是event处理的中间组件，可以介于一个`EventSinker`开始一个EventBroadcaster。基于同一个`EventBroadcaster`创建的`Recorder`可以在任何时间记录事件。Broadcaster负责将事件消费并存储到`Sinker`中。 32 | 33 | EventBroadcaster的接口如下所示： 34 | 35 | ```go 36 | // EventBroadcaster knows how to receive events and send them to any EventSink, watcher, or log. 37 | type EventBroadcaster interface { 38 | // StartEventWatcher starts sending events received from this EventBroadcaster to the given 39 | // event handler function. The return value can be ignored or used to stop recording, if 40 | // desired. 41 | //添加一个事件监听器 42 | StartEventWatcher(eventHandler func(*v1.Event)) watch.Interface 43 | 44 | // StartRecordingToSink starts sending events received from this EventBroadcaster to the given 45 | // sink. The return value can be ignored or used to stop recording, if desired. 46 | // 开始将数据写入到sinker中 47 | StartRecordingToSink(sink EventSink) watch.Interface 48 | 49 | // StartLogging starts sending events received from this EventBroadcaster to the given logging 50 | // function. The return value can be ignored or used to stop recording, if desired. 51 | //将事件写入到日志中 52 | StartLogging(logf func(format string, args ...interface{})) watch.Interface 53 | 54 | // StartStructuredLogging starts sending events received from this EventBroadcaster to the structured 55 | // logging function. The return value can be ignored or used to stop recording, if desired. 56 | // 按照structed形式记录事件 57 | StartStructuredLogging(verbosity klog.Level) watch.Interface 58 | 59 | // NewRecorder returns an EventRecorder that can be used to send events to this EventBroadcaster 60 | // with the event source set to the given event source. 61 | // 新建一个Recorder，后续基于此Recorder记录的事件都会写入到存储中 62 | NewRecorder(scheme *runtime.Scheme, source v1.EventSource) EventRecorder 63 | 64 | // Shutdown shuts down the broadcaster 65 | // 结束 66 | Shutdown() 67 | } 68 | ``` 69 | 70 | 具体的实现如下所示： 71 | 72 | ```go 73 | type eventBroadcasterImpl struct { 74 | *watch.Broadcaster // 事件广播器，recorder会将数据写入到这里，eventBroadercaster会开启单独的监听协程完成数据处理 75 | sleepDuration time.Duration //每次发送到sinker后失败需要等待的时间 76 | options CorrelatorOptions // 聚合与改正相关的参数 77 | } 78 | ``` 79 | 80 | ```go 81 | // StartEventWatcher starts sending events received from this EventBroadcaster to the given event handler function. 82 | // The return value can be ignored or used to stop recording, if desired. 83 | // 开始一个监听器 84 | func (e *eventBroadcasterImpl) StartEventWatcher(eventHandler func(*v1.Event)) watch.Interface { 85 | //新建一个监听器，这样底层的broadcater之间接收到数据后，就可以通知此watcher 86 | watcher := e.Watch() 87 | go func() { 88 | defer utilruntime.HandleCrash() 89 | for watchEvent := range watcher.ResultChan() { 90 | event, ok := watchEvent.Object.(*v1.Event) 91 | if !ok { 92 | // This is all local, so there's no reason this should 93 | // ever happen. 94 | continue 95 | } 96 | eventHandler(event) 97 | } 98 | }() 99 | return watcher 100 | } 101 | ``` 102 | 103 | 104 | ```go 105 | // StartRecordingToSink starts sending events received from the specified eventBroadcaster to the given sink. 106 | // The return value can be ignored or used to stop recording, if desired. 107 | // TODO: make me an object with parameterizable queue length and retry interval 108 | // 开始向Sinker中写入 109 | func (e *eventBroadcasterImpl) StartRecordingToSink(sink EventSink) watch.Interface { 110 | // 新建一个改写器 111 | eventCorrelator := NewEventCorrelatorWithOptions(e.options) 112 | return e.StartEventWatcher( 113 | func(event *v1.Event) { 114 | // 将数据写入到sinker中 115 | recordToSink(sink, event, eventCorrelator, e.sleepDuration) 116 | }) 117 | } 118 | ``` 119 | 120 | ```go 121 | func recordToSink(sink EventSink, event *v1.Event, eventCorrelator *EventCorrelator, sleepDuration time.Duration) { 122 | // Make a copy before modification, because there could be multiple listeners. 123 | // Events are safe to copy like this. 124 | eventCopy := *event 125 | event = &eventCopy 126 | // 基于新建的corrector处理下数据 127 | result, err := eventCorrelator.EventCorrelate(event) 128 | if err != nil { 129 | utilruntime.HandleError(err) 130 | } 131 | // 如果需要跳过，则直接跳过 132 | if result.Skip { 133 | return 134 | } 135 | tries := 0 136 | for { 137 | //开始写如数据到sinker 138 | if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) { 139 | //如果成功则直接返回 140 | break 141 | } 142 | //重试 143 | tries++ 144 | if tries >= maxTriesPerEvent { 145 | klog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) 146 | break 147 | } 148 | // Randomize the first sleep so that various clients won't all be 149 | // synced up if the master goes down. 150 | //第一次增加一些随机事件，这样大部分APIServer访问不会同时发生 151 | if tries == 1 { 152 | time.Sleep(time.Duration(float64(sleepDuration) * rand.Float64())) 153 | } else { 154 | time.Sleep(sleepDuration) 155 | } 156 | } 157 | } 158 | ``` 159 | 160 | 161 | ```go 162 | // recordEvent attempts to write event to a sink. It returns true if the event 163 | // was successfully recorded or discarded, false if it should be retried. 164 | // If updateExistingEvent is false, it creates a new event, otherwise it updates 165 | // existing event. 166 | // 尝试写数据到sinker中 167 | func recordEvent(sink EventSink, event *v1.Event, patch []byte, updateExistingEvent bool, eventCorrelator *EventCorrelator) bool { 168 | var newEvent *v1.Event 169 | var err error 170 | // 似乎是对于eventSeries的处理，但是没有找到series的打印入口 171 | if updateExistingEvent { 172 | newEvent, err = sink.Patch(event, patch) 173 | } 174 | // Update can fail because the event may have been removed and it no longer exists. 175 | if !updateExistingEvent || (updateExistingEvent && util.IsKeyNotFoundError(err)) { 176 | //如果不是更新操作，或者更新过程中发现错误，则新建 177 | // Making sure that ResourceVersion is empty on creation 178 | event.ResourceVersion = "" 179 | newEvent, err = sink.Create(event) 180 | } 181 | if err == nil { 182 | // we need to update our event correlator with the server returned state to handle name/resourceversion 183 | //否则更新 184 | eventCorrelator.UpdateState(newEvent) 185 | return true 186 | } 187 | 188 | // If we can't contact the server, then hold everything while we keep trying. 189 | // Otherwise, something about the event is malformed and we should abandon it. 190 | switch err.(type) { 191 | case *restclient.RequestConstructionError: 192 | // We will construct the request the same next time, so don't keep trying. 193 | klog.Errorf("Unable to construct event '%#v': '%v' (will not retry!)", event, err) 194 | return true 195 | case *errors.StatusError: 196 | if errors.IsAlreadyExists(err) { 197 | klog.V(5).Infof("Server rejected event '%#v': '%v' (will not retry!)", event, err) 198 | } else { 199 | klog.Errorf("Server rejected event '%#v': '%v' (will not retry!)", event, err) 200 | } 201 | return true 202 | case *errors.UnexpectedObjectError: 203 | // We don't expect this; it implies the server's response didn't match a 204 | // known pattern. Go ahead and retry. 205 | default: 206 | // This case includes actual http transport errors. Go ahead and retry. 207 | } 208 | klog.Errorf("Unable to write event: '%#v': '%v'(may retry after sleeping)", event, err) 209 | return false 210 | } 211 | ``` 212 | 213 | 214 | ### Recorder 215 | 216 | ```go 217 | //recorder 实现 218 | type recorderImpl struct { 219 | scheme *runtime.Scheme //scheme 用于解析某个对象GVR等信息 220 | source v1.EventSource // 事件的创造者 221 | *watch.Broadcaster //将消息发送到Broadcaster 222 | clock clock.Clock //时钟 223 | } 224 | 225 | //创建事件 226 | func (recorder *recorderImpl) generateEvent(object runtime.Object, annotations map[string]string, eventtype, reason, message string) { 227 | //创建索引对象 228 | ref, err := ref.GetReference(recorder.scheme, object) 229 | if err != nil { 230 | klog.Errorf("Could not construct reference to: '%#v' due to: '%v'. Will not report event: '%v' '%v' '%v'", object, err, eventtype, reason, message) 231 | return 232 | } 233 | 234 | // 只能是warninng或者normal 235 | if !util.ValidateEventType(eventtype) { 236 | klog.Errorf("Unsupported event type: '%v'", eventtype) 237 | return 238 | } 239 | 240 | //新建一个对象 241 | event := recorder.makeEvent(ref, annotations, eventtype, reason, message) 242 | //设置来源 243 | event.Source = recorder.source 244 | 245 | // NOTE: events should be a non-blocking operation, but we also need to not 246 | // put this in a goroutine, otherwise we'll race to write to a closed channel 247 | // when we go to shut down this broadcaster. Just drop events if we get overloaded, 248 | // and log an error if that happens (we've configured the broadcaster to drop 249 | // outgoing events anyway). 250 | //发送或者丢弃事件，调用底层broadcaster方法，触发整理逻辑 251 | if sent := recorder.ActionOrDrop(watch.Added, event); !sent { 252 | klog.Errorf("unable to record event: too many queued events, dropped event %#v", event) 253 | } 254 | } 255 | 256 | //发送一个事件 257 | func (recorder *recorderImpl) Event(object runtime.Object, eventtype, reason, message string) { 258 | recorder.generateEvent(object, nil, eventtype, reason, message) 259 | } 260 | 261 | func (recorder *recorderImpl) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { 262 | recorder.Event(object, eventtype, reason, fmt.Sprintf(messageFmt, args...)) 263 | } 264 | 265 | func (recorder *recorderImpl) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { 266 | recorder.generateEvent(object, annotations, eventtype, reason, fmt.Sprintf(messageFmt, args...)) 267 | } 268 | 269 | //创建一个事件对象 270 | func (recorder *recorderImpl) makeEvent(ref *v1.ObjectReference, annotations map[string]string, eventtype, reason, message string) *v1.Event { 271 | t := metav1.Time{Time: recorder.clock.Now()} 272 | namespace := ref.Namespace 273 | if namespace == "" { 274 | namespace = metav1.NamespaceDefault 275 | } 276 | return &v1.Event{ 277 | ObjectMeta: metav1.ObjectMeta{ 278 | Name: fmt.Sprintf("%v.%x", ref.Name, t.UnixNano()), 279 | Namespace: namespace, 280 | Annotations: annotations, 281 | }, 282 | InvolvedObject: *ref, 283 | Reason: reason, 284 | Message: message, 285 | FirstTimestamp: t, 286 | LastTimestamp: t, 287 | Count: 1, 288 | Type: eventtype, 289 | } 290 | } 291 | ``` 292 | 293 | 294 | ### Sinker 295 | 296 | ```go 297 | // EventSink knows how to store events (client.Client implements it.) 298 | // EventSink must respect the namespace that will be embedded in 'event'. 299 | // It is assumed that EventSink will return the same sorts of errors as 300 | // pkg/client's REST client. 301 | // 事件存储接口，可创建，可以更新，可删除 302 | type EventSink interface { 303 | Create(event *v1.Event) (*v1.Event, error) 304 | Update(event *v1.Event) (*v1.Event, error) 305 | Patch(oldEvent *v1.Event, data []byte) (*v1.Event, error) 306 | } 307 | ``` 308 | 309 | 310 | 可以基于`k8s.io/client-go/kubernetes/typed/core/v1`包的`EventSinkImpl`的实现 311 | 312 | ```go 313 | sinker := typedv1core.EventSinkImpl{Interface: client.CoreV1().Events("")} 314 | ``` 315 | 316 | 317 | ### EventCorrelator 318 | 319 | 可以对Event做聚合或者做限流。 320 | 321 | #### 限流器的实现 322 | 323 | ```go 324 | 325 | // getSpamKey builds unique event key based on source, involvedObject 326 | // 限流器的Key 327 | func getSpamKey(event *v1.Event) string { 328 | return strings.Join([]string{ 329 | event.Source.Component, 330 | event.Source.Host, 331 | event.InvolvedObject.Kind, 332 | event.InvolvedObject.Namespace, 333 | event.InvolvedObject.Name, 334 | string(event.InvolvedObject.UID), 335 | event.InvolvedObject.APIVersion, 336 | }, 337 | "") 338 | } 339 | 340 | // EventFilterFunc is a function that returns true if the event should be skipped 341 | type EventFilterFunc func(event *v1.Event) bool 342 | 343 | // EventSourceObjectSpamFilter is responsible for throttling 344 | // the amount of events a source and object can produce. 345 | //具体实现 346 | type EventSourceObjectSpamFilter struct { 347 | sync.RWMutex 348 | 349 | // the cache that manages last synced state 350 | // cache 351 | cache *lru.Cache 352 | 353 | // burst is the amount of events we allow per source + object 354 | burst int 355 | 356 | // qps is the refill rate of the token bucket in queries per second 357 | qps float32 358 | 359 | // clock is used to allow for testing over a time interval 360 | clock clock.Clock 361 | } 362 | 363 | // NewEventSourceObjectSpamFilter allows burst events from a source about an object with the specified qps refill. 364 | func NewEventSourceObjectSpamFilter(lruCacheSize, burst int, qps float32, clock clock.Clock) *EventSourceObjectSpamFilter { 365 | return &EventSourceObjectSpamFilter{ 366 | cache: lru.New(lruCacheSize), 367 | burst: burst, 368 | qps: qps, 369 | clock: clock, 370 | } 371 | } 372 | 373 | // spamRecord holds data used to perform spam filtering decisions. 374 | type spamRecord struct { 375 | // rateLimiter controls the rate of events about this object 376 | rateLimiter flowcontrol.RateLimiter 377 | } 378 | 379 | // Filter controls that a given source+object are not exceeding the allowed rate. 380 | func (f *EventSourceObjectSpamFilter) Filter(event *v1.Event) bool { 381 | var record spamRecord 382 | 383 | // controls our cached information about this event (source+object) 384 | // 拿到key 385 | eventKey := getSpamKey(event) 386 | 387 | // do we have a record of similar events in our cache? 388 | f.Lock() 389 | defer f.Unlock() 390 | value, found := f.cache.Get(eventKey) 391 | if found { 392 | record = value.(spamRecord) 393 | } 394 | 395 | // verify we have a rate limiter for this record 396 | if record.rateLimiter == nil { 397 | //令牌桶限流 398 | record.rateLimiter = flowcontrol.NewTokenBucketRateLimiterWithClock(f.qps, f.burst, f.clock) 399 | } 400 | 401 | // ensure we have available rate 402 | filter := !record.rateLimiter.TryAccept() 403 | 404 | // update the cache 405 | f.cache.Add(eventKey, record) 406 | 407 | return filter 408 | } 409 | ``` 410 | 411 | #### 聚合器 412 | 413 | ```go 414 | 415 | // EventAggregatorKeyFunc is responsible for grouping events for aggregation 416 | // It returns a tuple of the following: 417 | // aggregateKey - key the identifies the aggregate group to bucket this event 418 | // localKey - key that makes this event in the local group 419 | type EventAggregatorKeyFunc func(event *v1.Event) (aggregateKey string, localKey string) 420 | 421 | // EventAggregatorByReasonFunc aggregates events by exact match on event.Source, event.InvolvedObject, event.Type, 422 | // event.Reason, event.ReportingController and event.ReportingInstance 423 | //搞一个聚合key 424 | func EventAggregatorByReasonFunc(event *v1.Event) (string, string) { 425 | return strings.Join([]string{ 426 | event.Source.Component, 427 | event.Source.Host, 428 | event.InvolvedObject.Kind, 429 | event.InvolvedObject.Namespace, 430 | event.InvolvedObject.Name, 431 | string(event.InvolvedObject.UID), 432 | event.InvolvedObject.APIVersion, 433 | event.Type, 434 | event.Reason, 435 | event.ReportingController, 436 | event.ReportingInstance, 437 | }, 438 | ""), event.Message 439 | } 440 | 441 | // EventAggregatorMessageFunc is responsible for producing an aggregation message 442 | type EventAggregatorMessageFunc func(event *v1.Event) string 443 | 444 | // EventAggregratorByReasonMessageFunc returns an aggregate message by prefixing the incoming message 445 | // 聚合之后使用的消息 446 | func EventAggregatorByReasonMessageFunc(event *v1.Event) string { 447 | return "(combined from similar events): " + event.Message 448 | } 449 | 450 | // EventAggregator identifies similar events and aggregates them into a single event 451 | type EventAggregator struct { 452 | sync.RWMutex 453 | 454 | // The cache that manages aggregation state 455 | // cache中存储record 456 | cache *lru.Cache 457 | 458 | // The function that groups events for aggregation 459 | // 返回2个string，第一个作为cache外层的key，第二个作为record内部的key 460 | keyFunc EventAggregatorKeyFunc 461 | 462 | // The function that generates a message for an aggregate event 463 | // 当信息被聚合后，应该如何显示信息 464 | messageFunc EventAggregatorMessageFunc 465 | 466 | // The maximum number of events in the specified interval before aggregation occurs 467 | // record中多于多少个events时会聚合 468 | maxEvents uint 469 | 470 | // The amount of time in seconds that must transpire since the last occurrence of a similar event before it's considered new 471 | // event最大保存时间 472 | maxIntervalInSeconds uint 473 | 474 | // clock is used to allow for testing over a time interval 475 | clock clock.Clock 476 | } 477 | 478 | // NewEventAggregator returns a new instance of an EventAggregator 479 | func NewEventAggregator(lruCacheSize int, keyFunc EventAggregatorKeyFunc, messageFunc EventAggregatorMessageFunc, 480 | maxEvents int, maxIntervalInSeconds int, clock clock.Clock) *EventAggregator { 481 | return &EventAggregator{ 482 | cache: lru.New(lruCacheSize), //存储对象多少 483 | keyFunc: keyFunc,// 聚合key，localkey的产生方法 484 | messageFunc: messageFunc, // 当消息需要聚合时,聚合信息产生方法 485 | maxEvents: uint(maxEvents), //单个record event最多数量 486 | maxIntervalInSeconds: uint(maxIntervalInSeconds), // 最大事件缓存事件 487 | clock: clock, 488 | } 489 | } 490 | 491 | // aggregateRecord holds data used to perform aggregation decisions 492 | type aggregateRecord struct { 493 | // we track the number of unique local keys we have seen in the aggregate set to know when to actually aggregate 494 | // if the size of this set exceeds the max, we know we need to aggregate 495 | // record内部的多个local key 496 | localKeys sets.String 497 | // The last time at which the aggregate was recorded 498 | lastTimestamp metav1.Time 499 | } 500 | 501 | // EventAggregate checks if a similar event has been seen according to the 502 | // aggregation configuration (max events, max interval, etc) and returns: 503 | // 504 | // - The (potentially modified) event that should be created 505 | // - The cache key for the event, for correlation purposes. This will be set to 506 | // the full key for normal events, and to the result of 507 | // EventAggregatorMessageFunc for aggregate events. 508 | // 聚合方法 509 | func (e *EventAggregator) EventAggregate(newEvent *v1.Event) (*v1.Event, string) { 510 | now := metav1.NewTime(e.clock.Now()) 511 | var record aggregateRecord 512 | // eventKey is the full cache key for this event 513 | // 一个event的全部信息key，包括message和其他元数据 514 | eventKey := getEventKey(newEvent) 515 | // aggregateKey is for the aggregate event, if one is needed. 516 | //默认情况下aggregateKey 不包含 message，localKey为message，也就是说按照除了message的元数据做record分类，基于message做record内记录 517 | aggregateKey, localKey := e.keyFunc(newEvent) 518 | 519 | // Do we have a record of similar events in our cache? 520 | e.Lock() 521 | defer e.Unlock() 522 | //从缓存中获取记录 523 | value, found := e.cache.Get(aggregateKey) 524 | if found { 525 | record = value.(aggregateRecord) 526 | } 527 | 528 | // Is the previous record too old? If so, make a fresh one. Note: if we didn't 529 | // find a similar record, its lastTimestamp will be the zero value, so we 530 | // create a new one in that case. 531 | maxInterval := time.Duration(e.maxIntervalInSeconds) * time.Second 532 | interval := now.Time.Sub(record.lastTimestamp.Time) 533 | // 如果事件太长了，就当没有任何event发生过 534 | if interval > maxInterval { 535 | record = aggregateRecord{localKeys: sets.NewString()} 536 | } 537 | 538 | // Write the new event into the aggregation record and put it on the cache 539 | //加入一条localkey 540 | record.localKeys.Insert(localKey) 541 | // 更新时间 542 | record.lastTimestamp = now 543 | // 增加记录 544 | e.cache.Add(aggregateKey, record) 545 | 546 | // If we are not yet over the threshold for unique events, don't correlate them 547 | // 如果单个record中的localKey数量没有限流，则正常增加 548 | if uint(record.localKeys.Len()) < e.maxEvents { 549 | // eventKey为圈梁数据 550 | return newEvent, eventKey 551 | } 552 | 553 | // do not grow our local key set any larger than max 554 | // 删除任何一个,因为刚刚添加了一个，所以不会影响限流 555 | record.localKeys.PopAny() 556 | 557 | // create a new aggregate event, and return the aggregateKey as the cache key 558 | // (so that it can be overwritten.) 559 | // 新建一个对象 560 | eventCopy := &v1.Event{ 561 | ObjectMeta: metav1.ObjectMeta{ 562 | Name: fmt.Sprintf("%v.%x", newEvent.InvolvedObject.Name, now.UnixNano()), 563 | Namespace: newEvent.Namespace, 564 | }, 565 | Count: 1, // 将数值设置为1，表示后期需要不断更新操作 566 | FirstTimestamp: now, 567 | InvolvedObject: newEvent.InvolvedObject, 568 | LastTimestamp: now, 569 | Message: e.messageFunc(newEvent), 570 | Type: newEvent.Type, 571 | Reason: newEvent.Reason, 572 | Source: newEvent.Source, 573 | } 574 | // 返回，copyEvent,并返回聚合的key，这样可以和其他限流的对象公用一个事件，也就是后面会走更新操作 575 | return eventCopy, aggregateKey 576 | } 577 | ``` 578 | 579 | 580 | #### eventLogger 581 | 582 | 记录已经发生的事件，查看事件是否存在，如果已经存在了则尽力更新事件。 583 | 584 | ```go 585 | 586 | // eventLogger logs occurrences of an event 587 | type eventLogger struct { 588 | sync.RWMutex 589 | cache *lru.Cache 590 | clock clock.Clock 591 | } 592 | 593 | // newEventLogger observes events and counts their frequencies 594 | func newEventLogger(lruCacheEntries int, clock clock.Clock) *eventLogger { 595 | return &eventLogger{cache: lru.New(lruCacheEntries), clock: clock} 596 | } 597 | 598 | // eventObserve records an event, or updates an existing one if key is a cache hit 599 | // 查看key是否存在，如果已经存在，并且count>0 则生成patch需要的body 600 | func (e *eventLogger) eventObserve(newEvent *v1.Event, key string) (*v1.Event, []byte, error) { 601 | var ( 602 | patch []byte 603 | err error 604 | ) 605 | eventCopy := *newEvent 606 | event := &eventCopy 607 | 608 | e.Lock() 609 | defer e.Unlock() 610 | 611 | // Check if there is an existing event we should update 612 | // 获得记录 613 | lastObservation := e.lastEventObservationFromCache(key) 614 | 615 | // If we found a result, prepare a patch 616 | // 大于0, 则更新 617 | if lastObservation.count > 0 { 618 | // update the event based on the last observation so patch will work as desired 619 | event.Name = lastObservation.name 620 | event.ResourceVersion = lastObservation.resourceVersion 621 | event.FirstTimestamp = lastObservation.firstTimestamp 622 | event.Count = int32(lastObservation.count) + 1 623 | 624 | eventCopy2 := *event 625 | eventCopy2.Count = 0 626 | eventCopy2.LastTimestamp = metav1.NewTime(time.Unix(0, 0)) 627 | eventCopy2.Message = "" 628 | 629 | newData, _ := json.Marshal(event) 630 | oldData, _ := json.Marshal(eventCopy2) 631 | patch, err = strategicpatch.CreateTwoWayMergePatch(oldData, newData, event) 632 | } 633 | 634 | // record our new observation 635 | // 由于Record内的部分限流key，长时间不存在，增加过程会将老对象删除 636 | e.cache.Add( 637 | key, 638 | eventLog{ 639 | count: uint(event.Count), 640 | firstTimestamp: event.FirstTimestamp, 641 | name: event.Name, 642 | resourceVersion: event.ResourceVersion, 643 | }, 644 | ) 645 | return event, patch, err 646 | } 647 | 648 | // updateState updates its internal tracking information based on latest server state 649 | // 更新最新事件 650 | func (e *eventLogger) updateState(event *v1.Event) { 651 | key := getEventKey(event) 652 | e.Lock() 653 | defer e.Unlock() 654 | // record our new observation 655 | e.cache.Add( 656 | key, 657 | eventLog{ 658 | count: uint(event.Count), 659 | firstTimestamp: event.FirstTimestamp, 660 | name: event.Name, 661 | resourceVersion: event.ResourceVersion, 662 | }, 663 | ) 664 | } 665 | 666 | // lastEventObservationFromCache returns the event from the cache, reads must be protected via external lock 667 | func (e *eventLogger) lastEventObservationFromCache(key string) eventLog { 668 | value, ok := e.cache.Get(key) 669 | if ok { 670 | observationValue, ok := value.(eventLog) 671 | if ok { 672 | return observationValue 673 | } 674 | } 675 | return eventLog{} 676 | } 677 | 678 | ``` 679 | 680 | #### EventCorrelator 681 | 682 | ```go 683 | func NewEventCorrelator(clock clock.Clock) *EventCorrelator { 684 | cacheSize := maxLruCacheEntries 685 | spamFilter := NewEventSourceObjectSpamFilter(cacheSize, defaultSpamBurst, defaultSpamQPS, clock) 686 | return &EventCorrelator{ 687 | // 基于限流器做限流 688 | filterFunc: spamFilter.Filter, 689 | // 聚合器 690 | aggregator: NewEventAggregator( 691 | cacheSize, 692 | EventAggregatorByReasonFunc, 693 | EventAggregatorByReasonMessageFunc, 694 | defaultAggregateMaxEvents, 695 | defaultAggregateIntervalInSeconds, 696 | clock), 697 | // eventlogger 698 | logger: newEventLogger(cacheSize, clock), 699 | } 700 | } 701 | 702 | func NewEventCorrelatorWithOptions(options CorrelatorOptions) *EventCorrelator { 703 | optionsWithDefaults := populateDefaults(options) 704 | spamFilter := NewEventSourceObjectSpamFilter(optionsWithDefaults.LRUCacheSize, 705 | optionsWithDefaults.BurstSize, optionsWithDefaults.QPS, optionsWithDefaults.Clock) 706 | return &EventCorrelator{ 707 | filterFunc: spamFilter.Filter, 708 | aggregator: NewEventAggregator( 709 | optionsWithDefaults.LRUCacheSize, 710 | optionsWithDefaults.KeyFunc, 711 | optionsWithDefaults.MessageFunc, 712 | optionsWithDefaults.MaxEvents, 713 | optionsWithDefaults.MaxIntervalInSeconds, 714 | optionsWithDefaults.Clock), 715 | logger: newEventLogger(optionsWithDefaults.LRUCacheSize, optionsWithDefaults.Clock), 716 | } 717 | } 718 | 719 | // populateDefaults populates the zero value options with defaults 720 | // 增加默认值 721 | func populateDefaults(options CorrelatorOptions) CorrelatorOptions { 722 | if options.LRUCacheSize == 0 { 723 | options.LRUCacheSize = maxLruCacheEntries 724 | } 725 | if options.BurstSize == 0 { 726 | options.BurstSize = defaultSpamBurst 727 | } 728 | if options.QPS == 0 { 729 | options.QPS = defaultSpamQPS 730 | } 731 | if options.KeyFunc == nil { 732 | options.KeyFunc = EventAggregatorByReasonFunc 733 | } 734 | if options.MessageFunc == nil { 735 | options.MessageFunc = EventAggregatorByReasonMessageFunc 736 | } 737 | if options.MaxEvents == 0 { 738 | options.MaxEvents = defaultAggregateMaxEvents 739 | } 740 | if options.MaxIntervalInSeconds == 0 { 741 | options.MaxIntervalInSeconds = defaultAggregateIntervalInSeconds 742 | } 743 | if options.Clock == nil { 744 | options.Clock = clock.RealClock{} 745 | } 746 | return options 747 | } 748 | 749 | // EventCorrelate filters, aggregates, counts, and de-duplicates all incoming events 750 | func (c *EventCorrelator) EventCorrelate(newEvent *v1.Event) (*EventCorrelateResult, error) { 751 | if newEvent == nil { 752 | return nil, fmt.Errorf("event is nil") 753 | } 754 | // 拿到聚合key aggregateEvent, 755 | //如果没有发生限流，则aggregateEvent所谓原始event，ckey为event全部字段生成的key 756 | //如果生成限流，则aggregateEvent为聚合厚度额event，key为aggregateKey 757 | aggregateEvent, ckey := c.aggregator.EventAggregate(newEvent) 758 | // 查看事件是否存在，并且是否是聚合后的key，如果是聚合后的key，则生成patch需要更新的数据 759 | observedEvent, patch, err := c.logger.eventObserve(aggregateEvent, ckey) 760 | if c.filterFunc(observedEvent) { 761 | return &EventCorrelateResult{Skip: true}, nil 762 | } 763 | return &EventCorrelateResult{Event: observedEvent, Patch: patch}, err 764 | } 765 | ``` 766 | 767 | 768 | 769 | 770 | 771 | 772 | -------------------------------------------------------------------------------- /tools/event-record/event.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "go.uber.org/zap" 7 | v1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/api/errors" 9 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/client-go/kubernetes" 11 | "k8s.io/client-go/kubernetes/scheme" 12 | "k8s.io/client-go/tools/clientcmd" 13 | "k8s.io/client-go/tools/events" 14 | "time" 15 | ) 16 | 17 | const PodName = "test-lifecycle-pod" 18 | const ImageName = "busybox" 19 | const ServiceName = "ServiceName" 20 | const ContainerName = "busybox" 21 | const DefaultNamespace = "default" 22 | 23 | var terminationGracePeriod = int64(0) 24 | 25 | func main() { 26 | logger := zap.NewExample().Sugar() 27 | 28 | config, err := clientcmd.BuildConfigFromFlags("", "/Users/yulongxin/.kube/config") 29 | if err != nil { 30 | logger.Panicw("create rest-config failed", "err", err) 31 | } 32 | 33 | client, err := kubernetes.NewForConfig(config) 34 | if err != nil { 35 | logger.Panicw("create rest client failed", "errr", err) 36 | } 37 | 38 | sinker := events.EventSinkImpl{Interface: client.EventsV1()} 39 | 40 | //新建一个eventbroadcast 41 | eventBroadcast := events.NewBroadcaster(&sinker) 42 | closeCh := make(chan struct{}) 43 | // 开启eventboadcast，EventRecorder新建的信息会唤醒EventBroadcaster的go-routine，go-routine会消费信息，发送给EventSinker进而存储到kubernetes 44 | go eventBroadcast.StartRecordingToSink(closeCh) 45 | //新建一个record 46 | r := eventBroadcast.NewRecorder(scheme.Scheme, "test") 47 | pod, err := createPod(client) 48 | 49 | if err != nil { 50 | logger.Panicw("craete pod failed", "error", err) 51 | } 52 | logger.Infow("pod details", "name", pod.Name) 53 | //新建一个event，会到broadcaster的queue中，进而唤醒所有watch的go-routine，进而将信息发送到sinker中，存储到kubernetes 54 | r.Eventf(pod, pod, v1.EventTypeNormal, "test event record", "add", "message added %v", 1) 55 | r.Eventf(pod, pod, v1.EventTypeNormal, "test event record", "add", "message added %v", 2) 56 | 57 | //等到流程完成 58 | time.Sleep(time.Second) 59 | //关闭broadcster 60 | eventBroadcast.Shutdown() 61 | 62 | } 63 | 64 | func createPod(client *kubernetes.Clientset) (*v1.Pod, error) { 65 | pod, err := client.CoreV1().Pods(DefaultNamespace).Create(context.Background(), &v1.Pod{ 66 | ObjectMeta: meta_v1.ObjectMeta{ 67 | Name: PodName, 68 | Labels: map[string]string{ServiceName: PodName}, 69 | }, 70 | Spec: v1.PodSpec{ 71 | Containers: []v1.Container{ 72 | { 73 | Name: ContainerName, 74 | Image: ImageName, 75 | Command: []string{"/bin/sh", "-c", "while :; do date +%s ; sleep 1 ; done"}, 76 | TTY: true, 77 | }, 78 | }, 79 | TerminationGracePeriodSeconds: &terminationGracePeriod, 80 | }, 81 | }, meta_v1.CreateOptions{}) 82 | 83 | if err != nil && errors.IsAlreadyExists(err) { 84 | pod, err = client.CoreV1().Pods(DefaultNamespace).Get(context.Background(), PodName, meta_v1.GetOptions{}) 85 | if err != nil { 86 | return nil, err 87 | } 88 | fmt.Println(pod.Name) 89 | return pod, nil 90 | } 91 | return pod, err 92 | } 93 | -------------------------------------------------------------------------------- /tools/event-record/images/archtect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weenxin/client-go/c7db1f7d9a182fd5b9b3d303e10705ffa5834c3f/tools/event-record/images/archtect.png -------------------------------------------------------------------------------- /tools/event-record/record.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "go.uber.org/zap" 7 | v1 "k8s.io/api/core/v1" 8 | "k8s.io/apimachinery/pkg/api/errors" 9 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/client-go/kubernetes" 11 | "k8s.io/client-go/kubernetes/scheme" 12 | typedv1core "k8s.io/client-go/kubernetes/typed/core/v1" 13 | "k8s.io/client-go/tools/clientcmd" 14 | "k8s.io/client-go/tools/record" 15 | "time" 16 | ) 17 | 18 | const PodName = "test-lifecycle-pod" 19 | const ImageName = "busybox" 20 | const ServiceName = "ServiceName" 21 | const ContainerName = "busybox" 22 | const DefaultNamespace = "default" 23 | 24 | var terminationGracePeriod = int64(0) 25 | 26 | func main() { 27 | 28 | logger := zap.NewExample().Sugar() 29 | 30 | config, err := clientcmd.BuildConfigFromFlags("", "/Users/yulongxin/.kube/config") 31 | if err != nil { 32 | logger.Panicw("create rest-config failed", "err", err) 33 | } 34 | 35 | client, err := kubernetes.NewForConfig(config) 36 | if err != nil { 37 | logger.Panicw("create rest client failed", "errr", err) 38 | } 39 | 40 | //新建sinker 41 | sinker := typedv1core.EventSinkImpl{Interface: client.CoreV1().Events("")} 42 | 43 | // 创建修改器，event最多1条，模拟合并操作 44 | option := record.CorrelatorOptions{ 45 | MaxEvents: 2, 46 | } 47 | //创建broadcaster 48 | broadcaster := record.NewBroadcasterWithCorrelatorOptions(option) 49 | 50 | //写到APISserver 51 | go broadcaster.StartRecordingToSink(&sinker) 52 | // 写到日志 53 | go broadcaster.StartLogging(logger.Infof) 54 | // 事件监听 55 | go broadcaster.StartEventWatcher(func(event *v1.Event) { 56 | logger.Infow("watch message ", "event", event) 57 | }) 58 | 59 | // 增加一个Recorder 60 | recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{ 61 | Component: "test", 62 | Host: "weenxin", 63 | }) 64 | 65 | // 创建一个pod 66 | pod, err := createOrGetPod(client) 67 | 68 | if err != nil { 69 | logger.Panicw("craete pod failed", "error", err) 70 | } 71 | logger.Infow("pod details", "name", pod.Name) 72 | //新建一个event，会到broadcaster的queue中，进而唤醒所有watch的go-routine，进而将信息发送到sinker中，存储到kubernetes 73 | recorder.Eventf(pod, v1.EventTypeNormal, "test event record", "add", "message added %v", 1) 74 | //新建一个event，会到broadcaster的queue中，进而唤醒所有watch的go-routine，进而将信息发送到sinker中，存储到kubernetes 75 | recorder.Eventf(pod, v1.EventTypeNormal, "test event record", "add", "message added %v", 2) 76 | 77 | //等到流程完成 78 | time.Sleep(time.Second) 79 | //关闭broadcster 80 | broadcaster.Shutdown() 81 | 82 | } 83 | 84 | func createOrGetPod(client *kubernetes.Clientset) (*v1.Pod, error) { 85 | pod, err := client.CoreV1().Pods(DefaultNamespace).Create(context.Background(), &v1.Pod{ 86 | ObjectMeta: meta_v1.ObjectMeta{ 87 | Name: PodName, 88 | Labels: map[string]string{ServiceName: PodName}, 89 | }, 90 | Spec: v1.PodSpec{ 91 | Containers: []v1.Container{ 92 | { 93 | Name: ContainerName, 94 | Image: ImageName, 95 | Command: []string{"/bin/sh", "-c", "while :; do date +%s ; sleep 1 ; done"}, 96 | TTY: true, 97 | }, 98 | }, 99 | TerminationGracePeriodSeconds: &terminationGracePeriod, 100 | }, 101 | }, meta_v1.CreateOptions{}) 102 | 103 | if err != nil && errors.IsAlreadyExists(err) { 104 | pod, err = client.CoreV1().Pods(DefaultNamespace).Get(context.Background(), PodName, meta_v1.GetOptions{}) 105 | if err != nil { 106 | return nil, err 107 | } 108 | fmt.Println(pod.Name) 109 | return pod, nil 110 | } 111 | return pod, err 112 | } 113 | -------------------------------------------------------------------------------- /tools/leader-election/README.md: -------------------------------------------------------------------------------- 1 | # Leader-Election 2 | 3 | Kubernetes中大部分组件都是高可用的，比如Scheduler，Control-Manager等，这些组件实现高可用的方式是，一个节点作为Leader，其他组件作为备份节点。工作时，只有Leader在工作，备份组件以为在等待获取Leader身份，保证系统的高可用。 4 | 5 | client-go中有关于Leader-Election的相关代码。 6 | 7 | 实现逻辑为： 8 | - 定义一种锁定资源，支持Lease，ConfigMap，Endpoint 9 | - Leader节点，定时刷新租期，通过`Get` -> `Update` 最后时间的方式，持续获得租期 10 | - Slave节点，定时获取`Get`资源，如果Leader节点租期过了，就会`Update`抢占资源（多个Slave抢占节点，由于ResourceVersion的问题，只能有一个成功） 11 | 12 | 13 | ## 如何使用 14 | 15 | [leader-election.go](/tools/leader-election/leader-election.go) 16 | 17 | ## 代码实现 18 | 19 | 20 | ### 资源锁 21 | 22 | 23 | #### 定义 24 | 25 | ```go 26 | type LeaderElectionRecord struct { 27 | // HolderIdentity is the ID that owns the lease. If empty, no one owns this lease and 28 | // all callers may acquire. Versions of this library prior to Kubernetes 1.14 will not 29 | // attempt to acquire leases with empty identities and will wait for the full lease 30 | // interval to expire before attempting to reacquire. This value is set to empty when 31 | // a client voluntarily steps down. 32 | // Leader节点的标识 33 | HolderIdentity string `json:"holderIdentity"` 34 | //租期长度 35 | LeaseDurationSeconds int `json:"leaseDurationSeconds"` 36 | //获取时间 37 | AcquireTime metav1.Time `json:"acquireTime"` 38 | // 刷新时间 39 | RenewTime metav1.Time `json:"renewTime"` 40 | // Leader转换了几次 41 | LeaderTransitions int `json:"leaderTransitions"` 42 | } 43 | type Interface interface { 44 | // Get returns the LeaderElectionRecord 45 | // 获取资源信息, 并且设置资源的最新ResouceVersion 46 | Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) 47 | 48 | // Create attempts to create a LeaderElectionRecord 49 | // 创建对象 50 | Create(ctx context.Context, ler LeaderElectionRecord) error 51 | 52 | // Update will update and existing LeaderElectionRecord 53 | // 更新对象，基于ler做Spec的生成，但是ResourceVersion使用的是之前获取和缓存的对象 54 | Update(ctx context.Context, ler LeaderElectionRecord) error 55 | 56 | // RecordEvent is used to record events 57 | // 记录事件 58 | RecordEvent(string) 59 | 60 | // Identity will return the locks Identity 61 | // 节点的唯一身份标识 62 | Identity() string 63 | 64 | // Describe is used to convert details on current resource lock 65 | // into a string 66 | // 描述信息 67 | Describe() string 68 | } 69 | ``` 70 | 71 | #### 实现 72 | 73 | ```go 74 | type ResourceLockConfig struct { 75 | // Identity is the unique string identifying a lease holder across 76 | // all participants in an election. 77 | Identity string 78 | // EventRecorder is optional. 79 | EventRecorder EventRecorder 80 | 81 | type LeaseLock struct { 82 | // LeaseMeta should contain a Name and a Namespace of a 83 | // LeaseMeta object that the LeaderElector will attempt to lead. 84 | // Lease的Meta信息 85 | LeaseMeta metav1.ObjectMeta 86 | //添加更新使用的client 87 | Client coordinationv1client.LeasesGetter 88 | //当前节点的身份标识 89 | LockConfig ResourceLockConfig 90 | //缓存的Lease对象，当服务器的ResouceVersion与当前对象的ResouceVersion不一致时，会拒绝更新操作 91 | lease *coordinationv1.Lease 92 | } 93 | ``` 94 | 95 | 96 | #### 新建操作 97 | 98 | ```go 99 | 100 | // Manufacture will create a lock of a given type according to the input parameters 101 | func New(lockType string, ns string, name string, coreClient corev1.CoreV1Interface, coordinationClient coordinationv1.CoordinationV1Interface, rlc ResourceLockConfig) (Interface, error) { 102 | endpointsLock := &EndpointsLock{ 103 | EndpointsMeta: metav1.ObjectMeta{ 104 | Namespace: ns, 105 | Name: name, 106 | }, 107 | Client: coreClient, 108 | LockConfig: rlc, 109 | } 110 | configmapLock := &ConfigMapLock{ 111 | ConfigMapMeta: metav1.ObjectMeta{ 112 | Namespace: ns, 113 | Name: name, 114 | }, 115 | Client: coreClient, 116 | LockConfig: rlc, 117 | } 118 | leaseLock := &LeaseLock{ 119 | LeaseMeta: metav1.ObjectMeta{ 120 | Namespace: ns, 121 | Name: name, 122 | }, 123 | Client: coordinationClient, 124 | LockConfig: rlc, 125 | } 126 | switch lockType { 127 | case EndpointsResourceLock: 128 | return endpointsLock, nil 129 | case ConfigMapsResourceLock: 130 | return configmapLock, nil 131 | case LeasesResourceLock: 132 | return leaseLock, nil 133 | case EndpointsLeasesResourceLock: 134 | return &MultiLock{ 135 | Primary: endpointsLock, 136 | Secondary: leaseLock, 137 | }, nil 138 | case ConfigMapsLeasesResourceLock: 139 | return &MultiLock{ 140 | Primary: configmapLock, 141 | Secondary: leaseLock, 142 | }, nil 143 | default: 144 | return nil, fmt.Errorf("Invalid lock-type %s", lockType) 145 | } 146 | } 147 | 148 | // NewFromKubeconfig will create a lock of a given type according to the input parameters. 149 | // Timeout set for a client used to contact to Kubernetes should be lower than 150 | // RenewDeadline to keep a single hung request from forcing a leader loss. 151 | // Setting it to max(time.Second, RenewDeadline/2) as a reasonable heuristic. 152 | func NewFromKubeconfig(lockType string, ns string, name string, rlc ResourceLockConfig, kubeconfig *restclient.Config, renewDeadline time.Duration) (Interface, error) { 153 | // shallow copy, do not modify the kubeconfig 154 | config := *kubeconfig 155 | timeout := renewDeadline / 2 156 | if timeout < time.Second { 157 | timeout = time.Second 158 | } 159 | config.Timeout = timeout 160 | leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "leader-election")) 161 | return New(lockType, ns, name, leaderElectionClient.CoreV1(), leaderElectionClient.CoordinationV1(), rlc) 162 | } 163 | 164 | ``` 165 | 166 | ## LeaderElector 167 | 168 | #### 定义 169 | 170 | ```go 171 | type LeaderElectionConfig struct { 172 | // Lock is the resource that will be used for locking 173 | // 锁的对象，可以更新，增加，自身节点的id 174 | Lock rl.Interface 175 | 176 | // LeaseDuration is the duration that non-leader candidates will 177 | // wait to force acquire leadership. This is measured against time of 178 | // last observed ack. 179 | // 180 | // A client needs to wait a full LeaseDuration without observing a change to 181 | // the record before it can attempt to take over. When all clients are 182 | // shutdown and a new set of clients are started with different names against 183 | // the same leader record, they must wait the full LeaseDuration before 184 | // attempting to acquire the lease. Thus LeaseDuration should be as short as 185 | // possible (within your tolerance for clock skew rate) to avoid a possible 186 | // long waits in the scenario. 187 | // 188 | // Core clients default this value to 15 seconds. 189 | // 租期时间 190 | LeaseDuration time.Duration 191 | // RenewDeadline is the duration that the acting master will retry 192 | // refreshing leadership before giving up. 193 | // 194 | // Core clients default this value to 10 seconds. 195 | // 每次刷新租期，获取租期的Deadline 196 | RenewDeadline time.Duration 197 | // RetryPeriod is the duration the LeaderElector clients should wait 198 | // between tries of actions. 199 | // 200 | // Core clients default this value to 2 seconds. 201 | // 没隔多久执行一次租期续约 202 | RetryPeriod time.Duration 203 | 204 | // Callbacks are callbacks that are triggered during certain lifecycle 205 | // events of the LeaderElector 206 | // 租期变化的回调函数，比如成为leader，获取租期失败等等 207 | Callbacks LeaderCallbacks 208 | 209 | // WatchDog is the associated health checker 210 | // WatchDog may be null if its not needed/configured. 211 | WatchDog *HealthzAdaptor 212 | 213 | // ReleaseOnCancel should be set true if the lock should be released 214 | // when the run context is cancelled. If you set this to true, you must 215 | // ensure all code guarded by this lease has successfully completed 216 | // prior to cancelling the context, or you may have two processes 217 | // simultaneously acting on the critical path. 218 | // 当Leader节点主动退出时，释放租期 219 | ReleaseOnCancel bool 220 | 221 | // Name is the name of the resource lock for debugging 222 | Name string 223 | } 224 | 225 | // LeaderCallbacks are callbacks that are triggered during certain 226 | // lifecycle events of the LeaderElector. These are invoked asynchronously. 227 | // 228 | // possible future callbacks: 229 | // * OnChallenge() 230 | type LeaderCallbacks struct { 231 | // OnStartedLeading is called when a LeaderElector client starts leading 232 | // 当某个节点成为Leader时调用 233 | OnStartedLeading func(context.Context) 234 | // OnStoppedLeading is called when a LeaderElector client stops leading 235 | // 当前节点不再是Leader时调用 236 | OnStoppedLeading func() 237 | // OnNewLeader is called when the client observes a leader that is 238 | // not the previously observed leader. This includes the first observed 239 | // leader when the client starts. 240 | // 当新节点产生时调用 241 | OnNewLeader func(identity string) 242 | } 243 | 244 | // LeaderElector is a leader election client. 245 | type LeaderElector struct { 246 | // 配置，锁对象，获取租期的相关参数 247 | config LeaderElectionConfig 248 | // internal bookkeeping 249 | // 缓存监控到的选举记录 250 | observedRecord rl.LeaderElectionRecord 251 | // marshal后的字符数组 252 | observedRawRecord []byte 253 | // 最后观察时间 254 | observedTime time.Time 255 | // used to implement OnNewLeader(), may lag slightly from the 256 | // value observedRecord.HolderIdentity if the transition has 257 | // not yet been reported. 258 | // 上一次的Leader名称，如果本次获取的和这次不一样就会调用OnNewLeader 回调函数 259 | reportedLeader string 260 | 261 | // clock is wrapper around time to allow for less flaky testing 262 | clock clock.Clock 263 | 264 | metrics leaderMetricsAdapter 265 | } 266 | 267 | ``` 268 | 269 | 270 | #### 新建 271 | 272 | ```go 273 | // NewLeaderElector creates a LeaderElector from a LeaderElectionConfig 274 | func NewLeaderElector(lec LeaderElectionConfig) (*LeaderElector, error) { 275 | // 合法检查 276 | 277 | // 如果租期小于Deadline，配置有问题 278 | if lec.LeaseDuration <= lec.RenewDeadline { 279 | return nil, fmt.Errorf("leaseDuration must be greater than renewDeadline") 280 | } 281 | // 如果刷新时间小于retry时间 282 | if lec.RenewDeadline <= time.Duration(JitterFactor*float64(lec.RetryPeriod)) { 283 | return nil, fmt.Errorf("renewDeadline must be greater than retryPeriod*JitterFactor") 284 | } 285 | if lec.LeaseDuration < 1 { 286 | return nil, fmt.Errorf("leaseDuration must be greater than zero") 287 | } 288 | if lec.RenewDeadline < 1 { 289 | return nil, fmt.Errorf("renewDeadline must be greater than zero") 290 | } 291 | if lec.RetryPeriod < 1 { 292 | return nil, fmt.Errorf("retryPeriod must be greater than zero") 293 | } 294 | if lec.Callbacks.OnStartedLeading == nil { 295 | return nil, fmt.Errorf("OnStartedLeading callback must not be nil") 296 | } 297 | if lec.Callbacks.OnStoppedLeading == nil { 298 | return nil, fmt.Errorf("OnStoppedLeading callback must not be nil") 299 | } 300 | 301 | if lec.Lock == nil { 302 | return nil, fmt.Errorf("Lock must not be nil.") 303 | } 304 | le := LeaderElector{ 305 | config: lec, 306 | clock: clock.RealClock{}, 307 | metrics: globalMetricsFactory.newLeaderMetrics(), 308 | } 309 | le.metrics.leaderOff(le.config.Name) 310 | return &le, nil 311 | } 312 | 313 | ``` 314 | 315 | 316 | #### 核心时间 317 | 318 | ```go 319 | // RunOrDie starts a client with the provided config or panics if the config 320 | // fails to validate. RunOrDie blocks until leader election loop is 321 | // stopped by ctx or it has stopped holding the leader lease 322 | func RunOrDie(ctx context.Context, lec LeaderElectionConfig) { 323 | le, err := NewLeaderElector(lec) 324 | //如果配置有问题，直接返回错误 325 | if err != nil { 326 | panic(err) 327 | } 328 | // 向watchdog中注入Le 329 | if lec.WatchDog != nil { 330 | lec.WatchDog.SetLeaderElection(le) 331 | } 332 | // 开始跑 333 | le.Run(ctx) 334 | } 335 | 336 | // Run starts the leader election loop. Run will not return 337 | // before leader election loop is stopped by ctx or it has 338 | // stopped holding the leader lease 339 | func (le *LeaderElector) Run(ctx context.Context) { 340 | defer runtime.HandleCrash() 341 | defer func() { 342 | le.config.Callbacks.OnStoppedLeading() 343 | }() 344 | 345 | // 如果没有成功获取到租期则返回，函数退出只有一种可能，成功获取租期或者context被cancel 346 | if !le.acquire(ctx) { 347 | return // ctx signalled done 348 | } 349 | ctx, cancel := context.WithCancel(ctx) 350 | defer cancel() 351 | // 成功获取到了租期 352 | go le.config.Callbacks.OnStartedLeading(ctx) 353 | // 不断刷新租期 354 | le.renew(ctx) 355 | } 356 | 357 | // acquire loops calling tryAcquireOrRenew and returns true immediately when tryAcquireOrRenew succeeds. 358 | // Returns false if ctx signals done. 359 | func (le *LeaderElector) acquire(ctx context.Context) bool { 360 | ctx, cancel := context.WithCancel(ctx) 361 | defer cancel() 362 | succeeded := false 363 | desc := le.config.Lock.Describe() 364 | klog.Infof("attempting to acquire leader lease %v...", desc) 365 | // 每隔Retry时间段执行一次执行一次，知道context被取消 366 | wait.JitterUntil(func() { 367 | // 获取租期 368 | succeeded = le.tryAcquireOrRenew(ctx) 369 | // 更新最后获取的leader信息 370 | le.maybeReportTransition() 371 | // 如果没有获取租期，继续等待知道获取到租期 372 | if !succeeded { 373 | klog.V(4).Infof("failed to acquire lease %v", desc) 374 | return 375 | } 376 | // 成功获取租期， 377 | le.config.Lock.RecordEvent("became leader") 378 | le.metrics.leaderOn(le.config.Name) 379 | klog.Infof("successfully acquired lease %v", desc) 380 | // 停止Until函数执行 381 | cancel() 382 | }, le.config.RetryPeriod, JitterFactor, true, ctx.Done()) 383 | // 返回结果 384 | return succeeded 385 | } 386 | 387 | // tryAcquireOrRenew tries to acquire a leader lease if it is not already acquired, 388 | // else it tries to renew the lease if it has already been acquired. Returns true 389 | // on success else returns false. 390 | func (le *LeaderElector) tryAcquireOrRenew(ctx context.Context) bool { 391 | now := metav1.Now() 392 | leaderElectionRecord := rl.LeaderElectionRecord{ 393 | HolderIdentity: le.config.Lock.Identity(), 394 | LeaseDurationSeconds: int(le.config.LeaseDuration / time.Second), 395 | RenewTime: now, 396 | AcquireTime: now, 397 | } 398 | 399 | // 1. obtain or create the ElectionRecord 400 | // 从APIServer获取记录 401 | oldLeaderElectionRecord, oldLeaderElectionRawRecord, err := le.config.Lock.Get(ctx) 402 | if err != nil { 403 | if !errors.IsNotFound(err) { 404 | klog.Errorf("error retrieving resource lock %v: %v", le.config.Lock.Describe(), err) 405 | return false 406 | } 407 | // 之前没有则直接创建 408 | if err = le.config.Lock.Create(ctx, leaderElectionRecord); err != nil { 409 | klog.Errorf("error initially creating leader election record: %v", err) 410 | return false 411 | } 412 | le.observedRecord = leaderElectionRecord 413 | le.observedTime = le.clock.Now() 414 | // 成功获取租期 415 | return true 416 | } 417 | 418 | // 2. Record obtained, check the Identity & Time 419 | // 如果成功从APIServer中获取到了记录，则看下是否和之前不一样 , 增更新下 420 | if !bytes.Equal(le.observedRawRecord, oldLeaderElectionRawRecord) { 421 | le.observedRecord = *oldLeaderElectionRecord 422 | le.observedRawRecord = oldLeaderElectionRawRecord 423 | le.observedTime = le.clock.Now() 424 | } 425 | // 如果我不是Leader，并且在Leader的任期内，则什么都不做 426 | if len(oldLeaderElectionRecord.HolderIdentity) > 0 && 427 | le.observedTime.Add(le.config.LeaseDuration).After(now.Time) && 428 | !le.IsLeader() { 429 | klog.V(4).Infof("lock is held by %v and has not yet expired", oldLeaderElectionRecord.HolderIdentity) 430 | return false 431 | } 432 | 433 | // 3. We're going to try to update. The leaderElectionRecord is set to it's default 434 | // here. Let's correct it before updating. 435 | // 此时，要不就是没有leader了(所有Slave需要抢任期)，要不就是我就是leader(刷新任期) 436 | if le.IsLeader() { 437 | //刷新任期 438 | leaderElectionRecord.AcquireTime = oldLeaderElectionRecord.AcquireTime 439 | leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions 440 | } else { 441 | // 我不是任期，增加一次Leader转换 442 | leaderElectionRecord.LeaderTransitions = oldLeaderElectionRecord.LeaderTransitions + 1 443 | } 444 | 445 | // update the lock itself 446 | // 更新锁，多个Slave同时更新只会有一个成功，因为ResouceVersion有问题 447 | if err = le.config.Lock.Update(ctx, leaderElectionRecord); err != nil { 448 | klog.Errorf("Failed to update lock: %v", err) 449 | return false 450 | } 451 | 452 | //刷新，或者抢任期完成 453 | le.observedRecord = leaderElectionRecord 454 | le.observedTime = le.clock.Now() 455 | return true 456 | } 457 | 458 | // renew loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew fails or ctx signals done. 459 | func (le *LeaderElector) renew(ctx context.Context) { 460 | ctx, cancel := context.WithCancel(ctx) 461 | defer cancel() 462 | // 每隔 retry刷新下租期 463 | wait.Until(func() { 464 | timeoutCtx, timeoutCancel := context.WithTimeout(ctx, le.config.RenewDeadline) 465 | defer timeoutCancel() 466 | // 每隔Retry执行刷新下任期，直到timeout，或者成功刷新租期 467 | err := wait.PollImmediateUntil(le.config.RetryPeriod, func() (bool, error) { 468 | return le.tryAcquireOrRenew(timeoutCtx), nil 469 | }, timeoutCtx.Done()) 470 | 471 | le.maybeReportTransition() 472 | desc := le.config.Lock.Describe() 473 | if err == nil { 474 | klog.V(5).Infof("successfully renewed lease %v", desc) 475 | return 476 | } 477 | // 此时timeout了，无法刷新任期 478 | le.config.Lock.RecordEvent("stopped leading") 479 | le.metrics.leaderOff(le.config.Name) 480 | klog.Infof("failed to renew lease %v: %v", desc, err) 481 | // 任期结束，结束Util 482 | cancel() 483 | }, le.config.RetryPeriod, ctx.Done()) 484 | 485 | // if we hold the lease, give it up 486 | if le.config.ReleaseOnCancel { 487 | le.release() 488 | } 489 | } 490 | 491 | 492 | 493 | / GetLeader returns the identity of the last observed leader or returns the empty string if 494 | // no leader has yet been observed. 495 | func (le *LeaderElector) GetLeader() string { 496 | return le.observedRecord.HolderIdentity 497 | } 498 | 499 | // IsLeader returns true if the last observed leader was this client else returns false. 500 | func (le *LeaderElector) IsLeader() bool { 501 | // 当前节点和最后观察到的记录是否一致 502 | return le.observedRecord.HolderIdentity == le.config.Lock.Identity() 503 | } 504 | 505 | 506 | 507 | ``` -------------------------------------------------------------------------------- /tools/leader-election/leader-election.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "k8s.io/client-go/tools/clientcmd" 8 | "time" 9 | 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | clientset "k8s.io/client-go/kubernetes" 12 | "k8s.io/client-go/tools/leaderelection" 13 | "k8s.io/client-go/tools/leaderelection/resourcelock" 14 | "k8s.io/klog" 15 | ) 16 | 17 | var ( 18 | client *clientset.Clientset 19 | leaseLockName string 20 | leaseLockNamespace string 21 | instanceName string 22 | mode runMode 23 | changeModeCh = make(chan runMode) 24 | ) 25 | 26 | type runMode string 27 | 28 | const modeMaster = "master" 29 | const modeSlave = "slave" 30 | 31 | // 创建lock对象，使用LeaseLock 32 | func getNewLock(lockname, instanceName, namespace string) *resourcelock.LeaseLock { 33 | return &resourcelock.LeaseLock{ 34 | LeaseMeta: metav1.ObjectMeta{ 35 | Name: lockname, 36 | Namespace: namespace, 37 | }, 38 | Client: client.CoordinationV1(), 39 | LockConfig: resourcelock.ResourceLockConfig{ 40 | Identity: instanceName, 41 | }, 42 | } 43 | } 44 | 45 | // 干活 46 | func doStuff(changeModeCh chan runMode) { 47 | for { 48 | select { 49 | // 如果有模式变更立刻停止工作 50 | case theMode, ok := <-changeModeCh: 51 | if !ok { 52 | return 53 | } 54 | mode = theMode 55 | default: 56 | // 如果当前是leader角色，则干活 57 | if mode == modeMaster { 58 | klog.Info("working ...") 59 | time.Sleep(4 * time.Second) 60 | } 61 | } 62 | } 63 | } 64 | 65 | func runLeaderElection(lock *resourcelock.LeaseLock, ctx context.Context, instanceName string) { 66 | leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{ 67 | Lock: lock, // 制定锁的资源 68 | ReleaseOnCancel: true, //当cancel时，主动释放锁 69 | LeaseDuration: 15 * time.Second, // 每次获取15s的任期 70 | RenewDeadline: 10 * time.Second, // 71 | RetryPeriod: 2 * time.Second, 72 | Callbacks: leaderelection.LeaderCallbacks{ 73 | OnStartedLeading: func(c context.Context) { 74 | changeModeCh <- modeMaster 75 | klog.Info("a leader start leading .") 76 | }, 77 | OnStoppedLeading: func() { 78 | changeModeCh <- modeSlave 79 | klog.Info("I am no longer the leader, staying inactive.") 80 | }, 81 | OnNewLeader: func(leaderName string) { 82 | if leaderName == instanceName { 83 | klog.Info("I am leader!") 84 | return 85 | } 86 | klog.Infof("%s is new leader ", leaderName) 87 | }, 88 | }, 89 | }) 90 | } 91 | 92 | func main() { 93 | flag.StringVar(&leaseLockName, "lease-name", "", "Name of lease lock") 94 | flag.StringVar(&leaseLockNamespace, "lease-namespace", "default", "Name of lease lock namespace") 95 | flag.StringVar(&instanceName, "instance-name", "", "Name of instance") 96 | flag.Parse() 97 | 98 | if leaseLockName == "" { 99 | klog.Fatal("missing lease-name flag") 100 | } 101 | if leaseLockNamespace == "" { 102 | klog.Fatal("missing lease-namespace flag") 103 | } 104 | if instanceName == "" { 105 | klog.Fatal("missing instance-name flag") 106 | } 107 | 108 | config, err := clientcmd.BuildConfigFromFlags("", "/Users/yulongxin/.kube/config") 109 | if err != nil { 110 | klog.Fatalf(fmt.Sprintf("create client failed : %s", err)) 111 | } 112 | client = clientset.NewForConfigOrDie(config) 113 | 114 | ctx, cancel := context.WithCancel(context.Background()) 115 | defer cancel() 116 | 117 | go doStuff(changeModeCh) 118 | 119 | lock := getNewLock(leaseLockName, instanceName, leaseLockNamespace) 120 | runLeaderElection(lock, ctx, instanceName) 121 | } 122 | -------------------------------------------------------------------------------- /tools/port-forward/README.md: -------------------------------------------------------------------------------- 1 | # Port-forward 2 | 3 | 我们经常使用Kubectl `port-forward` 命令将Kubernetes中的服务forward到本地，那么是如何实现的呢？ 4 | 5 | ## 如何使用 6 | 7 | [port-forward.go](/tools/port-forward/port-forward.go) 8 | 9 | ## 代码逻辑 10 | 11 | ```go 12 | // New creates a new PortForwarder with localhost listen addresses. 13 | func New(dialer httpstream.Dialer, ports []string, stopChan <-chan struct{}, readyChan chan struct{}, out, errOut io.Writer) (*PortForwarder, error) { 14 | return NewOnAddresses(dialer, []string{"localhost"}, ports, stopChan, readyChan, out, errOut) 15 | } 16 | 17 | // NewOnAddresses creates a new PortForwarder with custom listen addresses. 18 | // dialer 知道如何拨号，服务端的信息存储在dialer中 19 | // 20 | func NewOnAddresses(dialer httpstream.Dialer, addresses []string, ports []string, stopChan <-chan struct{}, readyChan chan struct{}, out, errOut io.Writer) (*PortForwarder, error) { 21 | if len(addresses) == 0 { 22 | return nil, errors.New("you must specify at least 1 address") 23 | } 24 | parsedAddresses, err := parseAddresses(addresses) 25 | if err != nil { 26 | return nil, err 27 | } 28 | if len(ports) == 0 { 29 | return nil, errors.New("you must specify at least 1 port") 30 | } 31 | parsedPorts, err := parsePorts(ports) 32 | if err != nil { 33 | return nil, err 34 | } 35 | return &PortForwarder{ 36 | dialer: dialer, 37 | addresses: parsedAddresses, 38 | ports: parsedPorts, 39 | stopChan: stopChan, 40 | Ready: readyChan, 41 | out: out, 42 | errOut: errOut, 43 | }, nil 44 | } 45 | ``` 46 | 47 | ### ForwardPorts 48 | 49 | ```go 50 | func (pf *PortForwarder) ForwardPorts() error { 51 | defer pf.Close() 52 | 53 | var err error 54 | // 获取一个服务端连接 55 | pf.streamConn, _, err = pf.dialer.Dial(PortForwardProtocolV1Name) 56 | if err != nil { 57 | return fmt.Errorf("error upgrading connection: %s", err) 58 | } 59 | defer pf.streamConn.Close() 60 | 61 | return pf.forward() 62 | } 63 | ``` 64 | 65 | 66 | ```go 67 | // forward dials the remote host specific in req, upgrades the request, starts 68 | // listeners for each port specified in ports, and forwards local connections 69 | // to the remote host via streams. 70 | func (pf *PortForwarder) forward() error { 71 | var err error 72 | 73 | listenSuccess := false 74 | for i := range pf.ports { 75 | port := &pf.ports[i] 76 | // 监听并处理 77 | err = pf.listenOnPort(port) 78 | switch { 79 | case err == nil: 80 | listenSuccess = true 81 | default: 82 | if pf.errOut != nil { 83 | fmt.Fprintf(pf.errOut, "Unable to listen on port %d: %v\n", port.Local, err) 84 | } 85 | } 86 | } 87 | 88 | if !listenSuccess { 89 | return fmt.Errorf("unable to listen on any of the requested ports: %v", pf.ports) 90 | } 91 | 92 | if pf.Ready != nil { 93 | close(pf.Ready) 94 | } 95 | 96 | // wait for interrupt or conn closure 97 | select { 98 | case <-pf.stopChan: 99 | case <-pf.streamConn.CloseChan(): 100 | runtime.HandleError(errors.New("lost connection to pod")) 101 | } 102 | 103 | return nil 104 | } 105 | 106 | ``` 107 | 108 | 109 | ```go 110 | // listenOnPort delegates listener creation and waits for connections on requested bind addresses. 111 | // An error is raised based on address groups (default and localhost) and their failure modes 112 | func (pf *PortForwarder) listenOnPort(port *ForwardedPort) error { 113 | var errors []error 114 | failCounters := make(map[string]int, 2) 115 | successCounters := make(map[string]int, 2) 116 | for _, addr := range pf.addresses { 117 | //监听处理 118 | err := pf.listenOnPortAndAddress(port, addr.protocol, addr.address) 119 | if err != nil { 120 | errors = append(errors, err) 121 | failCounters[addr.failureMode]++ 122 | } else { 123 | successCounters[addr.failureMode]++ 124 | } 125 | } 126 | if successCounters["all"] == 0 && failCounters["all"] > 0 { 127 | return fmt.Errorf("%s: %v", "Listeners failed to create with the following errors", errors) 128 | } 129 | if failCounters["any"] > 0 { 130 | return fmt.Errorf("%s: %v", "Listeners failed to create with the following errors", errors) 131 | } 132 | return nil 133 | } 134 | 135 | ``` 136 | 137 | ```go 138 | // listenOnPortAndAddress delegates listener creation and waits for new connections 139 | // in the background f 140 | func (pf *PortForwarder) listenOnPortAndAddress(port *ForwardedPort, protocol string, address string) error { 141 | //监听端口 142 | listener, err := pf.getListener(protocol, address, port) 143 | if err != nil { 144 | return err 145 | } 146 | pf.listeners = append(pf.listeners, listener) 147 | // 等待连接并处理 148 | go pf.waitForConnection(listener, *port) 149 | return nil 150 | } 151 | ``` 152 | 153 | 154 | ```go 155 | // waitForConnection waits for new connections to listener and handles them in 156 | // the background. 157 | func (pf *PortForwarder) waitForConnection(listener net.Listener, port ForwardedPort) { 158 | for { 159 | conn, err := listener.Accept() 160 | if err != nil { 161 | // TODO consider using something like https://github.com/hydrogen18/stoppableListener? 162 | if !strings.Contains(strings.ToLower(err.Error()), "use of closed network connection") { 163 | runtime.HandleError(fmt.Errorf("error accepting connection on port %d: %v", port.Local, err)) 164 | } 165 | return 166 | } 167 | // 处理连接后的请求 168 | go pf.handleConnection(conn, port) 169 | } 170 | } 171 | ``` 172 | 173 | 174 | ```go 175 | // handleConnection copies data between the local connection and the stream to 176 | // the remote server. 177 | func (pf *PortForwarder) handleConnection(conn net.Conn, port ForwardedPort) { 178 | defer conn.Close() 179 | 180 | if pf.out != nil { 181 | fmt.Fprintf(pf.out, "Handling connection for %d\n", port.Local) 182 | } 183 | 184 | requestID := pf.nextRequestID() 185 | 186 | // create error stream 187 | headers := http.Header{} 188 | // 创建错误输出 189 | headers.Set(v1.StreamType, v1.StreamTypeError) 190 | headers.Set(v1.PortHeader, fmt.Sprintf("%d", port.Remote)) 191 | headers.Set(v1.PortForwardRequestIDHeader, strconv.Itoa(requestID)) 192 | errorStream, err := pf.streamConn.CreateStream(headers) 193 | if err != nil { 194 | runtime.HandleError(fmt.Errorf("error creating error stream for port %d -> %d: %v", port.Local, port.Remote, err)) 195 | return 196 | } 197 | // we're not writing to this stream 198 | // 只接收错误信息，但是不会发送信息 199 | errorStream.Close() 200 | 201 | errorChan := make(chan error) 202 | // 错误信息处理 203 | go func() { 204 | message, err := ioutil.ReadAll(errorStream) 205 | switch { 206 | case err != nil: 207 | errorChan <- fmt.Errorf("error reading from error stream for port %d -> %d: %v", port.Local, port.Remote, err) 208 | case len(message) > 0: 209 | errorChan <- fmt.Errorf("an error occurred forwarding %d -> %d: %v", port.Local, port.Remote, string(message)) 210 | } 211 | close(errorChan) 212 | }() 213 | 214 | // create data stream 215 | // 创建数据流 216 | headers.Set(v1.StreamType, v1.StreamTypeData) 217 | dataStream, err := pf.streamConn.CreateStream(headers) 218 | if err != nil { 219 | runtime.HandleError(fmt.Errorf("error creating forwarding stream for port %d -> %d: %v", port.Local, port.Remote, err)) 220 | return 221 | } 222 | 223 | localError := make(chan struct{}) 224 | remoteDone := make(chan struct{}) 225 | 226 | go func() { 227 | // Copy from the remote side to the local port. 228 | // 将请求发送到到输入 229 | if _, err := io.Copy(conn, dataStream); err != nil && !strings.Contains(err.Error(), "use of closed network connection") { 230 | runtime.HandleError(fmt.Errorf("error copying from remote stream to local connection: %v", err)) 231 | } 232 | 233 | // inform the select below that the remote copy is done 234 | close(remoteDone) 235 | }() 236 | 237 | go func() { 238 | // inform server we're not sending any more data after copy unblocks 239 | defer dataStream.Close() 240 | 241 | // Copy from the local port to the remote side. 242 | // 从远端输出 发送给请求方 243 | if _, err := io.Copy(dataStream, conn); err != nil && !strings.Contains(err.Error(), "use of closed network connection") { 244 | runtime.HandleError(fmt.Errorf("error copying from local connection to remote stream: %v", err)) 245 | // break out of the select below without waiting for the other copy to finish 246 | close(localError) 247 | } 248 | }() 249 | 250 | // wait for either a local->remote error or for copying from remote->local to finish 251 | select { 252 | case <-remoteDone: 253 | case <-localError: 254 | } 255 | 256 | // always expect something on errorChan (it may be nil) 257 | err = <-errorChan 258 | if err != nil { 259 | runtime.HandleError(err) 260 | } 261 | } 262 | ``` -------------------------------------------------------------------------------- /tools/port-forward/port-forward.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | clientset "k8s.io/client-go/kubernetes" 7 | "k8s.io/client-go/rest" 8 | "k8s.io/client-go/tools/clientcmd" 9 | "k8s.io/client-go/tools/portforward" 10 | "k8s.io/client-go/transport/spdy" 11 | "k8s.io/klog" 12 | "net/http" 13 | "net/url" 14 | "os" 15 | "os/signal" 16 | "syscall" 17 | ) 18 | 19 | // 博客地址：https://www.modb.pro/db/137716 20 | 21 | func main() { 22 | config, err := clientcmd.BuildConfigFromFlags("", "/Users/yulongxin/.kube/config") 23 | if err != nil { 24 | klog.Fatalf(fmt.Sprintf("create client failed : %s", err)) 25 | } 26 | client := clientset.NewForConfigOrDie(config) 27 | req := client.CoreV1().RESTClient().Post().Namespace("default"). 28 | Resource("pods").Name("nginx").SubResource("portforward") 29 | 30 | klog.Info(req.URL()) 31 | 32 | signals := make(chan os.Signal, 1) 33 | StopChannel := make(chan struct{}, 1) 34 | ReadyChannel := make(chan struct{}) 35 | 36 | defer signal.Stop(signals) 37 | 38 | signal.Notify(signals, os.Interrupt, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL) 39 | 40 | go func() { 41 | <-signals 42 | if StopChannel != nil { 43 | close(StopChannel) 44 | } 45 | }() 46 | 47 | if err := ForwardPorts("POST", req.URL(), config, StopChannel, ReadyChannel); err != nil { 48 | klog.Fatalln(err) 49 | } 50 | 51 | } 52 | 53 | func ForwardPorts(method string, url *url.URL, config *rest.Config, StopChannel, ReadyChannel chan struct{}) error { 54 | transport, upgrader, err := spdy.RoundTripperFor(config) 55 | if err != nil { 56 | return err 57 | } 58 | address := []string{"0.0.0.0"} 59 | ports := []string{"8080:80"} 60 | 61 | IOStreams := struct { 62 | // In think, os.Stdin 63 | In io.Reader 64 | // Out think, os.Stdout 65 | Out io.Writer 66 | // ErrOut think, os.Stderr 67 | ErrOut io.Writer 68 | }{os.Stdin, os.Stdout, os.Stderr} 69 | 70 | dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, method, url) 71 | fw, err := portforward.NewOnAddresses(dialer, address, ports, StopChannel, ReadyChannel, IOStreams.Out, IOStreams.ErrOut) 72 | if err != nil { 73 | return err 74 | } 75 | return fw.ForwardPorts() 76 | } 77 | -------------------------------------------------------------------------------- /tools/rest-client/README.md: -------------------------------------------------------------------------------- 1 | # Rest-client 2 | 3 | ## Kube-Config 4 | 5 | 在工作中，我们可能需要同时与多个Kubernetes集群交流，大部分情况下，我会通过一个Kubernetes config 管理集群信息的。那么Kubernetes Config中保存了那些内容呢 ？配置文件在程序中的结构是如何的呢 ？ 6 | 7 | ```go 8 | type Config struct { 9 | // Legacy field from pkg/api/types.go TypeMeta. 10 | // TODO(jlowdermilk): remove this after eliminating downstream dependencies. 11 | // +k8s:conversion-gen=false 12 | // +optional 13 | Kind string `json:"kind,omitempty"` 14 | // Legacy field from pkg/api/types.go TypeMeta. 15 | // TODO(jlowdermilk): remove this after eliminating downstream dependencies. 16 | // +k8s:conversion-gen=false 17 | // +optional 18 | APIVersion string `json:"apiVersion,omitempty"` 19 | // Preferences holds general information to be use for cli interactions 20 | Preferences Preferences `json:"preferences"` 21 | // Clusters is a map of referencable names to cluster configs 22 | // 集群信息，集群地址等信息 23 | Clusters map[string]*Cluster `json:"clusters"` 24 | // AuthInfos is a map of referencable names to user configs 25 | // 用户信息， 认证信息 26 | AuthInfos map[string]*AuthInfo `json:"users"` 27 | // Contexts is a map of referencable names to context configs 28 | // 将集群信息和认证信息组合在一起形成了一个context 29 | Contexts map[string]*Context `json:"contexts"` 30 | // CurrentContext is the name of the context that you would like to use by default 31 | //当前使用的context 32 | CurrentContext string `json:"current-context"` 33 | // Extensions holds additional information. This is useful for extenders so that reads and writes don't clobber unknown fields 34 | // +optional 35 | Extensions map[string]runtime.Object `json:"extensions,omitempty"` 36 | } 37 | 38 | ``` 39 | 40 | - 一个`config`由多个context组成 41 | - 一个`context` 由一个集群信息和用户认证信息组成 42 | - 集群信息和用户认证信息独立维护； 43 | 44 | 45 | ## Clientcmd 46 | 47 | 在`clientcmd`包中有关于kubeconfig相关的内容。包的主要用途是为了生成`rest.Config`. 48 | 49 | ```go 50 | func BuildConfigFromFlags(masterUrl, kubeconfigPath string) (*restclient.Config, error) { 51 | if kubeconfigPath == "" && masterUrl == "" { 52 | klog.Warning("Neither --kubeconfig nor --master was specified. Using the inClusterConfig. This might not work.") 53 | kubeconfig, err := restclient.InClusterConfig() 54 | if err == nil { 55 | return kubeconfig, nil 56 | } 57 | klog.Warning("error creating inClusterConfig, falling back to default config: ", err) 58 | } 59 | return NewNonInteractiveDeferredLoadingClientConfig( 60 | &ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, 61 | &ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterUrl}}).ClientConfig() 62 | } 63 | ``` 64 | 65 | ## rest 66 | 67 | rest包可以建立与Kubernetes连接的client。比如给予`clientcmd`建立的config，可以初始化一个全局的client。 68 | 69 | ```go 70 | client, err := kubernetes.NewForConfig(config) 71 | ``` 72 | 73 | 基于创建的client可以对kubernete做各种各样的操作比如建立一个pod。 74 | 75 | 76 | ### Create 77 | 78 | ```go 79 | //create a pod 80 | _, err = client.CoreV1().Pods(DefaultNamespace).Create(context.Background(), &v1.Pod{ 81 | ObjectMeta: meta_v1.ObjectMeta{ 82 | Name: PodName, 83 | Labels: map[string]string{ServiceName: PodName}, 84 | }, 85 | Spec: v1.PodSpec{ 86 | Containers: []v1.Container{ 87 | { 88 | Name: ContainerName, 89 | Image: ImageName, 90 | Command: []string{"/bin/sh", "-c", "while :; do date +%s ; sleep 1 ; done"}, 91 | }, 92 | }, 93 | }, 94 | }, 95 | meta_v1.CreateOptions{}, 96 | ) 97 | ``` 98 | 99 | ### Get 100 | 101 | ```go 102 | pod, err = client.CoreV1().Pods(DefaultNamespace).Get(context.Background(), PodName, meta_v1.GetOptions{}) 103 | if err != nil { 104 | panic(fmt.Sprintf("can not get pod , err : %s", err)) 105 | } 106 | ``` 107 | 108 | ### Update 109 | 110 | 111 | ```go 112 | pod.Spec.Containers[0].Image = "busybox:1.35.0" 113 | pod, err = client.CoreV1().Pods(DefaultNamespace).Update(context.Background(), pod, meta_v1.UpdateOptions{}) 114 | if err != nil { 115 | panic(fmt.Sprintf("can not upate pod, err : %s", err)) 116 | } 117 | ``` 118 | 119 | 120 | ### List Pod 121 | 122 | ```go 123 | //测试listPod 124 | pods, err := client.CoreV1().Pods(DefaultNamespace).List(context.Background(), meta_v1.ListOptions{ 125 | LabelSelector: labels.SelectorFromSet(map[string]string{ServiceName: PodName}).String(), 126 | }) 127 | fmt.Printf("pods count : %v \n", len(pods.Items)) 128 | 129 | for index, pod := range pods.Items { 130 | fmt.Printf("pods[%d].name is %s \n", index, pod.Name) 131 | } 132 | ``` 133 | 134 | ### Subsource/Log 135 | 136 | ```go 137 | 138 | logCloseCh := make(chan struct{}) 139 | if len(pods.Items) > 0 { 140 | pod := pods.Items[0] 141 | sinceSeconds := int64(10) 142 | s, err := client.CoreV1().Pods(DefaultNamespace).GetLogs(pod.Name, &v1.PodLogOptions{SinceSeconds: &sinceSeconds, Follow: true}).Stream(context.Background()) 143 | if err != nil { 144 | panic(fmt.Sprintf("get logs failed : %v", err)) 145 | } 146 | b := bufio.NewScanner(s) 147 | go wait.Until(func() { 148 | if b.Scan() { 149 | fmt.Println("log line : ", b.Text()) 150 | } 151 | if b.Err() != nil && b.Err() != io.EOF { 152 | panic(fmt.Sprintf("read from stream failed , error is : %v", b.Err())) 153 | } 154 | }, time.Millisecond*100, logCloseCh) 155 | 156 | time.Sleep(5 * time.Second) 157 | close(logCloseCh) 158 | time.Sleep(1 * time.Second) 159 | s.Close() 160 | } 161 | ``` 162 | 163 | 164 | ### Subresouce/Exec 165 | 166 | ```go 167 | if len(pods.Items) > 0 { 168 | execCloseCh := make(chan struct{}) 169 | 170 | req := client.CoreV1().RESTClient().Post().Namespace(DefaultNamespace).Resource("pods").Name(PodName). 171 | SubResource("exec").Param("container", pods.Items[0].Spec.Containers[0].Name).VersionedParams(&v1.PodExecOptions{ 172 | Stdin: true, 173 | Stdout: true, 174 | Stderr: true, 175 | TTY: true, 176 | Container: pods.Items[0].Name, 177 | Command: []string{"/bin/sh"}, 178 | }, scheme.ParameterCodec) 179 | 180 | exec, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, req.URL()) 181 | if err != nil { 182 | panic(fmt.Sprintf("new spdy executor failed :%s", err)) 183 | } 184 | 185 | go wait.Until(func() { 186 | err = exec.Stream(remotecommand.StreamOptions{ 187 | Stdin: os.Stdin, 188 | Stdout: os.Stdout, 189 | Stderr: os.Stderr, 190 | Tty: true, 191 | }) 192 | //if err != nil { 193 | // t.Fatalf("exec stream failed , err : %s", err) 194 | // } 195 | }, time.Millisecond*100, execCloseCh) 196 | 197 | time.Sleep(10 * time.Second) 198 | close(execCloseCh) 199 | time.Sleep(time.Second) 200 | } 201 | ``` 202 | 203 | 204 | 205 | 206 | 207 | -------------------------------------------------------------------------------- /tools/rest-client/client-pod.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | v1 "k8s.io/api/core/v1" 9 | meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/apimachinery/pkg/labels" 11 | "k8s.io/apimachinery/pkg/util/wait" 12 | "k8s.io/apimachinery/pkg/watch" 13 | "k8s.io/client-go/kubernetes" 14 | "k8s.io/client-go/kubernetes/scheme" 15 | "k8s.io/client-go/tools/clientcmd" 16 | "k8s.io/client-go/tools/remotecommand" 17 | "net/http" 18 | "os" 19 | "time" 20 | ) 21 | 22 | const PodName = "test-lifecycle-pod" 23 | const ImageName = "busybox" 24 | const ServiceName = "ServiceName" 25 | const ContainerName = "busybox" 26 | const DefaultNamespace = "default" 27 | 28 | func main() { 29 | config, err := clientcmd.BuildConfigFromFlags("", "/Users/yulongxin/.kube/config") 30 | if err != nil { 31 | panic(fmt.Sprintf("create client failed : %s", err)) 32 | } 33 | client, err := kubernetes.NewForConfig(config) 34 | 35 | //create a pod 36 | terminationGracePeriod := int64(0) 37 | pod, err := client.CoreV1().Pods(DefaultNamespace).Create(context.Background(), &v1.Pod{ 38 | ObjectMeta: meta_v1.ObjectMeta{ 39 | Name: PodName, 40 | Labels: map[string]string{ServiceName: PodName}, 41 | }, 42 | Spec: v1.PodSpec{ 43 | Containers: []v1.Container{ 44 | { 45 | Name: ContainerName, 46 | Image: ImageName, 47 | Command: []string{"/bin/sh", "-c", "while :; do date +%s ; sleep 1 ; done"}, 48 | TTY: true, 49 | }, 50 | }, 51 | TerminationGracePeriodSeconds: &terminationGracePeriod, 52 | }, 53 | }, 54 | meta_v1.CreateOptions{}, 55 | ) 56 | if err != nil { 57 | panic(fmt.Sprintf("can not create pod , err : %s", err)) 58 | } 59 | 60 | waitForPodReady(client) 61 | fmt.Println("pod has been created") 62 | 63 | //程序退出时，删除pod 64 | defer client.CoreV1().Pods(DefaultNamespace).Delete(context.Background(), PodName, meta_v1.DeleteOptions{}) 65 | 66 | //update pod 67 | pod, err = client.CoreV1().Pods(DefaultNamespace).Get(context.Background(), PodName, meta_v1.GetOptions{}) 68 | if err != nil { 69 | panic(fmt.Sprintf("can not get pod , err : %s", err)) 70 | } 71 | 72 | pod.Spec.Containers[0].Image = "busybox:1.35.0" 73 | pod, err = client.CoreV1().Pods(DefaultNamespace).Update(context.Background(), pod, meta_v1.UpdateOptions{}) 74 | if err != nil { 75 | panic(fmt.Sprintf("can not upate pod, err : %s", err)) 76 | } 77 | 78 | waitForPodReady(client) 79 | fmt.Println("pod has been updated") 80 | 81 | //list Pod 82 | pods, err := client.CoreV1().Pods(DefaultNamespace).List(context.Background(), meta_v1.ListOptions{ 83 | LabelSelector: labels.SelectorFromSet(map[string]string{ServiceName: PodName}).String(), 84 | }) 85 | fmt.Printf("pods count : %v \n", len(pods.Items)) 86 | 87 | for index, pod := range pods.Items { 88 | fmt.Printf("pods[%d].name is %s \n", index, pod.Name) 89 | } 90 | 91 | //get logs 92 | logCloseCh := make(chan struct{}) 93 | if len(pods.Items) > 0 { 94 | pod := pods.Items[0] 95 | sinceSeconds := int64(10) 96 | s, err := client.CoreV1().Pods(DefaultNamespace).GetLogs(pod.Name, &v1.PodLogOptions{SinceSeconds: &sinceSeconds, Follow: true}).Stream(context.Background()) 97 | if err != nil { 98 | panic(fmt.Sprintf("get logs failed : %v", err)) 99 | } 100 | b := bufio.NewScanner(s) 101 | go wait.Until(func() { 102 | if b.Scan() { 103 | fmt.Println("log line : ", b.Text()) 104 | } 105 | if b.Err() != nil && b.Err() != io.EOF { 106 | panic(fmt.Sprintf("read from stream failed , error is : %v", b.Err())) 107 | } 108 | }, time.Millisecond*100, logCloseCh) 109 | 110 | time.Sleep(5 * time.Second) 111 | close(logCloseCh) 112 | time.Sleep(1 * time.Second) 113 | s.Close() 114 | } 115 | 116 | //exec 117 | if len(pods.Items) > 0 { 118 | execCloseCh := make(chan struct{}) 119 | 120 | req := client.CoreV1().RESTClient().Post().Namespace(DefaultNamespace).Resource("pods").Name(PodName). 121 | SubResource("exec").Param("container", pods.Items[0].Spec.Containers[0].Name).VersionedParams(&v1.PodExecOptions{ 122 | Stdin: true, 123 | Stdout: true, 124 | Stderr: true, 125 | TTY: true, 126 | Container: pods.Items[0].Name, 127 | Command: []string{"/bin/sh"}, 128 | }, scheme.ParameterCodec) 129 | 130 | exec, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, req.URL()) 131 | if err != nil { 132 | panic(fmt.Sprintf("new spdy executor failed :%s", err)) 133 | } 134 | 135 | go wait.Until(func() { 136 | err = exec.Stream(remotecommand.StreamOptions{ 137 | Stdin: os.Stdin, 138 | Stdout: os.Stdout, 139 | Stderr: os.Stderr, 140 | Tty: true, 141 | }) 142 | if err != nil { 143 | panic(fmt.Sprintf("exec stream failed , err : %s", err)) 144 | } 145 | }, time.Millisecond*100, execCloseCh) 146 | 147 | time.Sleep(10 * time.Second) 148 | close(execCloseCh) 149 | time.Sleep(time.Second) 150 | } 151 | 152 | } 153 | 154 | func waitForPodReady(client *kubernetes.Clientset) { 155 | //测试watcher 156 | watcher, err := client.CoreV1().Pods(DefaultNamespace).Watch(context.Background(), meta_v1.ListOptions{ 157 | LabelSelector: labels.SelectorFromSet(map[string]string{ServiceName: PodName}).String(), 158 | }) 159 | if err != nil { 160 | panic(fmt.Sprintf("can not watch resource, err : %s", err)) 161 | } 162 | 163 | loop: 164 | for event := range watcher.ResultChan() { 165 | switch event.Type { 166 | case watch.Modified: 167 | thePod := event.Object.(*v1.Pod) 168 | for _, cond := range thePod.Status.Conditions { 169 | if cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue { 170 | watcher.Stop() 171 | fmt.Println("pod status is ready") 172 | break loop 173 | } 174 | } 175 | case watch.Added: 176 | 177 | case watch.Bookmark: 178 | 179 | default: 180 | panic(fmt.Sprintf("unexcepted event type : %v, %v", event.Type, event.Object)) 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /util/README.md: -------------------------------------------------------------------------------- 1 | # Util 包源码阅读 2 | 3 | - [WorkQueue](/util/workqueue/README.md) 声明式队列，延迟队列，限流等； 4 | - [flowControl](/util/flowcontrol/README.md) wait，backoff，限流等； 5 | - [Jsonpath](/util/jsonpath/README.md) kubectl jsonpath的使用和代码逻辑； 6 | - [Certs](/util/jsonpath/README.md) (没有搞清楚原理，还在整理中)； 7 | -------------------------------------------------------------------------------- /util/certs/README.md: -------------------------------------------------------------------------------- 1 | # SSL 2 | 3 | ## 证书使用过程 4 | 5 | ### CA产生证书 6 | 7 | `openssl req -out -new -newkey rsa:2048 -nodes -keyout private.key -out server.crt` 8 | 9 | ![create-csr](/ch3/images/create-crt.png) 10 | 11 | ### 启动Server 12 | 13 | `go run ./ch3/server/server.go --private-key=./private.key --public-cert=./server.crt` 14 | 15 | ### 访问server 16 | 17 | `curl https://cyfh.com:8080/ping` 18 | 19 | 由于证书不受信任，因此输出： 20 | 21 | ``` 22 | curl: (60) SSL certificate problem: self signed certificate 23 | More details here: https://curl.se/docs/sslcerts.html 24 | 25 | curl failed to verify the legitimacy of the server and therefore could not 26 | establish a secure connection to it. To learn more about this situation and 27 | how to fix it, please visit the web page mentioned above. 28 | 29 | ``` 30 | 31 | 指定cert， 运行` curl --cacert ./server.crt https://cyfh.com:8080/ping`，可以正常访问 32 | 33 | ``` 34 | pong 35 | ``` 36 | 37 | 将cert加入到信任cert中。 38 | 39 | ```bash 40 | sudo security add-trusted-cert \ 41 | -d -r trustRoot \ 42 | -k /Library/Keychains/System.keychain ./server.crt 43 | ``` 44 | 45 | 再次访问，不基于cert。` curl https://cyfh.com:8080/ping` 成功访问。 46 | 47 | 删除cert 48 | 49 | ``` 50 | sudo security remove-trusted-cert \ 51 | -d ./server.crt 52 | ``` 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /util/certs/cert_test.go: -------------------------------------------------------------------------------- 1 | package certs 2 | 3 | import ( 4 | cryptorand "crypto/rand" 5 | "crypto/rsa" 6 | "k8s.io/client-go/util/cert" 7 | "testing" 8 | ) 9 | 10 | const COMMON_NAME = "foo.example.com" 11 | func TestNewSelfSignedCACert(t *testing.T) { 12 | key, err := rsa.GenerateKey(cryptorand.Reader, 2048) 13 | if err != nil { 14 | t.Fatalf("rsa key failed to generate: %v", err) 15 | } 16 | selfSingedCert , err := cert.NewSelfSignedCACert(cert.Config{ 17 | CommonName: COMMON_NAME, 18 | Organization: nil, 19 | AltNames: cert.AltNames{}, 20 | Usages: nil, 21 | },key) 22 | for _, dns := range selfSingedCert.DNSNames { 23 | t.Log(dns) 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /util/certs/images/create-cert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weenxin/client-go/c7db1f7d9a182fd5b9b3d303e10705ffa5834c3f/util/certs/images/create-cert.png -------------------------------------------------------------------------------- /util/certs/images/create-csr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/weenxin/client-go/c7db1f7d9a182fd5b9b3d303e10705ffa5834c3f/util/certs/images/create-csr.png -------------------------------------------------------------------------------- /util/certs/key_test.go: -------------------------------------------------------------------------------- 1 | package certs 2 | 3 | import ( 4 | "crypto/rsa" 5 | "k8s.io/client-go/util/keyutil" 6 | "testing" 7 | ) 8 | 9 | func TestLoadCertsFromFile(t *testing.T) { 10 | prk,err := keyutil.PrivateKeyFromFile("testdata/ca-key.pem") 11 | if err != nil { 12 | t.Logf("parse private failed : %v", err) 13 | } 14 | prkRSA := prk.(*rsa.PrivateKey) 15 | t.Logf("primes : %v ", prkRSA.Primes) 16 | 17 | data,err := keyutil.MarshalPrivateKeyToPEM(prkRSA) 18 | 19 | if err != nil { 20 | t.Logf("MarshalPrivateKeyToPEM failed : %v", err) 21 | } 22 | 23 | 24 | 25 | prkRSA2,err := keyutil.ParsePrivateKeyPEM(data) 26 | if err != nil { 27 | t.Logf("ParsePrivateKeyPEM failed : %v", err) 28 | } 29 | 30 | t.Logf("should be equeal : %v", prkRSA.Equal(prkRSA2)) 31 | 32 | 33 | pks, err := keyutil.PublicKeysFromFile("testdata/ca.pem") 34 | if err != nil { 35 | t.Logf("parse public failed : %v", err) 36 | } 37 | for _, pk := range pks { 38 | pkRSA := pk.(*rsa.PublicKey) 39 | t.Logf("E: %d",pkRSA.E) 40 | } 41 | 42 | 43 | 44 | 45 | 46 | //pk := prk.(rsa.PrivateKey) 47 | //t.Logf("primes : %v ", prkRSA.Primes) 48 | 49 | 50 | } -------------------------------------------------------------------------------- /util/certs/server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/gin-gonic/gin" 6 | ) 7 | 8 | var ( 9 | privateKey = flag.String("private-key","./private.key","server private key") 10 | publicCert = flag.String("public-cert","server.crt","server public key") 11 | ) 12 | 13 | func main() { 14 | r := gin.Default() 15 | 16 | // Ping handler 17 | r.GET("/ping", func(c *gin.Context) { 18 | c.String(200, "pong") 19 | }) 20 | 21 | err := r.RunTLS(":8080",*publicCert,*privateKey) 22 | if err != nil { 23 | panic(err) 24 | } 25 | } -------------------------------------------------------------------------------- /util/certs/testdata/ca-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAzuZIAgG/xO9n2jUkcJPiIu6XMCmaRtC3ITuW8WNChg9ByuvL 3 | 891/OBmCCtPGWaAdmHEjb/BuP7hojV+AH13/SgtgO8QzdOvd/cl+rWtz4s5jxsgc 4 | +1jxjzOGx0/7rg21WMIiAJa3BgZGhWdhKeN9lX8V0wyCiIIcBHdqLqLLt4WsjAKK 5 | K22wkpKaVmCiLcQ+STp4ys0HTfJ5fFN8MpgR5mKyAbWtEP5JeC8H+7i2G8svEW1r 6 | ENP2Cp1yOXY26k9FImErkEScehAJCNIZUuxJ3e7LJ8RB5gN8rA4nXOQoFl8dZ/2u 7 | KOlmC9Ut/jENqdIhxHd8jaU+Uk6Nehd/DKmc/QIDAQABAoIBAEeyx6tiXclqSWv1 8 | 1I3qwYfndJ8tG5iExaGPm+xEwI3ihaNDeWcmD+sp88W6w9bVRMxKEMS+5gXQ6vJq 9 | 8ORt14UaUhwEDuzqvAAxrmN8hClJ6ou9V0XG+dbtPTyevBICECEVDWF02TIORKwA 10 | Okyg2ClRZAfCIw4ZQIpPs407vyr2WLmy+1K2UZ6PMvsVragY7BWdLbat8Q5PCuFx 11 | jtr31G0cYLFzZkKpU6Eu1xwR33ZYOSd5TDi/EDfw7h+WuNdZA4cLpsdv2ijeQi6l 12 | G1HCCjhkIiMpNcqdq8kpc40JqM3uY1slbZ83ep9LIIX99EIK4fg8RPhUY5yT3Zxa 13 | nBh3KyUCgYEA0fcl4Stis3TtX69Upci851+tNHcZuYogoaIJ5zy/qdZdgCvnGjyK 14 | ZpaUvRrgLkBbrusm8OYv8zwSV3hf+VTdgNJ868vJTavKG+6G2qs19C5NPFNT0+6k 15 | zH26M2auYzlFwC64f/J49J/EfJaFg2BdhspxIzb1tyPj9wNgjgIQpQMCgYEA/EMN 16 | dOpWQofUk7W/23XOZvgOVvYW+XJvKlMvzJ2q38iuexgkW3X2rD1iyRe6z797/uVm 17 | QbFoBXpt9EbdlRDGFVXGCrs+I4v0cJeuYg0LSuTR66EluAO1I/VSiTLOassM4frq 18 | YvhOcsQdOsa+nviiFoouPQ+/GG7//FOzedmEFf8CgYEAiJbpM4RP30KvsVGUFY1z 19 | JmmdecYJHCR5eey52I4yeUAU726gFvdPadGHzUWQlUYU3sPnP0vf4bIwlGqgtMi6 20 | mDVWQub9e+PHxUf0vogmv0TnxdjHbdWPq+AWo52m0LHPaGh8ae/JNApDfnFFReQI 21 | s8k6tJfqXUuh3215AKjjPxUCgYEA5qIvVWtCPkWNdxa+OFl2wfGcuqj3zExxklLH 22 | 6t1j8oG59FTYOL6bLlJZVN1bMMEzDpKRcRJPtmJ+0M1RnQ3e7HalP/59C2bTr3Ue 23 | PAsM/gZCXpnQsHLDVALg2QsYac8HEjyjuqSmpIrk5m36rcFIAC+Jos4YurARXpLw 24 | DkFuX+UCgYA8PdXJsKvHOzWAOn+87ToHtsUttc3vMp8cYGelnGs73z24yzoxBFCB 25 | ZPiX01bjBXyRiIlAGgkbf3FcgYYmCCHlcbzWwSofwx/oxAfLmJYfOnT4yJu7IeKO 26 | 1+zsRLlxJaaC6zolAu7JeX1gXtdzw3bruLhdtnNYGHuGoc1exo6fhw== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /util/certs/testdata/ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDoDCCAoigAwIBAgIUWH5jChZFisrINVzn75wezey/FtYwDQYJKoZIhvcNAQEL 3 | BQAwaDELMAkGA1UEBhMCVVMxDzANBgNVBAgTBk9yZWdvbjERMA8GA1UEBxMIUG9y 4 | dGxhbmQxEzARBgNVBAoTCkt1YmVybmV0ZXMxCzAJBgNVBAsTAkNBMRMwEQYDVQQD 5 | EwpLdWJlcm5ldGVzMB4XDTIyMDUxMzEwNDgwMFoXDTI3MDUxMjEwNDgwMFowaDEL 6 | MAkGA1UEBhMCVVMxDzANBgNVBAgTBk9yZWdvbjERMA8GA1UEBxMIUG9ydGxhbmQx 7 | EzARBgNVBAoTCkt1YmVybmV0ZXMxCzAJBgNVBAsTAkNBMRMwEQYDVQQDEwpLdWJl 8 | cm5ldGVzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzuZIAgG/xO9n 9 | 2jUkcJPiIu6XMCmaRtC3ITuW8WNChg9ByuvL891/OBmCCtPGWaAdmHEjb/BuP7ho 10 | jV+AH13/SgtgO8QzdOvd/cl+rWtz4s5jxsgc+1jxjzOGx0/7rg21WMIiAJa3BgZG 11 | hWdhKeN9lX8V0wyCiIIcBHdqLqLLt4WsjAKKK22wkpKaVmCiLcQ+STp4ys0HTfJ5 12 | fFN8MpgR5mKyAbWtEP5JeC8H+7i2G8svEW1rENP2Cp1yOXY26k9FImErkEScehAJ 13 | CNIZUuxJ3e7LJ8RB5gN8rA4nXOQoFl8dZ/2uKOlmC9Ut/jENqdIhxHd8jaU+Uk6N 14 | ehd/DKmc/QIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB 15 | /zAdBgNVHQ4EFgQUogJT6s8jf3Tkq25gowT79nVswzIwDQYJKoZIhvcNAQELBQAD 16 | ggEBAHX4YVoUPT2X3ZdyXiFv/E+Yp5VpGF8G3ChtZHMWoPox3rTbE4IKiQKMmX74 17 | R8yNpARs/Qj2F3tfXZ3CR/7F+XP+qqrz0uVeGGY1CaYgTP3fMIBZwUVYCAE/2L1P 18 | O4OGiFikIkFZsCR0H/pYqGAZCQKpPs88kr+DEy7Ui0HhAcCsb++NQlkoqUFpf5Nu 19 | p7A1zGRa6K1KrQwvsP6fl9DGEzItQqETqqP1au4lf1+XcCpXJy/Z3s9wGbN7mg2N 20 | zHPiz+jKp5Pd3Tb1H/WGebrbKH3881Yqi+cKqlVmbCf2Bgri6M5gXuLLtzreP0sH 21 | dvUzzWZ+zLF/F6lp7xbjmtnhpmo= 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /util/flowcontrol/README.md: -------------------------------------------------------------------------------- 1 | # Retry 包 2 | 3 | ## Wait 4 | 5 | package : `k8s.io/apimachinery/pkg/util/wait` 6 | 7 | ### 做什么 8 | 9 | 定时执行某类操作。 10 | - `给定context结束` 11 | - `给定stop chan结束` 12 | - 定时执行直到成功或者失败 13 | 14 | ### 如何使用 15 | [retry_test](/util/flowcontrol/retry_test.go) 16 | 17 | ### 如何实现 18 | #### Group 19 | 20 | ```go 21 | // 等同与WaitGroup 22 | type Group struct { 23 | wg sync.WaitGroup 24 | } 25 | // 等同与WaitGroup.Wait() 26 | func (g *Group) Wait() { 27 | g.wg.Wait() 28 | } 29 | 30 | //接收chan的包装 31 | func (g *Group) StartWithChannel(stopCh <-chan struct{}, f func(stopCh <-chan struct{})) { 32 | g.Start(func() { 33 | f(stopCh) 34 | }) 35 | } 36 | 37 | //接收context的包装 38 | func (g *Group) StartWithContext(ctx context.Context, f func(context.Context)) { 39 | g.Start(func() { 40 | f(ctx) 41 | }) 42 | } 43 | 44 | //开始运行f 45 | func (g *Group) Start(f func()) { 46 | g.wg.Add(1) 47 | go func() { 48 | defer g.wg.Done() 49 | f() 50 | }() 51 | } 52 | 53 | ``` 54 | 55 | #### Backoff 56 | 57 | ```go 58 | // Backoff holds parameters applied to a Backoff function. 59 | type Backoff struct { 60 | // 最初的duration 61 | Duration time.Duration 62 | // 每次backoff 都会对Duration * Factor 增加 backoff时间，最大到Cap，同时最大放大Steps次数 63 | Factor float64 64 | // 抖动率，在duration增加 Duration * Jitter * Rand（0,1） 65 | Jitter float64 66 | // 可以放大次数 67 | Steps int 68 | // 最大backoff时间 69 | Cap time.Duration 70 | } 71 | 72 | // 计算需要backoff时间 73 | func (b *Backoff) Step() time.Duration { 74 | //如果不需要在放大factor了 75 | if b.Steps < 1 { 76 | //如果需要加入抖动 77 | if b.Jitter > 0 { 78 | //增加抖动 79 | return Jitter(b.Duration, b.Jitter) 80 | } 81 | //返回duration 82 | return b.Duration 83 | } 84 | //次数减1 85 | b.Steps-- 86 | 87 | duration := b.Duration 88 | 89 | // calculate the next step 90 | //如果factor不为0 91 | if b.Factor != 0 { 92 | //计算duration 93 | b.Duration = time.Duration(float64(b.Duration) * b.Factor) 94 | //如果duration 大于 cap则设置为cap 95 | if b.Cap > 0 && b.Duration > b.Cap { 96 | b.Duration = b.Cap 97 | b.Steps = 0 98 | } 99 | } 100 | //如果需要抖动则加入抖动 101 | if b.Jitter > 0 { 102 | duration = Jitter(duration, b.Jitter) 103 | } 104 | return duration 105 | } 106 | //抖动函数 107 | func Jitter(duration time.Duration, maxFactor float64) time.Duration { 108 | if maxFactor <= 0.0 { 109 | maxFactor = 1.0 110 | } 111 | wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration)) 112 | return wait 113 | } 114 | 115 | ``` 116 | 117 | 118 | #### Until 119 | 120 | ```go 121 | 122 | // Forever 每period执行一下f 123 | func Forever(f func(), period time.Duration) { 124 | Until(f, period, NeverStop) 125 | } 126 | 127 | //Until 直到stopCh，每period执行一次函数 128 | func Until(f func(), period time.Duration, stopCh <-chan struct{}) { 129 | JitterUntil(f, period, 0.0, true, stopCh) 130 | } 131 | 132 | 133 | //每period执行一次，直到ctx结束 134 | func UntilWithContext(ctx context.Context, f func(context.Context), period time.Duration) { 135 | JitterUntilWithContext(ctx, f, period, 0.0, true) 136 | } 137 | 138 | 139 | //Sliding 指先使用Backoff计算backoff时间，还是先执行f， Sliding： 为true则后执行 140 | func NonSlidingUntil(f func(), period time.Duration, stopCh <-chan struct{}) { 141 | JitterUntil(f, period, 0.0, false, stopCh) 142 | } 143 | 144 | //接收context的函数包装 145 | func NonSlidingUntilWithContext(ctx context.Context, f func(context.Context), period time.Duration) { 146 | JitterUntilWithContext(ctx, f, period, 0.0, false) 147 | } 148 | 149 | // 每period 执行f，加入抖动因子，sliding为true则后计算backoff时间，否则先计算backoff时间 150 | func JitterUntil(f func(), period time.Duration, jitterFactor float64, sliding bool, stopCh <-chan struct{}) { 151 | BackoffUntil(f, NewJitteredBackoffManager(period, jitterFactor, &clock.RealClock{}), sliding, stopCh) 152 | } 153 | 154 | //具体的Until函数 155 | func BackoffUntil(f func(), backoff BackoffManager, sliding bool, stopCh <-chan struct{}) { 156 | var t clock.Timer 157 | for { 158 | //如果已经停止了，则直接结束 159 | select { 160 | case <-stopCh: 161 | return 162 | default: 163 | } 164 | 165 | //如果sliding为false则先计算backoff时间 166 | if !sliding { 167 | t = backoff.Backoff() 168 | } 169 | 170 | //执行函数 171 | func() { 172 | defer runtime.HandleCrash() 173 | f() 174 | }() 175 | 176 | if sliding { 177 | t = backoff.Backoff() 178 | } 179 | 180 | // NOTE: b/c there is no priority selection in golang 181 | // it is possible for this to race, meaning we could 182 | // trigger t.C and stopCh, and t.C select falls through. 183 | // In order to mitigate we re-check stopCh at the beginning 184 | // of every loop to prevent extra executions of f(). 185 | //select没有优先级可言，因此为了防止当stopCh和timer都发信号这种场景，我们在循环前 186 | //re-check一下，防止多执行一次f 187 | select { 188 | case <-stopCh: 189 | if !t.Stop() { 190 | <-t.C() 191 | } 192 | return 193 | case <-t.C(): 194 | } 195 | } 196 | } 197 | 198 | ``` 199 | 200 | #### ExponentialBackoff 201 | 202 | ```go 203 | 204 | 205 | //BackoffManager 提供了一个返回timer的interface，调用者应该backoff到Timer.C()返回信号，如果第一个backoff返回的timer还没有成功等待结束，又重新调用backoff，那么timer行为是无法预测的。 BackoffManager不是线程安全的 206 | type BackoffManager interface { 207 | Backoff() clock.Timer 208 | } 209 | // 指数backoff实现 210 | type exponentialBackoffManagerImpl struct { 211 | backoff *Backoff //backoff结构体 212 | backoffTimer clock.Timer // backoffManager返回的timer 213 | lastBackoffStart time.Time //最后一次开始时间 214 | initialBackoff time.Duration //最初的backoff Duration 215 | backoffResetDuration time.Duration //多长时间没有backoff，则reset一次 216 | clock clock.Clock //时钟 217 | } 218 | 219 | 220 | func NewExponentialBackoffManager(initBackoff, maxBackoff, resetDuration time.Duration, backoffFactor, jitter float64, c clock.Clock) BackoffManager { 221 | return &exponentialBackoffManagerImpl{ 222 | backoff: &Backoff{ 223 | Duration: initBackoff, //base时间 224 | Factor: backoffFactor, //每次指数增长倍数 225 | Jitter: jitter, //抖动因子 226 | 227 | // the current impl of wait.Backoff returns Backoff.Duration once steps are used up, which is not 228 | // what we ideally need here, we set it to max int and assume we will never use up the steps 229 | Steps: math.MaxInt32, //无限大的次数 230 | Cap: maxBackoff, //最大backoff 231 | }, 232 | backoffTimer: nil, 233 | initialBackoff: initBackoff, //初始时间段 234 | lastBackoffStart: c.Now(), //最后一次backoff时间 235 | backoffResetDuration: resetDuration, //多长时间没有调用backoff则reset一次 236 | clock: c, //时钟 237 | } 238 | } 239 | 240 | // 获得下一次backoff 241 | func (b *exponentialBackoffManagerImpl) getNextBackoff() time.Duration { 242 | //如果很长时间没有backoff了，则清零backoff时间 243 | if b.clock.Now().Sub(b.lastBackoffStart) > b.backoffResetDuration { 244 | b.backoff.Steps = math.MaxInt32 245 | b.backoff.Duration = b.initialBackoff 246 | } 247 | //设置最新backoff时间 248 | b.lastBackoffStart = b.clock.Now() 249 | return b.backoff.Step() 250 | } 251 | 252 | // Backoff implements BackoffManager.Backoff, it returns a timer so caller can block on the timer for exponential backoff. 253 | // The returned timer must be drained before calling Backoff() the second time 254 | func (b *exponentialBackoffManagerImpl) Backoff() clock.Timer { 255 | //如果没有创建过timer，则创建 256 | if b.backoffTimer == nil { 257 | b.backoffTimer = b.clock.NewTimer(b.getNextBackoff()) 258 | } else {//否则复用之前的timer，此时连续来两次调用backoff会返回同一个timer，同时等待一个timer.C()返回结果是随机的 259 | b.backoffTimer.Reset(b.getNextBackoff()) 260 | } 261 | return b.backoffTimer 262 | } 263 | 264 | 265 | //指数执行condition，最多执行backoff.steps次 266 | func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error { 267 | for backoff.Steps > 0 { 268 | if ok, err := runConditionWithCrashProtection(condition); err != nil || ok { 269 | return err 270 | } 271 | if backoff.Steps == 1 { 272 | break 273 | } 274 | time.Sleep(backoff.Step()) 275 | } 276 | return ErrWaitTimeout 277 | } 278 | 279 | ``` 280 | 281 | #### JitteredBackoffManager 282 | 283 | ```go 284 | 285 | type jitteredBackoffManagerImpl struct { 286 | clock clock.Clock //时钟 287 | duration time.Duration // bash时间 288 | jitter float64 //抖动因子 289 | backoffTimer clock.Timer //定时器 290 | } 291 | 292 | 293 | //NewJitteredBackoffManager 新建BackoffManager 294 | func NewJitteredBackoffManager(duration time.Duration, jitter float64, c clock.Clock) BackoffManager { 295 | return &jitteredBackoffManagerImpl{ 296 | clock: c, 297 | duration: duration, 298 | jitter: jitter, 299 | backoffTimer: nil, 300 | } 301 | } 302 | //时间 303 | func (j *jitteredBackoffManagerImpl) getNextBackoff() time.Duration { 304 | jitteredPeriod := j.duration 305 | if j.jitter > 0.0 { 306 | jitteredPeriod = Jitter(j.duration, j.jitter) 307 | } 308 | return jitteredPeriod 309 | } 310 | 311 | func (j *jitteredBackoffManagerImpl) Backoff() clock.Timer { 312 | //计算backoff时间 313 | backoff := j.getNextBackoff() 314 | //如果timer为空 315 | if j.backoffTimer == nil { 316 | //新建一个timer 317 | j.backoffTimer = j.clock.NewTimer(backoff) 318 | } else {//否则复用已有的timer 319 | j.backoffTimer.Reset(backoff) 320 | } 321 | return j.backoffTimer 322 | } 323 | 324 | ``` 325 | 326 | #### Poll 327 | 328 | 329 | ```go 330 | //返回一个函数，该函数返回一个chan，这个chan会定时收到事件 331 | //如果timeout为0 ，则需要通过ctx，关闭chan，防止go-routine泄漏 332 | func poller(interval, timeout time.Duration) WaitWithContextFunc { 333 | return WaitWithContextFunc(func(ctx context.Context) <-chan struct{} { 334 | ch := make(chan struct{}) 335 | 336 | go func() { 337 | defer close(ch) 338 | 339 | tick := time.NewTicker(interval) 340 | defer tick.Stop() 341 | 342 | var after <-chan time.Time 343 | if timeout != 0 { 344 | // time.After is more convenient, but it 345 | // potentially leaves timers around much longer 346 | // than necessary if we exit early. 347 | timer := time.NewTimer(timeout) 348 | after = timer.C 349 | defer timer.Stop() 350 | } 351 | 352 | for { 353 | select { 354 | case <-tick.C: 355 | // If the consumer isn't ready for this signal drop it and 356 | // check the other channels. 357 | select { 358 | case ch <- struct{}{}: 359 | default: 360 | } 361 | case <-after: 362 | return 363 | case <-ctx.Done(): 364 | return 365 | } 366 | } 367 | }() 368 | 369 | return ch 370 | }) 371 | } 372 | ``` 373 | 374 | ```go 375 | type ConditionWithContextFunc func(context.Context) (done bool, err error) 376 | //WaitForWithContext wait也就是poller驱动执行fn，直到ctx.Done 或者fn返回（true，nil） 377 | func WaitForWithContext(ctx context.Context, wait WaitWithContextFunc, fn ConditionWithContextFunc) error { 378 | waitCtx, cancel := context.WithCancel(context.Background()) 379 | defer cancel() 380 | c := wait(waitCtx) 381 | for { 382 | select { 383 | case _, open := <-c: 384 | ok, err := runConditionWithCrashProtectionWithContext(ctx, fn) 385 | if err != nil { 386 | return err 387 | } 388 | if ok { 389 | return nil 390 | } 391 | if !open { 392 | return ErrWaitTimeout 393 | } 394 | case <-ctx.Done(): 395 | // returning ctx.Err() will break backward compatibility 396 | return ErrWaitTimeout 397 | } 398 | } 399 | } 400 | 401 | //对于chan的封装，wait的封装 402 | func WaitFor(wait WaitFunc, fn ConditionFunc, done <-chan struct{}) error { 403 | ctx, cancel := contextForChannel(done) 404 | defer cancel() 405 | return WaitForWithContext(ctx, wait.WithContext(), fn.WithContext()) 406 | } 407 | 408 | //如果immediate为true，则立即执行一次，后面就基于wait驱动执行condition 409 | func poll(ctx context.Context, immediate bool, wait WaitWithContextFunc, condition ConditionWithContextFunc) error { 410 | if immediate { 411 | done, err := runConditionWithCrashProtectionWithContext(ctx, condition) 412 | if err != nil { 413 | return err 414 | } 415 | if done { 416 | return nil 417 | } 418 | } 419 | 420 | select { 421 | case <-ctx.Done(): 422 | // returning ctx.Err() will break backward compatibility 423 | return ErrWaitTimeout 424 | default: 425 | return WaitForWithContext(ctx, wait, condition) 426 | } 427 | } 428 | ``` 429 | 430 | #### ExponentialBackoffWithContext 431 | 432 | ```go 433 | //指数等待方式 434 | func ExponentialBackoffWithContext(ctx context.Context, backoff Backoff, condition ConditionFunc) error { 435 | for backoff.Steps > 0 { 436 | select { 437 | case <-ctx.Done(): 438 | return ctx.Err() 439 | default: 440 | } 441 | 442 | if ok, err := runConditionWithCrashProtection(condition); err != nil || ok { 443 | return err 444 | } 445 | 446 | if backoff.Steps == 1 { 447 | break 448 | } 449 | 450 | waitBeforeRetry := backoff.Step() 451 | select { 452 | case <-ctx.Done(): 453 | return ctx.Err() 454 | case <-time.After(waitBeforeRetry): 455 | } 456 | } 457 | 458 | return ErrWaitTimeout 459 | } 460 | 461 | ``` 462 | 463 | # flowcontrol 包 464 | 465 | ## Backoff 466 | 467 | ### 如何使用 468 | 469 | [TestBackoff](/util/flowcontrol/backoff_test.go) 470 | 471 | ### 实现逻辑 472 | 473 | 474 | #### 结构定义 475 | 476 | ```go 477 | 478 | type backoffEntry struct { 479 | backoff time.Duration 480 | lastUpdate time.Time 481 | } 482 | 483 | type Backoff struct { 484 | sync.RWMutex //锁，保护perItemBackoff 485 | Clock clock.Clock //时钟 486 | defaultDuration time.Duration // base时间 487 | maxDuration time.Duration // 最大时间 488 | perItemBackoff map[string]*backoffEntry //各个元素的backoff的情况 489 | rand *rand.Rand //rand种子 490 | 491 | // maxJitterFactor adds jitter to the exponentially backed off delay. 492 | // if maxJitterFactor is zero, no jitter is added to the delay in 493 | // order to maintain current behavior. 494 | maxJitterFactor float64 //抖动因子 495 | } 496 | ``` 497 | #### 对象构建 498 | 499 | ```go 500 | //用于测试 501 | func NewFakeBackOff(initial, max time.Duration, tc *testingclock.FakeClock) *Backoff { 502 | return newBackoff(tc, initial, max, 0.0) 503 | } 504 | 505 | //新建一个backoff， initial为初始时间， max为对最大backoff时间 506 | func NewBackOff(initial, max time.Duration) *Backoff { 507 | return NewBackOffWithJitter(initial, max, 0.0) 508 | } 509 | //initial为初始时间， max为对最大backoff时间， maxJitterFactor为抖动时间 510 | func NewFakeBackOffWithJitter(initial, max time.Duration, tc *testingclock.FakeClock, maxJitterFactor float64) *Backoff { 511 | return newBackoff(tc, initial, max, maxJitterFactor) 512 | } 513 | //initial为初始时间， max为对最大backoff时间， maxJitterFactor为抖动时间 514 | func NewBackOffWithJitter(initial, max time.Duration, maxJitterFactor float64) *Backoff { 515 | clock := clock.RealClock{} 516 | return newBackoff(clock, initial, max, maxJitterFactor) 517 | } 518 | 519 | func newBackoff(clock clock.Clock, initial, max time.Duration, maxJitterFactor float64) *Backoff { 520 | var random *rand.Rand 521 | if maxJitterFactor > 0 { 522 | random = rand.New(rand.NewSource(clock.Now().UnixNano())) 523 | } 524 | return &Backoff{ 525 | perItemBackoff: map[string]*backoffEntry{}, //保存对象backoff当前状态 526 | Clock: clock, 527 | defaultDuration: initial, 528 | maxDuration: max, 529 | maxJitterFactor: maxJitterFactor, 530 | rand: random, 531 | } 532 | } 533 | ``` 534 | 535 | ```go 536 | // 获取当前backoff时间，用于sleep，等待时间ok在执行具体动作 537 | func (p *Backoff) Get(id string) time.Duration { 538 | p.RLock() 539 | defer p.RUnlock() 540 | var delay time.Duration 541 | entry, ok := p.perItemBackoff[id] 542 | if ok { 543 | delay = entry.backoff 544 | } 545 | return delay 546 | } 547 | ``` 548 | 549 | ```go 550 | // 本次动作执行失败，会调用Next，增加backoff时间 551 | func (p *Backoff) Next(id string, eventTime time.Time) { 552 | p.Lock() 553 | defer p.Unlock() 554 | entry, ok := p.perItemBackoff[id] 555 | if !ok || hasExpired(eventTime, entry.lastUpdate, p.maxDuration) { 556 | entry = p.initEntryUnsafe(id) 557 | entry.backoff += p.jitter(entry.backoff) 558 | } else { 559 | delay := entry.backoff * 2 // exponential 560 | delay += p.jitter(entry.backoff) // add some jitter to the delay 561 | entry.backoff = time.Duration(integer.Int64Min(int64(delay), int64(p.maxDuration))) 562 | } 563 | entry.lastUpdate = p.Clock.Now() 564 | } 565 | ``` 566 | 567 | ```go 568 | //如果成功了，则reset该对象，reset backoff时间 569 | func (p *Backoff) Reset(id string) { 570 | p.Lock() 571 | defer p.Unlock() 572 | delete(p.perItemBackoff, id) 573 | } 574 | ``` 575 | 576 | ```go 577 | 578 | // Returns True if the elapsed time since eventTime is smaller than the current backoff window 579 | func (p *Backoff) IsInBackOffSince(id string, eventTime time.Time) bool { 580 | p.RLock() 581 | defer p.RUnlock() 582 | entry, ok := p.perItemBackoff[id] 583 | if !ok { 584 | return false 585 | } 586 | if hasExpired(eventTime, entry.lastUpdate, p.maxDuration) { 587 | return false 588 | } 589 | return p.Clock.Since(eventTime) < entry.backoff 590 | } 591 | 592 | // Returns True if time since lastupdate is less than the current backoff window. 593 | func (p *Backoff) IsInBackOffSinceUpdate(id string, eventTime time.Time) bool { 594 | p.RLock() 595 | defer p.RUnlock() 596 | entry, ok := p.perItemBackoff[id] 597 | if !ok { 598 | return false 599 | } 600 | if hasExpired(eventTime, entry.lastUpdate, p.maxDuration) { 601 | return false 602 | } 603 | return eventTime.Sub(entry.lastUpdate) < entry.backoff 604 | } 605 | 606 | ``` 607 | 608 | ```go 609 | //删除时间过长，不需要追踪的对象 610 | func (p *Backoff) GC() { 611 | p.Lock() 612 | defer p.Unlock() 613 | now := p.Clock.Now() 614 | for id, entry := range p.perItemBackoff { 615 | if now.Sub(entry.lastUpdate) > p.maxDuration*2 { 616 | // GC when entry has not been updated for 2*maxDuration 617 | delete(p.perItemBackoff, id) 618 | } 619 | } 620 | } 621 | 622 | //删除对象，有点向reset 623 | func (p *Backoff) DeleteEntry(id string) { 624 | p.Lock() 625 | defer p.Unlock() 626 | delete(p.perItemBackoff, id) 627 | } 628 | 629 | ``` 630 | 631 | ## Throttle 632 | 633 | ### 如何使用 634 | 635 | [TestThrottle](/util/flowcontrol/throttle_test.go) 636 | 637 | 638 | ### 实现逻辑 639 | 640 | #### 结构定义 641 | 642 | ```go 643 | //乐观锁，可以尽量尝试获取lock 644 | type PassiveRateLimiter interface { 645 | // TryAccept returns true if a token is taken immediately. Otherwise, 646 | // it returns false. 647 | TryAccept() bool 648 | // Stop stops the rate limiter, subsequent calls to CanAccept will return false 649 | Stop() 650 | // QPS returns QPS of this rate limiter 651 | QPS() float32 652 | } 653 | 654 | //悲观锁 655 | type RateLimiter interface { 656 | PassiveRateLimiter 657 | // Accept returns once a token becomes available. 658 | // goroutine 会sleep到可以获取到锁的时间 659 | Accept() 660 | // Wait returns nil if a token is taken before the Context is done. 661 | //等待到可以获取到令牌 662 | Wait(ctx context.Context) error 663 | } 664 | 665 | type tokenBucketPassiveRateLimiter struct { 666 | limiter *rate.Limiter //基于令牌桶实现 667 | qps float32 //令牌发送频率 668 | clock clock.PassiveClock //时钟 669 | } 670 | 671 | type tokenBucketRateLimiter struct { 672 | tokenBucketPassiveRateLimiter // 673 | clock Clock //需要等待，所以需要Sleep等待，clock.PassiveClock满足不了需求 674 | } 675 | ``` 676 | 677 | #### 新建对象 678 | 679 | ```go 680 | 681 | // NewTokenBucketRateLimiterWithClock is identical to NewTokenBucketRateLimiter 682 | // but allows an injectable clock, for testing. 683 | func NewTokenBucketRateLimiterWithClock(qps float32, burst int, c Clock) RateLimiter { 684 | limiter := rate.NewLimiter(rate.Limit(qps), burst) 685 | return newTokenBucketRateLimiterWithClock(limiter, c, qps) 686 | } 687 | 688 | // NewTokenBucketPassiveRateLimiterWithClock is similar to NewTokenBucketRateLimiterWithClock 689 | // except that it returns a PassiveRateLimiter which does not have Accept() and Wait() methods 690 | // and uses a PassiveClock. 691 | func NewTokenBucketPassiveRateLimiterWithClock(qps float32, burst int, c clock.PassiveClock) PassiveRateLimiter { 692 | limiter := rate.NewLimiter(rate.Limit(qps), burst) 693 | return newTokenBucketRateLimiterWithPassiveClock(limiter, c, qps) 694 | } 695 | 696 | func newTokenBucketRateLimiterWithClock(limiter *rate.Limiter, c Clock, qps float32) *tokenBucketRateLimiter { 697 | return &tokenBucketRateLimiter{ 698 | tokenBucketPassiveRateLimiter: *newTokenBucketRateLimiterWithPassiveClock(limiter, c, qps), 699 | clock: c, 700 | } 701 | } 702 | 703 | func newTokenBucketRateLimiterWithPassiveClock(limiter *rate.Limiter, c clock.PassiveClock, qps float32) *tokenBucketPassiveRateLimiter { 704 | return &tokenBucketPassiveRateLimiter{ 705 | limiter: limiter, 706 | qps: qps, 707 | clock: c, 708 | } 709 | } 710 | 711 | ``` 712 | 713 | #### 相关方法和实现 714 | 715 | ```go 716 | //stop需要基于limiter做stop，limiter还不支持，因此什么都不做，看起来还是有些问题的 717 | func (tbprl *tokenBucketPassiveRateLimiter) Stop() { 718 | 719 | } 720 | 721 | func (tbprl *tokenBucketPassiveRateLimiter) QPS() float32 { 722 | return tbprl.qps 723 | } 724 | //尝试获取，使用AllowN不等待 725 | func (tbprl *tokenBucketPassiveRateLimiter) TryAccept() bool { 726 | return tbprl.limiter.AllowN(tbprl.clock.Now(), 1) 727 | } 728 | 729 | // Accept will block until a token becomes available 730 | // goroutine等待 731 | func (tbrl *tokenBucketRateLimiter) Accept() { 732 | now := tbrl.clock.Now() 733 | tbrl.clock.Sleep(tbrl.limiter.ReserveN(now, 1).DelayFrom(now)) 734 | } 735 | //调用令牌桶的等待 736 | func (tbrl *tokenBucketRateLimiter) Wait(ctx context.Context) error { 737 | return tbrl.limiter.Wait(ctx) 738 | } 739 | 740 | ``` 741 | 742 | 743 | 744 | 745 | 746 | 747 | 748 | -------------------------------------------------------------------------------- /util/flowcontrol/backoff_test.go: -------------------------------------------------------------------------------- 1 | package flowcontrol 2 | 3 | import ( 4 | "testing" 5 | testclock "k8s.io/utils/clock/testing" 6 | "k8s.io/client-go/util/flowcontrol" 7 | "time" 8 | ) 9 | 10 | func TestFlowControlBackoff(t *testing.T) { 11 | id := "test1" 12 | tc := testclock.NewFakeClock(time.Now()) 13 | b := flowcontrol.NewFakeBackOff(time.Millisecond,time.Second,tc) 14 | 15 | //第一次做一件事情 16 | duration := b.Get(id) 17 | t.Logf("first time : %v ", duration) 18 | //失败了 19 | b.Next(id,time.Now()) 20 | //过了一段时间，重试下 21 | tc.Step(duration) 22 | duration = b.Get(id) 23 | t.Logf("second time , retry : %v ", duration) 24 | 25 | 26 | //又失败了 27 | b.Next(id,time.Now()) 28 | //过了一段时间继续重试 29 | tc.Step(duration) 30 | duration = b.Get(id) 31 | t.Logf("third time , retry : %v ", duration) 32 | 33 | //成功了 34 | b.Reset(id) 35 | 36 | //过了一段时间需要继续做这个事情 37 | duration = b.Get(id) 38 | t.Logf("first time : %v ", duration) 39 | //失败，则继续累加 40 | b.Next(id,time.Now()) 41 | duration = b.Get(id) 42 | t.Logf("second time , retry : %v ", duration) 43 | 44 | 45 | 46 | timer := tc.NewTimer(3 * time.Second) 47 | 48 | go func() { 49 | select { 50 | case <- timer.C(): 51 | //需要手动调用gc，才会删除map中的条目 52 | b.GC() 53 | } 54 | }() 55 | 56 | 57 | t.Logf("IsInBackOffSince : %v", b.IsInBackOffSince(id,tc.Now())) 58 | //最后一次更新时间（调用next时间）到eventTime，是否小于backoff时间窗口 59 | t.Logf("IsInBackOffSinceUpdate : %v", b.IsInBackOffSinceUpdate(id,tc.Now())) 60 | begin := tc.Now() 61 | 62 | tc.Step(3*time.Millisecond) 63 | //eventTime到当前时间，是否小于backoff时间窗口 64 | t.Logf("IsInBackOffSince : %v", b.IsInBackOffSince(id,begin)) 65 | //最后一次更新时间（调用next时间）到eventTime，是否小于backoff时间窗口 66 | t.Logf("IsInBackOffSinceUpdate : %v", b.IsInBackOffSinceUpdate(id,tc.Now()) ) 67 | 68 | 69 | tc.Step( 3 * time.Second ) //由于长时间没有重试则会将数据 70 | time.Sleep(1 * time.Millisecond) 71 | 72 | 73 | duration = b.Get(id) 74 | t.Logf("after gc dration is reset , %v",duration) 75 | } 76 | -------------------------------------------------------------------------------- /util/flowcontrol/retry_test.go: -------------------------------------------------------------------------------- 1 | package flowcontrol 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "k8s.io/apimachinery/pkg/util/wait" 7 | "k8s.io/client-go/util/retry" 8 | "k8s.io/utils/clock" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestGroup(t *testing.T) { 14 | //类似于WaitGroup，但是只支持执行一个函数 15 | g := wait.Group{} 16 | ctx ,cancel := context.WithTimeout(context.Background(),100*time.Millisecond) 17 | defer cancel() 18 | output := make(chan struct{}) 19 | g.StartWithContext(ctx, func(ctx context.Context) { 20 | select { 21 | case output<- struct{}{}: 22 | t.Logf("success add item") 23 | case <-ctx.Done(): 24 | t.Logf("context timeout") 25 | } 26 | 27 | }) 28 | select { 29 | case <-ctx.Done(): 30 | t.Log("context timeout") 31 | case <-output: 32 | t.Log("get item success") 33 | } 34 | //等待函数执行结束 35 | g.Wait() 36 | 37 | } 38 | 39 | func TestUntil(t *testing.T) { 40 | timer := clock.RealClock{}.NewTimer(100 * time.Millisecond) 41 | stop := make(chan struct{}) 42 | count := 0 43 | 44 | //直到stop收到信号为止 45 | go wait.Until(func() { 46 | t.Logf("count is : %v" ,count) 47 | count ++ 48 | },time.Millisecond * 10 ,stop ) 49 | 50 | select { 51 | case <-timer.C(): 52 | close(stop) 53 | } 54 | 55 | time.Sleep(time.Second) 56 | } 57 | 58 | func TestBackoffUntil(t *testing.T) { 59 | count := 0 60 | begin := time.Now() 61 | 62 | bm := wait.NewExponentialBackoffManager( 63 | 1 * time.Millisecond, // 1ms 为base 64 | 20 * time.Millisecond, //最大20ms 65 | time.Second, //1秒钟内没有backoff，则清空backoff时间 66 | 2, //放大因子 67 | 0.1, //抖动因子 68 | clock.RealClock{}) 69 | 70 | stop := make(chan struct{}) 71 | 72 | 73 | //相对于Util来说加入了backoff策略，对后端更加友好 74 | go wait.BackoffUntil(func() { 75 | t.Logf("count : %v , duration : %v ms", count,time.Now().Sub(begin).Round(time.Millisecond) ) 76 | count ++ 77 | },bm, 78 | false,//是否每隔一段时间执行操作【timer， proc， timer】，还是保证每隔间隔内执行操作即可【timer（proc），timer（proc）】 79 | // true : 【timer， proc， timer】 80 | //false : 【timer（proc），timer（proc） 81 | stop) 82 | 83 | 84 | 85 | time.Sleep(1 *time.Second) 86 | 87 | close(stop) 88 | 89 | t.Logf("stop BackoffUntil") 90 | time.Sleep(1 * time.Second) 91 | } 92 | 93 | 94 | func TestExponentialBackoff(t *testing.T){ 95 | //指数级别的回退 96 | bf := wait.Backoff{ 97 | Duration: time.Millisecond, //base 1ms 98 | Factor: 2, //每次放大2倍 99 | Jitter: 0.1, //0.1的抖动 100 | Steps: 5, //最多执行5次时间变更，后面就不变了 101 | Cap: 20 * time.Millisecond, //最多20ms 102 | } 103 | 104 | counter := 3 105 | begin := time.Now() 106 | err := wait.ExponentialBackoff(bf, func() (done bool, err error) { 107 | t.Logf("counter : %v, duration : %v", counter, time.Now().Sub(begin).Round(time.Millisecond)) 108 | counter -- 109 | if counter == 0 { 110 | return true ,nil 111 | } 112 | return false,nil 113 | }) 114 | 115 | if err != nil { 116 | t.Logf("run failed , err : %v", err) 117 | }else { 118 | t.Logf("run success, counter : %v",counter) 119 | } 120 | } 121 | 122 | 123 | 124 | func TestPool(t *testing.T){ 125 | counter := 3 126 | begin := time.Now() 127 | //每5毫秒执行一次，最多执行一秒钟，执行成功或者遇到错误为止 128 | err := wait.Poll( 5 * time.Millisecond , time.Second, func() (done bool, err error) { 129 | t.Logf("counter : %v, duration : %v", counter, time.Now().Sub(begin).Round(time.Millisecond)) 130 | counter -- 131 | if counter == 0 { 132 | return true ,nil 133 | } 134 | return false,nil 135 | }) 136 | 137 | if err != nil { 138 | t.Logf("run failed , err : %v", err) 139 | }else { 140 | t.Logf("run success, counter : %v",counter) 141 | } 142 | } 143 | 144 | func TestRetry(t *testing.T) { 145 | count := 2 146 | 147 | err := retry.OnError(retry.DefaultBackoff, 148 | //判断错误是否可重试 149 | func(err error) bool { 150 | return true 151 | }, 152 | //执行函数 153 | func() error { 154 | if count == 0 { 155 | return nil 156 | } 157 | count -- 158 | return errors.New("not zero") 159 | }, 160 | ) 161 | 162 | if err != nil { 163 | t.Logf("failed to set count to zero err: %v",err) 164 | }else { 165 | t.Logf("success set count to zero count : %v",count) 166 | } 167 | } 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /util/flowcontrol/throttle_test.go: -------------------------------------------------------------------------------- 1 | package flowcontrol 2 | 3 | import ( 4 | "context" 5 | "k8s.io/client-go/util/flowcontrol" 6 | "runtime" 7 | "sync" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | //悲观锁 14 | func TestThrottle(t *testing.T){ 15 | //QPS 100 ，桶中最多5个令牌 16 | r := flowcontrol.NewTokenBucketRateLimiter(100, 5) 17 | count := int32(0) 18 | var wg sync.WaitGroup 19 | ctx,cancel := context.WithCancel(context.Background()) 20 | process := runtime.GOMAXPROCS(0) 21 | wg.Add(process) 22 | //开启多个goroutine抢令牌，执行函数 23 | for i := 0 ; i < process ; i ++ { 24 | go func() { 25 | defer wg.Done() 26 | for { 27 | err := r.Wait(ctx) 28 | if err != nil { 29 | return 30 | }else { 31 | atomic.AddInt32(&count,1) 32 | } 33 | } 34 | 35 | }() 36 | } 37 | //r.Stop()//这个好像没有啥用 38 | 39 | //一秒钟之后看下结果 40 | time.Sleep(time.Second) 41 | cancel() 42 | wg.Wait() 43 | 44 | t.Logf("TestThrottle counter is : %v", count) 45 | } 46 | 47 | //乐观用法 48 | func TestThrottlePassive(t *testing.T){ 49 | //QPS 100 ，桶中最多5个令牌 50 | r := flowcontrol.NewTokenBucketRateLimiter(100, 5) 51 | count := int32(0) 52 | var wg sync.WaitGroup 53 | ctx,cancel := context.WithCancel(context.Background()) 54 | process := runtime.GOMAXPROCS(0) 55 | wg.Add(process) 56 | //开多个goroutine开始执行 57 | for i := 0 ; i < process ; i ++ { 58 | go func() { 59 | defer wg.Done() 60 | for { 61 | select { 62 | case <- ctx.Done(): 63 | return 64 | default: 65 | if r.TryAccept() { 66 | atomic.AddInt32(&count,1) 67 | } 68 | } 69 | } 70 | 71 | }() 72 | } 73 | //r.Stop()//这个好像没有啥用 74 | time.Sleep(time.Second) 75 | cancel() 76 | wg.Wait() 77 | 78 | t.Logf("TestPassiveThrottle counter is : %v", count) 79 | } 80 | -------------------------------------------------------------------------------- /util/jsonpath/README.md: -------------------------------------------------------------------------------- 1 | ## JsonPath 2 | 3 | ## 用例场景 4 | 5 | kubectl 支持jsonpath输出： 6 | 7 | ### 创建测试pod 8 | 9 | 创建一个pod: `kubectl create -f ./util/jsonpath/testdata/pod.yaml` 10 | 11 | 获取json数据: `kubectl get pod -o json` 12 | 13 | ### jsonpath获取结果 14 | 15 | 获取全部数据： `kubectl get pod test-jsonpath -o=jsonpath="{@}"` 16 | 17 | ### 只获取指定字段 18 | 19 | - `kubectl get pod test-jsonpath -o=jsonpath="{.kind}"` 20 | - `kubectl get pod test-jsonpath -o=jsonpath="{.metadata.name}"` 21 | - `kubectl get pod test-jsonpath -o=jsonpath="{['metadata.name']}"` : 将对象当作map使用 22 | 23 | ### 寻找所有子field 24 | 25 | - `kubectl get pod test-jsonpath -o=jsonpath="names : [{..name}] "` 26 | 27 | 会获取所有字段字段为name的结果： 28 | 29 | ``` 30 | names : [test-jsonpath kube-api-access-w9kjk kube-root-ca.crt nginx kube-api-access-w9kjk nginx] 31 | ``` 32 | 33 | ### 数组访问 34 | 35 | - 获取整个数组： `kubectl get pod test-jsonpath -o=jsonpath="{.spec.tolerations}"` 36 | 37 | ```json 38 | [ 39 | { 40 | "effect":"NoExecute", 41 | "key":"node.kubernetes.io/not-ready", 42 | "operator":"Exists", 43 | "tolerationSeconds":300 44 | }, 45 | { 46 | "effect":"NoExecute", 47 | "key":"node.kubernetes.io/unreachable", 48 | "operator":"Exists", 49 | "tolerationSeconds":300 50 | } 51 | ] 52 | ``` 53 | 54 | - 获取所有元素： `kubectl get pod test-jsonpath -o=jsonpath="{.spec.tolerations[*]}"` 55 | 56 | ```json 57 | { 58 | "effect":"NoExecute", 59 | "key":"node.kubernetes.io/not-ready", 60 | "operator":"Exists", 61 | "tolerationSeconds":300 62 | }, 63 | { 64 | "effect":"NoExecute", 65 | "key":"node.kubernetes.io/unreachable", 66 | "operator":"Exists", 67 | "tolerationSeconds":300 68 | } 69 | ``` 70 | 71 | 72 | - 获取第0个元素 `kubectl get pod test-jsonpath -o=jsonpath="{.spec.tolerations[0]}"` 73 | 74 | ```json 75 | { 76 | "effect":"NoExecute", 77 | "key":"node.kubernetes.io/not-ready", 78 | "operator":"Exists", 79 | "tolerationSeconds":300 80 | } 81 | ``` 82 | 83 | 获取第[0~2]元素: `kubectl get pod test-jsonpath -o=jsonpath="{.spec.tolerations[0:2]}"` 84 | 85 | ```json 86 | {"effect":"NoExecute","key":"node.kubernetes.io/not-ready","operator":"Exists","tolerationSeconds":300} {"effect":"NoExecute","key":"node.kubernetes.io/unreachable","operator":"Exists","tolerationSeconds":300} 87 | ``` 88 | 89 | 90 | ### 筛选 91 | 92 | - 获取所有effect为NoExecute的污点 ： `kubectl get pod test-jsonpath -o=jsonpath='{.spec.tolerations[?(@.effect=="NoExecute")]}'` 93 | 94 | ### 迭代 95 | 96 | 97 | - 迭代数组： `kubectl get pod test-jsonpath -o=jsonpath="{range .spec.tolerations[*]} effect:{.effect} , key:{.key} | {end}"` 98 | 99 | **注意.spec.tolerations[*] 如果是 .spec.tolerations {@}将会是数组本身** 100 | 101 | ``` 102 | effect:NoExecute , key:node.kubernetes.io/not-ready | effect:NoExecute , key:node.kubernetes.io/unreachable | 103 | ``` 104 | 105 | ### 引用执行字符串 106 | 107 | - `kubectl get pod test-jsonpath -o=jsonpath="{range .spec.tolerations[*]}effect:{.effect}{'\t'}key:{.key}{'\n'}{end}"` 108 | 109 | 会输出： 110 | 111 | ``` 112 | effect:NoExecute key:node.kubernetes.io/not-ready 113 | effect:NoExecute key:node.kubernetes.io/unreachable 114 | ``` 115 | 116 | ## 代码使用 117 | 118 | [code](/util/jsonpath/jsonpath_test.go) 119 | 120 | 121 | ## 实现细节 122 | 123 | `hello {range .items[*]}{.metadata.name}{"\t"}{end}` 124 | 125 | 机会解析出如下所示的树形列表 126 | 127 | ``` 128 | Root 129 | NodeText: hello 130 | NodeList 131 | NodeIdentifier: range 132 | NodeField: items 133 | NodeArray: [{0 false false} {0 false false} {0 false false}] 134 | NodeList 135 | NodeField: metadata 136 | NodeField: name 137 | NodeList 138 | NodeText 139 | NodeList 140 | NodeIdentifier: end 141 | ``` 142 | 143 | ### Parse 阶段 144 | 145 | 输入 `hello {range .items[*]}{.metadata.name}{"\t"}{end}` 146 | 147 | 基于`Root`对象开始`Parse` 148 | 149 | #### Parse Text对象 150 | 151 | ```go 152 | func (p *Parser) parseText(cur *ListNode) error { 153 | for { 154 | if strings.HasPrefix(p.input[p.pos:], leftDelim) { //找到了第一个'{' 155 | if p.pos > p.start { 156 | cur.append(newText(p.consumeText())) // 将父亲节点加入一个子节点，类型为Text，结果为推进过程中消耗的字符 157 | } 158 | return p.parseLeftDelim(cur) 159 | } 160 | if p.next() == eof { //没有找到一直推进 161 | break 162 | } 163 | } 164 | // Correctly reached EOF. 165 | if p.pos > p.start { 166 | cur.append(newText(p.consumeText())) 167 | } 168 | return nil 169 | } 170 | ``` 171 | 172 | 173 | 174 | #### ParseLeftDelim 175 | 176 | `{range .items[*]}{.metadata.name}{"\t"}{end}` 177 | 178 | ```go 179 | // parseLeftDelim scans the left delimiter, which is known to be present. 180 | func (p *Parser) parseLeftDelim(cur *ListNode) error { //cur为root节点 181 | p.pos += len(leftDelim) 182 | p.consumeText() //消耗掉`{` 183 | newNode := newList() //新建一个节点 184 | cur.append(newNode) //加入到root节点的子节点中 185 | cur = newNode 186 | return p.parseInsideAction(cur) //解析内部的action 187 | } 188 | ``` 189 | 190 | #### parseInsideAction 191 | 192 | `range .items[*]}{.metadata.name}{"\t"}{end}` 193 | 194 | ```go 195 | func (p *Parser) parseInsideAction(cur *ListNode) error { //当前节点为已经之前新建的节点 196 | prefixMap := map[string]func(*ListNode) error{ 197 | rightDelim: p.parseRightDelim, 198 | "[?(": p.parseFilter, 199 | "..": p.parseRecursive, 200 | } 201 | for prefix, parseFunc := range prefixMap { 202 | if strings.HasPrefix(p.input[p.pos:], prefix) { 203 | return parseFunc(cur) 204 | } 205 | } 206 | 207 | //此时是一个 range operation 208 | 209 | switch r := p.next(); { 210 | case r == eof || isEndOfLine(r): //不符合 211 | return fmt.Errorf("unclosed action") 212 | case r == ' ': //去除所有空格 213 | p.consumeText() 214 | case r == '@' || r == '$': //the current object, just pass it 215 | p.consumeText() 216 | case r == '[': //第三次将会遇到 217 | return p.parseArray(cur) 218 | case r == '"' || r == '\'': 219 | return p.parseQuote(cur, r) 220 | case r == '.': // 第二次解析时遇到了. 221 | return p.parseField(cur) 222 | case r == '+' || r == '-' || unicode.IsDigit(r): 223 | p.backup() 224 | return p.parseNumber(cur) 225 | case isAlphaNumeric(r): //是字符，开始解析 226 | p.backup() //将标识符退回 227 | return p.parseIdentifier(cur) 228 | default: 229 | return fmt.Errorf("unrecognized character in action: %#U", r) 230 | } 231 | return p.parseInsideAction(cur) 232 | } 233 | ``` 234 | 235 | 236 | 237 | ```go 238 | `range .items[*]}{.metadata.name}{"\t"}{end}` 239 | 240 | // parseIdentifier scans build-in keywords, like "range" "end" 241 | func (p *Parser) parseIdentifier(cur *ListNode) error { //当前节点还是新建的1号节点 242 | var r rune 243 | for { 244 | r = p.next() 245 | if isTerminator(r) { 246 | p.backup() 247 | break 248 | } 249 | } 250 | value := p.consumeText() //拿到range 251 | 252 | if isBool(value) { //不是true也不是false 253 | v, err := strconv.ParseBool(value) 254 | if err != nil { 255 | return fmt.Errorf("can not parse bool '%s': %s", value, err.Error()) 256 | } 257 | 258 | cur.append(newBool(v)) 259 | } else { 260 | cur.append(newIdentifier(value)) //1号节点增加了一个节点，为NodeIdentifier ： range 261 | } 262 | 263 | return p.parseInsideAction(cur) //继续解析内部action 264 | } 265 | ``` 266 | 267 | 268 | `items[*]}{.metadata.name}{"\t"}{end}` 269 | 270 | ```go 271 | // parseField scans a field until a terminator 272 | func (p *Parser) parseField(cur *ListNode) error { 273 | p.consumeText() //忽略. 274 | for p.advance() { //持续向前 275 | } 276 | value := p.consumeText() //拿到item 277 | if value == "*" { 278 | cur.append(newWildcard()) 279 | } else { 280 | cur.append(newField(strings.Replace(value, "\\", "", -1))) //将1号节点加入： NodeField： items 281 | } 282 | return p.parseInsideAction(cur) //继续处理 283 | } 284 | ``` 285 | 286 | `[*]}{.metadata.name}{"\t"}{end}` 287 | ```go 288 | case r == '[': //第三次将会遇到 289 | return p.parseArray(cur) 290 | ``` 291 | 292 | ```go 293 | // parseArray scans array index selection 294 | func (p *Parser) parseArray(cur *ListNode) error { 295 | Loop: 296 | for { 297 | switch p.next() { //持续推进 298 | case eof, '\n': 299 | return fmt.Errorf("unterminated array") 300 | case ']': //知道遇到第一个']' 301 | break Loop 302 | } 303 | } 304 | text := p.consumeText() //此时text为 '[*]' 305 | text = text[1 : len(text)-1] //去掉左右 [] 306 | if text == "*" { //text为 : 307 | text = ":" 308 | } 309 | 310 | //union operator 311 | strs := strings.Split(text, ",") //是否为union 312 | if len(strs) > 1 { 313 | union := []*ListNode{} 314 | for _, str := range strs { 315 | parser, err := parseAction("union", fmt.Sprintf("[%s]", strings.Trim(str, " "))) 316 | if err != nil { 317 | return err 318 | } 319 | union = append(union, parser.Root) 320 | } 321 | cur.append(newUnion(union)) 322 | return p.parseInsideAction(cur) 323 | } 324 | 325 | // dict key， 是否为字段访问操作 326 | value := dictKeyRex.FindStringSubmatch(text) //text内容是否为'字符串' ，如果是，匹配出内部结果：字符串 327 | if value != nil { 328 | parser, err := parseAction("arraydict", fmt.Sprintf(".%s", value[1])) 329 | if err != nil { 330 | return err 331 | } 332 | for _, node := range parser.Root.Nodes { 333 | cur.append(node) 334 | } 335 | return p.parseInsideAction(cur) 336 | } 337 | 338 | //slice operator 339 | value = sliceOperatorRex.FindStringSubmatch(text) //是否匹配slice中数据索引 340 | if value == nil { 341 | return fmt.Errorf("invalid array index %s", text) 342 | } 343 | value = value[1:] //第一个为字符串本身，1为第一个索引，2：为第二个，3为第三个 344 | params := [3]ParamsEntry{} 345 | for i := 0; i < 3; i++ { 346 | if value[i] != "" { 347 | if i > 0 { 348 | value[i] = value[i][1:] //去掉： 349 | } 350 | if i > 0 && value[i] == "" { 351 | params[i].Known = false 352 | } else { 353 | var err error 354 | params[i].Known = true 355 | params[i].Value, err = strconv.Atoi(value[i]) 356 | if err != nil { 357 | return fmt.Errorf("array index %s is not a number", value[i]) 358 | } 359 | } 360 | } else { 361 | if i == 1 { 362 | params[i].Known = true 363 | params[i].Value = params[0].Value + 1 364 | params[i].Derived = true 365 | } else { 366 | params[i].Known = false 367 | params[i].Value = 0 368 | } 369 | } 370 | } 371 | cur.append(newArray(params)) 372 | return p.parseInsideAction(cur) // 继续处理 373 | } 374 | ``` 375 | 376 | 377 | `}{.metadata.name}{"\t"}{end}` 378 | 379 | 380 | ```go 381 | prefixMap := map[string]func(*ListNode) error{ 382 | rightDelim: p.parseRightDelim, 383 | "[?(": p.parseFilter, 384 | "..": p.parseRecursive, 385 | } 386 | for prefix, parseFunc := range prefixMap { 387 | if strings.HasPrefix(p.input[p.pos:], prefix) { 388 | return parseFunc(cur) 389 | } 390 | } 391 | ``` 392 | 393 | 394 | ```go 395 | // parseRightDelim scans the right delimiter, which is known to be present. 396 | func (p *Parser) parseRightDelim(cur *ListNode) error { 397 | p.pos += len(rightDelim) 398 | p.consumeText() //消耗掉`}` 399 | return p.parseText(p.Root) //重新开始对跟对象渲染 400 | } 401 | ``` 402 | 403 | 此时还有`{.metadata.name}{"\t"}{end}` 404 | 405 | 406 | ### 执行阶段 407 | 408 | 409 | ```go 410 | func (j *JSONPath) FindResults(data interface{}) ([][]reflect.Value, error) { 411 | if j.parser == nil { 412 | return nil, fmt.Errorf("%s is an incomplete jsonpath template", j.name) 413 | } 414 | 415 | cur := []reflect.Value{reflect.ValueOf(data)} //构造统一处理参数 416 | nodes := j.parser.Root.Nodes //拿到所有根对象 417 | fullResult := [][]reflect.Value{} 418 | for i := 0; i < len(nodes); i++ { 419 | node := nodes[i] 420 | results, err := j.walk(cur, node) //挨个处理 421 | if err != nil { 422 | return nil, err 423 | } 424 | 425 | // encounter an end node, break the current block 426 | if j.endRange > 0 && j.endRange <= j.inRange { 427 | j.endRange-- 428 | j.lastEndNode = &nodes[i] 429 | break 430 | } 431 | // encounter a range node, start a range loop 432 | if j.beginRange > 0 { 433 | j.beginRange-- 434 | j.inRange++ 435 | if len(results) > 0 { 436 | for _, value := range results { 437 | j.parser.Root.Nodes = nodes[i+1:] 438 | nextResults, err := j.FindResults(value.Interface()) 439 | if err != nil { 440 | return nil, err 441 | } 442 | fullResult = append(fullResult, nextResults...) 443 | } 444 | } else { 445 | // If the range has no results, we still need to process the nodes within the range 446 | // so the position will advance to the end node 447 | j.parser.Root.Nodes = nodes[i+1:] 448 | _, err := j.FindResults(nil) 449 | if err != nil { 450 | return nil, err 451 | } 452 | } 453 | j.inRange-- 454 | 455 | // Fast forward to resume processing after the most recent end node that was encountered 456 | for k := i + 1; k < len(nodes); k++ { 457 | if &nodes[k] == j.lastEndNode { 458 | i = k 459 | break 460 | } 461 | } 462 | continue 463 | } 464 | fullResult = append(fullResult, results) 465 | } 466 | return fullResult, nil 467 | } 468 | ``` 469 | 470 | #### 渲染文字 471 | 472 | ``` 473 | Root 474 | NodeText: hello 475 | NodeList 476 | NodeIdentifier: range 477 | NodeField: items 478 | NodeArray: [{0 false false} {0 false false} {0 false false}] 479 | NodeList 480 | NodeField: metadata 481 | NodeField: name 482 | NodeList 483 | NodeText 484 | NodeList 485 | NodeIdentifier: end 486 | ``` 487 | 488 | ```go 489 | // walk visits tree rooted at the given node in DFS order 490 | func (j *JSONPath) walk(value []reflect.Value, node Node) ([]reflect.Value, error) { 491 | switch node := node.(type) { 492 | case *ListNode: 493 | return j.evalList(value, node) 494 | case *TextNode: 495 | return []reflect.Value{reflect.ValueOf(node.Text)}, nil //返回一个hello 496 | case *FieldNode: 497 | return j.evalField(value, node) 498 | case *ArrayNode: 499 | return j.evalArray(value, node) 500 | case *FilterNode: 501 | return j.evalFilter(value, node) 502 | case *IntNode: 503 | return j.evalInt(value, node) 504 | case *BoolNode: 505 | return j.evalBool(value, node) 506 | case *FloatNode: 507 | return j.evalFloat(value, node) 508 | case *WildcardNode: 509 | return j.evalWildcard(value, node) 510 | case *RecursiveNode: 511 | return j.evalRecursive(value, node) 512 | case *UnionNode: 513 | return j.evalUnion(value, node) 514 | case *IdentifierNode: 515 | return j.evalIdentifier(value, node) 516 | default: 517 | return value, fmt.Errorf("unexpected Node %v", node) 518 | } 519 | } 520 | ``` 521 | 522 | ### 渲染Range 523 | 524 | ``` 525 | NodeList 526 | NodeIdentifier: range 527 | NodeField: items 528 | NodeArray: [{0 false false} {0 false false} {0 false false}] 529 | NodeList 530 | NodeField: metadata 531 | NodeField: name 532 | NodeList 533 | NodeText 534 | NodeList 535 | NodeIdentifier: end 536 | ``` 537 | 538 | ```go 539 | node := nodes[i] //跟对象 540 | results, err := j.walk(cur, node) //Walk完成后，获取获取到了所有Item，并且beginRange = 1 541 | if err != nil { 542 | return nil, err 543 | } 544 | 545 | // encounter an end node, break the current block 546 | if j.endRange > 0 && j.endRange <= j.inRange { 547 | j.endRange-- 548 | j.lastEndNode = &nodes[i] 549 | break 550 | } 551 | // encounter a range node, start a range loop 552 | if j.beginRange > 0 { //beginRange 为整数的原因为支持多重循环 553 | j.beginRange-- 554 | j.inRange++ // 555 | if len(results) > 0 { 556 | for _, value := range results { //对每个元素循环操作，达到了循环的作用 557 | j.parser.Root.Nodes = nodes[i+1:] 558 | nextResults, err := j.FindResults(value.Interface()) //将所有元素都开始继续执行，此时因为beginRange = 0 ； 559 | //endRange = 0 ； inRange =1 ，所以后续的操作都不会涉及到循环操作。 560 | if err != nil { 561 | return nil, err 562 | } 563 | fullResult = append(fullResult, nextResults...) 564 | } 565 | } else { 566 | // If the range has no results, we still need to process the nodes within the range 567 | // so the position will advance to the end node 568 | j.parser.Root.Nodes = nodes[i+1:] 569 | _, err := j.FindResults(nil) 570 | if err != nil { 571 | return nil, err 572 | } 573 | } 574 | j.inRange-- //处理完成后接触循环 575 | 576 | // Fast forward to resume processing after the most recent end node that was encountered 577 | for k := i + 1; k < len(nodes); k++ { 578 | if &nodes[k] == j.lastEndNode { 579 | i = k 580 | break 581 | } 582 | } 583 | continue 584 | } 585 | fullResult = append(fullResult, results) //加入结果 586 | ``` 587 | 588 | 589 | #### 处理输出 590 | 591 | #### 结束 592 | 593 | ``` 594 | NodeList 595 | NodeIdentifier: end 596 | ``` 597 | 598 | ```go 599 | func (j *JSONPath) FindResults(data interface{}) ([][]reflect.Value, error) { 600 | if j.parser == nil { 601 | return nil, fmt.Errorf("%s is an incomplete jsonpath template", j.name) 602 | } 603 | 604 | cur := []reflect.Value{reflect.ValueOf(data)} 605 | nodes := j.parser.Root.Nodes 606 | fullResult := [][]reflect.Value{} 607 | for i := 0; i < len(nodes); i++ { 608 | node := nodes[i] 609 | results, err := j.walk(cur, node) 610 | if err != nil { 611 | return nil, err 612 | } 613 | 614 | // encounter an end node, break the current block 615 | if j.endRange > 0 && j.endRange <= j.inRange { //此时j.endRange = 1 ； j.inRange = 1 616 | j.endRange-- 617 | j.lastEndNode = &nodes[i] 618 | break 619 | } 620 | // encounter a range node, start a range loop 621 | if j.beginRange > 0 { // j.beginRange = 0 622 | j.beginRange-- 623 | j.inRange++ 624 | if len(results) > 0 { 625 | for _, value := range results { 626 | j.parser.Root.Nodes = nodes[i+1:] 627 | nextResults, err := j.FindResults(value.Interface()) 628 | if err != nil { 629 | return nil, err 630 | } 631 | fullResult = append(fullResult, nextResults...) 632 | } 633 | } else { 634 | // If the range has no results, we still need to process the nodes within the range 635 | // so the position will advance to the end node 636 | j.parser.Root.Nodes = nodes[i+1:] 637 | _, err := j.FindResults(nil) 638 | if err != nil { 639 | return nil, err 640 | } 641 | } 642 | j.inRange-- 643 | 644 | // Fast forward to resume processing after the most recent end node that was encountered 645 | for k := i + 1; k < len(nodes); k++ { 646 | if &nodes[k] == j.lastEndNode { 647 | i = k 648 | break 649 | } 650 | } 651 | continue 652 | } 653 | fullResult = append(fullResult, results) //result 为空 654 | } 655 | return fullResult, nil 656 | } 657 | 658 | ``` 659 | 660 | 661 | 662 | 663 | 664 | 665 | 666 | 667 | -------------------------------------------------------------------------------- /util/jsonpath/jsonpath_test.go: -------------------------------------------------------------------------------- 1 | package jsonpath 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "k8s.io/apimachinery/pkg/util/json" 7 | "k8s.io/client-go/util/jsonpath" 8 | "testing" 9 | ) 10 | 11 | func TestJsonpathUsage(t *testing.T){ 12 | templates := []string{"{@}","{@[0]}","{@[-1]}","{@[0:2]}","{@[0:4:2]}"} 13 | p := jsonpath.New("test") 14 | for _,template := range templates { 15 | err := p.Parse(template) 16 | if err != nil { 17 | t.Fatalf("expected nil error, got %v", err) 18 | } 19 | buf := new(bytes.Buffer) 20 | bools := []bool{true,false,true,false} 21 | err = p.Execute(buf,bools) 22 | if err != nil { 23 | t.Fatalf("expected nil error, got %v", err) 24 | } 25 | 26 | t.Logf("%v with %q parse result : %s", bools, template , buf.String()) 27 | } 28 | 29 | 30 | } 31 | 32 | type book struct { 33 | Category string 34 | Author string 35 | Title string 36 | Price float32 37 | } 38 | 39 | func (b book) String() string { 40 | return fmt.Sprintf("{Category: %s, Author: %s, Title: %s, Price: %v}", b.Category, b.Author, b.Title, b.Price) 41 | } 42 | 43 | type bicycle struct { 44 | Color string 45 | Price float32 46 | IsNew bool 47 | } 48 | 49 | type empName string 50 | type job string 51 | type store struct { 52 | Book []book 53 | Bicycle []bicycle 54 | Name string 55 | Labels map[string]int 56 | Employees map[empName]job 57 | } 58 | 59 | 60 | func TestStructInput(t *testing.T) { 61 | 62 | storeData := store{ 63 | Name: "jsonpath", 64 | Book: []book{ 65 | {"reference", "Nigel Rees", "Sayings of the Centurey", 8.95}, 66 | {"fiction", "Evelyn Waugh", "Sword of Honour", 12.99}, 67 | {"fiction", "Herman Melville", "Moby Dick", 8.99}, 68 | }, 69 | Bicycle: []bicycle{ 70 | {"red", 19.95, true}, 71 | {"green", 20.01, false}, 72 | }, 73 | Labels: map[string]int{ 74 | "engieer": 10, 75 | "web/html": 15, 76 | "k8s-app": 20, 77 | }, 78 | Employees: map[empName]job{ 79 | "jason": "manager", 80 | "dan": "clerk", 81 | }, 82 | } 83 | 84 | 85 | p := jsonpath.New("test Strut") 86 | templates := []string{ 87 | "hello {.Name}", 88 | "{.Labels.web/html}", 89 | "{.Book[*].Author}", 90 | "{range .Bicycle[*]}{ \"{\" }{ @.* }{ \"} \" }{end}"} 91 | for _,template := range templates { 92 | err := p.Parse(template) 93 | if err != nil { 94 | t.Fatalf("expected nil error, got %v", err) 95 | } 96 | buf := new(bytes.Buffer) 97 | err = p.Execute(buf,storeData) 98 | if err != nil { 99 | t.Fatalf("expected nil error, got %v", err) 100 | } 101 | 102 | t.Logf("%v with %q parse result : %s", storeData, template , buf.String()) 103 | } 104 | 105 | } 106 | 107 | func TestJSONInput(t *testing.T) { 108 | var pointsJSON = []byte(`[ 109 | {"id": "i1", "x":4, "y":-5}, 110 | {"id": "i2", "x":-2, "y":-5, "z":1}, 111 | {"id": "i3", "x": 8, "y": 3 }, 112 | {"id": "i4", "x": -6, "y": -1 }, 113 | {"id": "i5", "x": 0, "y": 2, "z": 1 }, 114 | {"id": "i6", "x": 1, "y": 4 } 115 | ]`) 116 | 117 | var pointsData interface{} 118 | err := json.Unmarshal(pointsJSON, &pointsData) 119 | if err != nil { 120 | t.Error(err) 121 | } 122 | templates := []string{ 123 | "{[?(@.z)].id}", // 存在字段z的筛选 124 | "{[0]['id']}", //第0个元素的id字段 125 | } 126 | p := jsonpath.New("test json") 127 | 128 | for _,template := range templates { 129 | err := p.Parse(template) 130 | if err != nil { 131 | t.Fatalf("expected nil error, got %v", err) 132 | } 133 | buf := new(bytes.Buffer) 134 | err = p.Execute(buf,pointsData) 135 | if err != nil { 136 | t.Fatalf("expected nil error, got %v", err) 137 | } 138 | 139 | t.Logf("%v with %q parse result : %s", pointsData, template , buf.String()) 140 | } 141 | } 142 | 143 | // TestKubernetes tests some use cases from kubernetes 144 | func TestKubernetes(t *testing.T) { 145 | var input = []byte(`{ 146 | "kind": "List", 147 | "items":[ 148 | { 149 | "kind":"None", 150 | "metadata":{ 151 | "name":"127.0.0.1", 152 | "labels":{ 153 | "kubernetes.io/hostname":"127.0.0.1" 154 | } 155 | }, 156 | "status":{ 157 | "capacity":{"cpu":"4"}, 158 | "ready": true, 159 | "addresses":[{"type": "LegacyHostIP", "address":"127.0.0.1"}] 160 | } 161 | }, 162 | { 163 | "kind":"None", 164 | "metadata":{ 165 | "name":"127.0.0.2", 166 | "labels":{ 167 | "kubernetes.io/hostname":"127.0.0.2" 168 | } 169 | }, 170 | "status":{ 171 | "capacity":{"cpu":"8"}, 172 | "ready": true, 173 | "addresses":[ 174 | {"type": "LegacyHostIP", "address":"127.0.0.2"}, 175 | {"type": "another", "address":"127.0.0.3"} 176 | ] 177 | } 178 | } 179 | ], 180 | "users":[ 181 | { 182 | "name": "myself", 183 | "user": {} 184 | }, 185 | { 186 | "name": "e2e", 187 | "user": {"username": "admin", "password": "secret"} 188 | } 189 | ] 190 | }`) 191 | var nodesData interface{} 192 | err := json.Unmarshal(input, &nodesData) 193 | if err != nil { 194 | t.Error(err) 195 | } 196 | 197 | p := jsonpath.New("test json") 198 | templates := []string{ 199 | "{range .items[*]}{.metadata.name}, {end}{.kind}", // 便利所有节点，输出metadata的名称，最后输出kind 200 | "{range .items[*]}{.metadata.name}{\"\\t\"}{end}", // 加入tab键 201 | "{.items[*].status.addresses[*].address}", //所有item的status的所有address的address 202 | "{range .items[*]}{range .status.addresses[*]}{.address}, {end}{end}",//输出所有节点address 203 | "{.items[*].metadata.name}",//所有item的名称 204 | `{.items[*]['metadata.name', 'status.capacity']}`, //所有item的名称和能力，先输出所有名称，再输出所有capacity 205 | `{range .items[*]}[{.metadata.name}, {.status.capacity}] {end}`,//一个一个元素的输出，先名称，再能力 206 | `{.users[?(@.name=="e2e")].user.password}`, //筛选出所有username为e2e的，然后提取密码 207 | `{.items[0].metadata.labels.kubernetes\.io/hostname}`, //获取第一个item的kubernetes.io/hostname的LabelValue值 208 | `{.items[?(@.metadata.labels.kubernetes\.io/hostname=="127.0.0.1")].kind}`,//基于item的kubernetes.io/hostname的LabelValue值筛选，然后获取kind 209 | `{.items[?(@..ready==true)].metadata.name}`,//获取所有ready为true的item，然后输出metadata的name 210 | } 211 | for _,template := range templates { 212 | err := p.Parse(template) 213 | if err != nil { 214 | t.Fatalf("expected nil error, got %v", err) 215 | } 216 | buf := new(bytes.Buffer) 217 | err = p.Execute(buf,nodesData) 218 | if err != nil { 219 | t.Fatalf("expected nil error, got %v", err) 220 | } 221 | 222 | t.Logf(" %q parse result : %s", template , buf.String()) 223 | } 224 | 225 | 226 | } 227 | 228 | -------------------------------------------------------------------------------- /util/jsonpath/testdata/pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: test-jsonpath 5 | spec: 6 | containers: 7 | - name: nginx 8 | image: nginx:1.14.2 9 | ports: 10 | - containerPort: 80 -------------------------------------------------------------------------------- /util/workqueue/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## WorkQueue 3 | 4 | K8s的采用声明式设计模式，与命令式不同的是，如果一个服务当前又3个节点，期望扩容为5个节点，两种设计模式的描述语句是不同的： 5 | - 声明式： 保证服务有5个工作节点 6 | - 命令式： 向服务增加两个节点 7 | 8 | 声明式的编程模式的优势在于，按照这种设计模式编写的模块和代码都是幂等的，所以在处理失败重试，`controller`节点挂掉等异常情况相对简单。 9 | 10 | `WorkQueue` 这一章主要聚焦在限流和事件队列上。 11 | 12 | - 限流器： 令牌桶的实现机制 13 | - 队列： 实现了一个生命式的工作队列，区分为Processing，Queue，Dirty，当同一个对象多次更新放入队列时，只会触发一次处理流程； 14 | - 条件变量： 一个不是很常用的加锁方式 15 | - 延时队列： 当当前情况不适合执行指定操作，或者当前失败需要后续重试时，可以稍后执行相关动作的队列实现； 16 | - 限流延时队列： 限流器与延时队列的组合 17 | - 并行函数： 分片并行执行Util函数 18 | 19 | 20 | 详细内容见： 21 | 22 | - [令牌桶](/ch1/rate.md) 23 | - [限流器](/ch1/default-rate-limiter.md) 24 | - [队列](/ch1/queue.md) 25 | - [条件变量](/ch1/cond.md) 26 | - [延时队列](/ch1/delaying-queue.md) 27 | - [限流延时队列](/ch1/delaying-queue.md) 28 | - [并行函数](/ch1/parallelizer.md) 29 | -------------------------------------------------------------------------------- /util/workqueue/clock.md: -------------------------------------------------------------------------------- 1 | # Clock 2 | 3 | ## Interface 4 | 5 | 为了方便测试做的一个对于Clock的一次封装 6 | 7 | ```go 8 | 9 | // PassiveClock allows for injecting fake or real clocks into code 10 | // that needs to read the current time but does not support scheduling 11 | // activity in the future. 12 | type PassiveClock interface { 13 | //当前时间 14 | Now() time.Time 15 | //当前时间与目标之间的时间差 16 | Since(time.Time) time.Duration 17 | } 18 | 19 | // Clock allows for injecting fake or real clocks into code that 20 | // needs to do arbitrary things based on time. 21 | type Clock interface { 22 | PassiveClock 23 | // After returns the channel of a new Timer. 24 | // This method does not allow to free/GC the backing timer before it fires. Use 25 | // NewTimer instead. 26 | After(d time.Duration) <-chan time.Time 27 | // NewTimer returns a new Timer. 28 | NewTimer(d time.Duration) Timer 29 | // Sleep sleeps for the provided duration d. 30 | // Consider making the sleep interruptible by using 'select' on a context channel and a timer channel. 31 | Sleep(d time.Duration) 32 | // Tick returns the channel of a new Ticker. 33 | // This method does not allow to free/GC the backing ticker. Use 34 | // NewTicker from WithTicker instead. 35 | Tick(d time.Duration) <-chan time.Time 36 | } 37 | 38 | // WithTicker allows for injecting fake or real clocks into code that 39 | // needs to do arbitrary things based on time. 40 | type WithTicker interface { 41 | Clock 42 | // NewTicker returns a new Ticker. 43 | NewTicker(time.Duration) Ticker 44 | } 45 | 46 | // WithDelayedExecution allows for injecting fake or real clocks into 47 | // code that needs to make use of AfterFunc functionality. 48 | type WithDelayedExecution interface { 49 | Clock 50 | // AfterFunc executes f in its own goroutine after waiting 51 | // for d duration and returns a Timer whose channel can be 52 | // closed by calling Stop() on the Timer. 53 | AfterFunc(d time.Duration, f func()) Timer 54 | } 55 | 56 | // WithTickerAndDelayedExecution allows for injecting fake or real clocks 57 | // into code that needs Ticker and AfterFunc functionality 58 | type WithTickerAndDelayedExecution interface { 59 | WithTicker 60 | // AfterFunc executes f in its own goroutine after waiting 61 | // for d duration and returns a Timer whose channel can be 62 | // closed by calling Stop() on the Timer. 63 | AfterFunc(d time.Duration, f func()) Timer 64 | } 65 | 66 | // Ticker defines the Ticker interface. 67 | type Ticker interface { 68 | C() <-chan time.Time 69 | Stop() 70 | } 71 | 72 | ``` 73 | 74 | ## RealClock 75 | 76 | 真实的Clock，大部分用的 `time`包的函数 77 | 78 | 79 | ```go 80 | 81 | // RealClock 是真的Clock 82 | type RealClock struct{} 83 | 84 | // Now 返回当前时间 85 | func (RealClock) Now() time.Time { 86 | return time.Now() 87 | } 88 | ``` 89 | 90 | ```go 91 | // 返回当前时间和目标时间的时间差 92 | func (RealClock) Since(ts time.Time) time.Duration { 93 | return time.Since(ts) 94 | } 95 | 96 | // 过多久之后返回一个信号 97 | func (RealClock) After(d time.Duration) <-chan time.Time { 98 | return time.After(d) 99 | } 100 | 101 | // 新建timer 102 | func (RealClock) NewTimer(d time.Duration) Timer { 103 | return &realTimer{ 104 | timer: time.NewTimer(d), 105 | } 106 | } 107 | 108 | // AfterFunc is the same as time.AfterFunc(d, f). 109 | func (RealClock) AfterFunc(d time.Duration, f func()) Timer { 110 | return &realTimer{ 111 | timer: time.AfterFunc(d, f), 112 | } 113 | } 114 | 115 | 116 | 117 | // Tick is the same as time.Tick(d) 118 | // This method does not allow to free/GC the backing ticker. Use 119 | // NewTicker instead. 120 | func (RealClock) Tick(d time.Duration) <-chan time.Time { 121 | return time.Tick(d) 122 | } 123 | 124 | // NewTicker returns a new Ticker. 125 | func (RealClock) NewTicker(d time.Duration) Ticker { 126 | return &realTicker{ 127 | ticker: time.NewTicker(d), 128 | } 129 | } 130 | 131 | // Sleep is the same as time.Sleep(d) 132 | // Consider making the sleep interruptible by using 'select' on a context channel and a timer channel. 133 | func (RealClock) Sleep(d time.Duration) { 134 | time.Sleep(d) 135 | } 136 | 137 | // Timer allows for injecting fake or real timers into code that 138 | // needs to do arbitrary things based on time. 139 | type Timer interface { 140 | C() <-chan time.Time 141 | Stop() bool 142 | Reset(d time.Duration) bool 143 | } 144 | 145 | var _ = Timer(&realTimer{}) 146 | 147 | // realTimer is backed by an actual time.Timer. 148 | type realTimer struct { 149 | timer *time.Timer 150 | } 151 | 152 | // C returns the underlying timer's channel. 153 | func (r *realTimer) C() <-chan time.Time { 154 | return r.timer.C 155 | } 156 | 157 | // Stop calls Stop() on the underlying timer. 158 | func (r *realTimer) Stop() bool { 159 | return r.timer.Stop() 160 | } 161 | 162 | // Reset calls Reset() on the underlying timer. 163 | func (r *realTimer) Reset(d time.Duration) bool { 164 | return r.timer.Reset(d) 165 | } 166 | 167 | type realTicker struct { 168 | ticker *time.Ticker 169 | } 170 | 171 | func (r *realTicker) C() <-chan time.Time { 172 | return r.ticker.C 173 | } 174 | 175 | func (r *realTicker) Stop() { 176 | r.ticker.Stop() 177 | } 178 | 179 | ``` 180 | 181 | -------------------------------------------------------------------------------- /util/workqueue/cond.md: -------------------------------------------------------------------------------- 1 | # Cond 2 | 3 | 条件变量使用举例 4 | 5 | 1. Type.Get //调用接口 6 | 2. q.cond.L.Lock //抢锁 7 | 3. for len(q.queue) == 0 && !q.shuttingDown {q.cond.Wait() } //获取到锁，并且队列长度不为0 8 | -> t := runtime_notifyListAdd(&c.notify) // 将goroutine增加到对象的等待队列中 9 | -> c.L.Unlock() //释放锁，让可以下一次抢锁 10 | -> runtime_notifyListWait(&c.notify, t)，// 等待资源，等待通知，降低资源使用； 11 | -> c.L.Lock() //继续抢锁 12 | 13 | -------------------------------------------------------------------------------- /util/workqueue/default-rate-limiter.md: -------------------------------------------------------------------------------- 1 | # Default Rate Limiter 2 | 3 | ## RateLimiter 4 | 5 | ```go 6 | type RateLimiter interface { 7 | // 执行当前对象，需要等待多久 8 | When(item interface{}) time.Duration 9 | // 不追踪该对象 10 | Forget(item interface{}) 11 | // 该对象已经重试了多少次 12 | NumRequeues(item interface{}) int 13 | } 14 | ``` 15 | 16 | 17 | ## DefaultControllerRateLimiter 18 | 19 | ```go 20 | // 默认controller限流器， 21 | func DefaultControllerRateLimiter() RateLimiter { 22 | //取最大等待时间 23 | return NewMaxOfRateLimiter( 24 | //失败指数等待 25 | NewItemExponentialFailureRateLimiter(5*time.Millisecond, 1000*time.Second), 26 | // 整个限流器通用，只对重试起效 27 | &BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, 28 | ) 29 | } 30 | ``` 31 | 32 | ## BucketRateLimiter 33 | 34 | ```go 35 | // 对于令牌桶的封装 36 | type BucketRateLimiter struct { 37 | *rate.Limiter 38 | } 39 | 40 | var _ RateLimiter = &BucketRateLimiter{} 41 | 42 | //按照令牌桶计算等待时间 43 | func (r *BucketRateLimiter) When(item interface{}) time.Duration { 44 | return r.Limiter.Reserve().Delay() 45 | } 46 | //多少次重新入队 47 | func (r *BucketRateLimiter) NumRequeues(item interface{}) int { 48 | return 0 49 | } 50 | //放弃对象 51 | func (r *BucketRateLimiter) Forget(item interface{}) { 52 | } 53 | ``` 54 | 55 | ## MaxOfRateLimiter 56 | 57 | ```go 58 | //最大限流器 59 | type MaxOfRateLimiter struct { 60 | limiters []RateLimiter 61 | } 62 | //取最大等待时间 63 | func (r *MaxOfRateLimiter) When(item interface{}) time.Duration { 64 | ret := time.Duration(0) 65 | for _, limiter := range r.limiters { 66 | curr := limiter.When(item) 67 | if curr > ret { 68 | ret = curr 69 | } 70 | } 71 | 72 | return ret 73 | } 74 | 75 | func NewMaxOfRateLimiter(limiters ...RateLimiter) RateLimiter { 76 | return &MaxOfRateLimiter{limiters: limiters} 77 | } 78 | 79 | func (r *MaxOfRateLimiter) NumRequeues(item interface{}) int { 80 | ret := 0 81 | //取所有最大重试次数中最大的 82 | for _, limiter := range r.limiters { 83 | curr := limiter.NumRequeues(item) 84 | if curr > ret { 85 | ret = curr 86 | } 87 | } 88 | 89 | return ret 90 | } 91 | 92 | func (r *MaxOfRateLimiter) Forget(item interface{}) { 93 | //所有都忽略 94 | for _, limiter := range r.limiters { 95 | limiter.Forget(item) 96 | } 97 | } 98 | ``` 99 | 100 | ## ItemExponentialFailureRateLimiter 101 | 102 | ```go 103 | // 每次失败都会指数增加delay时间 104 | type ItemExponentialFailureRateLimiter struct { 105 | failuresLock sync.Mutex //lock 106 | failures map[interface{}]int //统计失败次数 107 | 108 | baseDelay time.Duration //失败delay 109 | maxDelay time.Duration // 最大delay 110 | } 111 | 112 | //确保ItemExponentialFailureRateLimiter 实现了RateLimiter 接口 113 | var _ RateLimiter = &ItemExponentialFailureRateLimiter{} 114 | 115 | func NewItemExponentialFailureRateLimiter(baseDelay time.Duration, maxDelay time.Duration) RateLimiter { 116 | return &ItemExponentialFailureRateLimiter{ 117 | failures: map[interface{}]int{}, 118 | baseDelay: baseDelay, 119 | maxDelay: maxDelay, 120 | } 121 | } 122 | //默认的DefaultItemBasedRateLimiter， base时间为1毫秒，最大时间为1000s 123 | func DefaultItemBasedRateLimiter() RateLimiter { 124 | return NewItemExponentialFailureRateLimiter(time.Millisecond, 1000*time.Second) 125 | } 126 | 127 | //每次when都会认为是一次尝试，基于尝试次数计算delay时间 128 | func (r *ItemExponentialFailureRateLimiter) When(item interface{}) time.Duration { 129 | r.failuresLock.Lock() 130 | defer r.failuresLock.Unlock() 131 | 132 | //获取时间 133 | exp := r.failures[item] 134 | r.failures[item] = r.failures[item] + 1 135 | 136 | // 计算等待时间 137 | backoff := float64(r.baseDelay.Nanoseconds()) * math.Pow(2, float64(exp)) 138 | if backoff > math.MaxInt64 { 139 | return r.maxDelay 140 | } 141 | 142 | calculated := time.Duration(backoff) 143 | if calculated > r.maxDelay { 144 | return r.maxDelay 145 | } 146 | 147 | return calculated 148 | } 149 | 150 | //获取失败次数 151 | func (r *ItemExponentialFailureRateLimiter) NumRequeues(item interface{}) int { 152 | r.failuresLock.Lock() 153 | defer r.failuresLock.Unlock() 154 | 155 | //获取失败次数 156 | return r.failures[item] 157 | } 158 | 159 | //Forget 放弃该对象的记录 160 | func (r *ItemExponentialFailureRateLimiter) Forget(item interface{}) { 161 | r.failuresLock.Lock() 162 | defer r.failuresLock.Unlock() 163 | 164 | delete(r.failures, item) 165 | } 166 | ``` 167 | 168 | ## ItemFastSlowRateLimiter 169 | 170 | ```go 171 | 172 | // ItemFastSlowRateLimiter does a quick retry for a certain number of attempts, then a slow retry after that 173 | type ItemFastSlowRateLimiter struct { 174 | failuresLock sync.Mutex //锁 175 | failures map[interface{}]int //失败次数统计 176 | 177 | maxFastAttempts int //快速队列，重试次数 178 | fastDelay time.Duration //快速队列base 179 | slowDelay time.Duration //慢速队列base 180 | } 181 | 182 | //确保ItemFastSlowRateLimiter 实现了RateLimiter 接口 183 | var _ RateLimiter = &ItemFastSlowRateLimiter{} 184 | 185 | func NewItemFastSlowRateLimiter(fastDelay, slowDelay time.Duration, maxFastAttempts int) RateLimiter { 186 | return &ItemFastSlowRateLimiter{ 187 | failures: map[interface{}]int{}, 188 | fastDelay: fastDelay, 189 | slowDelay: slowDelay, 190 | maxFastAttempts: maxFastAttempts, 191 | } 192 | } 193 | 194 | func (r *ItemFastSlowRateLimiter) When(item interface{}) time.Duration { 195 | r.failuresLock.Lock() 196 | defer r.failuresLock.Unlock() 197 | 198 | r.failures[item] = r.failures[item] + 1 199 | 200 | //如果失败次数小于快速最大重试次数，则返回快速时间 201 | if r.failures[item] <= r.maxFastAttempts { 202 | return r.fastDelay 203 | } 204 | 205 | //否则返回慢速时间 206 | return r.slowDelay 207 | } 208 | 209 | // 返回重试次数 210 | func (r *ItemFastSlowRateLimiter) NumRequeues(item interface{}) int { 211 | r.failuresLock.Lock() 212 | defer r.failuresLock.Unlock() 213 | 214 | return r.failures[item] 215 | } 216 | 217 | //删除对象记录 218 | func (r *ItemFastSlowRateLimiter) Forget(item interface{}) { 219 | r.failuresLock.Lock() 220 | defer r.failuresLock.Unlock() 221 | 222 | delete(r.failures, item) 223 | } 224 | ``` 225 | 226 | ## NewWithMaxWaitRateLimiter 227 | 228 | ```go 229 | func NewWithMaxWaitRateLimiter(limiter RateLimiter, maxDelay time.Duration) RateLimiter { 230 | return &WithMaxWaitRateLimiter{limiter: limiter, maxDelay: maxDelay} 231 | } 232 | 233 | //基于最大limiter的时间和最大时间做比较 234 | func (w WithMaxWaitRateLimiter) When(item interface{}) time.Duration { 235 | delay := w.limiter.When(item) 236 | if delay > w.maxDelay { 237 | return w.maxDelay 238 | } 239 | 240 | return delay 241 | } 242 | // 忽略对象记录 243 | func (w WithMaxWaitRateLimiter) Forget(item interface{}) { 244 | w.limiter.Forget(item) 245 | } 246 | 247 | // 重新入队次数 248 | func (w WithMaxWaitRateLimiter) NumRequeues(item interface{}) int { 249 | return w.limiter.NumRequeues(item) 250 | } 251 | 252 | ``` 253 | 254 | -------------------------------------------------------------------------------- /util/workqueue/delaying-queue.md: -------------------------------------------------------------------------------- 1 | # Delaying Queue 2 | 3 | ## Interface 4 | 5 | ```go 6 | // DelayingInterface 支持将一个失败的对象重新加入到队列中重新处理 7 | type DelayingInterface interface { 8 | Interface 9 | // AddAfter adds an item to the workqueue after the indicated duration has passed 10 | AddAfter(item interface{}, duration time.Duration) 11 | } 12 | ``` 13 | 14 | ## 新建 15 | 16 | ```go 17 | // NewDelayingQueue constructs a new workqueue with delayed queuing ability. 18 | // NewDelayingQueue does not emit metrics. For use with a MetricsProvider, please use 19 | // NewNamedDelayingQueue instead. 20 | func NewDelayingQueue() DelayingInterface { 21 | return NewDelayingQueueWithCustomClock(clock.RealClock{}, "") 22 | } 23 | 24 | // NewDelayingQueueWithCustomQueue constructs a new workqueue with ability to 25 | // inject custom queue Interface instead of the default one 26 | func NewDelayingQueueWithCustomQueue(q Interface, name string) DelayingInterface { 27 | return newDelayingQueue(clock.RealClock{}, q, name) 28 | } 29 | 30 | // NewNamedDelayingQueue constructs a new named workqueue with delayed queuing ability 31 | func NewNamedDelayingQueue(name string) DelayingInterface { 32 | return NewDelayingQueueWithCustomClock(clock.RealClock{}, name) 33 | } 34 | 35 | // NewDelayingQueueWithCustomClock constructs a new named workqueue 36 | // with ability to inject real or fake clock for testing purposes 37 | func NewDelayingQueueWithCustomClock(clock clock.WithTicker, name string) DelayingInterface { 38 | return newDelayingQueue(clock, NewNamed(name), name) 39 | } 40 | ``` 41 | 42 | 43 | ## delayingType 延迟队列实现 44 | 45 | ```go 46 | // delayingType wraps an Interface and provides delayed re-enquing 47 | type delayingType struct { 48 | Interface //具体底层队列 49 | 50 | // clock tracks time for delayed firing 51 | clock clock.Clock //具体时钟 52 | 53 | // stopCh lets us signal a shutdown to the waiting loop 54 | stopCh chan struct{} // 停止chan 55 | // stopOnce guarantees we only signal shutdown a single time 56 | stopOnce sync.Once //确保只会被关闭一次，否则多次关闭chan会导致panic 57 | 58 | // heartbeat ensures we wait no more than maxWait before firing 59 | heartbeat clock.Ticker //定时唤醒 60 | 61 | // waitingForAddCh is a buffered channel that feeds waitingForAdd 62 | waitingForAddCh chan *waitFor //等待加入到队列的元素 63 | 64 | // metrics counts the number of retries 65 | metrics retryMetrics 66 | } 67 | ``` 68 | 69 | ```go 70 | // waitFor holds the data to add and the time it should be added 71 | type waitFor struct { 72 | data t //具体的数据 73 | readyAt time.Time //何时处理 74 | // index in the priority queue (heap) 75 | index int //在优先队列中的位置 76 | } 77 | ``` 78 | 79 | 80 | 81 | ## waitForPriorityQueue 82 | 83 | 构造一个小顶堆的优先队列，使用golang的源码包： `src/container/heap` 84 | 85 | heap对于Interface的定义如下所示： 86 | 87 | ```go 88 | 89 | type heap.Interface interface { 90 | sort.Interface 91 | Push(x any) // add x as element Len() 92 | Pop() any // remove and return element Len() - 1. 93 | } 94 | 95 | type sort.Interface interface { 96 | // Len is the number of elements in the collection. 97 | Len() int 98 | 99 | Less(i, j int) bool 100 | // Swap swaps the elements with indexes i and j. 101 | Swap(i, j int) 102 | } 103 | 104 | ``` 105 | 定义结构满足接口 106 | 107 | ```go 108 | type waitForPriorityQueue []*waitFor 109 | 110 | //实现sort.Interface 111 | func (pq waitForPriorityQueue) Len() int { 112 | return len(pq) 113 | } 114 | //实现sort.Interface 115 | func (pq waitForPriorityQueue) Less(i, j int) bool { 116 | return pq[i].readyAt.Before(pq[j].readyAt) 117 | } 118 | //实现sort.Interface 119 | func (pq waitForPriorityQueue) Swap(i, j int) { 120 | pq[i], pq[j] = pq[j], pq[i] 121 | pq[i].index = i 122 | pq[j].index = j 123 | } 124 | 125 | // 实现heap.Interface 126 | func (pq *waitForPriorityQueue) Push(x interface{}) { 127 | n := len(*pq) 128 | item := x.(*waitFor) 129 | item.index = n 130 | *pq = append(*pq, item) 131 | } 132 | 133 | // 实现heap.Interface 134 | func (pq *waitForPriorityQueue) Pop() interface{} { 135 | n := len(*pq) 136 | item := (*pq)[n-1] 137 | item.index = -1 138 | *pq = (*pq)[0:(n - 1)] 139 | return item 140 | } 141 | 142 | // 查看小顶堆的堆顶元素 143 | func (pq waitForPriorityQueue) Peek() interface{} { 144 | return pq[0] 145 | } 146 | 147 | ``` 148 | 149 | ## 关闭队列 150 | 151 | ```go 152 | // 关闭队列 153 | func (q *delayingType) ShutDown() { 154 | //可重入 155 | q.stopOnce.Do(func() { 156 | //关闭底层队列 157 | q.Interface.ShutDown() 158 | //发送关闭信号 159 | close(q.stopCh) 160 | //停止心跳 161 | q.heartbeat.Stop() 162 | }) 163 | } 164 | ``` 165 | 166 | ## 增加元素 167 | 168 | ```go 169 | // 过指定时间后，将对象加入到队列 170 | func (q *delayingType) AddAfter(item interface{}, duration time.Duration) { 171 | // 如果队列已经结束，则结束 172 | if q.ShuttingDown() { 173 | return 174 | } 175 | 176 | q.metrics.retry() 177 | 178 | // immediately add things with no delay 179 | // 如果需要立刻加入，则直接加入到底层队列 180 | if duration <= 0 { 181 | q.Add(item) 182 | return 183 | } 184 | 185 | //加入等待队列，或者等待结束 186 | select { 187 | case <-q.stopCh: 188 | // unblock if ShutDown() is called 189 | case q.waitingForAddCh <- &waitFor{data: item, readyAt: q.clock.Now().Add(duration)}: 190 | } 191 | } 192 | ``` 193 | 194 | ## waitingLoop 195 | 196 | ```go 197 | // waitingLoop runs until the workqueue is shutdown and keeps a check on the list of items to be added. 198 | func (q *delayingType) waitingLoop() { 199 | //处理异常 200 | defer utilruntime.HandleCrash() 201 | 202 | // Make a placeholder channel to use when there are no items in our list 203 | //当没有元素时使用 204 | never := make(<-chan time.Time) 205 | 206 | // Make a timer that expires when the item at the head of the waiting queue is ready 207 | //优先队列中最小等待时间 208 | var nextReadyAtTimer clock.Timer 209 | 210 | //优先队列，所有待加入对象都在这里 211 | waitingForQueue := &waitForPriorityQueue{} 212 | heap.Init(waitingForQueue) 213 | //多次加入同一个对象，需要去重就好 214 | waitingEntryByData := map[t]*waitFor{} 215 | 216 | for { 217 | //如果底层队列已经关闭则直接退出 218 | if q.Interface.ShuttingDown() { 219 | return 220 | } 221 | //获取当前时间 222 | now := q.clock.Now() 223 | 224 | // Add ready entries 225 | //如果队列中有元素 226 | for waitingForQueue.Len() > 0 { 227 | //从优先队列中找到最早需要处理的对象 228 | entry := waitingForQueue.Peek().(*waitFor) 229 | //如果还不需要处理，则进入等待队列 230 | if entry.readyAt.After(now) { 231 | break 232 | } 233 | //从优先队列中获取元素 234 | entry = heap.Pop(waitingForQueue).(*waitFor) 235 | //增加到底层队列 236 | q.Add(entry.data) 237 | //删除数据记录 238 | delete(waitingEntryByData, entry.data) 239 | } 240 | 241 | // Set up a wait for the first item's readyAt (if one exists) 242 | nextReadyAt := never 243 | //如果等待队中有元素，则定时唤醒loop 244 | if waitingForQueue.Len() > 0 { 245 | if nextReadyAtTimer != nil { 246 | nextReadyAtTimer.Stop() 247 | } 248 | entry := waitingForQueue.Peek().(*waitFor) 249 | nextReadyAtTimer = q.clock.NewTimer(entry.readyAt.Sub(now)) 250 | nextReadyAt = nextReadyAtTimer.C() 251 | } 252 | 253 | select { 254 | //结束了 255 | case <-q.stopCh: 256 | return 257 | //心跳 258 | case <-q.heartbeat.C(): 259 | // continue the loop, which will add ready items 260 | //下一个对象ready 261 | case <-nextReadyAt: 262 | // continue the loop, which will add ready items 263 | //又有新的元素加入到队列，那么就处理新元素，尝试将它加入到优先队列，或者直接加入底层的队列 264 | case waitEntry := <-q.waitingForAddCh: 265 | //如果需要延时加入 266 | if waitEntry.readyAt.After(q.clock.Now()) { 267 | insert(waitingForQueue, waitingEntryByData, waitEntry) 268 | } else { 269 | //不需要延时，直接加入 270 | q.Add(waitEntry.data) 271 | } 272 | 273 | drained := false 274 | //获取chan中的所有元素，一次性处理完 275 | for !drained { 276 | select { 277 | case waitEntry := <-q.waitingForAddCh: 278 | if waitEntry.readyAt.After(q.clock.Now()) { 279 | insert(waitingForQueue, waitingEntryByData, waitEntry) 280 | } else { 281 | q.Add(waitEntry.data) 282 | } 283 | default: 284 | drained = true 285 | } 286 | } 287 | } 288 | } 289 | } 290 | 291 | ``` 292 | 293 | ```go 294 | 295 | // 将对象加入到优先队列 296 | func insert(q *waitForPriorityQueue, knownEntries map[t]*waitFor, entry *waitFor) { 297 | // 如果优先队列已经有这个元素了 298 | existing, exists := knownEntries[entry.data] 299 | if exists { 300 | //并且新加入的元素更早，则需要更新下时间，并更新下优先队列 301 | if existing.readyAt.After(entry.readyAt) { 302 | existing.readyAt = entry.readyAt 303 | heap.Fix(q, existing.index) 304 | } 305 | return 306 | } 307 | 308 | //不存在，则将对象加入到优先队列，等待加入 309 | heap.Push(q, entry) 310 | //标记优先队列已经有这个对象了 311 | knownEntries[entry.data] = entry 312 | } 313 | ``` 314 | 315 | 316 | ## 测试 317 | 318 | ```go 319 | func TestSimpleQueue(t *testing.T) { 320 | //初始化FackClock 321 | fakeClock := testingclock.NewFakeClock(time.Now()) 322 | q := NewDelayingQueueWithCustomClock(fakeClock, "") 323 | 324 | first := "foo" 325 | //增加一个对象 326 | q.AddAfter(first, 50*time.Millisecond) //50秒后增加item 327 | //等待结束 328 | if err := waitForWaitingQueueToFill(q); err != nil { 329 | t.Fatalf("unexpected err: %v", err) 330 | } 331 | //由于现在时间未到，会被消费到优先队列中 332 | if q.Len() != 0 { 333 | t.Errorf("should not have added") 334 | } 335 | 336 | //向前推进60ms，此时会触发`nextReadyAt`timer，进而将对象加入到底层队列中 337 | fakeClock.Step(60 * time.Millisecond) 338 | 339 | if err := waitForAdded(q, 1); err != nil { 340 | t.Errorf("should have added") 341 | } 342 | //获取对象 343 | item, _ := q.Get() 344 | q.Done(item) 345 | 346 | //向前推进10s，导致heartbeat timer触发 347 | // step past the next heartbeat 348 | fakeClock.Step(10 * time.Second) 349 | 350 | //获取一下队列 351 | err := wait.Poll(1*time.Millisecond, 30*time.Millisecond, func() (done bool, err error) { 352 | if q.Len() > 0 { 353 | return false, fmt.Errorf("added to queue") 354 | } 355 | 356 | return false, nil 357 | }) 358 | //应该等待超时 359 | if err != wait.ErrWaitTimeout { 360 | t.Errorf("expected timeout, got: %v", err) 361 | } 362 | 363 | //队列长队应该为空 364 | if q.Len() != 0 { 365 | t.Errorf("should not have added") 366 | } 367 | } 368 | ``` 369 | 370 | 371 | 372 | 373 | 374 | -------------------------------------------------------------------------------- /util/workqueue/parallelizer.md: -------------------------------------------------------------------------------- 1 | # Parallelizer 2 | 3 | 并行执行指定工作： 4 | - 预先组织好工作队列，比如一个slice : `works := make([]int, 100)` 5 | - 分配10个worker开始工作，1号worker处理：0-9 ， 2号处理：10 - 19； 6 | 7 | 8 | ```go 9 | //定义函数模版 10 | type DoWorkPieceFunc func(piece int) 11 | 12 | //执行参数 13 | type options struct { 14 | chunkSize int //每个woker处理多少对象 15 | } 16 | 17 | type Options func(*options) 18 | 19 | //指定每个worker负责范围的大小 20 | func WithChunkSize(c int) func(*options) { 21 | return func(o *options) { 22 | o.chunkSize = c 23 | } 24 | } 25 | 26 | ``` 27 | 28 | 29 | ```go 30 | //并行执行工作 31 | func ParallelizeUntil(ctx context.Context, workers, pieces int, doWorkPiece DoWorkPieceFunc, opts ...Options) { 32 | if pieces == 0 { 33 | return 34 | } 35 | o := options{} 36 | for _, opt := range opts { 37 | opt(&o) 38 | } 39 | chunkSize := o.chunkSize 40 | if chunkSize < 1 { 41 | chunkSize = 1 42 | } 43 | 44 | chunks := ceilDiv(pieces, chunkSize) 45 | toProcess := make(chan int, chunks) 46 | //将分段信息加入到chan 47 | for i := 0; i < chunks; i++ { 48 | toProcess <- i 49 | } 50 | close(toProcess) 51 | 52 | var stop <-chan struct{} 53 | if ctx != nil { 54 | stop = ctx.Done() 55 | } 56 | //取最小的worker数量 57 | if chunks < workers { 58 | workers = chunks 59 | } 60 | wg := sync.WaitGroup{} 61 | wg.Add(workers) 62 | for i := 0; i < workers; i++ { 63 | go func() { 64 | defer utilruntime.HandleCrash() 65 | defer wg.Done() 66 | //一个一个pieces的处理对象 67 | for chunk := range toProcess { 68 | start := chunk * chunkSize 69 | end := start + chunkSize 70 | if end > pieces { 71 | end = pieces 72 | } 73 | for p := start; p < end; p++ { 74 | select { 75 | case <-stop: 76 | return 77 | default: 78 | doWorkPiece(p) 79 | } 80 | } 81 | } 82 | }() 83 | } 84 | wg.Wait() 85 | } 86 | 87 | ``` 88 | 89 | 90 | -------------------------------------------------------------------------------- /util/workqueue/queue.md: -------------------------------------------------------------------------------- 1 | # Queue 2 | 3 | ## Interface 4 | 5 | ```go 6 | type Interface interface { 7 | Add(item interface{}) //增加对象 8 | Len() int //队列长度 9 | Get() (item interface{}, shutdown bool) //获取对象 10 | Done(item interface{}) //结束对象 11 | ShutDown() //停止对象 12 | ShutDownWithDrain() 13 | ShuttingDown() bool 14 | } 15 | ``` 16 | 17 | ```go 18 | // New constructs a new work queue (see the package comment). 19 | func New() *Type { 20 | return NewNamed("") 21 | } 22 | //新建Named 23 | func NewNamed(name string) *Type { 24 | rc := clock.RealClock{} 25 | return newQueue( 26 | rc, 27 | globalMetricsFactory.newQueueMetrics(name, rc), //metric信息 28 | defaultUnfinishedWorkUpdatePeriod, 29 | ) 30 | } 31 | 32 | func newQueue(c clock.WithTicker, metrics queueMetrics, updatePeriod time.Duration) *Type { 33 | t := &Type{ 34 | clock: c, 35 | dirty: set{}, 36 | processing: set{}, 37 | cond: sync.NewCond(&sync.Mutex{}), 38 | metrics: metrics, 39 | unfinishedWorkUpdatePeriod: updatePeriod, 40 | } 41 | 42 | // Don't start the goroutine for a type of noMetrics so we don't consume 43 | // resources unnecessarily 44 | if _, ok := metrics.(noMetrics); !ok { 45 | go t.updateUnfinishedWorkLoop() 46 | } 47 | 48 | return t 49 | } 50 | ``` 51 | 52 | 53 | ## Type 54 | 55 | ```go 56 | // Type 是一个工作队列 57 | type Type struct { 58 | // queue defines the order in which we will work on items. Every 59 | // element of queue should be in the dirty set and not in the 60 | // processing set. 61 | //每个待工作元素应该在`dirty`中，但是不应该在processing中，queue也指定了执行顺序 62 | queue []t 63 | 64 | // dirty 定义所有待处理的对象 65 | dirty set 66 | 67 | // Things that are currently being processed are in the processing set. 68 | // These things may be simultaneously in the dirty set. When we finish 69 | // processing something and remove it from this set, we'll check if 70 | // it's in the dirty set, and if so, add it to the queue. 71 | // 正在被处理的对象，这些对象有可能也在dirty中，当我们处理完成对象后，会检查dirty中是否有该对象，如果有该对象则代表对象有更新，我们会将对象增加到queue中 72 | processing set 73 | 74 | cond *sync.Cond 75 | 76 | shuttingDown bool 77 | drain bool 78 | 79 | metrics queueMetrics 80 | 81 | unfinishedWorkUpdatePeriod time.Duration 82 | clock clock.WithTicker 83 | } 84 | ``` 85 | 86 | 87 | ```go 88 | type empty struct{} 89 | type t interface{} 90 | //set是map的封装 91 | type set map[t]empty 92 | 93 | func (s set) has(item t) bool { 94 | _, exists := s[item] 95 | return exists 96 | } 97 | 98 | func (s set) insert(item t) { 99 | s[item] = empty{} 100 | } 101 | 102 | func (s set) delete(item t) { 103 | delete(s, item) 104 | } 105 | 106 | func (s set) len() int { 107 | return len(s) 108 | } 109 | ``` 110 | 111 | ## Add 112 | 113 | ```go 114 | // Add 标示对象需要被处理 115 | func (q *Type) Add(item interface{}) { 116 | q.cond.L.Lock() 117 | defer q.cond.L.Unlock() 118 | //如果队列正在被关闭，则什么都不做 119 | if q.shuttingDown { 120 | return 121 | } 122 | //如果dirty中有对象，则什么都不做 123 | if q.dirty.has(item) { 124 | return 125 | } 126 | 127 | q.metrics.add(item) 128 | 129 | //将对象设置为dirty状态 130 | q.dirty.insert(item) 131 | //如果processing中有对象，则不需要做什么事情了 132 | if q.processing.has(item) { 133 | return 134 | } 135 | 136 | //增加对象到队列 137 | q.queue = append(q.queue, item) 138 | //通知一下目前已经有对象更新了 139 | q.cond.Signal() 140 | } 141 | 142 | ``` 143 | 144 | ```go 145 | //返回队列长度 146 | func (q *Type) Len() int { 147 | q.cond.L.Lock() 148 | defer q.cond.L.Unlock() 149 | return len(q.queue) 150 | } 151 | 152 | ``` 153 | 154 | 155 | ```go 156 | // Get blocks until it can return an item to be processed. If shutdown = true, 157 | // the caller should end their goroutine. You must call Done with item when you 158 | // have finished processing it. 159 | func (q *Type) Get() (item interface{}, shutdown bool) { 160 | q.cond.L.Lock() 161 | defer q.cond.L.Unlock() 162 | //如果队列长度为空，并且没有被关闭，则等待 163 | for len(q.queue) == 0 && !q.shuttingDown { 164 | q.cond.Wait() 165 | } 166 | //如果队列为空，则返回空对象，并且标示已经被关闭 167 | if len(q.queue) == 0 { 168 | // We must be shutting down. 169 | return nil, true 170 | } 171 | 172 | //拿第一个对象 173 | item = q.queue[0] 174 | // The underlying array still exists and reference this object, so the object will not be garbage collected. 175 | //释放对象 176 | q.queue[0] = nil 177 | //更新队列 178 | q.queue = q.queue[1:] 179 | 180 | q.metrics.get(item) 181 | 182 | //增加到正在处理列表中 183 | q.processing.insert(item) 184 | //从dirty中删除对象 185 | q.dirty.delete(item) 186 | 187 | //最终的结果是： processing：有item， dirty中没有对象， queue没有对象 188 | 189 | //返回对象 190 | return item, false 191 | } 192 | ``` 193 | 194 | ```go 195 | // Done marks item as done processing, and if it has been marked as dirty again 196 | // while it was being processed, it will be re-added to the queue for 197 | // re-processing. 198 | func (q *Type) Done(item interface{}) { 199 | q.cond.L.Lock() 200 | defer q.cond.L.Unlock() 201 | 202 | q.metrics.done(item) 203 | 204 | //删除对象 205 | q.processing.delete(item) 206 | //如果脏队列中有这个对象，则需要重新处理下这个对象 207 | if q.dirty.has(item) { 208 | q.queue = append(q.queue, item) 209 | //通知下 210 | q.cond.Signal() 211 | } else if q.processing.len() == 0 { //当队列中没有元素时，需要通知waitForProcessing 可以结束。 212 | q.cond.Signal() 213 | } 214 | } 215 | ``` 216 | 217 | ```go 218 | func (q *Type) ShutDown() { 219 | q.setDrain(false) 220 | //通知所有work，处理结束了 221 | q.shutdown() 222 | } 223 | 224 | // 忽略所有新的请求，等待所有对象被执行结束 225 | func (q *Type) ShutDownWithDrain() { 226 | q.setDrain(true) 227 | //通知所有worker可以停止了 228 | q.shutdown() 229 | //等待所有对象处理结束 230 | for q.isProcessing() && q.shouldDrain() { 231 | q.waitForProcessing() 232 | } 233 | } 234 | 235 | // isProcessing indicates if there are still items on the work queue being 236 | // processed. It's used to drain the work queue on an eventual shutdown. 237 | func (q *Type) isProcessing() bool { 238 | q.cond.L.Lock() 239 | defer q.cond.L.Unlock() 240 | return q.processing.len() != 0 241 | } 242 | 243 | //等待处理结束，依赖于Done函数当队列为空时的通知 244 | func (q *Type) waitForProcessing() { 245 | q.cond.L.Lock() 246 | defer q.cond.L.Unlock() 247 | // Ensure that we do not wait on a queue which is already empty, as that 248 | // could result in waiting for Done to be called on items in an empty queue 249 | // which has already been shut down, which will result in waiting 250 | // indefinitely. 251 | if q.processing.len() == 0 { 252 | return 253 | } 254 | q.cond.Wait() 255 | } 256 | 257 | func (q *Type) setDrain(shouldDrain bool) { 258 | q.cond.L.Lock() 259 | defer q.cond.L.Unlock() 260 | q.drain = shouldDrain 261 | } 262 | 263 | func (q *Type) shouldDrain() bool { 264 | q.cond.L.Lock() 265 | defer q.cond.L.Unlock() 266 | return q.drain 267 | } 268 | 269 | //结束队列 270 | func (q *Type) shutdown() { 271 | q.cond.L.Lock() 272 | defer q.cond.L.Unlock() 273 | q.shuttingDown = true 274 | q.cond.Broadcast() 275 | } 276 | 277 | func (q *Type) ShuttingDown() bool { 278 | q.cond.L.Lock() 279 | defer q.cond.L.Unlock() 280 | 281 | return q.shuttingDown 282 | } 283 | 284 | ``` 285 | 286 | 287 | ## 测试 288 | 289 | ```go 290 | func TestBasic(t *testing.T) { 291 | //table testing 292 | tests := []struct { 293 | queue *workqueue.Type 294 | queueShutDown func(workqueue.Interface) 295 | }{ 296 | { 297 | queue: workqueue.New(), //新建对象 298 | queueShutDown: workqueue.Interface.ShutDown, //获取函数地址 299 | }, 300 | { 301 | queue: workqueue.New(), //新建对象 302 | queueShutDown: workqueue.Interface.ShutDownWithDrain,//获取函数地址 303 | }, 304 | } 305 | for _, test := range tests { 306 | // If something is seriously wrong this test will never complete. 307 | 308 | // Start producers 309 | const producers = 50 310 | producerWG := sync.WaitGroup{} 311 | producerWG.Add(producers) 312 | for i := 0; i < producers; i++ { 313 | go func(i int) { 314 | defer producerWG.Done() 315 | for j := 0; j < 50; j++ { 316 | //增加对象，如果已经在dirty队列中，则直接忽略，所以多个producer会有非常多重复的 317 | test.queue.Add(i) 318 | time.Sleep(time.Millisecond) 319 | } 320 | }(i) 321 | } 322 | 323 | // Start consumers 324 | const consumers = 10 325 | consumerWG := sync.WaitGroup{} 326 | consumerWG.Add(consumers) 327 | for i := 0; i < consumers; i++ { 328 | go func(i int) { 329 | defer consumerWG.Done() 330 | for { 331 | //获取对象，如果此时有对象更新（相同对象重新入队），则将对象重新加入到queue中 332 | item, quit := test.queue.Get() 333 | if item == "added after shutdown!" { 334 | t.Errorf("Got an item added after shutdown.") 335 | } 336 | if quit { 337 | return 338 | } 339 | t.Logf("Worker %v: begin processing %v", i, item) 340 | time.Sleep(3 * time.Millisecond) 341 | t.Logf("Worker %v: done processing %v", i, item) 342 | test.queue.Done(item) 343 | } 344 | }(i) 345 | } 346 | 347 | producerWG.Wait() 348 | test.queueShutDown(test.queue) //使用方法（作用对象）的方式调用对象 349 | test.queue.Add("added after shutdown!") 350 | consumerWG.Wait() 351 | if test.queue.Len() != 0 { 352 | t.Errorf("Expected the queue to be empty, had: %v items", test.queue.Len()) 353 | } 354 | } 355 | } 356 | ``` 357 | -------------------------------------------------------------------------------- /util/workqueue/rate-limiting-queue.md: -------------------------------------------------------------------------------- 1 | # Rate limiting queue 2 | 3 | ## Interface 4 | 5 | ```go 6 | 7 | //有限流的延时队列 8 | type RateLimitingInterface interface { 9 | DelayingInterface 10 | 11 | // AddRateLimited 当限流允许时将对象加入到工作队列中 12 | AddRateLimited(item interface{}) 13 | 14 | 15 | // Forget 限流器忽略该对象，但是需要手动调用队列的Done才能从队列中忽略该对象 16 | Forget(item interface{}) 17 | 18 | // 重新入队次数 19 | NumRequeues(item interface{}) int 20 | } 21 | 22 | ``` 23 | 24 | 25 | ## 新建 26 | 27 | ```go 28 | // NewRateLimitingQueue 基于限流器构建一个延时队列 29 | func NewRateLimitingQueue(rateLimiter RateLimiter) RateLimitingInterface { 30 | return &rateLimitingType{ 31 | DelayingInterface: NewDelayingQueue(), 32 | rateLimiter: rateLimiter, 33 | } 34 | } 35 | 36 | func NewNamedRateLimitingQueue(rateLimiter RateLimiter, name string) RateLimitingInterface { 37 | return &rateLimitingType{ 38 | DelayingInterface: NewNamedDelayingQueue(name), 39 | rateLimiter: rateLimiter, 40 | } 41 | } 42 | ``` 43 | 44 | 45 | ## 方法 46 | 47 | ```go 48 | // 限流延时队列实现 49 | type rateLimitingType struct { 50 | DelayingInterface 51 | 52 | rateLimiter RateLimiter 53 | } 54 | 55 | // 实现接口 56 | func (q *rateLimitingType) AddRateLimited(item interface{}) { 57 | q.DelayingInterface.AddAfter(item, q.rateLimiter.When(item)) 58 | } 59 | 60 | //实现接口 61 | func (q *rateLimitingType) NumRequeues(item interface{}) int { 62 | return q.rateLimiter.NumRequeues(item) 63 | } 64 | //实现接口 65 | func (q *rateLimitingType) Forget(item interface{}) { 66 | q.rateLimiter.Forget(item) 67 | } 68 | ``` 69 | -------------------------------------------------------------------------------- /util/workqueue/rate.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ## 1.1 令牌桶 6 | 7 | [wiki](https://en.wikipedia.org/wiki/Token_bucket) 8 | 9 | ### 1.1.1 令牌桶基本结构 10 | 11 | ```go 12 | type Limiter struct { 13 | mu sync.Mutex // 互斥锁 14 | limit Limit //限流器，没秒钟按照这个频率下发令牌 15 | burst int //桶的最大令牌数量，超过这个数量的令牌将会被丢弃 16 | tokens float64 //当前令牌数量 17 | // last is the last time the limiter's tokens field was updated 18 | last time.Time //最后一次计算令牌的时间 19 | // lastEvent is the latest time of a rate-limited event (past or future) 20 | lastEvent time.Time //最后一次限流发生时间 21 | } 22 | ``` 23 | 24 | ```go 25 | // Limit 获取令牌颁发频率 26 | func (lim *Limiter) Limit() Limit { 27 | lim.mu.Lock() 28 | defer lim.mu.Unlock() 29 | return lim.limit 30 | } 31 | 32 | // Burst 获取桶最大数量 33 | // 一个最大令牌数的令牌桶不允许任何时间除非limit为无限 34 | func (lim *Limiter) Burst() int { 35 | lim.mu.Lock() 36 | defer lim.mu.Unlock() 37 | return lim.burst 38 | } 39 | ``` 40 | 41 | ```go 42 | // 消耗一个令牌 43 | func (lim *Limiter) Allow() bool { 44 | return lim.AllowN(time.Now(), 1) 45 | } 46 | 47 | // AllowN 报告当前是否有n个令牌立刻可得 48 | // 当你想要忽略超出限流的事件时，使用该方法； 49 | // 否则应该使用Reserve或者Wait方法. 50 | func (lim *Limiter) AllowN(now time.Time, n int) bool { 51 | return lim.reserveN(now, n, 0).ok 52 | } 53 | ``` 54 | 55 | 56 | 57 | ### 1.1.2 Reservation 58 | 59 | ```go 60 | type Reservation struct { 61 | ok bool // 是否有效 62 | lim *Limiter //限流器父亲 63 | tokens int //需要多少token 64 | timeToAct time.Time //何时可以出发这个事件 65 | // This is the Limit at reservation time, it can change later. 66 | limit Limit // 父限流器的令牌发送频率 67 | } 68 | ``` 69 | 70 | ```go 71 | //是否可以获得指定数量的token 72 | //如果OK返回false，则Delay将返回InfDuration，Cancel不做任何事情 73 | func (r *Reservation) OK() bool { 74 | return r.ok 75 | } 76 | 77 | //需要等待的时间 78 | func (r *Reservation) Delay() time.Duration { 79 | return r.DelayFrom(time.Now()) 80 | } 81 | 82 | // InfDuration is the duration returned by Delay when a Reservation is not OK. 83 | const InfDuration = time.Duration(1<<63 - 1) 84 | 85 | //DelayFrom 指定还需要多久可以执行相关动作，返回zero时间代表着相关动作可以立即执行，InfDuration代表永远无法提供指定数量的令牌 86 | func (r *Reservation) DelayFrom(now time.Time) time.Duration { 87 | if !r.ok { //如果不OK则返回无限长的时间 88 | return InfDuration 89 | } 90 | delay := r.timeToAct.Sub(now) 91 | if delay < 0 { 92 | return 0 93 | } 94 | return delay 95 | } 96 | 97 | // 取消Reservation，并返还已经获取的令牌 98 | func (r *Reservation) Cancel() { 99 | r.CancelAt(time.Now()) 100 | } 101 | 102 | // CancelAt 代表相关的动作将不会执行，尽量归还所有已经获取的令牌 103 | func (r *Reservation) CancelAt(now time.Time) { 104 | if !r.ok { //如果不OK不需要做任何事情 105 | return 106 | } 107 | 108 | 获取锁 109 | r.lim.mu.Lock() 110 | defer r.lim.mu.Unlock() 111 | 112 | //如果没有限流，或者没有获取任何token，或者事件应该在更早之前就发生，则不做任何事情 113 | if r.lim.limit == Inf || r.tokens == 0 || r.timeToAct.Before(now) { 114 | return 115 | } 116 | 117 | // calculate tokens to restore 118 | // The duration between lim.lastEvent and r.timeToAct tells us how many tokens were reserved 119 | // after r was obtained. These tokens should not be restored. 120 | // 由于limiter的令牌颁发频率可能发生变化，因此需按照 `r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct))` 而不是直接归还指定数量的token， 计算下需要规划多少token，此时token可能是负值的 121 | restoreTokens := float64(r.tokens) - r.limit.tokensFromDuration(r.lim.lastEvent.Sub(r.timeToAct)) 122 | //如果不需要返回任何token，则返回 123 | if restoreTokens <= 0 { 124 | return 125 | } 126 | // 更新下令牌数量（按照当前时间与最后一次更新时间的差） 127 | now, _, tokens := r.lim.advance(now) 128 | // 增加token 129 | tokens += restoreTokens 130 | if burst := float64(r.lim.burst); tokens > burst { 131 | tokens = burst 132 | } 133 | // update state 134 | r.lim.last = now 135 | r.lim.tokens = tokens 136 | if r.timeToAct == r.lim.lastEvent { //如果本次就是最后一个Reservation那么尝试计算上一个事件时间 137 | prevEvent := r.timeToAct.Add(r.limit.durationFromTokens(float64(-r.tokens))) 138 | if !prevEvent.Before(now) { 139 | r.lim.lastEvent = prevEvent 140 | } 141 | } 142 | } 143 | ``` 144 | 145 | ```go 146 | // Reserve is shorthand for ReserveN(time.Now(), 1). 147 | func (lim *Limiter) Reserve() *Reservation { 148 | return lim.ReserveN(time.Now(), 1) 149 | } 150 | 151 | // ReserveN returns a Reservation that indicates how long the caller must wait before n events happen. 152 | // The Limiter takes this Reservation into account when allowing future events. 153 | // The returned Reservation’s OK() method returns false if n exceeds the Limiter's burst size. 154 | // Usage example: 155 | // r := lim.ReserveN(time.Now(), 1) 156 | // if !r.OK() { 157 | // // Not allowed to act! Did you remember to set lim.burst to be > 0 ? 158 | // return 159 | // } 160 | // time.Sleep(r.Delay()) 161 | // Act() 162 | // Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events. 163 | // If you need to respect a deadline or cancel the delay, use Wait instead. 164 | // To drop or skip events exceeding rate limit, use Allow instead. 165 | func (lim *Limiter) ReserveN(now time.Time, n int) *Reservation { 166 | r := lim.reserveN(now, n, InfDuration) 167 | return &r 168 | } 169 | 170 | 171 | // reserveN is a helper method for AllowN, ReserveN, and WaitN. 172 | // maxFutureReserve specifies the maximum reservation wait duration allowed. 173 | // reserveN returns Reservation, not *Reservation, to avoid allocation in AllowN and WaitN. 174 | func (lim *Limiter) reserveN(now time.Time, n int, maxFutureReserve time.Duration) Reservation { 175 | //加锁 176 | lim.mu.Lock() 177 | defer lim.mu.Unlock() 178 | 179 | //如果不限流则直接返回 180 | if lim.limit == Inf { 181 | return Reservation{ 182 | ok: true, 183 | lim: lim, 184 | tokens: n, 185 | timeToAct: now, 186 | } 187 | } else if lim.limit == 0 { 188 | //限流但是不会定时颁发令牌 189 | var ok bool 190 | //如果桶的令牌数还够用 191 | if lim.burst >= n { 192 | ok = true 193 | lim.burst -= n 194 | } 195 | //返回结果 196 | return Reservation{ 197 | ok: ok, 198 | lim: lim, 199 | tokens: lim.burst, 200 | timeToAct: now, 201 | } 202 | } 203 | 204 | //更新令牌数量 205 | now, last, tokens := lim.advance(now) 206 | 207 | // Calculate the remaining number of tokens resulting from the request. 208 | //分配令牌 209 | tokens -= float64(n) 210 | 211 | // Calculate the wait duration 212 | // 计算等待时间 213 | var waitDuration time.Duration 214 | if tokens < 0 { 215 | waitDuration = lim.limit.durationFromTokens(-tokens) 216 | } 217 | 218 | // Decide result 219 | // 要求的令牌数量必须小于桶的最大值，并且最大等待时间必须小于最大等待时间 220 | ok := n <= lim.burst && waitDuration <= maxFutureReserve 221 | 222 | // Prepare reservation 223 | r := Reservation{ 224 | ok: ok, 225 | lim: lim, 226 | limit: lim.limit, 227 | } 228 | //如果成功获取令牌 229 | if ok { 230 | r.tokens = n 231 | //计算时间发生时间 232 | r.timeToAct = now.Add(waitDuration) 233 | } 234 | 235 | // Update state 236 | if ok { 237 | lim.last = now 238 | //tokens 可能是负数 239 | lim.tokens = tokens 240 | //更新最后分配时间 241 | lim.lastEvent = r.timeToAct 242 | } else { 243 | lim.last = last 244 | } 245 | 246 | return r 247 | } 248 | 249 | ``` 250 | 251 | ```go 252 | 253 | // Wait is shorthand for WaitN(ctx, 1). 254 | func (lim *Limiter) Wait(ctx context.Context) (err error) { 255 | return lim.WaitN(ctx, 1) 256 | } 257 | 258 | //WaitN 等待限流器有N个令牌，如果n大于 limiter的桶大小，context超时，或者context被取消则返回失败信息 259 | func (lim *Limiter) WaitN(ctx context.Context, n int) (err error) { 260 | lim.mu.Lock() 261 | burst := lim.burst 262 | limit := lim.limit 263 | lim.mu.Unlock() 264 | 265 | //如果要求令牌大于桶大小，并且限流器不是无限大则返回错误 266 | if n > burst && limit != Inf { 267 | return fmt.Errorf("rate: Wait(n=%d) exceeds limiter's burst %d", n, burst) 268 | } 269 | // Check if ctx is already cancelled 270 | select { 271 | case <-ctx.Done(): 272 | return ctx.Err() 273 | default: 274 | } 275 | // Determine wait limit 276 | now := time.Now() 277 | waitLimit := InfDuration 278 | if deadline, ok := ctx.Deadline(); ok { 279 | waitLimit = deadline.Sub(now) 280 | } 281 | // Reserve，获取reserve，如果在指定时间内不能提供数量的令牌将会返回失败 282 | r := lim.reserveN(now, n, waitLimit) 283 | if !r.ok { 284 | return fmt.Errorf("rate: Wait(n=%d) would exceed context deadline", n) 285 | } 286 | // Wait if necessary，计算等待时间 287 | delay := r.DelayFrom(now) 288 | if delay == 0 { 289 | return nil 290 | } 291 | t := time.NewTimer(delay) 292 | defer t.Stop() 293 | select { 294 | case <-t.C: 295 | // We can proceed. 296 | return nil 297 | case <-ctx.Done(): 298 | // Context was canceled before we could proceed. Cancel the 299 | // reservation, which may permit other events to proceed sooner. 300 | //如果context被取消，则规划令牌 301 | r.Cancel() 302 | return ctx.Err() 303 | } 304 | } 305 | 306 | ``` 307 | 308 | 309 | ```go 310 | // 计算到目标时间时会有多少token 311 | // advance requires that lim.mu is held.（可以在函数名中加上locked，如advanceLocked） 312 | func (lim *Limiter) advance(now time.Time) (newNow time.Time, newLast time.Time, newTokens float64) { 313 | last := lim.last 314 | if now.Before(last) { 315 | last = now 316 | } 317 | 318 | // Calculate the new number of tokens, due to time that passed. 319 | elapsed := now.Sub(last) 320 | delta := lim.limit.tokensFromDuration(elapsed) 321 | tokens := lim.tokens + delta 322 | if burst := float64(lim.burst); tokens > burst { 323 | tokens = burst 324 | } 325 | return now, last, tokens 326 | } 327 | 328 | // token和时间的换算 329 | func (limit Limit) durationFromTokens(tokens float64) time.Duration { 330 | if limit <= 0 { 331 | return InfDuration 332 | } 333 | seconds := tokens / float64(limit) 334 | return time.Duration(float64(time.Second) * seconds) 335 | } 336 | 337 | // 时间和token的换算 338 | func (limit Limit) tokensFromDuration(d time.Duration) float64 { 339 | if limit <= 0 { 340 | return 0 341 | } 342 | return d.Seconds() * float64(limit) 343 | } 344 | 345 | ``` 346 | 347 | 348 | 349 | 350 | 351 | 352 | -------------------------------------------------------------------------------- /util/workqueue/rate_test.go: -------------------------------------------------------------------------------- 1 | package workqueue 2 | 3 | import ( 4 | "context" 5 | "runtime" 6 | "sync" 7 | "sync/atomic" 8 | "testing" 9 | "golang.org/x/time/rate" 10 | "time" 11 | ) 12 | 13 | func TestAllow(t *testing.T) { 14 | 15 | lim := rate.NewLimiter(10,100) 16 | 17 | lim.AllowN(time.Now(),100) 18 | 19 | numOk := int32(0) 20 | 21 | timer := time.NewTimer(1 * time.Second) 22 | 23 | Looper: for { 24 | select { 25 | case <-timer.C: 26 | break Looper 27 | default: 28 | if lim.Allow(){ 29 | atomic.AddInt32(&numOk, 1) 30 | time.Sleep(1 * time.Millisecond) 31 | } 32 | } 33 | } 34 | 35 | t.Logf("rate.NewLimiter(10,100) will produce %d num ok", numOk) 36 | } 37 | 38 | func TestReserve(t *testing.T) { 39 | lim := rate.NewLimiter(10,100) 40 | time.Sleep(100 * time.Millisecond) 41 | r := lim.ReserveN(time.Now(),110) 42 | t.Logf("rate.NewLimiter(10,100) request more than bust will return : %v" , r.OK()) 43 | 44 | 45 | r = lim.ReserveN(time.Now(),100) 46 | t.Logf("ReserveN empty will return : %v" , r.DelayFrom(time.Now())) 47 | r = lim.ReserveN(time.Now(),10) 48 | t.Logf("ReserveN empty will return : %v" , r.DelayFrom(time.Now())) 49 | 50 | r2 := lim.ReserveN(time.Now(),5) 51 | t.Logf("ReserveN empty 2 will return : %v" , r2.DelayFrom(time.Now())) 52 | 53 | r.Cancel() 54 | r3 := lim.ReserveN(time.Now(),5) 55 | t.Logf("ReserveN empty 3 will return : %v" , r3.DelayFrom(time.Now())) 56 | } 57 | 58 | func TestWait(t *testing.T) { 59 | lim := rate.NewLimiter(10,100) 60 | 61 | numOk := int32(0) 62 | 63 | ctx , cancel := context.WithTimeout(context.Background(),1 * time.Second ) 64 | defer cancel() 65 | 66 | numProc := runtime.GOMAXPROCS(0) 67 | var wg sync.WaitGroup 68 | wg.Add(numProc) 69 | for i := 0 ; i < numProc ; i ++ { 70 | go func() { 71 | defer wg.Done() 72 | for { 73 | select { 74 | case <-ctx.Done(): 75 | return 76 | default: 77 | if err := lim.Wait(context.Background()) ; err != nil { 78 | t.Logf("wait limiter failed , err : %s",err) 79 | }else { 80 | atomic.AddInt32(&numOk,1 ) 81 | } 82 | time.Sleep(1 * time.Millisecond) 83 | } 84 | } 85 | }() 86 | } 87 | 88 | wg.Wait() 89 | 90 | t.Logf("after 1 second , %d works numOk is : %d", numProc ,numOk ) 91 | 92 | } 93 | --------------------------------------------------------------------------------