├── .gitignore ├── LICENSE ├── README.md ├── control ├── pkg ├── file │ └── file.go ├── logger │ ├── config.go │ ├── file.go │ ├── logger.go │ ├── logger_test.go │ ├── multi.go │ ├── syslog_unix.go │ └── syslog_unsupported.go ├── sys │ └── sys.go ├── tools │ └── signal.go └── watcher │ └── watcher.go ├── sga └── main.go └── sgd ├── config ├── config.go └── funcs.go ├── cron └── cron.go ├── main.go ├── system └── system.go └── timer ├── destruct.go ├── heartbeat.go ├── memory.go ├── model.go ├── request.go ├── response.go ├── sleep.go ├── timer.go ├── toolkit.go └── watcher.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.sw[po] 6 | 7 | # Folders 8 | _obj 9 | _test 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | *.test 25 | *.prof 26 | *.log 27 | *.tar.gz 28 | 29 | .DS_Store 30 | /var 31 | /log 32 | /tmp 33 | /build 34 | /.idea 35 | /uuid.url 36 | /sgd/sgd* 37 | /sga/sga* 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SGT 2 | 3 | ## 背景 4 | 5 | 此进程在滴滴云上用于管理机器上面的其他agent，比如监控的agent、安全的agent，管理主要是：安装、升级、卸载、查看启动状态，不做其他事情。省去客户手工安装其他agent的工作。 6 | 7 | ## 安装 8 | 9 | 虚机创建的时候会自动安装此进程，如需对存量虚机安装，可以执行： 10 | 11 | ``` 12 | curl -s http://mirrors.intra.didiyun.com/didiyun_resource/sgd-v1.sh | bash 13 | ``` 14 | 15 | 只能在滴滴云的虚机里运行这条指令，适用64位linux系统 16 | 17 | ## 资源占用 18 | 19 | 安装完成之后机器上会有sgd和sga两个进程，sgd内存占用小于10MB，承担管理其他agent的核心业务逻辑，sga内存占用小于4MB，是sgd进程的伴生进程，在sgd挂掉的时候负责将其拉起。cpu使用率小于1% 20 | 21 | ## 规范要求 22 | 23 | sgd管理的其他agent需要提供control脚本，打到tar.gz包里，control脚本需要具备可执行权限，支持这些参数：pid | version | start | stop | uninstall | install，sgd就是利用业务agent的control脚本来做管理的 24 | -------------------------------------------------------------------------------- /control: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CWD=$(cd $(dirname $0)/; pwd) 4 | 5 | start() 6 | { 7 | if [ $(ps aux|grep -v grep|grep '/usr/local/sgd/bin/sgd' -c) -gt 0 ]; then 8 | echo "already started" 9 | exit 0 10 | fi 11 | 12 | DEV=${SGD_DEV:-0} 13 | if [ "x$DEV" = "x1" ]; then 14 | # in dev 15 | $CWD/bin/sgd -s http://100.64.8.7:80 -d 16 | else 17 | # in prod 18 | mkdir -p $CWD/log 19 | nohup $CWD/bin/sgd -s http://100.64.8.7:80 &> $CWD/log/app.log & 20 | fi 21 | } 22 | 23 | stop() 24 | { 25 | DEV=${SGD_DEV:-0} 26 | if [ "x$DEV" = "x1" ]; then 27 | # in dev 28 | ps aux|grep -v grep|grep '/usr/local/sgd/bin/sgd'|awk '{print $1}'|xargs kill && echo "sgd killed" 29 | ps aux|grep -v grep|grep '/usr/local/sgd/bin/sga'|awk '{print $1}'|xargs kill && echo "sga killed" 30 | if [ $(ps aux|grep -v grep|grep '/usr/local/sgd/bin/sg' -c) -ne 0 ]; then 31 | echo "cannot stop sgd and sga" 32 | exit 1 33 | fi 34 | else 35 | # in prod 36 | ps aux|grep -v grep|grep '/usr/local/sgd/bin/sgd'|awk '{print $1}'|xargs kill && echo "sgd killed" 37 | fi 38 | } 39 | 40 | build() 41 | { 42 | cd $CWD/sga && go build 43 | cd $CWD/sgd && go build 44 | } 45 | 46 | pack() 47 | { 48 | goos=$1 49 | goarch=$2 50 | ver=$3 51 | 52 | rm -rf $CWD/build && mkdir -p $CWD/build/bin && cp $CWD/control $CWD/build 53 | 54 | cd $CWD/sga && GOOS=$goos GOARCH=$goarch go build && cp sga $CWD/build/bin/sga 55 | cd $CWD/sgd && GOOS=$goos GOARCH=$goarch go build && cp sgd $CWD/build/bin/sgd 56 | 57 | cd $CWD/build && tar zcvf $CWD/sgd-${ver}_${goos}_${goarch}.tar.gz control bin 58 | } 59 | 60 | release() 61 | { 62 | if [ ! -f uuid.url ]; then 63 | echo "file uuid.url not found" 64 | exit 1 65 | fi 66 | 67 | url=$(cat uuid.url | head -n 1) 68 | sed -i "s|UUID_URL|$url|g" sgd/config/config.go 69 | 70 | build 71 | 72 | v=$($CWD/sgd/sgd -v) 73 | pack linux 386 $v 74 | pack linux amd64 $v 75 | } 76 | 77 | case "$1" in 78 | start) 79 | start 80 | ;; 81 | stop) 82 | stop 83 | ;; 84 | build) 85 | build 86 | ;; 87 | release) 88 | release 89 | ;; 90 | *) 91 | echo $"Usage: $0 {start|stop|build|release}" 92 | esac -------------------------------------------------------------------------------- /pkg/file/file.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "bufio" 5 | "crypto/md5" 6 | "encoding/hex" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net/http" 11 | "os" 12 | "path" 13 | "path/filepath" 14 | ) 15 | 16 | // SelfPath gets compiled executable file absolute path 17 | func SelfPath() string { 18 | path, _ := filepath.Abs(os.Args[0]) 19 | return path 20 | } 21 | 22 | // get absolute filepath, based on built executable file 23 | func RealPath(fp string) (string, error) { 24 | if path.IsAbs(fp) { 25 | return fp, nil 26 | } 27 | wd, err := os.Getwd() 28 | return path.Join(wd, fp), err 29 | } 30 | 31 | // SelfDir gets compiled executable file directory 32 | func SelfDir() string { 33 | return filepath.Dir(SelfPath()) 34 | } 35 | 36 | // get filepath dir name 37 | func Dir(fp string) string { 38 | return path.Dir(fp) 39 | } 40 | 41 | // IsExist checks whether a file or directory exists. 42 | // It returns false when the file or directory does not exist. 43 | func IsExist(fp string) bool { 44 | _, err := os.Stat(fp) 45 | return err == nil || os.IsExist(err) 46 | } 47 | 48 | // mkdir dir if not exist 49 | func EnsureDir(fp string) error { 50 | return os.MkdirAll(fp, os.ModePerm) 51 | } 52 | 53 | // list dirs under dirPath 54 | func DirsUnder(dirPath string) ([]string, error) { 55 | if !IsExist(dirPath) { 56 | return []string{}, nil 57 | } 58 | 59 | fs, err := ioutil.ReadDir(dirPath) 60 | if err != nil { 61 | return []string{}, err 62 | } 63 | 64 | sz := len(fs) 65 | if sz == 0 { 66 | return []string{}, nil 67 | } 68 | 69 | ret := make([]string, 0, sz) 70 | for i := 0; i < sz; i++ { 71 | if fs[i].IsDir() { 72 | name := fs[i].Name() 73 | if name != "." && name != ".." { 74 | ret = append(ret, name) 75 | } 76 | } 77 | } 78 | 79 | return ret, nil 80 | } 81 | 82 | // Unlink delete file 83 | func Unlink(fp string) error { 84 | if !IsExist(fp) { 85 | return nil 86 | } 87 | return os.Remove(fp) 88 | } 89 | 90 | // IsFile checks whether the path is a file, 91 | // it returns false when it's a directory or does not exist. 92 | func IsFile(fp string) bool { 93 | f, e := os.Stat(fp) 94 | if e != nil { 95 | return false 96 | } 97 | return !f.IsDir() 98 | } 99 | 100 | func ReadBytes(cpath string) ([]byte, error) { 101 | if !IsExist(cpath) { 102 | return nil, fmt.Errorf("%s not exists", cpath) 103 | } 104 | 105 | if !IsFile(cpath) { 106 | return nil, fmt.Errorf("%s not file", cpath) 107 | } 108 | 109 | return ioutil.ReadFile(cpath) 110 | } 111 | 112 | func ReadString(cpath string) (string, error) { 113 | bs, err := ReadBytes(cpath) 114 | if err != nil { 115 | return "", err 116 | } 117 | 118 | return string(bs), nil 119 | } 120 | 121 | func WriteBytes(filePath string, b []byte) (int, error) { 122 | os.MkdirAll(path.Dir(filePath), os.ModePerm) 123 | fw, err := os.Create(filePath) 124 | if err != nil { 125 | return 0, err 126 | } 127 | defer fw.Close() 128 | return fw.Write(b) 129 | } 130 | 131 | func WriteString(filePath string, s string) (int, error) { 132 | return WriteBytes(filePath, []byte(s)) 133 | } 134 | 135 | func Download(toFile, url string) error { 136 | out, err := os.Create(toFile) 137 | if err != nil { 138 | return err 139 | } 140 | 141 | defer out.Close() 142 | 143 | resp, err := http.Get(url) 144 | if err != nil { 145 | return err 146 | } 147 | 148 | if resp.Body == nil { 149 | return fmt.Errorf("%s response body is nil", url) 150 | } 151 | 152 | defer resp.Body.Close() 153 | 154 | _, err = io.Copy(out, resp.Body) 155 | return err 156 | } 157 | 158 | func Md5File(file string) (string, error) { 159 | f, err := os.Open(file) 160 | if err != nil { 161 | return "", err 162 | } 163 | defer f.Close() 164 | 165 | r := bufio.NewReader(f) 166 | h := md5.New() 167 | 168 | _, err = io.Copy(h, r) 169 | if err != nil { 170 | return "", err 171 | } 172 | return hex.EncodeToString(h.Sum(nil)), nil 173 | } 174 | 175 | func Md5String(str string) string { 176 | hasher := md5.New() 177 | io.WriteString(hasher, str) 178 | return hex.EncodeToString(hasher.Sum(nil)) 179 | } 180 | -------------------------------------------------------------------------------- /pkg/logger/config.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type LogConfig struct { 9 | Type string // syslog/stderr/std/file 10 | Level string // DEBUG/INFO/WARNING/ERROR/FATAL 11 | SyslogPriority string // local0-7 12 | SyslogSeverity string 13 | FileName string 14 | FileRotateCount int 15 | FileRotateSize uint64 16 | FileFlushDuration time.Duration 17 | RotateByHour bool 18 | KeepHours uint // make sense when RotateByHour is T 19 | } 20 | 21 | func initFromConfig(log *Logger, 22 | sb *syslogBackend, 23 | fb *FileBackend, 24 | config LogConfig) error { 25 | 26 | if config.Type == "stderr" || config.Type == "std" { 27 | log.LogToStderr() 28 | log.SetSeverity(config.Level) 29 | return nil 30 | } 31 | 32 | var err error 33 | if config.Type == "syslog" { 34 | if sb, err = NewSyslogBackend(config.SyslogPriority, config.SyslogSeverity); err != nil { 35 | return err 36 | } 37 | log.SetLogging(config.Level, sb) 38 | } else if config.Type == "file" { 39 | if fb, err = NewFileBackend(config.FileName); err != nil { 40 | return err 41 | } 42 | log.SetLogging(config.Level, fb) 43 | fb.Rotate(config.FileRotateCount, config.FileRotateSize) 44 | fb.SetFlushDuration(config.FileFlushDuration) 45 | fb.SetRotateByHour(config.RotateByHour) 46 | fb.SetKeepHours(config.KeepHours) 47 | } else { 48 | return fmt.Errorf("unknown log type: %s", config.Type) 49 | } 50 | return nil 51 | } 52 | 53 | func Init(config LogConfig) error { 54 | return initFromConfig(&logging, sysback, fileback, config) 55 | } 56 | 57 | func NewLoggerFromConfig(config LogConfig) (Logger, error) { 58 | var log Logger 59 | var fb *FileBackend = nil 60 | var sb *syslogBackend = nil 61 | err := initFromConfig(&log, sb, fb, config) 62 | return log, err 63 | } 64 | -------------------------------------------------------------------------------- /pkg/logger/file.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "regexp" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | const ( 18 | bufferSize = 256 * 1024 19 | ) 20 | 21 | func getLastCheck(now time.Time) uint64 { 22 | return uint64(now.Year())*1000000 + uint64(now.Month())*10000 + uint64(now.Day())*100 + uint64(now.Hour()) 23 | } 24 | 25 | type syncBuffer struct { 26 | *bufio.Writer 27 | file *os.File 28 | count uint64 29 | cur int 30 | filePath string 31 | parent *FileBackend 32 | } 33 | 34 | func (self *syncBuffer) Sync() error { 35 | return self.file.Sync() 36 | } 37 | 38 | func (self *syncBuffer) close() { 39 | self.Flush() 40 | self.Sync() 41 | self.file.Close() 42 | } 43 | 44 | func (self *syncBuffer) write(b []byte) { 45 | if !self.parent.rotateByHour && self.parent.maxSize > 0 && self.parent.rotateNum > 0 && self.count+uint64(len(b)) >= self.parent.maxSize { 46 | os.Rename(self.filePath, self.filePath+fmt.Sprintf(".%03d", self.cur)) 47 | self.cur++ 48 | if self.cur >= self.parent.rotateNum { 49 | self.cur = 0 50 | } 51 | self.count = 0 52 | } 53 | self.count += uint64(len(b)) 54 | self.Writer.Write(b) 55 | } 56 | 57 | type FileBackend struct { 58 | mu sync.Mutex 59 | dir string //directory for log files 60 | files [numSeverity]syncBuffer 61 | flushInterval time.Duration 62 | rotateNum int 63 | maxSize uint64 64 | fall bool 65 | rotateByHour bool 66 | lastCheck uint64 67 | reg *regexp.Regexp // for rotatebyhour log del... 68 | keepHours uint // keep how many hours old, only make sense when rotatebyhour is T 69 | } 70 | 71 | func (self *FileBackend) Flush() { 72 | self.mu.Lock() 73 | defer self.mu.Unlock() 74 | for i := 0; i < numSeverity; i++ { 75 | self.files[i].Flush() 76 | self.files[i].Sync() 77 | } 78 | 79 | } 80 | 81 | func (self *FileBackend) close() { 82 | self.Flush() 83 | } 84 | 85 | func (self *FileBackend) flushDaemon() { 86 | for { 87 | time.Sleep(self.flushInterval) 88 | self.Flush() 89 | } 90 | } 91 | 92 | func shouldDel(fileName string, left uint) bool { 93 | // tag should be like 2016071114 94 | tagInt, err := strconv.Atoi(strings.Split(fileName, ".")[2]) 95 | if err != nil { 96 | return false 97 | } 98 | 99 | point := time.Now().Unix() - int64(left*3600) 100 | 101 | if getLastCheck(time.Unix(point, 0)) > uint64(tagInt) { 102 | return true 103 | } 104 | 105 | return false 106 | 107 | } 108 | 109 | func (self *FileBackend) rotateByHourDaemon() { 110 | for { 111 | time.Sleep(time.Second * 1) 112 | 113 | if self.rotateByHour { 114 | check := getLastCheck(time.Now()) 115 | if self.lastCheck < check { 116 | for i := 0; i < numSeverity; i++ { 117 | os.Rename(self.files[i].filePath, self.files[i].filePath+fmt.Sprintf(".%d", self.lastCheck)) 118 | } 119 | self.lastCheck = check 120 | } 121 | 122 | // also check log dir to del overtime files 123 | files, err := ioutil.ReadDir(self.dir) 124 | if err == nil { 125 | for _, file := range files { 126 | // exactly match, then we 127 | if file.Name() == self.reg.FindString(file.Name()) && 128 | shouldDel(file.Name(), self.keepHours) { 129 | os.Remove(filepath.Join(self.dir, file.Name())) 130 | } 131 | } 132 | } 133 | } 134 | } 135 | } 136 | 137 | func (self *FileBackend) monitorFiles() { 138 | for range time.NewTicker(time.Second * 5).C { 139 | for i := 0; i < numSeverity; i++ { 140 | fileName := path.Join(self.dir, severityName[i]+".log") 141 | if _, err := os.Stat(fileName); err != nil && os.IsNotExist(err) { 142 | if f, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644); err == nil { 143 | self.mu.Lock() 144 | self.files[i].close() 145 | self.files[i].Writer = bufio.NewWriterSize(f, bufferSize) 146 | self.files[i].file = f 147 | self.mu.Unlock() 148 | } 149 | } 150 | } 151 | } 152 | } 153 | 154 | func (self *FileBackend) Log(s Severity, msg []byte) { 155 | self.mu.Lock() 156 | switch s { 157 | case FATAL: 158 | self.files[FATAL].write(msg) 159 | case ERROR: 160 | self.files[ERROR].write(msg) 161 | case WARNING: 162 | self.files[WARNING].write(msg) 163 | case INFO: 164 | self.files[INFO].write(msg) 165 | case DEBUG: 166 | self.files[DEBUG].write(msg) 167 | } 168 | if self.fall && s < INFO { 169 | self.files[INFO].write(msg) 170 | } 171 | self.mu.Unlock() 172 | if s == FATAL { 173 | self.Flush() 174 | } 175 | } 176 | 177 | func (self *FileBackend) Rotate(rotateNum1 int, maxSize1 uint64) { 178 | self.rotateNum = rotateNum1 179 | self.maxSize = maxSize1 180 | } 181 | 182 | func (self *FileBackend) SetRotateByHour(rotateByHour bool) { 183 | self.rotateByHour = rotateByHour 184 | if self.rotateByHour { 185 | self.lastCheck = getLastCheck(time.Now()) 186 | } else { 187 | self.lastCheck = 0 188 | } 189 | } 190 | 191 | func (self *FileBackend) SetKeepHours(hours uint) { 192 | self.keepHours = hours 193 | } 194 | 195 | func (self *FileBackend) Fall() { 196 | self.fall = true 197 | } 198 | 199 | func (self *FileBackend) SetFlushDuration(t time.Duration) { 200 | if t >= time.Second { 201 | self.flushInterval = t 202 | } else { 203 | self.flushInterval = time.Second 204 | } 205 | } 206 | func NewFileBackend(dir string) (*FileBackend, error) { 207 | if err := os.MkdirAll(dir, 0755); err != nil { 208 | return nil, err 209 | } 210 | var fb FileBackend 211 | fb.dir = dir 212 | for i := 0; i < numSeverity; i++ { 213 | fileName := path.Join(dir, severityName[i]+".log") 214 | f, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 215 | if err != nil { 216 | return nil, err 217 | } 218 | 219 | count := uint64(0) 220 | stat, err := f.Stat() 221 | if err == nil { 222 | count = uint64(stat.Size()) 223 | } 224 | fb.files[i] = syncBuffer{ 225 | Writer: bufio.NewWriterSize(f, bufferSize), 226 | file: f, 227 | filePath: fileName, 228 | parent: &fb, 229 | count: count, 230 | } 231 | 232 | } 233 | // default 234 | fb.flushInterval = time.Second * 3 235 | fb.rotateNum = 20 236 | fb.maxSize = 1024 * 1024 * 1024 237 | fb.rotateByHour = false 238 | fb.lastCheck = 0 239 | // init reg to match files 240 | // ONLY cover this centry... 241 | fb.reg = regexp.MustCompile("(INFO|ERROR|WARNING|DEBUG|FATAL)\\.log\\.20[0-9]{8}") 242 | fb.keepHours = 24 * 7 243 | 244 | go fb.flushDaemon() 245 | go fb.monitorFiles() 246 | go fb.rotateByHourDaemon() 247 | return &fb, nil 248 | } 249 | 250 | func Rotate(rotateNum1 int, maxSize1 uint64) { 251 | if fileback != nil { 252 | fileback.Rotate(rotateNum1, maxSize1) 253 | } 254 | } 255 | 256 | func Fall() { 257 | if fileback != nil { 258 | fileback.Fall() 259 | } 260 | } 261 | 262 | func SetFlushDuration(t time.Duration) { 263 | if fileback != nil { 264 | fileback.SetFlushDuration(t) 265 | } 266 | 267 | } 268 | 269 | func SetRotateByHour(rotateByHour bool) { 270 | if fileback != nil { 271 | fileback.SetRotateByHour(rotateByHour) 272 | } 273 | } 274 | 275 | func SetKeepHours(hours uint) { 276 | if fileback != nil { 277 | fileback.SetKeepHours(hours) 278 | } 279 | } 280 | -------------------------------------------------------------------------------- /pkg/logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "runtime" 8 | "strings" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | type Severity int 14 | 15 | const ( 16 | FATAL Severity = iota 17 | ERROR 18 | WARNING 19 | INFO 20 | DEBUG 21 | ) 22 | 23 | var severityName = []string{ 24 | FATAL: "FATAL", 25 | ERROR: "ERROR", 26 | WARNING: "WARNING", 27 | INFO: "INFO", 28 | DEBUG: "DEBUG", 29 | } 30 | 31 | const ( 32 | numSeverity = 5 33 | ) 34 | 35 | type Backend interface { 36 | Log(s Severity, msg []byte) 37 | close() 38 | } 39 | 40 | type stdBackend struct{} 41 | 42 | func (self *stdBackend) Log(s Severity, msg []byte) { 43 | os.Stdout.Write(msg) 44 | } 45 | 46 | func (self *stdBackend) close() {} 47 | 48 | type Logger struct { 49 | s Severity 50 | backend Backend 51 | mu sync.Mutex 52 | 53 | freeList *buffer 54 | freeListMu sync.Mutex 55 | 56 | logToStderr bool 57 | } 58 | 59 | //resued buffer for fast format the output string 60 | type buffer struct { 61 | bytes.Buffer 62 | tmp [64]byte 63 | next *buffer 64 | } 65 | 66 | func (self *Logger) getBuffer() *buffer { 67 | self.freeListMu.Lock() 68 | b := self.freeList 69 | if b != nil { 70 | self.freeList = b.next 71 | } 72 | self.freeListMu.Unlock() 73 | if b == nil { 74 | b = new(buffer) 75 | } else { 76 | b.next = nil 77 | b.Reset() 78 | } 79 | return b 80 | } 81 | 82 | // Some custom tiny helper functions to print the log header efficiently. 83 | const digits = "0123456789" 84 | 85 | // twoDigits formats a zero-prefixed two-digit integer at buf.tmp[i]. 86 | func (buf *buffer) twoDigits(i, d int) { 87 | buf.tmp[i+1] = digits[d%10] 88 | d /= 10 89 | buf.tmp[i] = digits[d%10] 90 | } 91 | 92 | // nDigits formats an n-digit integer at buf.tmp[i], 93 | // padding with pad on the left. 94 | // It assumes d >= 0. 95 | func (buf *buffer) nDigits(n, i, d int, pad byte) { 96 | j := n - 1 97 | for ; j >= 0 && d > 0; j-- { 98 | buf.tmp[i+j] = digits[d%10] 99 | d /= 10 100 | } 101 | for ; j >= 0; j-- { 102 | buf.tmp[i+j] = pad 103 | } 104 | } 105 | 106 | // someDigits formats a zero-prefixed variable-width integer at buf.tmp[i]. 107 | func (buf *buffer) someDigits(i, d int) int { 108 | // Print into the top, then copy down. We know there's space for at least 109 | // a 10-digit number. 110 | j := len(buf.tmp) 111 | for { 112 | j-- 113 | buf.tmp[j] = digits[d%10] 114 | d /= 10 115 | if d == 0 { 116 | break 117 | } 118 | } 119 | return copy(buf.tmp[i:], buf.tmp[j:]) 120 | } 121 | 122 | func (self *Logger) putBuffer(b *buffer) { 123 | if b.Len() >= 256 { 124 | // Let big buffers die a natural death. 125 | return 126 | } 127 | self.freeListMu.Lock() 128 | b.next = self.freeList 129 | self.freeList = b 130 | self.freeListMu.Unlock() 131 | } 132 | 133 | func (self *Logger) formatHeader(s Severity, file string, line int) *buffer { 134 | now := time.Now() 135 | if line < 0 { 136 | line = 0 // not a real line number, but acceptable to someDigits 137 | } 138 | buf := self.getBuffer() 139 | 140 | // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. 141 | // It's worth about 3X. Fprintf is hard. 142 | year, month, day := now.Date() 143 | hour, minute, second := now.Clock() 144 | //2015-06-16 12:00:35 ERROR test.go:12 ... 145 | buf.nDigits(4, 0, year, '0') 146 | buf.tmp[4] = '-' 147 | buf.twoDigits(5, int(month)) 148 | buf.tmp[7] = '-' 149 | buf.twoDigits(8, day) 150 | buf.tmp[10] = ' ' 151 | buf.twoDigits(11, hour) 152 | buf.tmp[13] = ':' 153 | buf.twoDigits(14, minute) 154 | buf.tmp[16] = ':' 155 | buf.twoDigits(17, second) 156 | buf.tmp[19] = '.' 157 | buf.nDigits(6, 20, now.Nanosecond()/1000, '0') 158 | buf.tmp[26] = ' ' 159 | buf.Write(buf.tmp[:27]) 160 | buf.WriteString(severityName[s]) 161 | buf.WriteByte(' ') 162 | buf.WriteString(file) 163 | buf.tmp[0] = ':' 164 | n := buf.someDigits(1, line) 165 | buf.tmp[n+1] = ' ' 166 | buf.Write(buf.tmp[:n+2]) 167 | return buf 168 | } 169 | 170 | func (self *Logger) header(s Severity, depth int) *buffer { 171 | _, file, line, ok := runtime.Caller(3 + depth) 172 | if !ok { 173 | file = "???" 174 | line = 1 175 | } else { 176 | dirs := strings.Split(file, "/") 177 | if len(dirs) >= 2 { 178 | file = dirs[len(dirs)-2] + "/" + dirs[len(dirs)-1] 179 | } else { 180 | file = dirs[len(dirs)-1] 181 | } 182 | } 183 | return self.formatHeader(s, file, line) 184 | } 185 | 186 | func (self *Logger) print(s Severity, args ...interface{}) { 187 | self.printDepth(s, 1, args...) 188 | } 189 | 190 | func (self *Logger) printf(s Severity, format string, args ...interface{}) { 191 | self.printfDepth(s, 1, format, args...) 192 | } 193 | 194 | func (self *Logger) printDepth(s Severity, depth int, args ...interface{}) { 195 | if self.s < s { 196 | return 197 | } 198 | buf := self.header(s, depth) 199 | fmt.Fprint(buf, args...) 200 | if buf.Bytes()[buf.Len()-1] != '\n' { 201 | buf.WriteByte('\n') 202 | } 203 | self.output(s, buf) 204 | } 205 | 206 | func (self *Logger) printfDepth(s Severity, depth int, format string, args ...interface{}) { 207 | if self.s < s { 208 | return 209 | } 210 | buf := self.header(s, depth) 211 | fmt.Fprintf(buf, format, args...) 212 | if buf.Bytes()[buf.Len()-1] != '\n' { 213 | buf.WriteByte('\n') 214 | } 215 | self.output(s, buf) 216 | } 217 | 218 | func (self *Logger) printfSimple(format string, args ...interface{}) { 219 | buf := self.getBuffer() 220 | fmt.Fprintf(buf, format, args...) 221 | if buf.Bytes()[buf.Len()-1] != '\n' { 222 | buf.WriteByte('\n') 223 | } 224 | self.output(INFO, buf) 225 | } 226 | 227 | func (self *Logger) output(s Severity, buf *buffer) { 228 | if self.s < s { 229 | return 230 | } 231 | if self.logToStderr { 232 | os.Stderr.Write(buf.Bytes()) 233 | } else { 234 | self.backend.Log(s, buf.Bytes()) 235 | } 236 | if s == FATAL { 237 | trace := stacks(true) 238 | os.Stderr.Write(trace) 239 | os.Exit(255) 240 | } 241 | self.putBuffer(buf) 242 | } 243 | 244 | func stacks(all bool) []byte { 245 | // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. 246 | n := 10000 247 | if all { 248 | n = 100000 249 | } 250 | var trace []byte 251 | for i := 0; i < 5; i++ { 252 | trace = make([]byte, n) 253 | nbytes := runtime.Stack(trace, all) 254 | if nbytes < len(trace) { 255 | return trace[:nbytes] 256 | } 257 | n *= 2 258 | } 259 | return trace 260 | } 261 | 262 | /*--------------------------logger public functions--------------------------*/ 263 | 264 | func NewLogger(level interface{}, backend Backend) *Logger { 265 | l := new(Logger) 266 | l.SetSeverity(level) 267 | l.backend = backend 268 | return l 269 | } 270 | 271 | func (l *Logger) SetSeverity(level interface{}) { 272 | if s, ok := level.(Severity); ok { 273 | l.s = s 274 | } else { 275 | if s, ok := level.(string); ok { 276 | for i, name := range severityName { 277 | if name == s { 278 | l.s = Severity(i) 279 | } 280 | } 281 | } 282 | } 283 | } 284 | 285 | func (l *Logger) Close() { 286 | if l.backend != nil { 287 | l.backend.close() 288 | } 289 | } 290 | 291 | func (l *Logger) LogToStderr() { 292 | l.logToStderr = true 293 | } 294 | 295 | func (l *Logger) Debug(args ...interface{}) { 296 | l.print(DEBUG, args...) 297 | } 298 | 299 | func (l *Logger) Debugf(format string, args ...interface{}) { 300 | l.printf(DEBUG, format, args...) 301 | } 302 | 303 | func (l *Logger) Info(args ...interface{}) { 304 | l.print(INFO, args...) 305 | } 306 | 307 | func (l *Logger) Infof(format string, args ...interface{}) { 308 | l.printf(INFO, format, args...) 309 | } 310 | 311 | func (l *Logger) Warning(args ...interface{}) { 312 | l.print(WARNING, args...) 313 | } 314 | 315 | func (l *Logger) Warningf(format string, args ...interface{}) { 316 | l.printf(WARNING, format, args...) 317 | } 318 | 319 | func (l *Logger) Error(args ...interface{}) { 320 | l.print(ERROR, args...) 321 | } 322 | 323 | func (l *Logger) Errorf(format string, args ...interface{}) { 324 | l.printf(ERROR, format, args...) 325 | } 326 | 327 | func (l *Logger) Fatal(args ...interface{}) { 328 | l.print(FATAL, args...) 329 | } 330 | 331 | func (l *Logger) Fatalf(format string, args ...interface{}) { 332 | l.printf(FATAL, format, args...) 333 | } 334 | 335 | func (l *Logger) SetLogging(level interface{}, backend Backend) { 336 | l.SetSeverity(level) 337 | l.backend = backend 338 | } 339 | 340 | ///////////////////////////////////////////////////////////////// 341 | // depth version, only a low level api 342 | func (l *Logger) LogDepth(s Severity, depth int, format string, args ...interface{}) { 343 | l.printfDepth(s, depth+1, format, args...) 344 | } 345 | 346 | func (l *Logger) PrintfSimple(format string, args ...interface{}) { 347 | l.printfSimple(format, args...) 348 | } 349 | 350 | /*---------------------------------------------------------------------------*/ 351 | 352 | var logging Logger 353 | var fileback *FileBackend = nil 354 | var sysback *syslogBackend = nil 355 | 356 | func init() { 357 | SetLogging(DEBUG, &stdBackend{}) 358 | } 359 | 360 | func SetLogging(level interface{}, backend Backend) { 361 | logging.SetLogging(level, backend) 362 | } 363 | 364 | func SetSeverity(level interface{}) { 365 | logging.SetSeverity(level) 366 | } 367 | 368 | func Close() { 369 | logging.Close() 370 | } 371 | 372 | func LogToStderr() { 373 | logging.LogToStderr() 374 | } 375 | 376 | /*-----------------------------public functions------------------------------*/ 377 | 378 | func Debug(args ...interface{}) { 379 | logging.print(DEBUG, args...) 380 | } 381 | 382 | func Debugf(format string, args ...interface{}) { 383 | logging.printf(DEBUG, format, args...) 384 | } 385 | 386 | func Info(args ...interface{}) { 387 | logging.print(INFO, args...) 388 | } 389 | 390 | func Infof(format string, args ...interface{}) { 391 | logging.printf(INFO, format, args...) 392 | } 393 | 394 | func Warning(args ...interface{}) { 395 | logging.print(WARNING, args...) 396 | } 397 | 398 | func Warningf(format string, args ...interface{}) { 399 | logging.printf(WARNING, format, args...) 400 | } 401 | 402 | func Error(args ...interface{}) { 403 | logging.print(ERROR, args...) 404 | } 405 | 406 | func Errorf(format string, args ...interface{}) { 407 | logging.printf(ERROR, format, args...) 408 | } 409 | 410 | func Fatal(args ...interface{}) { 411 | logging.print(FATAL, args...) 412 | } 413 | 414 | func Fatalf(format string, args ...interface{}) { 415 | logging.printf(FATAL, format, args...) 416 | } 417 | 418 | func LogDepth(s Severity, depth int, format string, args ...interface{}) { 419 | logging.printfDepth(s, depth+1, format, args...) 420 | } 421 | 422 | func Printf(format string, args ...interface{}) { 423 | logging.printfSimple(format, args...) 424 | } 425 | 426 | func GetLogger() *Logger { 427 | return &logging 428 | } 429 | -------------------------------------------------------------------------------- /pkg/logger/logger_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func InfoHelperDepth(format string, args ...interface{}) { 10 | LogDepth(INFO, 0, format, args...) 11 | } 12 | 13 | func InfoHelper(format string, args ...interface{}) { 14 | Infof(format, args...) 15 | } 16 | 17 | func TestStdout(t *testing.T) { 18 | Info("stdout") 19 | } 20 | 21 | func TestDefaultFileBackend(t *testing.T) { 22 | var conf LogConfig 23 | conf.Type = "file" 24 | conf.Level = "DEBUG" 25 | conf.FileName = "/tmp/dlog-test/defaultFileBackend" 26 | conf.FileRotateSize = 1024 * 1024 * 1024 27 | conf.FileRotateCount = 20 28 | //conf.FileFlushDuration = time.Second * 1 29 | 30 | Init(conf) 31 | Info("Info") 32 | Infof("%s", "Infof") 33 | Warning("Warning") 34 | Warningf("%s", "Warningf") 35 | Debug("Debug") 36 | Debugf("%s", "Debugf") 37 | Error("Error") 38 | Errorf("%s", "Errorf") 39 | Close() 40 | } 41 | 42 | func TestFileBackend(t *testing.T) { 43 | var conf LogConfig 44 | conf.Type = "file" 45 | conf.Level = "DEBUG" 46 | conf.FileName = "/tmp/dlog-test/fileBackend" 47 | conf.FileRotateSize = 1024 * 1024 * 1024 48 | conf.FileRotateCount = 20 49 | //conf.FileFlushDuration = time.Second * 1 50 | 51 | log, err := NewLoggerFromConfig(conf) 52 | if err != nil { 53 | fmt.Println("err when call NewLoggerFromConfig: [%s]", err.Error()) 54 | return 55 | } 56 | 57 | log.Info("Info") 58 | log.Infof("%s", "Infof") 59 | log.Warning("Warning") 60 | log.Warningf("%s", "Warningf") 61 | log.Debug("Debug") 62 | log.Debugf("%s", "Debugf") 63 | log.Error("Error") 64 | log.Errorf("%s", "Errorf") 65 | log.Close() 66 | } 67 | 68 | func TestBothFileBackend(t *testing.T) { 69 | var defaultConf LogConfig 70 | defaultConf.Type = "file" 71 | defaultConf.Level = "DEBUG" 72 | defaultConf.FileName = "/tmp/dlog-test/bothFileBackend/defaultFile" 73 | defaultConf.FileRotateSize = 1024 * 1024 * 1024 74 | defaultConf.FileRotateCount = 20 75 | Init(defaultConf) 76 | 77 | var conf1 LogConfig 78 | conf1.Type = "file" 79 | conf1.Level = "DEBUG" 80 | conf1.FileName = "/tmp/dlog-test/bothFileBackend/file1" 81 | conf1.FileRotateSize = 1024 * 1024 * 1024 82 | conf1.FileRotateCount = 20 83 | log1, err := NewLoggerFromConfig(conf1) 84 | if err != nil { 85 | fmt.Println("err when call NewLoggerFromConfig: [%s]", err.Error()) 86 | return 87 | } 88 | 89 | var conf2 LogConfig 90 | conf2.Type = "file" 91 | conf2.Level = "DEBUG" 92 | conf2.FileName = "/tmp/dlog-test/bothFileBackend/file2" 93 | conf2.FileRotateSize = 1024 * 1024 * 1024 94 | conf2.FileRotateCount = 20 95 | log2, err := NewLoggerFromConfig(conf2) 96 | if err != nil { 97 | fmt.Println("err when call NewLoggerFromConfig: [%s]", err.Error()) 98 | return 99 | } 100 | 101 | Info("Info") 102 | Infof("%s", "Infof") 103 | log1.Info("Info") 104 | log1.Infof("%s", "Infof") 105 | log2.Info("Info") 106 | log2.Infof("%s", "Infof") 107 | 108 | Warning("Warning") 109 | Warningf("%s", "Warningf") 110 | log1.Warning("Warning") 111 | log1.Warningf("%s", "Warningf") 112 | log2.Warning("Warning") 113 | log2.Warningf("%s", "Warningf") 114 | 115 | Debug("Debug") 116 | Debugf("%s", "Debugf") 117 | log1.Debug("Debug") 118 | log1.Debugf("%s", "Debugf") 119 | log2.Debug("Debug") 120 | log2.Debugf("%s", "Debugf") 121 | 122 | Error("Error") 123 | Errorf("%s", "Errorf") 124 | log1.Error("Error") 125 | log1.Errorf("%s", "Errorf") 126 | log2.Error("Error") 127 | log2.Errorf("%s", "Errorf") 128 | 129 | Close() 130 | log1.Close() 131 | log2.Close() 132 | } 133 | 134 | func TestSysLogBackend(t *testing.T) { 135 | b, err := NewSyslogBackend("local3", "mohawk") 136 | if err != nil { 137 | panic(err) 138 | } 139 | defer Close() 140 | SetLogging("DEBUG", b) 141 | Debug("Debug") 142 | Errorf("%d %s", 123, "error") 143 | Info("Info") 144 | time.Sleep(time.Second * 2) 145 | } 146 | 147 | /* 148 | func TestMultiBackend(t *testing.T) { 149 | b1, err := NewFileBackend("/tmp/dlog-test") 150 | if err != nil { 151 | panic(err) 152 | } 153 | defer Close() 154 | b2, err := NewSyslogBackend(syslog.LOG_LOCAL3, "dlog-test") 155 | if err != nil { 156 | panic(err) 157 | } 158 | m, _ := NewMultiBackend(b1, b2) 159 | SetLogging(INFO, m) 160 | Info("test multi") 161 | } 162 | */ 163 | -------------------------------------------------------------------------------- /pkg/logger/multi.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | type multiBackend struct { 4 | bes []Backend 5 | } 6 | 7 | func NewMultiBackend(bes ...Backend) (*multiBackend, error) { 8 | var b multiBackend 9 | b.bes = bes 10 | return &b, nil 11 | } 12 | 13 | func (self *multiBackend) Log(s Severity, msg []byte) { 14 | for _, be := range self.bes { 15 | be.Log(s, msg) 16 | } 17 | } 18 | 19 | func (self *multiBackend) close() { 20 | for _, be := range self.bes { 21 | be.close() 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pkg/logger/syslog_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin freebsd openbsd solaris 2 | 3 | package logger 4 | 5 | import ( 6 | "fmt" 7 | "log/syslog" 8 | "os" 9 | ) 10 | 11 | type syslogBackend struct { 12 | writer [numSeverity]*syslog.Writer 13 | buf [numSeverity]chan []byte 14 | } 15 | 16 | var SyslogPriorityMap = map[string]syslog.Priority{ 17 | "local0": syslog.LOG_LOCAL0, 18 | "local1": syslog.LOG_LOCAL1, 19 | "local2": syslog.LOG_LOCAL2, 20 | "local3": syslog.LOG_LOCAL3, 21 | "local4": syslog.LOG_LOCAL4, 22 | "local5": syslog.LOG_LOCAL5, 23 | "local6": syslog.LOG_LOCAL6, 24 | "local7": syslog.LOG_LOCAL7, 25 | } 26 | 27 | var pmap = []syslog.Priority{syslog.LOG_EMERG, syslog.LOG_ERR, syslog.LOG_WARNING, syslog.LOG_INFO, syslog.LOG_DEBUG} 28 | 29 | func NewSyslogBackend(priorityStr string, tag string) (*syslogBackend, error) { 30 | priority, ok := SyslogPriorityMap[priorityStr] 31 | if !ok { 32 | return nil, fmt.Errorf("unknown syslog priority: %s", priorityStr) 33 | } 34 | var err error 35 | var b syslogBackend 36 | for i := 0; i < numSeverity; i++ { 37 | b.writer[i], err = syslog.New(priority|pmap[i], tag) 38 | if err != nil { 39 | return nil, err 40 | } 41 | b.buf[i] = make(chan []byte, 1<<16) 42 | } 43 | b.log() 44 | return &b, nil 45 | } 46 | 47 | func DialSyslogBackend(network, raddr string, priority syslog.Priority, tag string) (*syslogBackend, error) { 48 | var err error 49 | var b syslogBackend 50 | for i := 0; i < numSeverity; i++ { 51 | b.writer[i], err = syslog.Dial(network, raddr, priority|pmap[i], tag+severityName[i]) 52 | if err != nil { 53 | return nil, err 54 | } 55 | b.buf[i] = make(chan []byte, 1<<16) 56 | } 57 | b.log() 58 | return &b, nil 59 | } 60 | 61 | func (self *syslogBackend) Log(s Severity, msg []byte) { 62 | msg1 := make([]byte, len(msg)) 63 | copy(msg1, msg) 64 | switch s { 65 | case FATAL: 66 | self.tryPutInBuf(FATAL, msg1) 67 | case ERROR: 68 | self.tryPutInBuf(ERROR, msg1) 69 | case WARNING: 70 | self.tryPutInBuf(WARNING, msg1) 71 | case INFO: 72 | self.tryPutInBuf(INFO, msg1) 73 | case DEBUG: 74 | self.tryPutInBuf(DEBUG, msg1) 75 | } 76 | } 77 | 78 | func (self *syslogBackend) close() { 79 | for i := 0; i < numSeverity; i++ { 80 | self.writer[i].Close() 81 | } 82 | } 83 | 84 | func (self *syslogBackend) tryPutInBuf(s Severity, msg []byte) { 85 | select { 86 | case self.buf[s] <- msg: 87 | default: 88 | os.Stderr.Write(msg) 89 | } 90 | } 91 | 92 | func (self *syslogBackend) log() { 93 | for i := 0; i < numSeverity; i++ { 94 | go func(index int) { 95 | for { 96 | msg := <-self.buf[index] 97 | self.writer[index].Write(msg[27:]) 98 | } 99 | }(i) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pkg/logger/syslog_unsupported.go: -------------------------------------------------------------------------------- 1 | // +build windows plan9 netbsd 2 | 3 | package logger 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | type syslogBackend struct { 10 | } 11 | 12 | func NewSyslogBackend(priorityStr string, tag string) (*syslogBackend, error) { 13 | return nil, fmt.Errorf("Platform does not support syslog") 14 | } 15 | 16 | func (self *syslogBackend) Log(s Severity, msg []byte) { 17 | } 18 | 19 | func (self *syslogBackend) close() { 20 | } 21 | 22 | func (self *syslogBackend) tryPutInBuf(s Severity, msg []byte) { 23 | } 24 | 25 | func (self *syslogBackend) log() { 26 | } 27 | -------------------------------------------------------------------------------- /pkg/sys/sys.go: -------------------------------------------------------------------------------- 1 | package sys 2 | 3 | import ( 4 | "bytes" 5 | "os/exec" 6 | "strings" 7 | "syscall" 8 | "time" 9 | ) 10 | 11 | func CmdOutString(name string, arg ...string) (string, error) { 12 | bs, err := CmdOutBytes(name, arg...) 13 | if err != nil { 14 | return "", err 15 | } 16 | 17 | return string(bs), nil 18 | } 19 | 20 | func CmdOutBytes(name string, arg ...string) ([]byte, error) { 21 | cmd := exec.Command(name, arg...) 22 | return cmd.CombinedOutput() 23 | } 24 | 25 | func CmdOutTrim(name string, arg ...string) (out string, err error) { 26 | out, err = CmdOutString(name, arg...) 27 | if err != nil { 28 | return 29 | } 30 | 31 | return strings.TrimSpace(string(out)), nil 32 | } 33 | 34 | func CmdRun(name string, arg ...string) error { 35 | cmd := exec.Command(name, arg...) 36 | return cmd.Run() 37 | } 38 | 39 | // CmdRunT Command run with timeout 40 | func CmdRunT(timeout time.Duration, name string, arg ...string) (output string, err error, istimeout bool) { 41 | cmd := exec.Command(name, arg...) 42 | cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} 43 | 44 | var b bytes.Buffer 45 | cmd.Stdout = &b 46 | cmd.Stderr = &b 47 | 48 | cmd.Start() 49 | err, istimeout = WrapTimeout(cmd, timeout) 50 | output = b.String() 51 | 52 | return 53 | } 54 | 55 | func WrapTimeout(cmd *exec.Cmd, timeout time.Duration) (error, bool) { 56 | var err error 57 | 58 | done := make(chan error) 59 | go func() { 60 | done <- cmd.Wait() 61 | }() 62 | 63 | select { 64 | case <-time.After(timeout): 65 | go func() { 66 | <-done // allow goroutine to exit 67 | }() 68 | 69 | // IMPORTANT: cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} is necessary before cmd.Start() 70 | err = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) 71 | return err, true 72 | case err = <-done: 73 | return err, false 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/tools/signal.go: -------------------------------------------------------------------------------- 1 | // +build !plan9 2 | 3 | package tools 4 | 5 | import ( 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | ) 10 | 11 | func OnInterrupt(fn func()) { 12 | signalChan := make(chan os.Signal, 1) 13 | signal.Ignore(syscall.SIGHUP) 14 | signal.Notify(signalChan, 15 | os.Interrupt, 16 | os.Kill, 17 | syscall.SIGALRM, 18 | syscall.SIGINT, 19 | syscall.SIGTERM, 20 | ) 21 | go func() { 22 | for _ = range signalChan { 23 | fn() 24 | os.Exit(0) 25 | } 26 | }() 27 | } 28 | -------------------------------------------------------------------------------- /pkg/watcher/watcher.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | type Watcher struct { 14 | Pid int 15 | Exe string 16 | Dur time.Duration 17 | Dev bool 18 | } 19 | 20 | func NewWatcher(exe string, dur time.Duration) *Watcher { 21 | return &Watcher{ 22 | Exe: exe, 23 | Dur: dur, 24 | } 25 | } 26 | 27 | func (w *Watcher) Start(f func()) { 28 | // clean deleted process 29 | fs, err := ioutil.ReadDir("/proc") 30 | if err != nil { 31 | if w.Dev { 32 | log.Println("ERR: cannot read /proc:", err) 33 | } 34 | return 35 | } 36 | 37 | sz := len(fs) 38 | for i := 0; i < sz; i++ { 39 | if !fs[i].IsDir() { 40 | continue 41 | } 42 | 43 | name := fs[i].Name() 44 | pid, err := strconv.Atoi(name) 45 | if err != nil { 46 | continue 47 | } 48 | 49 | exe := fmt.Sprintf("/proc/%d/exe", pid) 50 | if !IsExist(exe) { 51 | continue 52 | } 53 | 54 | target, err := os.Readlink(exe) 55 | if err == nil && strings.Contains(target, w.Exe) && strings.Contains(target, "deleted") { 56 | proc, err := os.FindProcess(pid) 57 | if err != nil { 58 | if w.Dev { 59 | log.Printf("ERR: cannot find process[pid:%d]: %v", pid, err) 60 | } 61 | continue 62 | } 63 | 64 | err = proc.Kill() 65 | if err != nil && w.Dev { 66 | log.Printf("ERR: cannot kill process[pid:%d]: %v", pid, err) 67 | } 68 | } 69 | } 70 | 71 | for { 72 | time.Sleep(w.Dur) 73 | w.check(f) 74 | } 75 | } 76 | 77 | func (w *Watcher) check(f func()) { 78 | defer func() { 79 | if err := recover(); err != nil { 80 | if w.Dev { 81 | log.Println("PANIC:", err) 82 | } 83 | return 84 | } 85 | }() 86 | 87 | if w.Pid > 0 { 88 | target, err := os.Readlink(fmt.Sprintf("/proc/%d/exe", w.Pid)) 89 | if err == nil && target == w.Exe { 90 | return 91 | } 92 | } 93 | 94 | fs, err := ioutil.ReadDir("/proc") 95 | if err != nil { 96 | if w.Dev { 97 | log.Println("ERR: cannot read /proc:", err) 98 | } 99 | return 100 | } 101 | 102 | sz := len(fs) 103 | for i := 0; i < sz; i++ { 104 | if !fs[i].IsDir() { 105 | continue 106 | } 107 | 108 | name := fs[i].Name() 109 | pid, err := strconv.Atoi(name) 110 | if err != nil { 111 | continue 112 | } 113 | 114 | exe := fmt.Sprintf("/proc/%d/exe", pid) 115 | if !IsExist(exe) { 116 | continue 117 | } 118 | 119 | target, err := os.Readlink(exe) 120 | if err == nil && target == w.Exe { 121 | w.Pid = pid 122 | return 123 | } 124 | } 125 | 126 | // w.Exe not found 127 | f() 128 | } 129 | 130 | func IsExist(fp string) bool { 131 | _, err := os.Stat(fp) 132 | return err == nil || os.IsExist(err) 133 | } 134 | -------------------------------------------------------------------------------- /sga/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "time" 7 | 8 | "sgt/pkg/sys" 9 | "sgt/pkg/watcher" 10 | ) 11 | 12 | var Dev bool 13 | 14 | func init() { 15 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 16 | 17 | isdev := flag.Bool("d", false, "in devolopment mode or not") 18 | flag.Parse() 19 | 20 | Dev = *isdev 21 | } 22 | 23 | func main() { 24 | sgdbin := "/usr/local/sgd/bin/sgd" 25 | ctlfile := "/usr/local/sgd/control" 26 | 27 | w := watcher.NewWatcher(sgdbin, time.Duration(7)*time.Second) 28 | w.Dev = Dev 29 | w.Start(func() { 30 | output, err, istimeout := sys.CmdRunT(time.Duration(5)*time.Second, ctlfile, "start") 31 | if istimeout { 32 | log.Printf("cannot start %s, timeout", sgdbin) 33 | return 34 | } 35 | 36 | if err != nil { 37 | log.Printf("cannot start %s, error: %v, output: %s", sgdbin, err, output) 38 | } 39 | }) 40 | } 41 | -------------------------------------------------------------------------------- /sgd/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | const ( 4 | Ver = "1.1.5" 5 | Url = "UUID_URL" 6 | 7 | LogFileSize = 1024 * 1024 * 5 8 | LogFileNum = 3 9 | 10 | Started = 1 11 | Stopped = 0 12 | ) 13 | 14 | var ( 15 | Dev = false 16 | Srv = "" 17 | UUID = "" 18 | Cwd = "" 19 | 20 | LogDir = "" 21 | AgsDir = "" 22 | TarDir = "" 23 | ) 24 | -------------------------------------------------------------------------------- /sgd/config/funcs.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "path" 9 | "time" 10 | 11 | "sgt/pkg/file" 12 | ) 13 | 14 | func Init() { 15 | Cwd = file.Dir(file.SelfDir()) 16 | if Dev { 17 | log.Printf("INF: workdir -> %s", Cwd) 18 | log.Printf("INF: version -> %s", Ver) 19 | } 20 | 21 | LogDir = path.Join(Cwd, "log") 22 | AgsDir = path.Join(Cwd, "ags") 23 | TarDir = path.Join(Cwd, "tar") 24 | 25 | uuid, err := GetUUID() 26 | if err == nil { 27 | SetUUID(uuid) 28 | return 29 | } 30 | 31 | for i := 0; i < 3; i++ { 32 | time.Sleep(time.Second) 33 | uuid, err = GetUUID() 34 | if err == nil { 35 | SetUUID(uuid) 36 | return 37 | } 38 | } 39 | 40 | log.Fatalf("FAT: cannot get uuid") 41 | } 42 | 43 | func SetUUID(uuid string) { 44 | if Dev { 45 | log.Printf("INF: uuid -> %s", uuid) 46 | } 47 | UUID = uuid 48 | } 49 | 50 | func GetUUID() (string, error) { 51 | client := http.Client{ 52 | Timeout: time.Second, 53 | } 54 | 55 | res, err := client.Get(Url) 56 | if err != nil { 57 | return "", fmt.Errorf("cannot dial uuid url: %v", err) 58 | } 59 | 60 | defer res.Body.Close() 61 | 62 | bs, err := ioutil.ReadAll(res.Body) 63 | if err != nil { 64 | return "", fmt.Errorf("cannot read uuid body: %v", err) 65 | } 66 | 67 | return string(bs), nil 68 | } 69 | -------------------------------------------------------------------------------- /sgd/cron/cron.go: -------------------------------------------------------------------------------- 1 | package cron 2 | 3 | import ( 4 | "log" 5 | 6 | "sgt/pkg/file" 7 | ) 8 | 9 | const sgd_cron_content = `SHELL=/bin/bash 10 | PATH=/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:~/bin 11 | MAILTO=root 12 | 13 | * * * * * root timeout 3 /usr/local/sgd/control start &>/dev/null 14 | ` 15 | 16 | func Init() { 17 | err := file.EnsureDir("/etc/cron.d") 18 | if err != nil { 19 | log.Printf("ERR: cannot exec mkdir -p /etc/cron.d: %v", err) 20 | return 21 | } 22 | 23 | _, err = file.WriteString("/etc/cron.d/sgd", sgd_cron_content) 24 | if err != nil { 25 | log.Printf("ERR: cannot write sgd cron: %v", err) 26 | return 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /sgd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path" 9 | 10 | "sgt/pkg/logger" 11 | "sgt/pkg/tools" 12 | "sgt/sgd/config" 13 | "sgt/sgd/cron" 14 | "sgt/sgd/system" 15 | "sgt/sgd/timer" 16 | ) 17 | 18 | func init() { 19 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 20 | 21 | server := flag.String("s", "http://127.0.0.1:8030", "server address") 22 | version := flag.Bool("v", false, "show version") 23 | help := flag.Bool("h", false, "help") 24 | isdev := flag.Bool("d", false, "in devolopment mode or not") 25 | puuid := flag.Bool("u", false, "print vm uuid") 26 | flag.Parse() 27 | 28 | handleVersion(*version) 29 | handleHelp(*help) 30 | handleServer(*server) 31 | handleIsdev(*isdev) 32 | handlePrint(*puuid) 33 | } 34 | 35 | func main() { 36 | cron.Init() 37 | config.Init() 38 | 39 | logDir := path.Join(config.Cwd, "log") 40 | lb, err := logger.NewFileBackend(logDir) 41 | if err != nil { 42 | log.Fatalln("FAT: cannot init logger:", err) 43 | } 44 | 45 | logger.SetLogging("ERROR", lb) 46 | lb.Rotate(config.LogFileNum, config.LogFileSize) 47 | 48 | defer func() { 49 | logger.Close() 50 | }() 51 | 52 | tools.OnInterrupt(func() { 53 | logger.Close() 54 | os.Exit(0) 55 | }) 56 | 57 | system.Init() 58 | timer.Init() 59 | 60 | timer.Sleep() 61 | 62 | go timer.Heartbeat() 63 | go timer.SgaWatch() 64 | 65 | select {} 66 | } 67 | 68 | func handleVersion(displayVersion bool) { 69 | if displayVersion { 70 | fmt.Println(config.Ver) 71 | os.Exit(0) 72 | } 73 | } 74 | 75 | func handleHelp(displayHelp bool) { 76 | if displayHelp { 77 | flag.Usage() 78 | os.Exit(0) 79 | } 80 | } 81 | 82 | func handleServer(addr string) { 83 | config.Srv = addr 84 | } 85 | 86 | func handleIsdev(isdev bool) { 87 | config.Dev = isdev 88 | } 89 | 90 | func handlePrint(pu bool) { 91 | if pu { 92 | uuid, err := config.GetUUID() 93 | if err != nil { 94 | os.Exit(1) 95 | } 96 | 97 | fmt.Println(uuid) 98 | os.Exit(0) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /sgd/system/system.go: -------------------------------------------------------------------------------- 1 | package system 2 | 3 | import ( 4 | "hash/crc32" 5 | "math/rand" 6 | "os" 7 | "time" 8 | 9 | "sgt/sgd/config" 10 | ) 11 | 12 | func Init() { 13 | rand.Seed(time.Now().UnixNano() + int64(os.Getpid()+os.Getppid()) + int64(crc32.ChecksumIEEE([]byte(config.UUID)))) 14 | } 15 | -------------------------------------------------------------------------------- /sgd/timer/destruct.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | 11 | "sgt/pkg/file" 12 | ) 13 | 14 | func destruct() { 15 | err := file.Unlink("/etc/cron.d/sgd") 16 | if err != nil { 17 | log.Println("cannot del /etc/cron.d/sgd:", err) 18 | } 19 | 20 | fs, err := ioutil.ReadDir("/proc") 21 | if err != nil { 22 | log.Println("cannot read /proc:", err) 23 | os.Exit(1) 24 | } 25 | 26 | sz := len(fs) 27 | for i := 0; i < sz; i++ { 28 | if !fs[i].IsDir() { 29 | continue 30 | } 31 | 32 | name := fs[i].Name() 33 | pid, err := strconv.Atoi(name) 34 | if err != nil { 35 | continue 36 | } 37 | 38 | exe := fmt.Sprintf("/proc/%d/exe", pid) 39 | if !file.IsExist(exe) { 40 | continue 41 | } 42 | 43 | target, err := os.Readlink(exe) 44 | if err == nil && strings.Contains(target, "/usr/local/sgd/bin/sga") { 45 | proc, err := os.FindProcess(pid) 46 | if err != nil { 47 | continue 48 | } 49 | 50 | err = proc.Kill() 51 | if err != nil { 52 | log.Printf("ERR: cannot kill process[pid:%d]: %v", pid, err) 53 | } 54 | } 55 | } 56 | 57 | os.Exit(1) 58 | } 59 | -------------------------------------------------------------------------------- /sgd/timer/heartbeat.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "time" 10 | 11 | "sgt/pkg/logger" 12 | "sgt/sgd/config" 13 | ) 14 | 15 | func Heartbeat() { 16 | dur := time.Duration(3) * time.Second 17 | url := fmt.Sprintf("%s/heartbeat", config.Srv) 18 | for { 19 | heartbeat(url) 20 | time.Sleep(dur) 21 | } 22 | } 23 | 24 | func heartbeat(url string) { 25 | defer func() { 26 | if err := recover(); err != nil { 27 | logger.Error(err) 28 | return 29 | } 30 | }() 31 | 32 | req, err := makeHeartbeatRequest() 33 | if err != nil { 34 | logger.Error("cannot make heartbeat request: ", err) 35 | return 36 | } 37 | 38 | bs, err := json.Marshal(req) 39 | if err != nil { 40 | logger.Error("cannot marshal heartbeat request: ", err) 41 | return 42 | } 43 | 44 | if config.Dev { 45 | log.Println("INF: heartbeat request ->", string(bs)) 46 | } 47 | 48 | res, err := httpcli.Post(url, "application/json", bytes.NewBuffer(bs)) 49 | if err != nil { 50 | logger.Error("cannot dial heartbeat server: ", err) 51 | return 52 | } 53 | 54 | if res.StatusCode != 200 { 55 | logger.Error("heartbeat server return statuscode: ", res.StatusCode) 56 | return 57 | } 58 | 59 | if res.Body == nil { 60 | logger.Error("heartbeat server response body is nil") 61 | return 62 | } 63 | 64 | defer res.Body.Close() 65 | 66 | bs, err = ioutil.ReadAll(res.Body) 67 | if err != nil { 68 | logger.Error("cannot read heartbeat server response: ", err) 69 | return 70 | } 71 | 72 | if config.Dev { 73 | log.Println("INF: heartbeat response ->", string(bs)) 74 | } 75 | 76 | var reply HeartbeatResponse 77 | err = json.Unmarshal(bs, &reply) 78 | if err != nil { 79 | logger.Error("cannot unmarshal heartbeat server response: ", err) 80 | return 81 | } 82 | 83 | handleHeartbeatResponse(reply) 84 | 85 | checkMem() 86 | } 87 | -------------------------------------------------------------------------------- /sgd/timer/memory.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "log" 5 | "runtime" 6 | 7 | "sgt/sgd/config" 8 | ) 9 | 10 | const ( 11 | max_mem uint64 = 30 * 1024 * 1024 12 | ) 13 | 14 | func checkMem() { 15 | m := &runtime.MemStats{} 16 | runtime.ReadMemStats(m) 17 | 18 | if config.Dev { 19 | log.Printf("INF: mem use: %dMB", m.HeapSys/1024/1024) 20 | } 21 | 22 | if m.HeapSys > max_mem { 23 | log.Fatalf("FAT: mem use: %dMB, overload", m.HeapSys/1024/1024) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /sgd/timer/model.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | type AgentStat struct { 4 | Name string 5 | Ver string 6 | Pid int 7 | Cmdline string 8 | Stat int 9 | } 10 | 11 | // Stat: 0: stopped, 1: started 12 | type AgSt struct { 13 | Name string `json:"n"` 14 | Ver string `json:"v"` 15 | Stat int `json:"s"` 16 | } 17 | 18 | type HeartbeatRequest struct { 19 | UUID string `json:"id"` 20 | SgdVer string `json:"sgv"` 21 | Ags []AgSt `json:"ags"` 22 | } 23 | 24 | type AgVer struct { 25 | Name string `json:"n"` 26 | Ver string `json:"v"` 27 | } 28 | 29 | type HeartbeatResponse struct { 30 | Error string `json:"err"` 31 | SgdVer string `json:"sgv"` 32 | Ags []AgVer `json:"ags"` 33 | } 34 | 35 | type AgVerSlice []AgVer 36 | 37 | func (s AgVerSlice) Len() int { 38 | return len(s) 39 | } 40 | func (s AgVerSlice) Swap(i, j int) { 41 | s[i], s[j] = s[j], s[i] 42 | } 43 | func (s AgVerSlice) Less(i, j int) bool { 44 | return s[i].Name < s[j].Name 45 | } 46 | 47 | var ( 48 | agentStats = make(map[string]AgentStat) 49 | lastAgentDirs = "NULL" 50 | lastAgentVers = "NULL" 51 | ) 52 | -------------------------------------------------------------------------------- /sgd/timer/request.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sort" 7 | "strings" 8 | 9 | "sgt/pkg/file" 10 | "sgt/sgd/config" 11 | ) 12 | 13 | func makeHeartbeatRequest() (HeartbeatRequest, error) { 14 | req := HeartbeatRequest{ 15 | UUID: config.UUID, 16 | SgdVer: config.Ver, 17 | } 18 | 19 | err := updateAgentStats() 20 | if err != nil { 21 | return req, err 22 | } 23 | 24 | cnt := len(agentStats) 25 | ags := make([]AgSt, 0, cnt) 26 | for name, stat := range agentStats { 27 | ags = append(ags, AgSt{ 28 | Name: name, 29 | Ver: stat.Ver, 30 | Stat: stat.Stat, 31 | }) 32 | } 33 | 34 | req.Ags = ags 35 | 36 | return req, nil 37 | } 38 | 39 | func updateAgentStats() error { 40 | agentDirs, err := file.DirsUnder(config.AgsDir) 41 | if err != nil { 42 | return fmt.Errorf("cannot list %s, error: %v", config.AgsDir, err) 43 | } 44 | 45 | sort.Strings(agentDirs) 46 | 47 | newAgentDirs := strings.Join(agentDirs, "") 48 | if lastAgentDirs != newAgentDirs { 49 | if err = collectAgents(agentDirs); err != nil { 50 | return err 51 | } 52 | lastAgentDirs = newAgentDirs 53 | return nil 54 | } 55 | 56 | changed := false 57 | for name := range agentStats { 58 | // if agent stopped, start it 59 | if agentStats[name].Stat == config.Stopped { 60 | agentStart(name) 61 | changed = true 62 | } 63 | } 64 | 65 | if changed { 66 | return collectAgents(agentDirs) 67 | } 68 | 69 | changed = false 70 | for name := range agentStats { 71 | cmdline, err := file.ReadString(fmt.Sprintf("/proc/%d/cmdline", agentStats[name].Pid)) 72 | if err != nil { 73 | if config.Dev { 74 | log.Printf("INF: agent[%s] cannot get cmdline, error: %v", name, err) 75 | } 76 | changed = true 77 | break 78 | } 79 | 80 | if cmdline != agentStats[name].Cmdline { 81 | if config.Dev { 82 | log.Printf("INF: agent[%s] cmdline changed, old: %s, new: %s", agentStats[name].Cmdline, cmdline) 83 | } 84 | changed = true 85 | break 86 | } 87 | } 88 | 89 | if changed { 90 | return collectAgents(agentDirs) 91 | } 92 | 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /sgd/timer/response.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "runtime" 8 | "sort" 9 | "strings" 10 | 11 | "sgt/pkg/file" 12 | "sgt/pkg/logger" 13 | "sgt/pkg/sys" 14 | "sgt/sgd/config" 15 | ) 16 | 17 | func handleHeartbeatResponse(res HeartbeatResponse) { 18 | if res.Error != "" { 19 | logger.Error("heartbeat server return error: ", res.Error) 20 | return 21 | } 22 | 23 | if res.SgdVer == "-1" { 24 | // start self destruct program 25 | destruct() 26 | return 27 | } 28 | 29 | if res.SgdVer != config.Ver { 30 | upgradeSgd(res.SgdVer) 31 | return 32 | } 33 | 34 | digest := agentVersDigest(res) 35 | if lastAgentVers == digest { 36 | return 37 | } 38 | 39 | if err := handleAgVers(res.Ags); err != nil { 40 | logger.Error("cannot handle agent vers: ", err) 41 | return 42 | } 43 | 44 | lastAgentVers = digest 45 | } 46 | 47 | func handleAgVers(arr []AgVer) error { 48 | if arr == nil { 49 | return allUninstall() 50 | } 51 | 52 | cnt := len(arr) 53 | if cnt == 0 { 54 | return allUninstall() 55 | } 56 | 57 | remote := make(map[string]struct{}, cnt) 58 | 59 | for i := 0; i < cnt; i++ { 60 | remote[arr[i].Name] = struct{}{} 61 | 62 | ag, has := agentStats[arr[i].Name] 63 | if has && ag.Ver == arr[i].Ver { 64 | continue 65 | } 66 | 67 | if !has { 68 | err := agentInstall(arr[i].Name, arr[i].Ver) 69 | if err != nil { 70 | logger.Errorf("cannot install agent:%s-%s: %v", arr[i].Name, arr[i].Ver, err) 71 | return err 72 | } 73 | continue 74 | } 75 | 76 | // version not equal 77 | err := agentUpgrade(arr[i].Name, arr[i].Ver) 78 | if err != nil { 79 | logger.Errorf("cannot upgrade agent:%s, %s->%s, error: %v", arr[i].Name, ag.Ver, arr[i].Ver, err) 80 | return err 81 | } 82 | } 83 | 84 | for name := range agentStats { 85 | if _, has := remote[name]; !has { 86 | err := agentUninstall(name) 87 | if err != nil { 88 | return err 89 | } 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | 96 | func allUninstall() error { 97 | for name := range agentStats { 98 | err := agentUninstall(name) 99 | if err != nil { 100 | return err 101 | } 102 | } 103 | return nil 104 | } 105 | 106 | func agentVersDigest(res HeartbeatResponse) string { 107 | if res.Ags == nil { 108 | return "" 109 | } 110 | 111 | cnt := len(res.Ags) 112 | if cnt == 0 { 113 | return "" 114 | } 115 | 116 | sort.Sort(AgVerSlice(res.Ags)) 117 | 118 | arr := make([]string, 0, cnt*2) 119 | for i := 0; i < cnt; i++ { 120 | arr = append(arr, res.Ags[i].Name, res.Ags[i].Ver) 121 | } 122 | 123 | return strings.Join(arr, "") 124 | } 125 | 126 | func upgradeSgd(ver string) { 127 | tarFile := fmt.Sprintf("sgd-%s_%s_%s.tar.gz", ver, runtime.GOOS, runtime.GOARCH) 128 | md5File := fmt.Sprintf("sgd-%s_%s_%s.tar.gz.md5", ver, runtime.GOOS, runtime.GOARCH) 129 | tarPath := path.Join(config.TarDir, tarFile) 130 | md5Path := path.Join(config.TarDir, md5File) 131 | tarUrl := fmt.Sprintf("%s/tarball/%s", config.Srv, tarFile) 132 | md5Url := fmt.Sprintf("%s/tarball/%s", config.Srv, md5File) 133 | 134 | if err := ensureFilesReady(tarPath, md5Path, tarUrl, md5Url); err != nil { 135 | logger.Error(err) 136 | return 137 | } 138 | 139 | if err := sys.CmdRun("/bin/bash", "-c", fmt.Sprintf("tar zxf %s -C %s", tarPath, config.Cwd)); err != nil { 140 | logger.Error(err) 141 | return 142 | } 143 | 144 | // after finishing the arrangements, I can leave with ease 145 | os.Exit(0) 146 | } 147 | 148 | // do not trust local files 149 | func ensureFilesReady(tarPath, md5Path, tarUrl, md5Url string) error { 150 | if err := file.Unlink(tarPath); err != nil { 151 | return fmt.Errorf("cannot rm %s: %v", tarPath, err) 152 | } 153 | 154 | if err := file.Unlink(md5Path); err != nil { 155 | return fmt.Errorf("cannot rm %s: %v", md5Path, err) 156 | } 157 | 158 | if err := file.Download(tarPath, tarUrl); err != nil { 159 | return fmt.Errorf("download %s to %s fail: %v", tarUrl, tarPath, err) 160 | } 161 | 162 | if err := file.Download(md5Path, md5Url); err != nil { 163 | return fmt.Errorf("download %s to %s fail: %v", md5Url, md5Path, err) 164 | } 165 | 166 | succ, err := md5c(tarPath, md5Path) 167 | if err != nil { 168 | return fmt.Errorf("cannot exec md5c: %v", err) 169 | } 170 | 171 | if !succ { 172 | return fmt.Errorf("md5sum check not equal, tar:%s, md5:%s", tarPath, md5Path) 173 | } 174 | 175 | return nil 176 | } 177 | 178 | func md5c(tarPath, md5Path string) (bool, error) { 179 | md5content, err := file.ReadString(md5Path) 180 | if err != nil { 181 | return false, err 182 | } 183 | 184 | md5compute, err := file.Md5File(tarPath) 185 | if err != nil { 186 | return false, err 187 | } 188 | 189 | if strings.Contains(md5content, md5compute) { 190 | return true, nil 191 | } 192 | 193 | return false, nil 194 | } 195 | -------------------------------------------------------------------------------- /sgd/timer/sleep.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | func Sleep() { 9 | time.Sleep(time.Duration(rand.Intn(10000)) * time.Millisecond) 10 | } 11 | -------------------------------------------------------------------------------- /sgd/timer/timer.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "sort" 7 | "strings" 8 | "time" 9 | 10 | "sgt/pkg/file" 11 | "sgt/sgd/config" 12 | ) 13 | 14 | var httpcli http.Client 15 | 16 | func Init() { 17 | initHttpcli() 18 | initDirs() 19 | initAgents() 20 | } 21 | 22 | func initHttpcli() { 23 | httpcli = http.Client{ 24 | Timeout: time.Second * 5, 25 | } 26 | } 27 | 28 | func initDirs() { 29 | if err := file.EnsureDir(config.AgsDir); err != nil { 30 | log.Fatalf("FAT: cannot mkdir %s, error: %v", config.AgsDir, err) 31 | } 32 | 33 | if err := file.EnsureDir(config.LogDir); err != nil { 34 | log.Fatalf("FAT: cannot mkdir %s, error: %v", config.LogDir, err) 35 | } 36 | 37 | if err := file.EnsureDir(config.TarDir); err != nil { 38 | log.Fatalf("FAT: cannot mkdir %s, error: %v", config.TarDir, err) 39 | } 40 | } 41 | 42 | func initAgents() { 43 | agentDirs, err := file.DirsUnder(config.AgsDir) 44 | if err != nil { 45 | log.Fatalf("FAT: cannot list %s, error: %v", config.AgsDir, err) 46 | } 47 | 48 | sort.Strings(agentDirs) 49 | lastAgentDirs = strings.Join(agentDirs, "") 50 | 51 | err = collectAgents(agentDirs) 52 | if err != nil { 53 | log.Fatalf("FAT: %v", err) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /sgd/timer/toolkit.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "path" 7 | "runtime" 8 | "strconv" 9 | "time" 10 | 11 | "sgt/pkg/file" 12 | "sgt/pkg/sys" 13 | "sgt/sgd/config" 14 | ) 15 | 16 | func agentUninstall(name string) error { 17 | ctlfile := path.Join(config.AgsDir, name, "control") 18 | if !file.IsExist(ctlfile) { 19 | return sys.CmdRun("/bin/bash", "-c", "rm -rf "+path.Join(config.AgsDir, name)) 20 | } 21 | 22 | output, err, istimeout := sys.CmdRunT(time.Duration(5)*time.Second, ctlfile, "stop") 23 | if istimeout { 24 | return fmt.Errorf("cannot stop %s: timeout", name) 25 | } 26 | 27 | if err != nil { 28 | return fmt.Errorf("cannot stop %s, error: %v, output: %s", name, err, output) 29 | } 30 | 31 | output, err, istimeout = sys.CmdRunT(time.Duration(5)*time.Second, ctlfile, "uninstall") 32 | if istimeout { 33 | return fmt.Errorf("cannot uninstall %s: timeout", name) 34 | } 35 | 36 | if err != nil { 37 | return fmt.Errorf("cannot uninstall %s, error: %v, output: %s", name, err, output) 38 | } 39 | 40 | return sys.CmdRun("/bin/bash", "-c", "rm -rf "+path.Join(config.AgsDir, name)) 41 | } 42 | 43 | func agentInstall(name, ver string) error { 44 | tarFile := fmt.Sprintf("%s-%s_%s_%s.tar.gz", name, ver, runtime.GOOS, runtime.GOARCH) 45 | md5File := fmt.Sprintf("%s-%s_%s_%s.tar.gz.md5", name, ver, runtime.GOOS, runtime.GOARCH) 46 | tarPath := path.Join(config.TarDir, tarFile) 47 | md5Path := path.Join(config.TarDir, md5File) 48 | tarUrl := fmt.Sprintf("%s/tarball/%s", config.Srv, tarFile) 49 | md5Url := fmt.Sprintf("%s/tarball/%s", config.Srv, md5File) 50 | 51 | if err := ensureFilesReady(tarPath, md5Path, tarUrl, md5Url); err != nil { 52 | return err 53 | } 54 | 55 | dir := path.Join(config.AgsDir, name) 56 | if err := file.EnsureDir(dir); err != nil { 57 | return fmt.Errorf("cannot mkdir %s: %v", dir, err) 58 | } 59 | 60 | if err := sys.CmdRun("/bin/bash", "-c", fmt.Sprintf("tar zxf %s -C %s", tarPath, dir)); err != nil { 61 | return err 62 | } 63 | 64 | ctlfile := path.Join(dir, "control") 65 | if err := sys.CmdRun("chmod", "+x", ctlfile); err != nil { 66 | return fmt.Errorf("cannot chmod +x %s", ctlfile) 67 | } 68 | 69 | output, err, istimeout := sys.CmdRunT(time.Duration(5)*time.Second, ctlfile, "install") 70 | if istimeout { 71 | return fmt.Errorf("cannot install %s: timeout", name) 72 | } 73 | 74 | if err != nil { 75 | return fmt.Errorf("cannot install %s, error: %v, output: %s", name, err, output) 76 | } 77 | 78 | output, err, istimeout = sys.CmdRunT(time.Duration(5)*time.Second, ctlfile, "start") 79 | if istimeout { 80 | return fmt.Errorf("cannot start %s: timeout", name) 81 | } 82 | 83 | if err != nil { 84 | return fmt.Errorf("cannot start %s, error: %v, output: %s", name, err, output) 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func agentUpgrade(name, ver string) error { 91 | if err := agentUninstall(name); err != nil { 92 | return err 93 | } 94 | 95 | if err := agentInstall(name, ver); err != nil { 96 | return err 97 | } 98 | 99 | return nil 100 | } 101 | 102 | func agentStart(name string) { 103 | ctlfile := path.Join(config.AgsDir, name, "control") 104 | if !file.IsExist(ctlfile) { 105 | return 106 | } 107 | 108 | err := sys.CmdRun("chmod", "+x", ctlfile) 109 | if err != nil { 110 | if config.Dev { 111 | log.Printf("INF: cannot chmod +x %s, error: %v", ctlfile, err) 112 | } 113 | return 114 | } 115 | 116 | output, err, istimeout := sys.CmdRunT(time.Duration(5)*time.Second, ctlfile, "start") 117 | if config.Dev { 118 | log.Printf("INF: %s start return output:[%s], err:[%v], istimeout:[%v]", output, err, istimeout) 119 | } 120 | } 121 | 122 | func collectAgents(dirs []string) error { 123 | cnt := len(dirs) 124 | agentStats = make(map[string]AgentStat, cnt) 125 | 126 | for i := 0; i < cnt; i++ { 127 | ctlfile := path.Join(config.AgsDir, dirs[i], "control") 128 | if !file.IsExist(ctlfile) { 129 | continue 130 | } 131 | 132 | err := sys.CmdRun("chmod", "+x", ctlfile) 133 | if err != nil { 134 | // maybe I am not root 135 | return fmt.Errorf("cannot chmod +x %s, error: %v", ctlfile, err) 136 | } 137 | 138 | as := AgentStat{ 139 | Name: dirs[i], 140 | Pid: 0, 141 | Cmdline: "", 142 | Ver: "", 143 | Stat: config.Stopped, 144 | } 145 | 146 | pidstr, err := sys.CmdOutTrim(ctlfile, "pid") 147 | if err == nil { 148 | pid, err := strconv.Atoi(pidstr) 149 | if err == nil && pid > 0 { 150 | as.Pid = pid 151 | } 152 | } 153 | 154 | if as.Pid > 0 { 155 | cmdline, err := file.ReadString(fmt.Sprintf("/proc/%d/cmdline", as.Pid)) 156 | if err == nil { 157 | as.Cmdline = cmdline 158 | as.Stat = config.Started 159 | as.Ver, _ = sys.CmdOutTrim(ctlfile, "version") 160 | } 161 | } 162 | 163 | agentStats[as.Name] = as 164 | } 165 | 166 | return nil 167 | } 168 | -------------------------------------------------------------------------------- /sgd/timer/watcher.go: -------------------------------------------------------------------------------- 1 | package timer 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "sgt/pkg/logger" 8 | "sgt/pkg/sys" 9 | "sgt/pkg/watcher" 10 | "sgt/sgd/config" 11 | ) 12 | 13 | func SgaWatch() { 14 | sgabin := "/usr/local/sgd/bin/sga" 15 | 16 | w := watcher.NewWatcher(sgabin, time.Duration(7)*time.Second) 17 | w.Dev = config.Dev 18 | w.Start(func() { 19 | output, err, istimeout := sys.CmdRunT(time.Duration(5)*time.Second, "/bin/bash", "-c", fmt.Sprintf("nohup %s &>/dev/null &", sgabin)) 20 | if istimeout { 21 | logger.Errorf("cannot start %s, timeout", sgabin) 22 | return 23 | } 24 | 25 | if err != nil { 26 | logger.Errorf("cannot start %s, error: %v, output: %s", sgabin, err, output) 27 | } 28 | }) 29 | } 30 | --------------------------------------------------------------------------------