├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── events ├── doc.go ├── events.go ├── events_test.go └── memory.go ├── executor ├── command.go └── command_test.go ├── failover └── retry │ ├── doc.go │ ├── retry.go │ └── retry_test.go ├── file ├── file.go ├── file_test.go ├── zip.go └── zip_test.go ├── fpm ├── command.go ├── doc.go ├── process.go ├── process_test.go ├── signal_unix.go └── signal_windows.go ├── go.mod ├── go.sum ├── jsonutils ├── flatten.go └── flatten_test.go ├── jwt ├── jwt.go └── jwt_test.go ├── misc └── assert.go ├── network └── ip.go ├── next ├── fastcgi.go ├── fcgiclient.go ├── handler.go ├── readme.md ├── response.go └── rule.go ├── period_job ├── doc.go ├── job.go └── job_test.go ├── pidfile ├── pidfile.go ├── pidfile_darwin.go ├── pidfile_unix.go └── pidfile_windows.go ├── process ├── doc.go ├── manager.go ├── manager_test.go ├── process.go ├── process_unix.go ├── process_windows.go └── program.go └── sql └── extracter └── sql.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.dll 3 | *.so 4 | *.dylib 5 | *.test 6 | *.out 7 | .glide/ 8 | .idea/ 9 | .vscode/ 10 | vendor/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.11 5 | - 1.12 6 | - tip 7 | 8 | env: 9 | - GO111MODULE=on 10 | 11 | matrix: 12 | fast_finish: true 13 | 14 | install: 15 | - go get golang.org/x/tools/cmd/cover 16 | - go get github.com/mattn/goveralls 17 | 18 | script: 19 | - go vet -x ./... 20 | - go test -v -race ./... 21 | - go test -race -coverprofile=coverage.txt -covermode=atomic 22 | 23 | after_success: 24 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 管宜尧 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-toolkit 2 | 3 | [![Build Status](https://www.travis-ci.org/mylxsw/go-toolkit.svg?branch=master)](https://www.travis-ci.org/mylxsw/go-toolkit) 4 | [![codecov](https://codecov.io/gh/mylxsw/go-toolkit/branch/master/graph/badge.svg)](https://codecov.io/gh/mylxsw/go-toolkit) 5 | 6 | 7 | Golang 工具集，平时在开发中常用的一些库或者封装的函数 8 | 9 | 文档目前没有写，简略版本的API文档在这里 [https://godoc.org/github.com/mylxsw/go-toolkit](https://godoc.org/github.com/mylxsw/go-toolkit)。 10 | -------------------------------------------------------------------------------- /events/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package events 实现了应用中的事件机制，用于业务逻辑解耦。 3 | 4 | 例如 5 | 6 | eventManager := events.NewEventManager(events.NewMemoryEventStore(false)) 7 | eventManager.Listen(func(evt UserCreatedEvent) { 8 | t.Logf("user created: id=%s, name=%s", evt.ID, evt.UserName) 9 | 10 | if evt.ID != "111" { 11 | t.Error("test failed") 12 | } 13 | }) 14 | 15 | eventManager.Publish(UserCreatedEvent{ 16 | ID: "111", 17 | UserName: "李逍遥", 18 | }) 19 | 20 | */ 21 | package events 22 | -------------------------------------------------------------------------------- /events/events.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sync" 7 | ) 8 | 9 | // Listener is a event listener 10 | type Listener interface{} 11 | 12 | // EventStore is a interface for event store 13 | type EventStore interface { 14 | Listen(eventName string, listener Listener) 15 | Publish(eventName string, evt interface{}) 16 | SetManager(*EventManager) 17 | } 18 | 19 | // EventManager is a manager for event dispatch 20 | type EventManager struct { 21 | store EventStore 22 | lock sync.RWMutex 23 | } 24 | 25 | // NewEventManager create a eventManager 26 | func NewEventManager(store EventStore) *EventManager { 27 | manager := &EventManager{ 28 | store: store, 29 | } 30 | 31 | store.SetManager(manager) 32 | 33 | return manager 34 | } 35 | 36 | // Listen create a relation from event to listners 37 | func (em *EventManager) Listen(listeners ...Listener) { 38 | em.lock.Lock() 39 | defer em.lock.Unlock() 40 | 41 | for _, listener := range listeners { 42 | listenerType := reflect.TypeOf(listener) 43 | if listenerType.Kind() != reflect.Func { 44 | panic("listener must be a function") 45 | } 46 | 47 | if listenerType.NumIn() != 1 { 48 | panic("listener must be a function with only one arguemnt") 49 | } 50 | 51 | if listenerType.In(0).Kind() != reflect.Struct { 52 | panic("listener must be a function with only on argument of type struct") 53 | } 54 | 55 | em.store.Listen(fmt.Sprintf("%s", listenerType.In(0)), listener) 56 | } 57 | } 58 | 59 | // Publish a event 60 | func (em *EventManager) Publish(evt interface{}) { 61 | em.lock.RLock() 62 | defer em.lock.RUnlock() 63 | 64 | em.store.Publish(fmt.Sprintf("%s", reflect.TypeOf(evt)), evt) 65 | } 66 | 67 | // Call trigger listener to execute 68 | func (em *EventManager) Call(evt interface{}, listener Listener) { 69 | reflect.ValueOf(listener).Call([]reflect.Value{reflect.ValueOf(evt)}) 70 | } 71 | -------------------------------------------------------------------------------- /events/events_test.go: -------------------------------------------------------------------------------- 1 | package events_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mylxsw/go-toolkit/events" 7 | ) 8 | 9 | type UserCreatedEvent struct { 10 | ID string 11 | UserName string 12 | } 13 | 14 | type UserUpdatedEvent struct { 15 | ID string 16 | } 17 | 18 | func TestPublishEvent(t *testing.T) { 19 | eventManager := events.NewEventManager(events.NewMemoryEventStore(false)) 20 | eventManager.Listen(func(evt UserCreatedEvent) { 21 | t.Logf("user created: id=%s, name=%s", evt.ID, evt.UserName) 22 | 23 | if evt.ID != "111" { 24 | t.Error("test failed") 25 | } 26 | }) 27 | 28 | eventManager.Listen(func(evt UserUpdatedEvent) { 29 | t.Logf("user updated-1: id=%s", evt.ID) 30 | 31 | if evt.ID != "121" { 32 | t.Error("test failed") 33 | } 34 | }) 35 | 36 | eventManager.Listen(func(evt UserUpdatedEvent) { 37 | t.Logf("user updated-2: id=%s", evt.ID) 38 | 39 | if evt.ID != "121" { 40 | t.Error("test failed") 41 | } 42 | }) 43 | 44 | eventManager.Publish(UserCreatedEvent{ 45 | ID: "111", 46 | UserName: "李逍遥", 47 | }) 48 | 49 | eventManager.Publish(UserUpdatedEvent{ 50 | ID: "121", 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /events/memory.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | // MemoryEventStore is a event store for sync operations 4 | type MemoryEventStore struct { 5 | async bool 6 | listeners map[string][]Listener 7 | manager *EventManager 8 | } 9 | 10 | // NewMemoryEventStore create a sync event store 11 | func NewMemoryEventStore(async bool) *MemoryEventStore { 12 | return &MemoryEventStore{ 13 | async: async, 14 | listeners: make(map[string][]Listener), 15 | } 16 | } 17 | 18 | // Listen add a listener to a event 19 | func (eventStore *MemoryEventStore) Listen(evtType string, listener Listener) { 20 | if _, ok := eventStore.listeners[evtType]; !ok { 21 | eventStore.listeners[evtType] = make([]Listener, 0) 22 | } 23 | 24 | eventStore.listeners[evtType] = append(eventStore.listeners[evtType], listener) 25 | } 26 | 27 | // Publish publish a event 28 | func (eventStore *MemoryEventStore) Publish(evtType string, evt interface{}) { 29 | if listeners, ok := eventStore.listeners[evtType]; ok { 30 | for _, listener := range listeners { 31 | if eventStore.async { 32 | go eventStore.manager.Call(evt, listener) 33 | } else { 34 | eventStore.manager.Call(evt, listener) 35 | } 36 | } 37 | } 38 | } 39 | 40 | // SetManager event manager 41 | func (eventStore *MemoryEventStore) SetManager(manager *EventManager) { 42 | eventStore.manager = manager 43 | } 44 | -------------------------------------------------------------------------------- /executor/command.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os/exec" 10 | "strings" 11 | "sync" 12 | ) 13 | 14 | const outputChanSize = 1000 15 | 16 | // Command 命令行命令 17 | type Command struct { 18 | Executable string 19 | Args []string 20 | 21 | init func(cmd *exec.Cmd) error 22 | 23 | output chan Output 24 | stdout string 25 | stderr string 26 | 27 | lock sync.Mutex 28 | } 29 | 30 | // Output 命令式输出 31 | type Output struct { 32 | Type OutputType 33 | Content string 34 | } 35 | 36 | // OutputType Job输出类型 37 | type OutputType int 38 | 39 | const ( 40 | // Stdout 标准输出 41 | Stdout OutputType = iota 42 | // Stderr 标准错误输出 43 | Stderr 44 | ) 45 | 46 | // String 输出类型的字符串表示 47 | func (outputType OutputType) String() string { 48 | switch outputType { 49 | case Stdout: 50 | return "stdout" 51 | case Stderr: 52 | return "stderr" 53 | } 54 | 55 | return "" 56 | } 57 | 58 | // New 创建一个新的命令 59 | func New(executable string, args ...string) *Command { 60 | return &Command{ 61 | Executable: executable, 62 | Args: args, 63 | } 64 | } 65 | 66 | // Init initialize the command 67 | // you can set cmd properties in init callback 68 | // such as 69 | // cmd.SysProcAttr = &syscall.SysProcAttr{ 70 | // Setpgid: true, 71 | // } 72 | func (command *Command) Init(init func(cmd *exec.Cmd) error) { 73 | command.init = init 74 | } 75 | 76 | // Run 执行命令 77 | func (command *Command) Run(ctx context.Context) (bool, error) { 78 | defer command.close() 79 | 80 | cmd := exec.CommandContext(ctx, command.Executable, command.Args...) 81 | if command.init != nil { 82 | if err := command.init(cmd); err != nil { 83 | return false, err 84 | } 85 | } 86 | 87 | stdout, err := cmd.StdoutPipe() 88 | if err != nil { 89 | return false, fmt.Errorf("can not open stdout pipe: %s", err.Error()) 90 | } 91 | 92 | stderr, err := cmd.StderrPipe() 93 | if err != nil { 94 | return false, fmt.Errorf("can not open stderr pipe: %s", err.Error()) 95 | } 96 | 97 | if err = cmd.Start(); err != nil { 98 | return false, fmt.Errorf("can not start command: %s", err.Error()) 99 | } 100 | 101 | var wg sync.WaitGroup 102 | wg.Add(2) 103 | 104 | go func() { 105 | defer wg.Done() 106 | _ = command.bindOutputChan(&stdout, Stdout) 107 | }() 108 | 109 | go func() { 110 | defer wg.Done() 111 | _ = command.bindOutputChan(&stderr, Stderr) 112 | }() 113 | 114 | if err = cmd.Wait(); err != nil { 115 | return false, fmt.Errorf("wait for command failed: %s", err.Error()) 116 | } 117 | 118 | wg.Wait() 119 | 120 | if cmd.ProcessState.Success() { 121 | return true, nil 122 | } 123 | 124 | return false, errors.New("command execute finished, but an error occured") 125 | } 126 | 127 | // StdoutString 命令执行后标准输出 128 | func (command *Command) StdoutString() string { 129 | return command.stdout 130 | } 131 | 132 | // StderrString 命令执行后标准错误输出 133 | func (command *Command) StderrString() string { 134 | return command.stderr 135 | } 136 | 137 | // OpenOutputChan 打开输出channel 138 | func (command *Command) OpenOutputChan() <-chan Output { 139 | command.lock.Lock() 140 | defer command.lock.Unlock() 141 | 142 | if command.output == nil { 143 | command.output = make(chan Output, outputChanSize) 144 | } 145 | 146 | return command.output 147 | } 148 | 149 | func (command *Command) close() { 150 | command.lock.Lock() 151 | defer command.lock.Unlock() 152 | 153 | if command.output != nil { 154 | close(command.output) 155 | command.output = nil 156 | } 157 | } 158 | 159 | func (command *Command) bindOutputChan(input *io.ReadCloser, outputType OutputType) error { 160 | reader := bufio.NewReader(*input) 161 | for { 162 | line, err := reader.ReadString('\n') 163 | if err != nil || io.EOF == err { 164 | if err != io.EOF { 165 | return fmt.Errorf("read output failed: %s", err.Error()) 166 | } 167 | break 168 | } 169 | 170 | if outputType == Stdout { 171 | command.stdout += line 172 | } else { 173 | command.stderr += line 174 | } 175 | 176 | command.lock.Lock() 177 | outputIsNotNil := command.output != nil 178 | command.lock.Unlock() 179 | 180 | if outputIsNotNil { 181 | command.output <- Output{ 182 | Type: outputType, 183 | Content: strings.TrimRight(line, "\n"), 184 | } 185 | } 186 | } 187 | 188 | return nil 189 | } 190 | -------------------------------------------------------------------------------- /executor/command_test.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | ) 8 | 9 | func TestExecutor(t *testing.T) { 10 | testCommand(t, "ifconfig") 11 | testCommand(t, "ifconfig", "-a") 12 | 13 | testCommandWithOutputChan(t, "ifconfig") 14 | testCommandWithOutputChan(t, "ifconfig", "-a") 15 | 16 | //testCommand(t, "ping", "-c", "4", "baidu.com") 17 | //testCommandWithOutputChan(t, "ping", "-c", "2", "yunsom.com") 18 | } 19 | 20 | func testCommandWithOutputChan(t *testing.T, executable string, args ...string) { 21 | cmd := New(executable, args...) 22 | 23 | var wg sync.WaitGroup 24 | wg.Add(1) 25 | go func(outputChan <-chan Output) { 26 | defer wg.Done() 27 | for out := range outputChan { 28 | t.Logf("%s -> %s", out.Type.String(), out.Content) 29 | } 30 | }(cmd.OpenOutputChan()) 31 | 32 | if _, err := cmd.Run(context.TODO()); err != nil { 33 | t.Errorf("command execute failed: %s", err.Error()) 34 | } else { 35 | t.Log("test ok") 36 | } 37 | 38 | wg.Wait() 39 | } 40 | 41 | func testCommand(t *testing.T, executable string, args ...string) { 42 | cmd := New(executable, args...) 43 | 44 | if _, err := cmd.Run(context.TODO()); err != nil { 45 | t.Errorf("command %s execute failed: %s", executable, err.Error()) 46 | } else { 47 | t.Log("test ok") 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /failover/retry/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package retry 实现了一个错误重试函数。当函数执行出错时，能够自动根据设置的重试次数进行重试 3 | 4 | retryTimes, err := Retry(func(rt int) error { 5 | fmt.Printf("%d retry execute time: %s\n", rt, time.Now().String()) 6 | if rt == 2 { 7 | return nil 8 | } 9 | 10 | return errors.New("test error") 11 | }, 3).Run() 12 | */ 13 | package retry 14 | -------------------------------------------------------------------------------- /failover/retry/retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // retryLatter 用于重试失败的操作 9 | func retryLatter(f func(), delaySecond int) { 10 | fin := make(chan struct{}) 11 | time.AfterFunc(time.Duration(delaySecond)*time.Second, func() { 12 | defer func() { fin <- struct{}{} }() 13 | f() 14 | }) 15 | 16 | <-fin 17 | } 18 | 19 | // Retryer 重试器 20 | type Retryer struct { 21 | f func(retryTimes int) error 22 | retryTimes int 23 | maxRetryTimes int 24 | err error 25 | successFunc func(retryTimes int) 26 | failedFunc func(err error) 27 | finishFunc func(retryTimes int, err error) bool 28 | } 29 | 30 | func (r *Retryer) retryFunc() { 31 | r.retryTimes++ 32 | if r.err = callWithRecover(r.retryTimes-1, r.f); r.err != nil { 33 | if r.retryTimes < r.maxRetryTimes+1 { 34 | retryLatter(r.retryFunc, r.retryTimes) 35 | } 36 | } 37 | } 38 | 39 | // Retry 自动重试函数 40 | func Retry(f func(retryTimes int) error, max int) *Retryer { 41 | return &Retryer{ 42 | f: f, 43 | maxRetryTimes: max, 44 | retryTimes: 0, 45 | successFunc: func(retryTimes int) {}, 46 | failedFunc: func(err error) {}, 47 | finishFunc: func(retryTimes int, err error) bool { 48 | return false 49 | }, 50 | } 51 | } 52 | 53 | // Success 注册执行成功后置函数 54 | func (r *Retryer) Success(successFunc func(retryTimes int)) *Retryer { 55 | r.successFunc = successFunc 56 | 57 | return r 58 | } 59 | 60 | // Failed 注册执行失败后置函数（重试最大次数后，仍然失败才调用） 61 | func (r *Retryer) Failed(failedFunc func(err error)) *Retryer { 62 | r.failedFunc = failedFunc 63 | 64 | return r 65 | } 66 | 67 | // Finished 注册无论成功失败，最终执行完毕后执行的后置函数 68 | func (r *Retryer) Finished(finishFunc func(retryTimes int, err error) bool) *Retryer { 69 | r.finishFunc = finishFunc 70 | 71 | return r 72 | } 73 | 74 | // Run 同步的方式运行 75 | func (r *Retryer) Run() (int, error) { 76 | r.retryFunc() 77 | if !r.finishFunc(r.retryTimes, r.err) { 78 | if r.err != nil { 79 | r.failedFunc(r.err) 80 | } else { 81 | r.successFunc(r.retryTimes) 82 | } 83 | } 84 | 85 | return r.retryTimes, r.err 86 | } 87 | 88 | // RunAsync 异步方式运行 89 | func (r *Retryer) RunAsync() <-chan struct{} { 90 | fin := make(chan struct{}) 91 | 92 | // 由于异步执行模式下，f函数先同步执行了一次 93 | // 让重试次数在同步和异步模式下均表现如一 94 | r.retryTimes++ 95 | 96 | // 异步执行模式下，确保第一次执行是同步的，失败时才异步去重试 97 | if r.err = callWithRecover(r.retryTimes-1, r.f); r.err == nil { 98 | go func() { fin <- struct{}{} }() 99 | } else { 100 | 101 | go func() { 102 | _, _ = r.Run() 103 | fin <- struct{}{} 104 | }() 105 | } 106 | 107 | return fin 108 | } 109 | 110 | func callWithRecover(retryTime int, f func(retryTime int) error) (err error) { 111 | defer func() { 112 | if err2 := recover(); err2 != nil { 113 | err = fmt.Errorf("%v", err2) 114 | } 115 | }() 116 | return f(retryTime) 117 | } 118 | -------------------------------------------------------------------------------- /failover/retry/retry_test.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestRetryPanic(t *testing.T) { 11 | retryTimes, err := Retry(func(retryTimes int) error { 12 | if retryTimes < 1 { 13 | panic("sorry") 14 | } 15 | return nil 16 | }, 2).Run() 17 | if err != nil { 18 | t.Errorf("still error: %s", err.Error()) 19 | } 20 | 21 | if retryTimes != 2 { 22 | t.Errorf("test failed, expect %d, got %d", 2, retryTimes) 23 | } 24 | } 25 | 26 | func TestRetryLatter(t *testing.T) { 27 | fmt.Println("current time: " + time.Now().String()) 28 | 29 | retryTimes, err := Retry(func(rt int) error { 30 | fmt.Printf("%d retry execute time: %s\n", rt, time.Now().String()) 31 | if rt < 1 { 32 | return errors.New("test error") 33 | } 34 | return nil 35 | }, 3).Run() 36 | 37 | if err != nil { 38 | t.Errorf("still error: %s", err.Error()) 39 | } 40 | 41 | if retryTimes != 2 { 42 | t.Errorf("test failed, expect %d, got %d", 2, retryTimes) 43 | } 44 | 45 | fmt.Printf("retry %d times\n", retryTimes) 46 | 47 | succeed := false 48 | <-Retry(func(rt int) error { 49 | fmt.Printf("%d retry execute time: %s\n", rt, time.Now().String()) 50 | if rt < 2 { 51 | return errors.New("test error") 52 | } 53 | return nil 54 | }, 3).Success(func(rt int) { 55 | fmt.Printf("retry %d times\n", rt) 56 | succeed = true 57 | }).Failed(func(err error) { 58 | fmt.Println("still error: " + err.Error()) 59 | succeed = false 60 | }).RunAsync() 61 | 62 | 63 | if !succeed { 64 | t.Errorf("test failed") 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /file/file.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | ) 9 | 10 | // Exist 判断文件是否存在 11 | func Exist(path string) bool { 12 | _, err := os.Stat(path) 13 | if err != nil && os.IsNotExist(err) { 14 | return false 15 | } 16 | 17 | return true 18 | } 19 | 20 | // Size return file size for path 21 | func Size(path string) int64 { 22 | stat, err := os.Stat(path) 23 | if err != nil { 24 | return 0 25 | } 26 | 27 | return stat.Size() 28 | } 29 | 30 | // InsertSuffix insert a suffix to filepath 31 | func InsertSuffix(src string, suffix string) string { 32 | ext := path.Ext(src) 33 | 34 | return fmt.Sprintf("%s%s%s", src[:len(src)-len(ext)], suffix, ext) 35 | } 36 | 37 | // ReplaceExt replace ext for src 38 | func ReplaceExt(src string, ext string) string { 39 | ext1 := path.Ext(src) 40 | 41 | return fmt.Sprintf("%s%s", src[:len(src)-len(ext1)], ext) 42 | } 43 | 44 | // FileGetContents reads entire file into a string 45 | func FileGetContents(filename string) (string, error) { 46 | res, err := ioutil.ReadFile(filename) 47 | return string(res), err 48 | } 49 | -------------------------------------------------------------------------------- /file/file_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestFileExist(t *testing.T) { 9 | if !Exist("file.go") { 10 | t.Errorf("check file existence failed") 11 | } 12 | 13 | if Exist("not_exist_file") { 14 | t.Errorf("check file existence failed") 15 | } 16 | } 17 | 18 | func TestFileSize(t *testing.T) { 19 | fileSize := Size("zip.go") 20 | fmt.Printf("file size is %.2f K", float64(fileSize)/1024) 21 | 22 | if fileSize <= 0 { 23 | t.Errorf("check file size failed") 24 | } 25 | } 26 | 27 | func TestInsertSuffix(t *testing.T) { 28 | f1 := "/file/path/to/file.txt" 29 | if "/file/path/to/file@x1.txt" != InsertSuffix(f1, "@x1") { 30 | t.Errorf("test failed") 31 | } 32 | } 33 | 34 | func TestReplaceExt(t *testing.T) { 35 | f1 := "/file/path/to/file.txt" 36 | if ReplaceExt(f1, ".orm.go") != "/file/path/to/file.orm.go" { 37 | t.Errorf("test failed") 38 | } 39 | } -------------------------------------------------------------------------------- /file/zip.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "archive/zip" 5 | "fmt" 6 | "io" 7 | "os" 8 | ) 9 | 10 | // ZipFile is a file wrapper contains filename and path 11 | type ZipFile struct { 12 | Name string `json:"name"` 13 | Path string `json:"path"` 14 | } 15 | 16 | // CreateZipArchiveFile create a zip archive file from files 17 | func CreateZipArchiveFile(saveAs string, files []ZipFile, ignoreError bool) (err error) { 18 | defer func() { 19 | if err != nil { 20 | os.Remove(saveAs) 21 | } 22 | }() 23 | 24 | saveAsFile, err := os.Create(saveAs) 25 | if err != nil { 26 | return fmt.Errorf("can not create destination zip archive file: %s", err.Error()) 27 | } 28 | 29 | defer saveAsFile.Close() 30 | 31 | _, err = CreateZipArchive(saveAsFile, files, ignoreError) 32 | return 33 | } 34 | 35 | // CreateZipArchiveFileWithIgnored create a zip archive file from files, and return all ignored files 36 | func CreateZipArchiveFileWithIgnored(saveAs string, files []ZipFile, ignoreError bool) (ignoredFiles []ZipFile, err error) { 37 | defer func() { 38 | if err != nil { 39 | os.Remove(saveAs) 40 | } 41 | }() 42 | 43 | saveAsFile, err := os.Create(saveAs) 44 | if err != nil { 45 | return ignoredFiles, fmt.Errorf("can not create destination zip archive file: %s", err.Error()) 46 | } 47 | 48 | defer saveAsFile.Close() 49 | 50 | return CreateZipArchive(saveAsFile, files, ignoreError) 51 | } 52 | 53 | // CreateZipArchive create a zip archive 54 | func CreateZipArchive(w io.Writer, files []ZipFile, ignoreError bool) ([]ZipFile, error) { 55 | zipWriter := zip.NewWriter(w) 56 | defer zipWriter.Close() 57 | 58 | var ignoredFiles = make([]ZipFile, 0) 59 | for _, file := range files { 60 | if err := addFileToZipArchive(zipWriter, file); err != nil { 61 | ignoredFiles = append(ignoredFiles, file) 62 | 63 | if !ignoreError { 64 | return ignoredFiles, err 65 | } 66 | } 67 | } 68 | 69 | return ignoredFiles, nil 70 | } 71 | 72 | // addFileToZipArchive add a file to zip archive 73 | func addFileToZipArchive(zipWriter *zip.Writer, file ZipFile) error { 74 | 75 | zipfile, err := os.Open(file.Path) 76 | if err != nil { 77 | return fmt.Errorf("can not open %s: %s", file.Path, err.Error()) 78 | } 79 | 80 | defer zipfile.Close() 81 | 82 | info, err := zipfile.Stat() 83 | if err != nil { 84 | return fmt.Errorf("can not get file state for %s: %s", file.Path, err.Error()) 85 | } 86 | 87 | header, err := zip.FileInfoHeader(info) 88 | if err != nil { 89 | return fmt.Errorf("create zip file header failed for %s: %s", file.Path, err.Error()) 90 | } 91 | 92 | header.Name = file.Name 93 | header.Method = zip.Deflate 94 | 95 | writer, err := zipWriter.CreateHeader(header) 96 | if err != nil { 97 | return fmt.Errorf("create header failed for %s: %s", file.Path, err.Error()) 98 | } 99 | 100 | _, err = io.Copy(writer, zipfile) 101 | if err != nil { 102 | return fmt.Errorf("copy file content failed for %s: %s", file.Path, err.Error()) 103 | } 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /file/zip_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestCreateZipArchiveFile(t *testing.T) { 9 | 10 | files := []ZipFile{ 11 | {Name: "file_test.go", Path: "file_test.go"}, 12 | {Name: "file.go", Path: "file.go"}, 13 | {Name: "zip.go", Path: "zip.go"}, 14 | } 15 | 16 | zipfile := "xxx.zip" 17 | 18 | if err := CreateZipArchiveFile(zipfile, files, false); err != nil { 19 | t.Errorf("test failed: %s", err.Error()) 20 | } 21 | 22 | os.Remove(zipfile) 23 | } 24 | -------------------------------------------------------------------------------- /fpm/command.go: -------------------------------------------------------------------------------- 1 | package fpm 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "sync" 7 | 8 | "github.com/mylxsw/asteria/log" 9 | ) 10 | 11 | // Fpm FPM进程管理对象 12 | type Fpm struct { 13 | process *Process // phpfpm进程管理器 14 | started bool // 标识进程是否已经启动（停止） 15 | config *Config // 进程配置 16 | lock sync.Mutex 17 | } 18 | 19 | // Config Meta Fpm配置 20 | type Config struct { 21 | FpmBin string // phpfpm 可执行文件路径 22 | PhpConfigFile string // php.ini配置文件 23 | FpmConfigDir string // php-fpm.conf配置文件目录 24 | WorkDir string // 工作目录 25 | User string // 执行用户 26 | Group string // 执行用户组 27 | ErrorLogFile string // fpm错误日志 28 | SlowLogFile string // fpm 慢查询日志 29 | PIDFile string // fpm pid file 30 | SocketFile string // fpm socket file 31 | FpmConfigFile string // fpm config file 32 | 33 | PM string // fpm进程管理方式 34 | MaxChildren string // fpm最大子进程数目 35 | StartServers string // fpm启动时进程数目 36 | MinSpareServers string // fpm最小空闲进程数数目 37 | MaxSpareServers string // fpm最大空闲进程数目 38 | SlowlogTimeout string // fpm慢请求日志超时时间 39 | OutputHandler func(typ LogType, msg string) // php-fpm 标准输出，标准错误输出处理器 40 | } 41 | 42 | // NewFpm 创建一个PFM实例 43 | func NewFpm(config *Config) *Fpm { 44 | 45 | errorLog := config.ErrorLogFile 46 | if errorLog == "" { 47 | errorLog = filepath.Join(config.WorkDir, config.FpmConfigDir, "php-fpm.error.log") 48 | } 49 | 50 | slowLog := config.SlowLogFile 51 | if slowLog == "" { 52 | slowLog = filepath.Join(config.WorkDir, config.FpmConfigDir, "php-fpm.slow.log") 53 | } 54 | 55 | pidFile := config.PIDFile 56 | if pidFile == "" { 57 | pidFile = filepath.Join(config.WorkDir, config.FpmConfigDir, "php-fpm.pid") 58 | } 59 | 60 | socketFile := config.SocketFile 61 | if socketFile == "" { 62 | socketFile = filepath.Join(config.WorkDir, config.FpmConfigDir, "php-fpm.sock") 63 | } 64 | 65 | fpmConfigFile := config.FpmConfigFile 66 | if fpmConfigFile == "" { 67 | fpmConfigFile = filepath.Join(config.WorkDir, config.FpmConfigDir, "php-fpm.conf") 68 | } 69 | 70 | process := NewProcess(Meta{ 71 | FpmBin: config.FpmBin, 72 | PidFile: pidFile, 73 | ErrorLog: errorLog, 74 | Listen: socketFile, 75 | FpmConfigFile: fpmConfigFile, 76 | SlowLog: slowLog, 77 | PhpConfigFile: config.PhpConfigFile, 78 | User: config.User, 79 | Group: config.Group, 80 | PM: config.PM, 81 | MaxChildren: config.MaxChildren, 82 | StartServers: config.StartServers, 83 | MinSpareServers: config.MinSpareServers, 84 | MaxSpareServers: config.MaxSpareServers, 85 | SlowlogTimeout: config.SlowlogTimeout, 86 | OutputHandler: config.OutputHandler, 87 | }) 88 | 89 | // 更新/创建配置文件 90 | process.UpdateConfigFile(process.Meta.FpmConfigFile) 91 | 92 | return &Fpm{ 93 | config: config, 94 | process: process, 95 | } 96 | } 97 | 98 | // start 启动fpm master进程 99 | func (fpm *Fpm) start() error { 100 | // 先尝试关闭已经存在的fpm（当前项目相关的） 101 | _ = CloseExistProcess(fpm.process.GetProcessMeta().PidFile) 102 | 103 | fpm.lock.Lock() 104 | defer fpm.lock.Unlock() 105 | 106 | err := fpm.process.Start() 107 | fpm.started = true 108 | if err != nil { 109 | return fmt.Errorf("php-fpm process start failed: %s", err.Error()) 110 | } 111 | 112 | return nil 113 | } 114 | 115 | // Loop 循环检测fpm master是否挂掉，挂掉后自动重新启动 116 | func (fpm *Fpm) Loop(ok chan struct{}) error { 117 | if err := fpm.start(); err != nil { 118 | return err 119 | } 120 | 121 | log.Debugf("master process for php-fpm has started with pid=%d", fpm.process.PID()) 122 | ok <- struct{}{} 123 | 124 | for { 125 | if err := fpm.process.Wait(); err != nil { 126 | log.Errorf("php-fpm process has stopped: %v", err) 127 | } 128 | 129 | // 如果进程未启动（已经手动关闭），则退出循环 130 | if func() bool { 131 | fpm.lock.Lock() 132 | defer fpm.lock.Unlock() 133 | 134 | return !fpm.started 135 | }() { 136 | break 137 | } 138 | 139 | // 进程启动状态下，异常退出后重启进程 140 | if err := fpm.start(); err != nil { 141 | log.Errorf("php-fpm process start failed: %v", err) 142 | continue 143 | } 144 | 145 | log.Debugf("master process for php-fpm has restarted with pid=%d", fpm.process.PID()) 146 | } 147 | 148 | return nil 149 | } 150 | 151 | // Reload reload php-fpm process 152 | func (fpm *Fpm) Reload() error { 153 | log.Debug("reload php-fpm process") 154 | return fpm.process.Reload() 155 | } 156 | 157 | // Kill 停止fpm进程 158 | func (fpm *Fpm) Kill() error { 159 | fpm.lock.Lock() 160 | defer fpm.lock.Unlock() 161 | 162 | defer func() { 163 | if err := recover(); err != nil { 164 | log.Errorf("kill fpm process failed: %v", err) 165 | } 166 | }() 167 | 168 | err := fpm.process.Kill() 169 | if err != nil { 170 | log.Warningf("kill fpm process failed: %s", err.Error()) 171 | return err 172 | } 173 | 174 | fpm.started = false 175 | 176 | return nil 177 | } 178 | 179 | // GetNetworkAddress 获取监听的网络类型和地址 180 | func (fpm *Fpm) GetNetworkAddress() (network, address string) { 181 | fpm.lock.Lock() 182 | defer fpm.lock.Unlock() 183 | 184 | if !fpm.started { 185 | return "", "" 186 | } 187 | return fpm.process.Address() 188 | } 189 | -------------------------------------------------------------------------------- /fpm/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package fpm 实现了php-fpm进程管理。 3 | */ 4 | package fpm 5 | -------------------------------------------------------------------------------- /fpm/process.go: -------------------------------------------------------------------------------- 1 | package fpm 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "net" 8 | "os" 9 | "os/exec" 10 | "regexp" 11 | "strings" 12 | "time" 13 | 14 | "github.com/mylxsw/go-toolkit/file" 15 | "github.com/mylxsw/go-toolkit/pidfile" 16 | "gopkg.in/ini.v1" 17 | ) 18 | 19 | // Process PHP-FPM进程管理器 20 | type Process struct { 21 | Meta Meta 22 | cmd *exec.Cmd 23 | } 24 | 25 | type LogType string 26 | 27 | const ( 28 | LogTypeStdout LogType = "stdout" 29 | LogTypeStderr LogType = "stderr" 30 | ) 31 | 32 | // Meta 进程运行配置信息 33 | type Meta struct { 34 | FpmBin string 35 | PidFile string 36 | ErrorLog string 37 | SlowLog string 38 | Listen string 39 | FpmConfigFile string 40 | PhpConfigFile string 41 | User string 42 | Group string 43 | PM string // fpm进程管理方式 44 | MaxChildren string // fpm最大子进程数目 45 | StartServers string // fpm启动时进程数目 46 | MinSpareServers string // fpm最小空闲进程数数目 47 | MaxSpareServers string // fpm最大空闲进程数目 48 | SlowlogTimeout string // fpm慢请求日志超时时间 49 | OutputHandler func(typ LogType, msg string) 50 | } 51 | 52 | // NewProcess creates a new process descriptor 53 | func NewProcess(meta Meta) *Process { 54 | return &Process{ 55 | Meta: meta, 56 | } 57 | } 58 | 59 | // GetProcessMeta get process configuration info 60 | func (proc *Process) GetProcessMeta() Meta { 61 | return proc.Meta 62 | } 63 | 64 | // UpdateConfigFile 更新或创建fpm配置文件 65 | func (proc *Process) UpdateConfigFile(configFile string) { 66 | f := loadIniFromFile(configFile) 67 | 68 | _, _ = f.Section("global").NewKey("pid", proc.Meta.PidFile) 69 | _, _ = f.Section("global").NewKey("error_log", proc.Meta.ErrorLog) 70 | _, _ = f.Section("www").NewKey("listen", proc.Meta.Listen) 71 | _, _ = f.Section("www").NewKey("slowlog", proc.Meta.SlowLog) 72 | _, _ = f.Section("www").NewKey("pm", proc.Meta.PM) 73 | _, _ = f.Section("www").NewKey("pm.max_children", proc.Meta.MaxChildren) 74 | _, _ = f.Section("www").NewKey("pm.start_servers", proc.Meta.StartServers) 75 | _, _ = f.Section("www").NewKey("pm.min_spare_servers", proc.Meta.MinSpareServers) 76 | _, _ = f.Section("www").NewKey("pm.max_spare_servers", proc.Meta.MaxSpareServers) 77 | _, _ = f.Section("www").NewKey("request_slowlog_timeout", proc.Meta.SlowlogTimeout) 78 | 79 | if proc.Meta.User != "" { 80 | _, _ = f.Section("www").NewKey("user", proc.Meta.User) 81 | } 82 | if proc.Meta.Group != "" { 83 | _, _ = f.Section("www").NewKey("group", proc.Meta.Group) 84 | } 85 | 86 | if err := f.SaveTo(configFile); err != nil { 87 | panic(err) 88 | } 89 | } 90 | 91 | // Start 启动php-fpm主进程 92 | func (proc *Process) Start() (err error) { 93 | args := []string{ 94 | proc.Meta.FpmBin, 95 | "--fpm-config", 96 | proc.Meta.FpmConfigFile, 97 | "--nodaemonize", 98 | "--allow-to-run-as-root", 99 | } 100 | // look for php.ini file 101 | if proc.Meta.PhpConfigFile == "" { 102 | args = append(args, "-n") 103 | } else { 104 | args = append(args, "-c", proc.Meta.PhpConfigFile) 105 | } 106 | 107 | // generate extended information for debugger/profiler 108 | // args = append(args, "-e") 109 | 110 | proc.cmd = &exec.Cmd{ 111 | Path: proc.Meta.FpmBin, 112 | Args: args, 113 | } 114 | 115 | if proc.Meta.OutputHandler != nil { 116 | stdoutPipe, _ := proc.cmd.StdoutPipe() 117 | stderrPipe, _ := proc.cmd.StderrPipe() 118 | 119 | go proc.consoleLog(LogTypeStdout, &stdoutPipe) 120 | go proc.consoleLog(LogTypeStderr, &stderrPipe) 121 | } 122 | 123 | if err := proc.cmd.Start(); err != nil { 124 | return err 125 | } 126 | 127 | // wait until the service is connectable 128 | // or time out 129 | select { 130 | case <-proc.waitConn(): 131 | // do nothing 132 | case <-time.After(time.Second * 10): 133 | // wait 10 seconds or timeout 134 | err = fmt.Errorf("time out") 135 | } 136 | 137 | return 138 | } 139 | 140 | // PID get process pid 141 | func (proc *Process) PID() int { 142 | return proc.cmd.Process.Pid 143 | } 144 | 145 | // waitConn test whether process is ok 146 | func (proc *Process) waitConn() <-chan net.Conn { 147 | chanConn := make(chan net.Conn) 148 | go func() { 149 | for { 150 | if conn, err := net.Dial(proc.Address()); err != nil { 151 | time.Sleep(time.Millisecond * 2) 152 | } else { 153 | chanConn <- conn 154 | break 155 | } 156 | } 157 | }() 158 | return chanConn 159 | } 160 | 161 | // Address 返回监听方式和地址 162 | func (proc *Process) Address() (network, address string) { 163 | reIP := regexp.MustCompile("^(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})\\:(\\d{2,5}$)") 164 | rePort := regexp.MustCompile("^(\\d+)$") 165 | switch { 166 | case reIP.MatchString(proc.Meta.Listen): 167 | network = "tcp" 168 | address = proc.Meta.Listen 169 | case rePort.MatchString(proc.Meta.Listen): 170 | network = "tcp" 171 | address = ":" + proc.Meta.Listen 172 | default: 173 | network = "unix" 174 | address = proc.Meta.Listen 175 | } 176 | return 177 | } 178 | 179 | // Kill kill the process 180 | func (proc *Process) Kill() error { 181 | if proc.cmd == nil || proc.cmd.Process == nil { 182 | return fmt.Errorf("fpm process is not initialized") 183 | } 184 | 185 | return proc.cmd.Process.Signal(os.Interrupt) 186 | } 187 | 188 | // Wait wait for process to exit 189 | func (proc *Process) Wait() (err error) { 190 | return proc.cmd.Wait() 191 | } 192 | 193 | func (proc *Process) consoleLog(typ LogType, input *io.ReadCloser) error { 194 | reader := bufio.NewReader(*input) 195 | for { 196 | line, err := reader.ReadString('\n') 197 | if err != nil || io.EOF == err { 198 | if err != io.EOF { 199 | return fmt.Errorf("process fpm output failed: %s", err.Error()) 200 | } 201 | break 202 | } 203 | 204 | proc.Meta.OutputHandler(typ, strings.Trim(line, "\n")) 205 | } 206 | 207 | return nil 208 | } 209 | 210 | // CloseExistProcess close php-fpm process already exist for this project 211 | func CloseExistProcess(pidfileName string) error { 212 | pid, _ := pidfile.ReadPIDFile(pidfileName) 213 | if pid > 0 { 214 | process, _ := os.FindProcess(pid) 215 | _ = process.Signal(os.Interrupt) 216 | } 217 | 218 | return nil 219 | } 220 | 221 | func loadIniFromFile(configFile string) *ini.File { 222 | var f *ini.File 223 | if !file.Exist(configFile) { 224 | f = ini.Empty() 225 | _, _ = f.NewSection("global") 226 | _, _ = f.NewSection("www") 227 | } else { 228 | f2, err := ini.Load(configFile) 229 | if err != nil { 230 | panic(err) 231 | } 232 | 233 | f = f2 234 | } 235 | 236 | return f 237 | } 238 | -------------------------------------------------------------------------------- /fpm/process_test.go: -------------------------------------------------------------------------------- 1 | package fpm 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestProcess(t *testing.T) { 9 | process := NewProcess(Meta{ 10 | FpmBin: "/Users/mylxsw/.phpbrew/php/php-7.2.4/sbin/php-fpm", 11 | PidFile: "/tmp/fpm.pid", 12 | ErrorLog: "/tmp/fpm-error.log", 13 | SlowLog: "/tmp/fpm-slow.log", 14 | Listen: "/tmp/fpm.sock", 15 | FpmConfigFile: "/tmp/fpm.ini", 16 | PhpConfigFile: "/Users/mylxsw/.phpbrew/php/php-7.2.4/etc/php.ini", 17 | PM: "dynamic", 18 | MaxChildren: "10", 19 | StartServers: "4", // fpm启动时进程数目 20 | MinSpareServers: "4", // fpm最小空闲进程数数目 21 | MaxSpareServers: "8", // fpm最大空闲进程数目 22 | SlowlogTimeout: "3s", // fpm慢请求日志超时时间 23 | }) 24 | 25 | process.UpdateConfigFile("/tmp/fpm.ini") 26 | 27 | if err := process.Start(); err != nil { 28 | t.Fatal("无法启动fpm进程") 29 | } 30 | 31 | go func() { 32 | for { 33 | if err := process.Wait(); err != nil { 34 | t.Logf("process exited: %s", err.Error()) 35 | } 36 | 37 | t.Logf("process exited without error") 38 | 39 | if err := process.Start(); err != nil { 40 | t.Fatal("无法启动fpm进程") 41 | } 42 | 43 | t.Logf("new process started with pid=%d", process.PID()) 44 | } 45 | }() 46 | 47 | time.Sleep(3 * time.Second) 48 | process.Kill() 49 | 50 | time.Sleep(3 * time.Second) 51 | process.Kill() 52 | 53 | time.Sleep(3 * time.Second) 54 | process.Kill() 55 | 56 | time.Sleep(10 * time.Second) 57 | process.Kill() 58 | } 59 | 60 | func TestCloseExistProcess(t *testing.T) { 61 | CloseExistProcess("/Users/mylxsw/codes/golang/src/git.yunsom.cn/golang/php-server/tmp/s1/.fpm/php-fpm.pid") 62 | } 63 | -------------------------------------------------------------------------------- /fpm/signal_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package fpm 4 | 5 | import "syscall" 6 | 7 | // Reload reload php-fpm process 8 | func (proc *Process) Reload() error { 9 | return proc.cmd.Process.Signal(syscall.SIGUSR2) 10 | } 11 | -------------------------------------------------------------------------------- /fpm/signal_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package fpm 4 | 5 | import "fmt" 6 | 7 | // Reload reload php-fpm process 8 | func (proc *Process) Reload() error { 9 | return fmt.Errorf("windows not suppert reload signal(USR2)") 10 | } 11 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mylxsw/go-toolkit 2 | 3 | require ( 4 | github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 5 | github.com/dgrijalva/jwt-go v3.2.0+incompatible 6 | github.com/fatih/color v1.7.0 7 | github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect 8 | github.com/gorilla/context v1.1.1 9 | github.com/gorilla/mux v1.7.3 10 | github.com/mattn/go-colorable v0.1.2 // indirect 11 | github.com/mylxsw/asteria v0.0.0-20190730075526-1867e6bc4dbe 12 | github.com/mylxsw/coll v0.0.0-20190810120926-a7a6f0f4bae8 13 | github.com/mylxsw/container v0.0.0-20191208075953-c8ee6e3238cc 14 | github.com/smartystreets/assertions v1.0.1 // indirect 15 | github.com/stretchr/objx v0.2.0 // indirect 16 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect 17 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect 18 | golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect 19 | golang.org/x/text v0.3.2 // indirect 20 | golang.org/x/tools v0.0.0-20190802220118-1d1727260058 // indirect 21 | gopkg.in/ini.v1 v1.44.2 22 | ) 23 | 24 | go 1.13 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 h1:D21IyuvjDCshj1/qq+pCNd3VZOAEI9jy6Bi131YlXgI= 2 | github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 7 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 8 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= 9 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 10 | github.com/go-ini/ini v1.39.3 h1:y2UyknTfDmqZcBqdAHMt3zib4YT33TVtM6ABVrRVXQ0= 11 | github.com/go-ini/ini v1.39.3/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 12 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 13 | github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= 14 | github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 15 | github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 16 | github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= 17 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 18 | github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= 19 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 20 | github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= 21 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 22 | github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= 23 | github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 24 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 25 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 26 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 27 | github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= 28 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 29 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 30 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 31 | github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= 32 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 33 | github.com/mylxsw/asteria v0.0.0-20190717134529-a075d8075eec h1:C/cSpKJ2BbK1mo+dpVRDIFAaNd8spm5dBXwAPAjhHek= 34 | github.com/mylxsw/asteria v0.0.0-20190717134529-a075d8075eec/go.mod h1:Y7V0jLcTZeKt8bwscoeeofOVwX0krR1nm66sovdLP/o= 35 | github.com/mylxsw/asteria v0.0.0-20190723090831-c02d4e1dd24e h1:cVIuKoOC00LsMnP2dEIkyGZ+Mug/ta5LTjBZKN8ine4= 36 | github.com/mylxsw/asteria v0.0.0-20190723090831-c02d4e1dd24e/go.mod h1:yKtYUYKkYe2xOB6qqHW+NnoHd6zBFRk72NS/8V/dgwk= 37 | github.com/mylxsw/asteria v0.0.0-20190730075526-1867e6bc4dbe h1:JNV6IpUt1FnPkGFPB+FwfeC4jeb//yZ6WdO1/JKwRhY= 38 | github.com/mylxsw/asteria v0.0.0-20190730075526-1867e6bc4dbe/go.mod h1:yKtYUYKkYe2xOB6qqHW+NnoHd6zBFRk72NS/8V/dgwk= 39 | github.com/mylxsw/coll v0.0.0-20190810120926-a7a6f0f4bae8 h1:TtxSw54bx34zGgs7Y/VisH/sD5HOlopHbSavxTOzgko= 40 | github.com/mylxsw/coll v0.0.0-20190810120926-a7a6f0f4bae8/go.mod h1:Ugpjgv7bOSn1NXiPNHl92DdCGP2siWk50irFSyI+Hf8= 41 | github.com/mylxsw/container v0.0.0-20190810122756-1fd99a90f7c6 h1:wA8rXGj4ajvUHOPY3LHp1MQlB1v46J24PyaPG7MmxE8= 42 | github.com/mylxsw/container v0.0.0-20190810122756-1fd99a90f7c6/go.mod h1:v2QwNL+V2nI1o7naopTXXalpa1Y6b5E8lCwwANiYfyc= 43 | github.com/mylxsw/container v0.0.0-20191208075953-c8ee6e3238cc h1:xBh4hQSO+fcWa/bFLOjxO4huSgofBltDJ3KXPiZlHGo= 44 | github.com/mylxsw/container v0.0.0-20191208075953-c8ee6e3238cc/go.mod h1:v2QwNL+V2nI1o7naopTXXalpa1Y6b5E8lCwwANiYfyc= 45 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 46 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 47 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 48 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 49 | github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= 50 | github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c h1:Ho+uVpkel/udgjbwB5Lktg9BtvJSh2DT0Hi6LPSyI2w= 51 | github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= 52 | github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 53 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 54 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 55 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 56 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 57 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 58 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 59 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 60 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 61 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 62 | golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 63 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 64 | golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe h1:ay7inWg28/GEO1erz2KR0ywSgsw4yPHUw1egz2vGcN0= 65 | golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 66 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 67 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 68 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 69 | golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 70 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 71 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 72 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 73 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 74 | golang.org/x/tools v0.0.0-20190802220118-1d1727260058/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= 75 | gopkg.in/ini.v1 v1.39.3 h1:+LGDwGPQXrK1zLmDY5GMdgX7uNvs4iS+9fIRAGaDBbg= 76 | gopkg.in/ini.v1 v1.39.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 77 | gopkg.in/ini.v1 v1.44.2 h1:N6kNUPqiIyxP+s/aINPzRvNpcTVV30qLC0t6ZjZFlUU= 78 | gopkg.in/ini.v1 v1.44.2/go.mod h1:M3Cogqpuv0QCi3ExAY5V4uOt4qb/R3xZubo9m8lK5wg= 79 | -------------------------------------------------------------------------------- /jsonutils/flatten.go: -------------------------------------------------------------------------------- 1 | package jsonutils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | ) 9 | 10 | // JSONUtils a json utils object 11 | type JSONUtils struct { 12 | message []byte 13 | maxLevel int 14 | obj interface{} 15 | skipSimpleArray bool 16 | } 17 | 18 | // KvPair a kv pair 19 | type KvPair struct { 20 | Key string 21 | Value string 22 | } 23 | 24 | // New create a new json utils object and parse json to object 25 | func New(message []byte, maxLevel int, skipSimpleArray bool) (*JSONUtils, error) { 26 | var obj interface{} 27 | if err := json.Unmarshal(message, &obj); err != nil { 28 | return nil, err 29 | } 30 | 31 | return &JSONUtils{ 32 | message: message, 33 | obj: obj, 34 | maxLevel: maxLevel, 35 | skipSimpleArray: skipSimpleArray, 36 | }, nil 37 | } 38 | 39 | // ToKvPairsArray convert to an array with all kv pair 40 | func (ju *JSONUtils) ToKvPairsArray() []KvPair { 41 | return ju.createKvPairs(ju.obj, 1) 42 | } 43 | 44 | // ToKvPairs convert to a map with kv 45 | func (ju *JSONUtils) ToKvPairs() map[string]string { 46 | kvPairs := make(map[string]string) 47 | for _, kv := range ju.ToKvPairsArray() { 48 | kvPairs[kv.Key] = kv.Value 49 | } 50 | 51 | return kvPairs 52 | } 53 | 54 | func (ju *JSONUtils) createKvPairs(obj interface{}, level int) []KvPair { 55 | kvPairs := make([]KvPair, 0) 56 | 57 | objValue := reflect.ValueOf(obj) 58 | if !objValue.IsValid() { 59 | return kvPairs 60 | } 61 | 62 | objType := objValue.Type() 63 | 64 | switch objType.Kind() { 65 | case reflect.Map: 66 | for _, key := range objValue.MapKeys() { 67 | keyStr := fmt.Sprintf("%s", key) 68 | value := objValue.MapIndex(key).Interface() 69 | 70 | subValues := ju.recursiveSubValue(keyStr, value, level+1) 71 | if len(subValues) == 0 { 72 | kvPairs = append(kvPairs, KvPair{ 73 | Key: keyStr, 74 | Value: "(null)", 75 | }) 76 | } 77 | for _, kv := range subValues { 78 | kvPairs = append(kvPairs, kv) 79 | } 80 | } 81 | case reflect.Slice, reflect.Array: 82 | for i := 0; i < objValue.Len(); i++ { 83 | keyStr := fmt.Sprintf("[%d]", i) 84 | value := objValue.Index(i).Interface() 85 | 86 | subValues := ju.recursiveSubValue(keyStr, value, level+1) 87 | if len(subValues) == 0 { 88 | kvPairs = append(kvPairs, KvPair{ 89 | Key: keyStr, 90 | Value: "(null)", 91 | }) 92 | } 93 | 94 | for _, kv := range subValues { 95 | kvPairs = append(kvPairs, kv) 96 | } 97 | } 98 | default: 99 | } 100 | 101 | return kvPairs 102 | } 103 | 104 | func (ju *JSONUtils) recursiveSubValue(keyStr string, value interface{}, level int) []KvPair { 105 | kvPairs := make([]KvPair, 0) 106 | reflectValue := reflect.ValueOf(value) 107 | if !reflectValue.IsValid() { 108 | return kvPairs 109 | } 110 | 111 | valueType := reflectValue.Type().Kind() 112 | 113 | switch valueType { 114 | case reflect.Slice, reflect.Map, reflect.Array: 115 | if ju.maxLevel > 0 && level > ju.maxLevel { 116 | valueJSON, _ := json.Marshal(value) 117 | kvPairs = append(kvPairs, KvPair{ 118 | Key: keyStr, 119 | Value: string(valueJSON), 120 | }) 121 | 122 | break 123 | } 124 | 125 | // skip simple array 126 | if reflectValue.Len() > 0 && ju.skipSimpleArray && valueType != reflect.Map { 127 | subValue := reflect.ValueOf(reflectValue.Index(0).Interface()) 128 | subValueKind := subValue.Kind() 129 | if subValueKind != reflect.Map && subValueKind != reflect.Slice && subValueKind != reflect.Array { 130 | valueJSON, _ := json.Marshal(value) 131 | kvPairs = append(kvPairs, KvPair{ 132 | Key: keyStr, 133 | Value: string(valueJSON), 134 | }) 135 | 136 | break 137 | } 138 | } 139 | 140 | walkSub := ju.createKvPairs(value, level) 141 | // if sub value is empty, set value to [] or {} 142 | if len(walkSub) == 0 { 143 | emptyValue := "[]" 144 | if valueType == reflect.Map { 145 | emptyValue = "{}" 146 | } 147 | 148 | kvPairs = append(kvPairs, KvPair{ 149 | Key: keyStr, 150 | Value: emptyValue, 151 | }) 152 | } 153 | 154 | for _, kv := range walkSub { 155 | kvPairs = append(kvPairs, KvPair{ 156 | Key: keyStr + "." + kv.Key, 157 | Value: kv.Value, 158 | }) 159 | } 160 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 161 | kvPairs = append(kvPairs, KvPair{ 162 | Key: keyStr, 163 | Value: fmt.Sprintf("%d", value), 164 | }) 165 | case reflect.Float32, reflect.Float64: 166 | kvPairs = append(kvPairs, KvPair{ 167 | Key: keyStr, 168 | Value: strconv.FormatFloat(value.(float64), 'f', -1, 64), 169 | }) 170 | default: 171 | kvPairs = append(kvPairs, KvPair{ 172 | Key: keyStr, 173 | Value: fmt.Sprintf("%s", value), 174 | }) 175 | } 176 | 177 | return kvPairs 178 | } 179 | -------------------------------------------------------------------------------- /jsonutils/flatten_test.go: -------------------------------------------------------------------------------- 1 | package jsonutils_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/mylxsw/go-toolkit/jsonutils" 9 | ) 10 | 11 | var message1 = `{ 12 | "message": "ack_confirm", 13 | "context": { 14 | "msg": "层级消息内容", 15 | "sms": { 16 | "id": 44444, 17 | "app_id": 1, 18 | "template_params": { 19 | "username": "李逍遥", 20 | "password": "lixiaoyao", 21 | "gender": null, 22 | "created_at": "2018-11-12 13:47:55" 23 | }, 24 | "status": 1, 25 | "created_at": "2018-11-12 13:47:55", 26 | "updated_at": "2018-11-14 13:49:04" 27 | }, 28 | "ack": { 29 | "msg": "😄", 30 | "code": "6460" 31 | }, 32 | "file": "/webroot/your/project/Test.php:322" 33 | }, 34 | "level": null, 35 | "level_name": "ERROR", 36 | "channel": "custom_cmd", 37 | "datetime": "2018-11-16 13:51:01", 38 | "extra": { 39 | "ref": "5bee5ac564a71bbb33cai2jkk" 40 | } 41 | }` 42 | 43 | var message2 = `{ 44 | "message": null, 45 | "context": { 46 | "msg": null, 47 | "reason": "unknown", 48 | "extra": { 49 | "numbers": [], 50 | "numbers2": [1, 2, 3, 4, 5], 51 | "users": ["user1", "user2"], 52 | "mix": ["string1", 45], 53 | "nulls": [null, null, 5], 54 | "user": {} 55 | } 56 | } 57 | }` 58 | 59 | var message3 = `{ 60 | "messages": [ 61 | { 62 | "id": 123, 63 | "key": "xxx" 64 | }, 65 | { 66 | "id": 444, 67 | "key": "yyyy" 68 | } 69 | ] 70 | }` 71 | 72 | var message4 = `[ 73 | { 74 | "key": 123 75 | }, 76 | { 77 | "key": 444 78 | } 79 | ]` 80 | 81 | var message5 = `[13, 44, 55, 66]` 82 | 83 | func TestToKvPairs(t *testing.T) { 84 | ju, err := jsonutils.New([]byte(message1), 0, false) 85 | if err != nil { 86 | t.Errorf("parse json failed: %s", err.Error()) 87 | } 88 | 89 | kvPairs := ju.ToKvPairs() 90 | if len(kvPairs) == 0 { 91 | t.Error("convert to kv pairs failed") 92 | } 93 | 94 | if len(kvPairs) != 19 { 95 | t.Errorf("kv pairs not matched") 96 | } 97 | 98 | } 99 | 100 | func TestToKvPairsArray(t *testing.T) { 101 | ju, err := jsonutils.New([]byte(message1), 0, false) 102 | if err != nil { 103 | t.Errorf("parse json failed: %s", err.Error()) 104 | } 105 | 106 | kvPairs := ju.ToKvPairsArray() 107 | if len(kvPairs) != 19 { 108 | t.Errorf("kv pairs not matched") 109 | } 110 | } 111 | 112 | func TestNullValue(t *testing.T) { 113 | ju, err := jsonutils.New([]byte(message2), 0, false) 114 | if err != nil { 115 | t.Errorf("parse json failed: %s", err.Error()) 116 | } 117 | 118 | pairs := ju.ToKvPairs() 119 | if v, ok := pairs["context.msg"]; !ok || v != "(null)" { 120 | t.Errorf("kv pairs with null value test failed") 121 | } 122 | 123 | if v, ok := pairs["message"]; !ok || v != "(null)" { 124 | t.Errorf("kv pairs with null value test failed") 125 | } 126 | 127 | // for k, v := range pairs { 128 | // fmt.Printf("%s: %s\n", k, v) 129 | // } 130 | } 131 | 132 | func TestKvPairsWithLevelLimit(t *testing.T) { 133 | ju, err := jsonutils.New([]byte(message1), 2, false) 134 | if err != nil { 135 | t.Errorf("parse json failed: %s", err.Error()) 136 | } 137 | 138 | pairs := ju.ToKvPairsArray() 139 | for _, kv := range pairs { 140 | // fmt.Printf("%s : %s\n", kv.Key, kv.Value) 141 | if len(strings.Split(kv.Key, ".")) > 2 { 142 | t.Error("test kv pairs with level limit failed") 143 | } 144 | } 145 | } 146 | func TestNullValueSkipSimpleValue(t *testing.T) { 147 | ju, err := jsonutils.New([]byte(message2), 0, true) 148 | if err != nil { 149 | t.Errorf("parse json failed: %s", err.Error()) 150 | } 151 | 152 | pairs := ju.ToKvPairs() 153 | if v, ok := pairs["context.msg"]; !ok || v != "(null)" { 154 | t.Errorf("kv pairs with null value test failed") 155 | } 156 | 157 | if v, ok := pairs["message"]; !ok || v != "(null)" { 158 | t.Errorf("kv pairs with null value test failed") 159 | } 160 | 161 | // for k, v := range pairs { 162 | // fmt.Printf("%s: %s\n", k, v) 163 | // } 164 | } 165 | 166 | func TestComplexArrayValue(t *testing.T) { 167 | ju, err := jsonutils.New([]byte(message3), 2, true) 168 | if err != nil { 169 | t.Errorf("parse json failed: %s", err.Error()) 170 | } 171 | 172 | pairs := ju.ToKvPairsArray() 173 | // for _, kv := range pairs { 174 | // fmt.Printf("%s : %s\n", kv.Key, kv.Value) 175 | // } 176 | 177 | if len(pairs) != 2 { 178 | t.Errorf("kv pairs test failed for complex array") 179 | } 180 | } 181 | 182 | func TestRootArrayType(t *testing.T) { 183 | ju, err := jsonutils.New([]byte(message4), 2, true) 184 | if err != nil { 185 | t.Errorf("parse json failed: %s", err.Error()) 186 | } 187 | 188 | pairs := ju.ToKvPairsArray() 189 | 190 | for _, kv := range pairs { 191 | fmt.Printf("%s: %s\n", kv.Key, kv.Value) 192 | } 193 | } 194 | 195 | func TestRootArrayType2(t *testing.T) { 196 | ju, err := jsonutils.New([]byte(message5), 1, true) 197 | if err != nil { 198 | t.Errorf("parse json failed: %s", err.Error()) 199 | } 200 | 201 | pairs := ju.ToKvPairsArray() 202 | 203 | // for _, kv := range pairs { 204 | // fmt.Printf("%s: %s\n", kv.Key, kv.Value) 205 | // } 206 | 207 | if len(pairs) <= 0 { 208 | t.Errorf("test failed") 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /jwt/jwt.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "time" 7 | 8 | jwtlib "github.com/dgrijalva/jwt-go" 9 | ) 10 | 11 | // ErrTokenInvalid token不合法错误 12 | var ErrTokenInvalid = errors.New("Jwt token invalid") 13 | 14 | // Token token对象 15 | type Token struct { 16 | key string 17 | } 18 | 19 | // Claims payload内容 20 | type Claims map[string]interface{} 21 | 22 | // New 创建一个Token 23 | func New(key string) *Token { 24 | 25 | return &Token{ 26 | key: key, 27 | } 28 | } 29 | 30 | // CreateToken 创建Token 31 | func (jt *Token) CreateToken(payloads Claims, expire time.Duration) string { 32 | 33 | // 设置标准payload，指定有效时间范围 34 | // 签署时间允许误差60秒 35 | if expire != 0 { 36 | payloads["exp"] = time.Now().Add(expire * time.Second).Unix() 37 | } 38 | payloads["iat"] = time.Now().Add(-60 * time.Second).Unix() 39 | 40 | token := jwtlib.NewWithClaims(jwtlib.SigningMethodHS256, jwtlib.MapClaims(payloads)) 41 | str, err := token.SignedString([]byte(jt.key)) 42 | if err != nil { 43 | return "" 44 | } 45 | 46 | return str 47 | } 48 | 49 | // ParseToken 解析Token 50 | func (jt *Token) ParseToken(tokenString string) (Claims, error) { 51 | token, err := jwtlib.Parse(tokenString, func(token *jwtlib.Token) (interface{}, error) { 52 | if _, ok := token.Method.(*jwtlib.SigningMethodHMAC); !ok { 53 | return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) 54 | } 55 | 56 | return []byte(jt.key), nil 57 | }) 58 | 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | if claims, ok := token.Claims.(jwtlib.MapClaims); ok && token.Valid { 64 | return Claims(claims), nil 65 | } 66 | 67 | return nil, ErrTokenInvalid 68 | } 69 | -------------------------------------------------------------------------------- /jwt/jwt_test.go: -------------------------------------------------------------------------------- 1 | package jwt 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var key = "123456" 8 | 9 | func TestCreateToken(t *testing.T) { 10 | token := New(key) 11 | tokenString := token.CreateToken(Claims{ 12 | "uid": 12344, 13 | }, -600) 14 | 15 | if tokenString == "" { 16 | t.Errorf("Create token failed") 17 | } 18 | } 19 | 20 | func TestParseToken(t *testing.T) { 21 | 22 | var uid int = 123456 23 | 24 | { 25 | // 检查token可用 26 | token := New(key) 27 | 28 | claims, err := token.ParseToken(token.CreateToken(Claims{ 29 | "uid": uid, 30 | }, 500)) 31 | if err != nil { 32 | t.Errorf("Parse token failed: %v", err) 33 | } 34 | 35 | if int(claims["uid"].(float64)) != uid { 36 | t.Errorf("Uid not match") 37 | } 38 | 39 | } 40 | // 不合法的token 41 | 42 | // 1. key错误 43 | { 44 | token := New(key) 45 | tokenStr := token.CreateToken(Claims{ 46 | "uid": uid, 47 | }, 500) 48 | 49 | token2 := New("aabbcc") 50 | _, err := token2.ParseToken(tokenStr) 51 | if err == nil || err.Error() != "signature is invalid" { 52 | t.Errorf("Check invalid token failed") 53 | } 54 | } 55 | 56 | // 2. token 过期 57 | { 58 | token := New(key) 59 | tokenStr := token.CreateToken(Claims{ 60 | "uid": uid, 61 | }, -5000) 62 | 63 | _, err := token.ParseToken(tokenStr) 64 | if err == nil || err.Error() != "Token is expired" { 65 | t.Errorf("Check invalid token failed") 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /misc/assert.go: -------------------------------------------------------------------------------- 1 | package misc 2 | 3 | func AssertError(err error) { 4 | if err != nil { 5 | panic(err) 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /network/ip.go: -------------------------------------------------------------------------------- 1 | package network 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | // IsPublicIP 判断ip是否是公网IP 8 | func IsPublicIP(ip net.IP) bool { 9 | if ip.IsLoopback() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast() { 10 | return false 11 | } 12 | if ip4 := ip.To4(); ip4 != nil { 13 | switch true { 14 | case ip4[0] == 10: 15 | return false 16 | case ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31: 17 | return false 18 | case ip4[0] == 192 && ip4[1] == 168: 19 | return false 20 | default: 21 | return true 22 | } 23 | } 24 | return false 25 | } 26 | 27 | // IsIPv4 判断是否是IPv4 28 | func IsIPv4(ip net.IP) bool { 29 | return ip.To4() != nil 30 | } 31 | 32 | // GetLanIPs 获取局域网本地IP地址列表 33 | func GetLanIPs() (ips []string, err error) { 34 | addrs, err := net.InterfaceAddrs() 35 | if err != nil { 36 | return 37 | } 38 | 39 | for _, address := range addrs { 40 | if ipnet, ok := address.(*net.IPNet); ok && 41 | !ipnet.IP.IsLoopback() && 42 | !ipnet.IP.IsLinkLocalMulticast() && 43 | !ipnet.IP.IsLinkLocalUnicast() && 44 | IsIPv4(ipnet.IP) && 45 | !IsPublicIP(ipnet.IP) { 46 | 47 | ips = append(ips, ipnet.IP.String()) 48 | } 49 | } 50 | 51 | return 52 | } 53 | -------------------------------------------------------------------------------- /next/fastcgi.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Light Code Labs, LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fastcgi has middleware that acts as a FastCGI client. Requests 16 | // that get forwarded to FastCGI stop the middleware execution chain. 17 | // The most common use for this package is to serve PHP websites via php-fpm. 18 | 19 | package next 20 | 21 | import ( 22 | "context" 23 | "errors" 24 | "io" 25 | "net" 26 | "net/http" 27 | "os" 28 | "path" 29 | "path/filepath" 30 | "strconv" 31 | "strings" 32 | "sync/atomic" 33 | "time" 34 | ) 35 | 36 | // Handler is a middleware type that can handle requests as a FastCGI client. 37 | type Handler struct { 38 | Rules []Rule 39 | Root string 40 | FileSys http.FileSystem 41 | OverrideHostHeader string 42 | 43 | // These are sent to CGI scripts in env variables 44 | SoftwareName string 45 | SoftwareVersion string 46 | ServerName string 47 | ServerPort string 48 | } 49 | 50 | // ServeHTTP satisfies the httpserver.Handler interface. 51 | func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { 52 | for _, rule := range h.Rules { 53 | // The path must also be allowed (not ignored). 54 | if !rule.AllowedPath(r.URL.Path) { 55 | continue 56 | } 57 | 58 | // In addition to matching the path, a request must meet some 59 | // other criteria before being proxied as FastCGI. For example, 60 | // we probably want to exclude static assets (CSS, JS, images...) 61 | // but we also want to be flexible for the script we proxy to. 62 | 63 | fpath := rule.IndexFiles[0] + r.URL.Path 64 | //fpath := rule.Path 65 | 66 | if idx, ok := IndexFile(h.FileSys, fpath, rule.IndexFiles); ok { 67 | fpath = idx 68 | // Index file present. 69 | // If request path cannot be split, return error. 70 | if !rule.canSplit(fpath) { 71 | return http.StatusInternalServerError, ErrIndexMissingSplit 72 | } 73 | } else { 74 | // No index file present. 75 | // If request path cannot be split, ignore request. 76 | if !rule.canSplit(fpath) { 77 | continue 78 | } 79 | } 80 | 81 | // These criteria work well in this order for PHP sites 82 | if !h.exists(fpath) || fpath[len(fpath)-1] == '/' || strings.HasSuffix(fpath, rule.Ext) { 83 | 84 | // Create environment for CGI script 85 | env, err := h.buildEnv(r, rule, fpath) 86 | if err != nil { 87 | return http.StatusInternalServerError, err 88 | } 89 | 90 | // Connect to FastCGI gateway 91 | address, err := rule.Address() 92 | if err != nil { 93 | return http.StatusBadGateway, err 94 | } 95 | network, address := parseAddress(address) 96 | 97 | ctx := context.Background() 98 | if rule.ConnectTimeout > 0 { 99 | var cancel context.CancelFunc 100 | ctx, cancel = context.WithTimeout(ctx, rule.ConnectTimeout) 101 | defer cancel() 102 | } 103 | 104 | fcgiBackend, err := DialContext(ctx, network, address) 105 | if err != nil { 106 | return http.StatusBadGateway, err 107 | } 108 | defer fcgiBackend.Close() 109 | 110 | // read/write timeouts 111 | if err := fcgiBackend.SetReadTimeout(rule.ReadTimeout); err != nil { 112 | return http.StatusInternalServerError, err 113 | } 114 | if err := fcgiBackend.SetSendTimeout(rule.SendTimeout); err != nil { 115 | return http.StatusInternalServerError, err 116 | } 117 | 118 | var resp *http.Response 119 | 120 | var contentLength int64 121 | // if ContentLength is already set 122 | if r.ContentLength > 0 { 123 | contentLength = r.ContentLength 124 | } else { 125 | contentLength, _ = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) 126 | } 127 | switch r.Method { 128 | case "HEAD": 129 | resp, err = fcgiBackend.Head(env) 130 | case "GET": 131 | resp, err = fcgiBackend.Get(env, r.Body, contentLength) 132 | case "OPTIONS": 133 | resp, err = fcgiBackend.Options(env) 134 | default: 135 | resp, err = fcgiBackend.Post(env, r.Method, r.Header.Get("Content-Type"), r.Body, contentLength) 136 | } 137 | 138 | if resp != nil && resp.Body != nil { 139 | defer resp.Body.Close() 140 | } 141 | 142 | if err != nil { 143 | if err, ok := err.(net.Error); ok && err.Timeout() { 144 | return http.StatusGatewayTimeout, err 145 | } else if err != io.EOF { 146 | return http.StatusBadGateway, err 147 | } 148 | } 149 | 150 | // Write response header 151 | writeHeader(w, resp) 152 | 153 | // Write the response body 154 | _, err = io.Copy(w, resp.Body) 155 | if err != nil { 156 | return http.StatusBadGateway, err 157 | } 158 | 159 | // Log any stderr output from upstream 160 | if fcgiBackend.stderr.Len() != 0 { 161 | // Remove trailing newline, error logger already does this. 162 | err = LogError(strings.TrimSuffix(fcgiBackend.stderr.String(), "\n")) 163 | } 164 | 165 | // Normally we would return the status code if it is an error status (>= 400), 166 | // however, upstream FastCGI apps don't know about our contract and have 167 | // probably already written an error page. So we just return 0, indicating 168 | // that the response body is already written. However, we do return any 169 | // error value so it can be logged. 170 | // Note that the proxy middleware works the same way, returning status=0. 171 | return 0, err 172 | } 173 | } 174 | 175 | return 0, nil 176 | } 177 | 178 | // parseAddress returns the network and address of fcgiAddress. 179 | // The first string is the network, "tcp" or "unix", implied from the scheme and address. 180 | // The second string is fcgiAddress, with scheme prefixes removed. 181 | // The two returned strings can be used as parameters to the Dial() function. 182 | func parseAddress(fcgiAddress string) (string, string) { 183 | // check if address has tcp scheme explicitly set 184 | if strings.HasPrefix(fcgiAddress, "tcp://") { 185 | return "tcp", fcgiAddress[len("tcp://"):] 186 | } 187 | // check if address has fastcgi scheme explicitly set 188 | if strings.HasPrefix(fcgiAddress, "fastcgi://") { 189 | return "tcp", fcgiAddress[len("fastcgi://"):] 190 | } 191 | // check if unix socket 192 | if trim := strings.HasPrefix(fcgiAddress, "unix"); strings.HasPrefix(fcgiAddress, "/") || trim { 193 | if trim { 194 | return "unix", fcgiAddress[len("unix:"):] 195 | } 196 | return "unix", fcgiAddress 197 | } 198 | // default case, a plain tcp address with no scheme 199 | return "tcp", fcgiAddress 200 | } 201 | 202 | func writeHeader(w http.ResponseWriter, r *http.Response) { 203 | for key, vals := range r.Header { 204 | for _, val := range vals { 205 | w.Header().Add(key, val) 206 | } 207 | } 208 | w.WriteHeader(r.StatusCode) 209 | } 210 | 211 | func (h Handler) exists(path string) bool { 212 | if _, err := os.Stat(h.Root + path); err == nil { 213 | return true 214 | } 215 | return false 216 | } 217 | 218 | // buildEnv returns a set of CGI environment variables for the request. 219 | func (h Handler) buildEnv(r *http.Request, rule Rule, fpath string) (map[string]string, error) { 220 | var env map[string]string 221 | 222 | // Separate remote IP and port; more lenient than net.SplitHostPort 223 | var ip, port string 224 | if idx := strings.LastIndex(r.RemoteAddr, ":"); idx > -1 { 225 | ip = r.RemoteAddr[:idx] 226 | port = r.RemoteAddr[idx+1:] 227 | } else { 228 | ip = r.RemoteAddr 229 | } 230 | 231 | // Remove [] from IPv6 addresses 232 | ip = strings.Replace(ip, "[", "", 1) 233 | ip = strings.Replace(ip, "]", "", 1) 234 | 235 | // Split path in preparation for env variables. 236 | // Previous rule.canSplit checks ensure this can never be -1. 237 | splitPos := rule.splitPos(fpath) 238 | 239 | // Request has the extension; path was split successfully 240 | docURI := fpath[:splitPos+len(rule.SplitPath)] 241 | pathInfo := fpath[splitPos+len(rule.SplitPath):] 242 | scriptName := fpath 243 | 244 | // Strip PATH_INFO from SCRIPT_NAME 245 | scriptName = strings.TrimSuffix(scriptName, pathInfo) 246 | 247 | // SCRIPT_FILENAME is the absolute path of SCRIPT_NAME 248 | scriptFilename := filepath.Join(rule.Root, scriptName) 249 | 250 | // Add vhost path prefix to scriptName. Otherwise, some PHP software will 251 | // have difficulty discovering its URL. 252 | // pathPrefix, _ := r.Context().Value(caddy.CtxKey("path_prefix")).(string) 253 | // scriptName = path.Join(pathPrefix, scriptName) 254 | 255 | requestScheme := "http" 256 | if r.TLS != nil { 257 | requestScheme = "https" 258 | } 259 | 260 | hostHeader := r.Host 261 | if h.OverrideHostHeader != "" { 262 | hostHeader = h.OverrideHostHeader 263 | } 264 | 265 | // Some variables are unused but cleared explicitly to prevent 266 | // the parent environment from interfering. 267 | env = map[string]string{ 268 | // Variables defined in CGI 1.1 spec 269 | "AUTH_TYPE": "", // Not used 270 | "CONTENT_LENGTH": r.Header.Get("Content-Length"), 271 | "CONTENT_TYPE": r.Header.Get("Content-Type"), 272 | "GATEWAY_INTERFACE": "CGI/1.1", 273 | "PATH_INFO": pathInfo, 274 | "QUERY_STRING": r.URL.RawQuery, 275 | "REMOTE_ADDR": ip, 276 | "REMOTE_HOST": ip, // For speed, remote host lookups disabled 277 | "REMOTE_PORT": port, 278 | "REMOTE_IDENT": "", // Not used 279 | "REQUEST_METHOD": r.Method, 280 | "REQUEST_SCHEME": requestScheme, 281 | "SERVER_NAME": h.ServerName, 282 | "SERVER_PORT": h.ServerPort, 283 | "SERVER_PROTOCOL": r.Proto, 284 | "SERVER_SOFTWARE": h.SoftwareName + "/" + h.SoftwareVersion, 285 | 286 | // Other variables 287 | "DOCUMENT_ROOT": rule.Root, 288 | "DOCUMENT_URI": docURI, 289 | "HTTP_HOST": hostHeader, // added here, since not always part of headers 290 | "REQUEST_URI": r.RequestURI, 291 | "SCRIPT_FILENAME": scriptFilename, 292 | "SCRIPT_NAME": scriptName, 293 | } 294 | 295 | // compliance with the CGI specification requires that 296 | // PATH_TRANSLATED should only exist if PATH_INFO is defined. 297 | // Info: https://www.ietf.org/rfc/rfc3875 Page 14 298 | if env["PATH_INFO"] != "" { 299 | env["PATH_TRANSLATED"] = filepath.Join(rule.Root, pathInfo) // Info: http://www.oreilly.com/openbook/cgi/ch02_04.html 300 | } 301 | 302 | // Add all HTTP headers to env variables 303 | for field, val := range r.Header { 304 | header := strings.ToUpper(field) 305 | header = headerNameReplacer.Replace(header) 306 | env["HTTP_"+header] = strings.Join(val, ", ") 307 | } 308 | return env, nil 309 | } 310 | 311 | // Rule represents a FastCGI handling rule. 312 | // It is parsed from the fastcgi directive in the Caddyfile, see setup.go. 313 | type Rule struct { 314 | // The base path to match. Required. 315 | Path string 316 | 317 | // upstream load balancer 318 | balancer 319 | 320 | // Always process files with this extension with fastcgi. 321 | Ext string 322 | 323 | // Use this directory as the fastcgi root directory. Defaults to the root 324 | // directory of the parent virtual host. 325 | Root string 326 | 327 | // The path in the URL will be split into two, with the first piece ending 328 | // with the value of SplitPath. The first piece will be assumed as the 329 | // actual resource (CGI script) name, and the second piece will be set to 330 | // PATH_INFO for the CGI script to use. 331 | SplitPath string 332 | 333 | // If the URL ends with '/' (which indicates a directory), these index 334 | // files will be tried instead. 335 | IndexFiles []string 336 | 337 | // Environment Variables 338 | EnvVars [][2]string 339 | 340 | // Ignored paths 341 | IgnoredSubPaths []string 342 | 343 | // The duration used to set a deadline when connecting to an upstream. 344 | ConnectTimeout time.Duration 345 | 346 | // The duration used to set a deadline when reading from the FastCGI server. 347 | ReadTimeout time.Duration 348 | 349 | // The duration used to set a deadline when sending to the FastCGI server. 350 | SendTimeout time.Duration 351 | } 352 | 353 | // balancer is a fastcgi upstream load balancer. 354 | type balancer interface { 355 | // Address picks an upstream address from the 356 | // underlying load balancer. 357 | Address() (string, error) 358 | } 359 | 360 | // roundRobin is a round robin balancer for fastcgi upstreams. 361 | type roundRobin struct { 362 | // Known Go bug: https://golang.org/pkg/sync/atomic/#pkg-note-BUG 363 | // must be first field for 64 bit alignment 364 | // on x86 and arm. 365 | index int64 366 | addresses []string 367 | } 368 | 369 | func (r *roundRobin) Address() (string, error) { 370 | index := atomic.AddInt64(&r.index, 1) % int64(len(r.addresses)) 371 | return r.addresses[index], nil 372 | } 373 | 374 | // canSplit checks if path can split into two based on rule.SplitPath. 375 | func (r Rule) canSplit(path string) bool { 376 | return r.splitPos(path) >= 0 377 | } 378 | 379 | // splitPos returns the index where path should be split 380 | // based on rule.SplitPath. 381 | func (r Rule) splitPos(path string) int { 382 | return strings.Index(strings.ToLower(path), strings.ToLower(r.SplitPath)) 383 | } 384 | 385 | // AllowedPath checks if requestPath is not an ignored path. 386 | func (r Rule) AllowedPath(requestPath string) bool { 387 | return true 388 | } 389 | 390 | var ( 391 | headerNameReplacer = strings.NewReplacer(" ", "_", "-", "_") 392 | // ErrIndexMissingSplit describes an index configuration error. 393 | ErrIndexMissingSplit = errors.New("configured index file(s) must include split value") 394 | ) 395 | 396 | // LogError is a non fatal error that allows requests to go through. 397 | type LogError string 398 | 399 | // Error satisfies error interface. 400 | func (l LogError) Error() string { 401 | return string(l) 402 | } 403 | 404 | // IndexFile looks for a file in /root/fpath/indexFile for each string 405 | // in indexFiles. If an index file is found, it returns the root-relative 406 | // path to the file and true. If no index file is found, empty string 407 | // and false is returned. fpath must end in a forward slash '/' 408 | // otherwise no index files will be tried (directory paths must end 409 | // in a forward slash according to HTTP). 410 | // 411 | // All paths passed into and returned from this function use '/' as the 412 | // path separator, just like URLs. IndexFle handles path manipulation 413 | // internally for systems that use different path separators. 414 | func IndexFile(root http.FileSystem, fpath string, indexFiles []string) (string, bool) { 415 | if fpath[len(fpath)-1] != '/' || root == nil { 416 | return "", false 417 | } 418 | for _, indexFile := range indexFiles { 419 | // func (http.FileSystem).Open wants all paths separated by "/", 420 | // regardless of operating system convention, so use 421 | // path.Join instead of filepath.Join 422 | fp := path.Join(fpath, indexFile) 423 | f, err := root.Open(fp) 424 | if err == nil { 425 | f.Close() 426 | return fp, true 427 | } 428 | } 429 | return "", false 430 | } 431 | -------------------------------------------------------------------------------- /next/fcgiclient.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Light Code Labs, LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fastcgi has middleware that acts as a FastCGI client. Requests 16 | // that get forwarded to FastCGI stop the middleware execution chain. 17 | // The most common use for this package is to serve PHP websites via php-fpm. 18 | 19 | package next 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "context" 25 | "encoding/binary" 26 | "errors" 27 | "io" 28 | "io/ioutil" 29 | "mime/multipart" 30 | "net" 31 | "net/http" 32 | "net/http/httputil" 33 | "net/textproto" 34 | "net/url" 35 | "os" 36 | "path/filepath" 37 | "strconv" 38 | "strings" 39 | "sync" 40 | "time" 41 | ) 42 | 43 | // FCGIListenSockFileno describes listen socket file number. 44 | const FCGIListenSockFileno uint8 = 0 45 | 46 | // FCGIHeaderLen describes header length. 47 | const FCGIHeaderLen uint8 = 8 48 | 49 | // Version1 describes the version. 50 | const Version1 uint8 = 1 51 | 52 | // FCGINullRequestID describes the null request ID. 53 | const FCGINullRequestID uint8 = 0 54 | 55 | // FCGIKeepConn describes keep connection mode. 56 | const FCGIKeepConn uint8 = 1 57 | 58 | const ( 59 | // BeginRequest is the begin request flag. 60 | BeginRequest uint8 = iota + 1 61 | // AbortRequest is the abort request flag. 62 | AbortRequest 63 | // EndRequest is the end request flag. 64 | EndRequest 65 | // Params is the parameters flag. 66 | Params 67 | // Stdin is the standard input flag. 68 | Stdin 69 | // Stdout is the standard output flag. 70 | Stdout 71 | // Stderr is the standard error flag. 72 | Stderr 73 | // Data is the data flag. 74 | Data 75 | // GetValues is the get values flag. 76 | GetValues 77 | // GetValuesResult is the get values result flag. 78 | GetValuesResult 79 | // UnknownType is the unknown type flag. 80 | UnknownType 81 | // MaxType is the maximum type flag. 82 | MaxType = UnknownType 83 | ) 84 | 85 | const ( 86 | // Responder is the responder flag. 87 | Responder uint8 = iota + 1 88 | // Authorizer is the authorizer flag. 89 | Authorizer 90 | // Filter is the filter flag. 91 | Filter 92 | ) 93 | 94 | const ( 95 | // RequestComplete is the completed request flag. 96 | RequestComplete uint8 = iota 97 | // CantMultiplexConns is the multiplexed connections flag. 98 | CantMultiplexConns 99 | // Overloaded is the overloaded flag. 100 | Overloaded 101 | // UnknownRole is the unknown role flag. 102 | UnknownRole 103 | ) 104 | 105 | const ( 106 | // MaxConns is the maximum connections flag. 107 | MaxConns string = "MAX_CONNS" 108 | // MaxRequests is the maximum requests flag. 109 | MaxRequests string = "MAX_REQS" 110 | // MultiplexConns is the multiplex connections flag. 111 | MultiplexConns string = "MPXS_CONNS" 112 | ) 113 | 114 | const ( 115 | maxWrite = 65500 // 65530 may work, but for compatibility 116 | maxPad = 255 117 | ) 118 | 119 | type header struct { 120 | Version uint8 121 | Type uint8 122 | ID uint16 123 | ContentLength uint16 124 | PaddingLength uint8 125 | Reserved uint8 126 | } 127 | 128 | // for padding so we don't have to allocate all the time 129 | // not synchronized because we don't care what the contents are 130 | var pad [maxPad]byte 131 | 132 | func (h *header) init(recType uint8, reqID uint16, contentLength int) { 133 | h.Version = 1 134 | h.Type = recType 135 | h.ID = reqID 136 | h.ContentLength = uint16(contentLength) 137 | h.PaddingLength = uint8(-contentLength & 7) 138 | } 139 | 140 | type record struct { 141 | h header 142 | rbuf []byte 143 | } 144 | 145 | func (rec *record) read(r io.Reader) (buf []byte, err error) { 146 | if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil { 147 | return 148 | } 149 | if rec.h.Version != 1 { 150 | err = errors.New("fcgi: invalid header version") 151 | return 152 | } 153 | if rec.h.Type == EndRequest { 154 | err = io.EOF 155 | return 156 | } 157 | n := int(rec.h.ContentLength) + int(rec.h.PaddingLength) 158 | if len(rec.rbuf) < n { 159 | rec.rbuf = make([]byte, n) 160 | } 161 | if _, err = io.ReadFull(r, rec.rbuf[:n]); err != nil { 162 | return 163 | } 164 | buf = rec.rbuf[:int(rec.h.ContentLength)] 165 | 166 | return 167 | } 168 | 169 | // FCGIClient implements a FastCGI client, which is a standard for 170 | // interfacing external applications with Web servers. 171 | type FCGIClient struct { 172 | mutex sync.Mutex 173 | rwc io.ReadWriteCloser 174 | h header 175 | buf bytes.Buffer 176 | stderr bytes.Buffer 177 | keepAlive bool 178 | reqID uint16 179 | readTimeout time.Duration 180 | sendTimeout time.Duration 181 | } 182 | 183 | // DialWithDialerContext connects to the fcgi responder at the specified network address, using custom net.Dialer 184 | // and a context. 185 | // See func net.Dial for a description of the network and address parameters. 186 | func DialWithDialerContext(ctx context.Context, network, address string, dialer net.Dialer) (fcgi *FCGIClient, err error) { 187 | var conn net.Conn 188 | conn, err = dialer.DialContext(ctx, network, address) 189 | if err != nil { 190 | return 191 | } 192 | 193 | fcgi = &FCGIClient{ 194 | rwc: conn, 195 | keepAlive: false, 196 | reqID: 1, 197 | } 198 | 199 | return 200 | } 201 | 202 | // DialContext is like Dial but passes ctx to dialer.Dial. 203 | func DialContext(ctx context.Context, network, address string) (fcgi *FCGIClient, err error) { 204 | return DialWithDialerContext(ctx, network, address, net.Dialer{}) 205 | } 206 | 207 | // Dial connects to the fcgi responder at the specified network address, using default net.Dialer. 208 | // See func net.Dial for a description of the network and address parameters. 209 | func Dial(network, address string) (fcgi *FCGIClient, err error) { 210 | return DialContext(context.Background(), network, address) 211 | } 212 | 213 | // Close closes fcgi connnection 214 | func (c *FCGIClient) Close() { 215 | c.rwc.Close() 216 | } 217 | 218 | func (c *FCGIClient) writeRecord(recType uint8, content []byte) (err error) { 219 | c.mutex.Lock() 220 | defer c.mutex.Unlock() 221 | c.buf.Reset() 222 | c.h.init(recType, c.reqID, len(content)) 223 | if err := binary.Write(&c.buf, binary.BigEndian, c.h); err != nil { 224 | return err 225 | } 226 | if _, err := c.buf.Write(content); err != nil { 227 | return err 228 | } 229 | if _, err := c.buf.Write(pad[:c.h.PaddingLength]); err != nil { 230 | return err 231 | } 232 | _, err = c.rwc.Write(c.buf.Bytes()) 233 | return err 234 | } 235 | 236 | func (c *FCGIClient) writeBeginRequest(role uint16, flags uint8) error { 237 | b := [8]byte{byte(role >> 8), byte(role), flags} 238 | return c.writeRecord(BeginRequest, b[:]) 239 | } 240 | 241 | func (c *FCGIClient) writeEndRequest(appStatus int, protocolStatus uint8) error { 242 | b := make([]byte, 8) 243 | binary.BigEndian.PutUint32(b, uint32(appStatus)) 244 | b[4] = protocolStatus 245 | return c.writeRecord(EndRequest, b) 246 | } 247 | 248 | func (c *FCGIClient) writePairs(recType uint8, pairs map[string]string) error { 249 | w := newWriter(c, recType) 250 | b := make([]byte, 8) 251 | nn := 0 252 | for k, v := range pairs { 253 | m := 8 + len(k) + len(v) 254 | if m > maxWrite { 255 | // param data size exceed 65535 bytes" 256 | vl := maxWrite - 8 - len(k) 257 | v = v[:vl] 258 | } 259 | n := encodeSize(b, uint32(len(k))) 260 | n += encodeSize(b[n:], uint32(len(v))) 261 | m = n + len(k) + len(v) 262 | if (nn + m) > maxWrite { 263 | w.Flush() 264 | nn = 0 265 | } 266 | nn += m 267 | if _, err := w.Write(b[:n]); err != nil { 268 | return err 269 | } 270 | if _, err := w.WriteString(k); err != nil { 271 | return err 272 | } 273 | if _, err := w.WriteString(v); err != nil { 274 | return err 275 | } 276 | } 277 | w.Close() 278 | return nil 279 | } 280 | 281 | func encodeSize(b []byte, size uint32) int { 282 | if size > 127 { 283 | size |= 1 << 31 284 | binary.BigEndian.PutUint32(b, size) 285 | return 4 286 | } 287 | b[0] = byte(size) 288 | return 1 289 | } 290 | 291 | // bufWriter encapsulates bufio.Writer but also closes the underlying stream when 292 | // Closed. 293 | type bufWriter struct { 294 | closer io.Closer 295 | *bufio.Writer 296 | } 297 | 298 | func (w *bufWriter) Close() error { 299 | if err := w.Writer.Flush(); err != nil { 300 | w.closer.Close() 301 | return err 302 | } 303 | return w.closer.Close() 304 | } 305 | 306 | func newWriter(c *FCGIClient, recType uint8) *bufWriter { 307 | s := &streamWriter{c: c, recType: recType} 308 | w := bufio.NewWriterSize(s, maxWrite) 309 | return &bufWriter{s, w} 310 | } 311 | 312 | // streamWriter abstracts out the separation of a stream into discrete records. 313 | // It only writes maxWrite bytes at a time. 314 | type streamWriter struct { 315 | c *FCGIClient 316 | recType uint8 317 | } 318 | 319 | func (w *streamWriter) Write(p []byte) (int, error) { 320 | nn := 0 321 | for len(p) > 0 { 322 | n := len(p) 323 | if n > maxWrite { 324 | n = maxWrite 325 | } 326 | if err := w.c.writeRecord(w.recType, p[:n]); err != nil { 327 | return nn, err 328 | } 329 | nn += n 330 | p = p[n:] 331 | } 332 | return nn, nil 333 | } 334 | 335 | func (w *streamWriter) Close() error { 336 | // send empty record to close the stream 337 | return w.c.writeRecord(w.recType, nil) 338 | } 339 | 340 | type streamReader struct { 341 | c *FCGIClient 342 | buf []byte 343 | } 344 | 345 | func (w *streamReader) Read(p []byte) (n int, err error) { 346 | 347 | if len(p) > 0 { 348 | if len(w.buf) == 0 { 349 | 350 | // filter outputs for error log 351 | for { 352 | rec := &record{} 353 | var buf []byte 354 | buf, err = rec.read(w.c.rwc) 355 | if err != nil { 356 | return 357 | } 358 | // standard error output 359 | if rec.h.Type == Stderr { 360 | w.c.stderr.Write(buf) 361 | continue 362 | } 363 | w.buf = buf 364 | break 365 | } 366 | } 367 | 368 | n = len(p) 369 | if n > len(w.buf) { 370 | n = len(w.buf) 371 | } 372 | copy(p, w.buf[:n]) 373 | w.buf = w.buf[n:] 374 | } 375 | 376 | return 377 | } 378 | 379 | // Do made the request and returns a io.Reader that translates the data read 380 | // from fcgi responder out of fcgi packet before returning it. 381 | func (c *FCGIClient) Do(p map[string]string, req io.Reader) (r io.Reader, err error) { 382 | err = c.writeBeginRequest(uint16(Responder), 0) 383 | if err != nil { 384 | return 385 | } 386 | 387 | err = c.writePairs(Params, p) 388 | if err != nil { 389 | return 390 | } 391 | 392 | body := newWriter(c, Stdin) 393 | if req != nil { 394 | io.Copy(body, req) 395 | } 396 | body.Close() 397 | 398 | r = &streamReader{c: c} 399 | return 400 | } 401 | 402 | // clientCloser is a io.ReadCloser. It wraps a io.Reader with a Closer 403 | // that closes FCGIClient connection. 404 | type clientCloser struct { 405 | *FCGIClient 406 | io.Reader 407 | } 408 | 409 | func (f clientCloser) Close() error { return f.rwc.Close() } 410 | 411 | // Request returns a HTTP Response with Header and Body 412 | // from fcgi responder 413 | func (c *FCGIClient) Request(p map[string]string, req io.Reader) (resp *http.Response, err error) { 414 | r, err := c.Do(p, req) 415 | if err != nil { 416 | return 417 | } 418 | 419 | rb := bufio.NewReader(r) 420 | tp := textproto.NewReader(rb) 421 | resp = new(http.Response) 422 | 423 | // Parse the response headers. 424 | mimeHeader, err := tp.ReadMIMEHeader() 425 | if err != nil && err != io.EOF { 426 | return 427 | } 428 | resp.Header = http.Header(mimeHeader) 429 | 430 | if resp.Header.Get("Status") != "" { 431 | statusParts := strings.SplitN(resp.Header.Get("Status"), " ", 2) 432 | resp.StatusCode, err = strconv.Atoi(statusParts[0]) 433 | if err != nil { 434 | return 435 | } 436 | if len(statusParts) > 1 { 437 | resp.Status = statusParts[1] 438 | } 439 | 440 | } else { 441 | resp.StatusCode = http.StatusOK 442 | } 443 | 444 | // TODO: fixTransferEncoding ? 445 | resp.TransferEncoding = resp.Header["Transfer-Encoding"] 446 | resp.ContentLength, _ = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) 447 | 448 | if chunked(resp.TransferEncoding) { 449 | resp.Body = clientCloser{c, httputil.NewChunkedReader(rb)} 450 | } else { 451 | resp.Body = clientCloser{c, ioutil.NopCloser(rb)} 452 | } 453 | return 454 | } 455 | 456 | // Get issues a GET request to the fcgi responder. 457 | func (c *FCGIClient) Get(p map[string]string, body io.Reader, l int64) (resp *http.Response, err error) { 458 | 459 | p["REQUEST_METHOD"] = "GET" 460 | p["CONTENT_LENGTH"] = strconv.FormatInt(l, 10) 461 | 462 | return c.Request(p, body) 463 | } 464 | 465 | // Head issues a HEAD request to the fcgi responder. 466 | func (c *FCGIClient) Head(p map[string]string) (resp *http.Response, err error) { 467 | 468 | p["REQUEST_METHOD"] = "HEAD" 469 | p["CONTENT_LENGTH"] = "0" 470 | 471 | return c.Request(p, nil) 472 | } 473 | 474 | // Options issues an OPTIONS request to the fcgi responder. 475 | func (c *FCGIClient) Options(p map[string]string) (resp *http.Response, err error) { 476 | 477 | p["REQUEST_METHOD"] = "OPTIONS" 478 | p["CONTENT_LENGTH"] = "0" 479 | 480 | return c.Request(p, nil) 481 | } 482 | 483 | // Post issues a POST request to the fcgi responder. with request body 484 | // in the format that bodyType specified 485 | func (c *FCGIClient) Post(p map[string]string, method string, bodyType string, body io.Reader, l int64) (resp *http.Response, err error) { 486 | if p == nil { 487 | p = make(map[string]string) 488 | } 489 | 490 | p["REQUEST_METHOD"] = strings.ToUpper(method) 491 | 492 | if len(p["REQUEST_METHOD"]) == 0 || p["REQUEST_METHOD"] == "GET" { 493 | p["REQUEST_METHOD"] = "POST" 494 | } 495 | 496 | p["CONTENT_LENGTH"] = strconv.FormatInt(l, 10) 497 | if len(bodyType) > 0 { 498 | p["CONTENT_TYPE"] = bodyType 499 | } else { 500 | p["CONTENT_TYPE"] = "application/x-www-form-urlencoded" 501 | } 502 | 503 | return c.Request(p, body) 504 | } 505 | 506 | // PostForm issues a POST to the fcgi responder, with form 507 | // as a string key to a list values (url.Values) 508 | func (c *FCGIClient) PostForm(p map[string]string, data url.Values) (resp *http.Response, err error) { 509 | body := bytes.NewReader([]byte(data.Encode())) 510 | return c.Post(p, "POST", "application/x-www-form-urlencoded", body, int64(body.Len())) 511 | } 512 | 513 | // PostFile issues a POST to the fcgi responder in multipart(RFC 2046) standard, 514 | // with form as a string key to a list values (url.Values), 515 | // and/or with file as a string key to a list file path. 516 | func (c *FCGIClient) PostFile(p map[string]string, data url.Values, file map[string]string) (resp *http.Response, err error) { 517 | buf := &bytes.Buffer{} 518 | writer := multipart.NewWriter(buf) 519 | bodyType := writer.FormDataContentType() 520 | 521 | for key, val := range data { 522 | for _, v0 := range val { 523 | err = writer.WriteField(key, v0) 524 | if err != nil { 525 | return 526 | } 527 | } 528 | } 529 | 530 | for key, val := range file { 531 | fd, e := os.Open(val) 532 | if e != nil { 533 | return nil, e 534 | } 535 | defer fd.Close() 536 | 537 | part, e := writer.CreateFormFile(key, filepath.Base(val)) 538 | if e != nil { 539 | return nil, e 540 | } 541 | _, err = io.Copy(part, fd) 542 | if err != nil { 543 | return 544 | } 545 | } 546 | 547 | err = writer.Close() 548 | if err != nil { 549 | return 550 | } 551 | 552 | return c.Post(p, "POST", bodyType, buf, int64(buf.Len())) 553 | } 554 | 555 | // SetReadTimeout sets the read timeout for future calls that read from the 556 | // fcgi responder. A zero value for t means no timeout will be set. 557 | func (c *FCGIClient) SetReadTimeout(t time.Duration) error { 558 | if conn, ok := c.rwc.(net.Conn); ok && t != 0 { 559 | return conn.SetReadDeadline(time.Now().Add(t)) 560 | } 561 | return nil 562 | } 563 | 564 | // SetSendTimeout sets the read timeout for future calls that send data to 565 | // the fcgi responder. A zero value for t means no timeout will be set. 566 | func (c *FCGIClient) SetSendTimeout(t time.Duration) error { 567 | if conn, ok := c.rwc.(net.Conn); ok && t != 0 { 568 | return conn.SetWriteDeadline(time.Now().Add(t)) 569 | } 570 | return nil 571 | } 572 | 573 | // Checks whether chunked is part of the encodings stack 574 | func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } 575 | -------------------------------------------------------------------------------- /next/handler.go: -------------------------------------------------------------------------------- 1 | package next 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net/http" 7 | "path/filepath" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/mylxsw/asteria/log" 12 | ) 13 | 14 | // CreateHTTPHandler create a http handler for request processing 15 | func CreateHTTPHandler(config *Config) http.Handler { 16 | rootDir := filepath.Dir(config.EndpointFile) 17 | 18 | handler := Handler{ 19 | Rules: config.Rules, 20 | Root: rootDir, 21 | FileSys: http.Dir(rootDir), 22 | SoftwareName: config.SoftwareName, 23 | SoftwareVersion: config.SoftwareVersion, 24 | ServerName: config.ServerIP, 25 | ServerPort: strconv.Itoa(config.ServerPort), 26 | OverrideHostHeader: config.OverrideHostHeader, 27 | } 28 | 29 | return &HTTPHandler{ 30 | handler: handler, 31 | config: config, 32 | errorLogHandler: config.ErrorLogHandler, 33 | } 34 | } 35 | 36 | // RequestLogHandler request log handler func 37 | type RequestLogHandler func(rc *RequestContext) 38 | type ErrorResponseHandler func(w http.ResponseWriter, r *http.Request, code int, err error) 39 | type ErrorLogHandler func(err error) 40 | 41 | // Config config object for create a handler 42 | type Config struct { 43 | EndpointFile string 44 | ServerIP string 45 | ServerPort int 46 | SoftwareName string 47 | SoftwareVersion string 48 | RequestLogHandler RequestLogHandler 49 | ErrorResponseHandler ErrorResponseHandler 50 | ErrorLogHandler ErrorLogHandler 51 | Rules []Rule 52 | OverrideHostHeader string 53 | } 54 | 55 | // HTTPHandler http request handler wrapper 56 | type HTTPHandler struct { 57 | handler Handler 58 | errorLogHandler ErrorLogHandler 59 | config *Config 60 | } 61 | 62 | // RequestContext request context information 63 | type RequestContext struct { 64 | UA string 65 | Method string 66 | Referer string 67 | Headers http.Header 68 | URI string 69 | Body string 70 | Consume float64 71 | Code int 72 | Error string 73 | } 74 | 75 | // ToMap convert the requestContext to a map 76 | func (rc *RequestContext) ToMap() map[string]interface{} { 77 | return map[string]interface{}{ 78 | "ua": rc.UA, 79 | "method": rc.Method, 80 | "referer": rc.Referer, 81 | "headers": rc.Headers, 82 | "uri": rc.URI, 83 | "body": rc.Body, 84 | "consume": rc.Consume, 85 | "code": rc.Code, 86 | "error": rc.Error, 87 | } 88 | } 89 | 90 | // ServeHTTP implements http.Handler interface 91 | func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 92 | var statusCode = 200 93 | respWriter := NewResponseWriter(w, func(code int) { 94 | statusCode = code 95 | }) 96 | 97 | body, _ := ioutil.ReadAll(r.Body) 98 | _ = r.Body.Close() 99 | r.Body = ioutil.NopCloser(bytes.NewBuffer(body)) 100 | 101 | var err error 102 | defer func(startTime time.Time) { 103 | consume := time.Now().Sub(startTime) 104 | if h.config.RequestLogHandler != nil { 105 | go func() { 106 | defer func() { 107 | if err := recover(); err != nil { 108 | log.Errorf("request log handler has some error: %v", err) 109 | } 110 | }() 111 | 112 | errorMsg := "" 113 | if err != nil { 114 | errorMsg = err.Error() 115 | } 116 | 117 | h.config.RequestLogHandler(&RequestContext{ 118 | UA: r.UserAgent(), 119 | Method: r.Method, 120 | Referer: r.Referer(), 121 | Headers: r.Header, 122 | URI: r.RequestURI, 123 | Body: string(body), 124 | Consume: consume.Seconds(), 125 | Code: statusCode, 126 | Error: errorMsg, 127 | }) 128 | }() 129 | } 130 | 131 | }(time.Now()) 132 | 133 | code, err := h.handler.ServeHTTP(respWriter, r) 134 | if err != nil { 135 | if code == 0 && h.errorLogHandler != nil { 136 | h.errorLogHandler(err) 137 | } else { 138 | log.Errorf("request failed, code=%d, err=%s", code, err.Error()) 139 | } 140 | } 141 | 142 | if code != 0 { 143 | respWriter.WriteHeader(code) 144 | if h.config.ErrorResponseHandler != nil { 145 | h.config.ErrorResponseHandler(respWriter, r, code, err) 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /next/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # Next 3 | 4 | 该模块实现了与php-fpm进程通信的功能，将http请求转发给php-fpm进程处理。 5 | 6 | // CreateHandler 创建服务处理器 7 | func CreateHandler(params *Config, network, address string) http.Handler { 8 | fpmAddr := address 9 | if network == "unix" { 10 | fpmAddr = fmt.Sprintf("%s:%s", network, address) 11 | } 12 | 13 | rootDir := filepath.Dir(params.EndpointFile) 14 | 15 | conf := next.Config{ 16 | EndpointFile: params.EndpointFile, 17 | ServerIP: params.ServiceIP, 18 | ServerPort: params.ServicePort, 19 | SoftwareName: "php-server", 20 | SoftwareVersion: "0.0.1", 21 | Rules: []next.Rule{next.NewPHPRule(rootDir, []string{fpmAddr})}, 22 | RequestLogHandler: func(rc *next.RequestContext) { 23 | var message bytes.Buffer 24 | if err := params.AccessLogTemplate.Execute(&message, rc); err != nil { 25 | log.Module("server").Errorf("invalid log format: %s", err.Error()) 26 | } else { 27 | if params.Debug { 28 | log.Module("server.request"). 29 | WithContext(rc.ToMap()).Debugf(message.String()) 30 | } else { 31 | log.Module("server.request").Debugf(message.String()) 32 | } 33 | } 34 | }, 35 | } 36 | 37 | return next.CreateHttpHandler(&conf) 38 | } 39 | 40 | 41 | http.HandleFunc("/", CreateHandler(config, network, address).ServeHTTP) 42 | srv := &http.Server{Handler: http.DefaultServeMux} 43 | go func() { 44 | if err := srv.Serve(listener); err != nil { 45 | log.Debugf("The http server has stopped: %v", err) 46 | } 47 | }() 48 | 49 | 50 | > 该模块大部分代码是从Caddy中提取出来的，参考 https://github.com/mholt/caddy/tree/master/caddyhttp/fastcgi -------------------------------------------------------------------------------- /next/response.go: -------------------------------------------------------------------------------- 1 | package next 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | // ResponseWriter is a response wrapper 8 | type ResponseWriter struct { 9 | http.ResponseWriter 10 | writeCodeListener func(code int) 11 | } 12 | 13 | // NewResponseWriter create a new ResposneWriter wrapper 14 | func NewResponseWriter(res http.ResponseWriter, writeCodeListener func(code int)) *ResponseWriter { 15 | return &ResponseWriter{res, writeCodeListener} 16 | } 17 | 18 | // Header atisfy the http.ResponseWriter interface 19 | func (w ResponseWriter) Header() http.Header { 20 | return w.ResponseWriter.Header() 21 | } 22 | 23 | // Write send content to response 24 | func (w ResponseWriter) Write(data []byte) (int, error) { 25 | return w.ResponseWriter.Write(data) 26 | } 27 | 28 | // WriteHeader send a response code to client 29 | func (w ResponseWriter) WriteHeader(statusCode int) { 30 | w.writeCodeListener(statusCode) 31 | w.ResponseWriter.WriteHeader(statusCode) 32 | } 33 | -------------------------------------------------------------------------------- /next/rule.go: -------------------------------------------------------------------------------- 1 | package next 2 | 3 | // NewPHPRule create a rule for php 4 | func NewPHPRule(rootDir string, addresses []string) Rule { 5 | return Rule{ 6 | Root: rootDir, 7 | Path: "/", 8 | balancer: &roundRobin{addresses: addresses, index: -1}, 9 | Ext: ".php", 10 | SplitPath: ".php", 11 | IndexFiles: []string{"index.php"}, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /period_job/doc.go: -------------------------------------------------------------------------------- 1 | /** 2 | Package period_job 实现了周期性任务的定时触发执行。 3 | 4 | manager := period_job.NewManager(ctx, cc) 5 | 6 | job1 := &DemoJob{} 7 | job2 := &DemoJob{} 8 | 9 | manager.Run("Test", job1, 10*time.Millisecond) 10 | manager.Run("Test2", job2, 20*time.Millisecond) 11 | 12 | manager.Wait() 13 | 14 | */ 15 | package period_job 16 | -------------------------------------------------------------------------------- /period_job/job.go: -------------------------------------------------------------------------------- 1 | package period_job 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/mylxsw/asteria/log" 9 | "github.com/mylxsw/container" 10 | ) 11 | 12 | // Job is a interface for a job 13 | type Job interface { 14 | Handle() 15 | } 16 | 17 | // Manager 周期性任务管理器 18 | type Manager struct { 19 | container container.Container 20 | ctx context.Context 21 | pauseJobs map[string]bool 22 | lock sync.RWMutex 23 | 24 | wg sync.WaitGroup 25 | } 26 | 27 | // NewManager 创建一个Manager 28 | func NewManager(ctx context.Context, cc container.Container) *Manager { 29 | return &Manager{ 30 | container: cc, 31 | ctx: ctx, 32 | pauseJobs: make(map[string]bool), 33 | } 34 | } 35 | 36 | // Paused return whether the named job has been paused 37 | func (jm *Manager) Paused(name string) bool { 38 | jm.lock.RLock() 39 | defer jm.lock.RUnlock() 40 | 41 | paused, _ := jm.pauseJobs[name] 42 | return paused 43 | } 44 | 45 | func (jm *Manager) Pause(name string, pause bool) { 46 | jm.lock.Lock() 47 | defer jm.lock.Unlock() 48 | 49 | jm.pauseJobs[name] = pause 50 | } 51 | 52 | // Run 启动周期性任务循环 53 | func (jm *Manager) Run(name string, job Job, interval time.Duration) { 54 | log.Debugf("Job %s running...", name) 55 | 56 | jm.wg.Add(1) 57 | 58 | go func() { 59 | globalTicker := time.NewTicker(interval) 60 | defer func() { 61 | globalTicker.Stop() 62 | jm.wg.Done() 63 | }() 64 | 65 | for { 66 | select { 67 | case <-globalTicker.C: 68 | if jm.Paused(name) { 69 | continue 70 | } 71 | 72 | func() { 73 | defer func() { 74 | if err := recover(); err != nil { 75 | log.Errorf("Job %s has some error：%s", name, err) 76 | } 77 | }() 78 | if err := jm.container.Resolve(job.Handle); err != nil { 79 | log.Errorf("Job %s failed: %s", name, err) 80 | } 81 | }() 82 | case <-jm.ctx.Done(): 83 | log.Debugf("Job %s stopped", name) 84 | return 85 | } 86 | } 87 | }() 88 | } 89 | 90 | // Wait 等待所有任务结束 91 | func (jm *Manager) Wait() { 92 | jm.wg.Wait() 93 | } 94 | -------------------------------------------------------------------------------- /period_job/job_test.go: -------------------------------------------------------------------------------- 1 | package period_job_test 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/mylxsw/container" 10 | "github.com/mylxsw/go-toolkit/period_job" 11 | ) 12 | 13 | type DemoJob struct { 14 | lock sync.Mutex 15 | counter int 16 | } 17 | 18 | func (job *DemoJob) Handle() { 19 | job.lock.Lock() 20 | defer job.lock.Unlock() 21 | 22 | job.counter++ 23 | } 24 | 25 | func (job *DemoJob) Count() int { 26 | job.lock.Lock() 27 | defer job.lock.Unlock() 28 | 29 | return job.counter 30 | } 31 | 32 | func TestJob(t *testing.T) { 33 | cc := container.New() 34 | 35 | ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) 36 | defer cancel() 37 | 38 | manager := period_job.NewManager(ctx, cc) 39 | 40 | job1 := &DemoJob{} 41 | job2 := &DemoJob{} 42 | 43 | manager.Run("Test", job1, 10*time.Millisecond) 44 | manager.Run("Test2", job2, 20*time.Millisecond) 45 | 46 | manager.Wait() 47 | 48 | if job1.Count() <= job2.Count() { 49 | t.Error("test failed") 50 | } 51 | 52 | if job1.Count() < 5 { 53 | t.Error("test failed") 54 | } 55 | 56 | if job2.Count() < 3 { 57 | t.Error("test failed") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pidfile/pidfile.go: -------------------------------------------------------------------------------- 1 | // Package pidfile provides structure and helper functions to create and remove 2 | // PID file. A PID file is usually a file used to store the process ID of a 3 | // running process. 4 | package pidfile 5 | 6 | import ( 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | // PIDFile is a file used to store the process ID of a running process. 15 | type PIDFile struct { 16 | path string 17 | } 18 | 19 | func checkPIDFileAlreadyExists(path string) error { 20 | if pidByte, err := ioutil.ReadFile(path); err == nil { 21 | pidString := strings.TrimSpace(string(pidByte)) 22 | if pid, err := strconv.Atoi(pidString); err == nil { 23 | if processExists(pid) { 24 | return fmt.Errorf("pid [%s] file found: %s", string(pidByte), path) 25 | } 26 | } 27 | 28 | } 29 | return nil 30 | } 31 | 32 | // New creates a PIDfile using the specified path. 33 | func New(path string) (*PIDFile, error) { 34 | if err := checkPIDFileAlreadyExists(path); err != nil { 35 | return nil, err 36 | } 37 | if err := ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", os.Getpid())), 0644); err != nil { 38 | return nil, err 39 | } 40 | 41 | return &PIDFile{path: path}, nil 42 | } 43 | 44 | // Remove removes the PIDFile. 45 | func (file PIDFile) Remove() error { 46 | if err := os.Remove(file.path); err != nil { 47 | return err 48 | } 49 | return nil 50 | } 51 | 52 | // ReadPIDFile 读取pid文件 53 | func ReadPIDFile(pidfile string) (pid int, err error) { 54 | f, err := os.Open(pidfile) 55 | if err != nil { 56 | return 57 | } 58 | 59 | b, err := ioutil.ReadAll(f) 60 | if err != nil { 61 | return 62 | } 63 | 64 | pid64, err := strconv.ParseInt(string(b), 10, 64) 65 | pid = int(pid64) 66 | return 67 | } 68 | -------------------------------------------------------------------------------- /pidfile/pidfile_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | 3 | package pidfile 4 | 5 | import ( 6 | "syscall" 7 | ) 8 | 9 | func processExists(pid int) bool { 10 | // OS X does not have a proc filesystem. 11 | // Use kill -0 pid to judge if the process exists. 12 | err := syscall.Kill(pid, 0) 13 | if err != nil { 14 | return false 15 | } 16 | 17 | return true 18 | } 19 | -------------------------------------------------------------------------------- /pidfile/pidfile_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows,!darwin 2 | 3 | package pidfile 4 | 5 | import ( 6 | "os" 7 | "path/filepath" 8 | "strconv" 9 | ) 10 | 11 | func processExists(pid int) bool { 12 | if _, err := os.Stat(filepath.Join("/proc", strconv.Itoa(pid))); err == nil { 13 | return true 14 | } 15 | return false 16 | } 17 | -------------------------------------------------------------------------------- /pidfile/pidfile_windows.go: -------------------------------------------------------------------------------- 1 | package pidfile 2 | 3 | import "syscall" 4 | 5 | const ( 6 | processQueryLimitedInformation = 0x1000 7 | 8 | stillActive = 259 9 | ) 10 | 11 | func processExists(pid int) bool { 12 | h, err := syscall.OpenProcess(processQueryLimitedInformation, false, uint32(pid)) 13 | if err != nil { 14 | return false 15 | } 16 | var c uint32 17 | err = syscall.GetExitCodeProcess(h, &c) 18 | syscall.Close(h) 19 | if err != nil { 20 | return c == stillActive 21 | } 22 | return true 23 | } 24 | -------------------------------------------------------------------------------- /process/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package process 是一个进程管理器，类似于supervisor，能够监控进程的运行状态，自动重启失败进程。 3 | 4 | manager := NewManager(3*time.Second, func(logType OutputType, line string, process *Process) { 5 | color.Green("%s[%s] => %s\n", process.GetName(), logType, line) 6 | }) 7 | manager.AddProgram("test", "/bin/sleep 1", 5, "") 8 | manager.AddProgram("prometheus", "/bin/echo Hello", 1, "") 9 | 10 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 11 | defer cancel() 12 | 13 | manager.Watch(ctx) 14 | */ 15 | package process 16 | -------------------------------------------------------------------------------- /process/manager.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/mylxsw/asteria/log" 8 | ) 9 | 10 | // Manager is process manager 11 | type Manager struct { 12 | programs map[string]*Program 13 | restartProcess chan *Process 14 | closeTimeout time.Duration 15 | processOutputFunc OutputHandler 16 | } 17 | 18 | // NewManager create a new process manager 19 | func NewManager(closeTimeout time.Duration, processOutputFunc OutputHandler) *Manager { 20 | return &Manager{ 21 | programs: make(map[string]*Program), 22 | closeTimeout: closeTimeout, 23 | processOutputFunc: processOutputFunc, 24 | } 25 | } 26 | 27 | // AddProgram add a new program to manager 28 | func (manager *Manager) AddProgram(name string, command string, procNum int, username string) { 29 | manager.programs[name] = NewProgram(name, command, username, procNum).initProcesses(manager.processOutputFunc) 30 | } 31 | 32 | // Watch start watch process 33 | func (manager *Manager) Watch(ctx context.Context) { 34 | 35 | manager.restartProcess = make(chan *Process) 36 | defer func() { 37 | // close restartProcess channel to prevent goroutine leak 38 | close(manager.restartProcess) 39 | }() 40 | 41 | for _, program := range manager.programs { 42 | for _, proc := range program.processes { 43 | go manager.startProcess(proc, 0) 44 | } 45 | } 46 | 47 | for { 48 | select { 49 | case process := <-manager.restartProcess: 50 | go manager.startProcess(process, process.retryDelayTime()) 51 | case <-ctx.Done(): 52 | log.Debug("it's time to close all processes...") 53 | for _, program := range manager.programs { 54 | for _, proc := range program.processes { 55 | proc.stop(manager.closeTimeout) 56 | } 57 | } 58 | return 59 | } 60 | } 61 | } 62 | 63 | func (manager *Manager) startProcess(process *Process, delay time.Duration) { 64 | if delay > 0 { 65 | log.Debugf("process %s will start after %.2fs", process.GetName(), delay.Seconds()) 66 | } 67 | 68 | process.lock.Lock() 69 | defer process.lock.Unlock() 70 | 71 | process.timer = time.AfterFunc(delay, func() { 72 | process.removeTimer() 73 | 74 | defer func() { 75 | if err := recover(); err != nil { 76 | // do nothing 77 | } 78 | }() 79 | 80 | log.Debugf("process %s starting...", process.GetName()) 81 | restartSignal := <-process.start() 82 | 83 | manager.restartProcess <- restartSignal 84 | }) 85 | 86 | } 87 | 88 | // Programs return all programs 89 | func (manager *Manager) Programs() map[string]*Program { 90 | return manager.programs 91 | } 92 | -------------------------------------------------------------------------------- /process/manager_test.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/fatih/color" 9 | ) 10 | 11 | func TestManager(t *testing.T) { 12 | 13 | manager := NewManager(3*time.Second, func(logType OutputType, line string, process *Process) { 14 | color.Green("%s[%s] => %s\n", process.GetName(), logType, line) 15 | }) 16 | manager.AddProgram("test", "/bin/sleep 1", 5, "") 17 | manager.AddProgram("prometheus", "/bin/echo Hello", 1, "") 18 | 19 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 20 | defer cancel() 21 | 22 | manager.Watch(ctx) 23 | } 24 | 25 | //func printInspections(manager *Manager) { 26 | // var template = "%-8s %-10s %-10s %-10s %-10s %-10s %-10s %-20s %-20s\n" 27 | // fmt.Println() 28 | // fmt.Printf(color.New(color.BgBlue).Sprint(template), "pid", "name", "running", "alive", "status", "user", "tried", "uptime", "command") 29 | // for _, program := range manager.Programs() { 30 | // for _, insp := range program.inspections() { 31 | // pid := "-" 32 | // if insp.PID > 0 { 33 | // pid = strconv.Itoa(insp.PID) 34 | // } 35 | // 36 | // uptime := "-" 37 | // if !insp.Uptime.IsZero() { 38 | // uptime = insp.Uptime.Format("2006-01-02 15:04:05") 39 | // } 40 | // 41 | // runningState := strconv.FormatBool(insp.IsRunning) 42 | // if insp.IsRunning { 43 | // runningState = color.GreenString("%-10s", "ok") 44 | // } else { 45 | // runningState = color.RedString("%-10s", "failed") 46 | // } 47 | // 48 | // fmt.Printf( 49 | // template, 50 | // pid, 51 | // strWithDefault(insp.Name, "-"), 52 | // runningState, 53 | // strWithDefault(fmt.Sprintf("%.4fs", insp.AliveTime), "-"), 54 | // strWithDefault(insp.Status, "-"), 55 | // strWithDefault(insp.User, "-"), 56 | // strWithDefault(strconv.Itoa(insp.TriedCount), "-"), 57 | // uptime, 58 | // strWithDefault(fmt.Sprintf("%s %s", insp.Command, insp.Args), "-"), 59 | // ) 60 | // } 61 | // } 62 | // 63 | // fmt.Println() 64 | //} 65 | // 66 | //func strWithDefault(str string, defaultVal string) string { 67 | // if str == "" { 68 | // return defaultVal 69 | // } 70 | // 71 | // return str 72 | //} 73 | -------------------------------------------------------------------------------- /process/process.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "os/user" 10 | "strings" 11 | "sync" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/mylxsw/asteria/log" 16 | ) 17 | 18 | // OutputType command output type 19 | type OutputType string 20 | 21 | const ( 22 | // LogTypeStderr stderr output 23 | LogTypeStderr = OutputType("stderr") 24 | // LogTypeStdout stdout output 25 | LogTypeStdout = OutputType("stdout") 26 | ) 27 | 28 | // OutputHandler process output handler 29 | type OutputHandler func(logType OutputType, line string, process *Process) 30 | 31 | // Process is a program instance 32 | type Process struct { 33 | name string // process name 34 | command string // the command to execute 35 | args []string // the arguments for command 36 | user string // the user to run the command 37 | uid string // the user id to run the command 38 | pid int 39 | 40 | *exec.Cmd 41 | stat chan *Process 42 | lastAliveTime time.Duration 43 | timer *time.Timer 44 | lock sync.Mutex 45 | outputHandler OutputHandler 46 | lastErrorMessage string // last error message 47 | } 48 | 49 | // GetPID get process pid 50 | func (process *Process) GetPID() int { 51 | process.lock.Lock() 52 | defer process.lock.Unlock() 53 | 54 | return process.pid 55 | } 56 | 57 | // SetPID update process pid 58 | func (process *Process) SetPID(pid int) { 59 | process.lock.Lock() 60 | defer process.lock.Unlock() 61 | 62 | process.pid = pid 63 | } 64 | 65 | // GetName get process name 66 | func (process *Process) GetName() string { 67 | process.lock.Lock() 68 | defer process.lock.Unlock() 69 | 70 | return process.name 71 | } 72 | 73 | // GetUser get process user 74 | func (process *Process) GetUser() string { 75 | process.lock.Lock() 76 | defer process.lock.Unlock() 77 | 78 | return process.user 79 | } 80 | 81 | // GetCommand get command 82 | func (process *Process) GetCommand() string { 83 | process.lock.Lock() 84 | defer process.lock.Unlock() 85 | 86 | return process.command 87 | } 88 | 89 | // GetArgs get command args 90 | func (process *Process) GetArgs() []string { 91 | process.lock.Lock() 92 | defer process.lock.Unlock() 93 | 94 | return process.args 95 | } 96 | 97 | // GetLastErrorMessage get last error message 98 | func (process *Process) GetLastErrorMessage() string { 99 | process.lock.Lock() 100 | defer process.lock.Unlock() 101 | 102 | return process.lastErrorMessage 103 | } 104 | 105 | // SetLastErrorMessage update last error message 106 | func (process *Process) SetLastErrorMessage(msg string) { 107 | process.lock.Lock() 108 | defer process.lock.Unlock() 109 | 110 | process.lastErrorMessage = msg 111 | } 112 | 113 | // NewProcess create a new process 114 | func NewProcess(name string, command string, args []string, username string) *Process { 115 | process := Process{ 116 | name: name, 117 | command: command, 118 | args: args, 119 | user: username, 120 | stat: make(chan *Process), 121 | } 122 | 123 | // need root privilege to set user or group, because setuid and setgid are privileged calls 124 | if username != "" { 125 | sysUser, err := user.Lookup(username) 126 | if err != nil { 127 | log.Warningf("lookup user %s failed: %s", username, err.Error()) 128 | } else { 129 | process.uid = sysUser.Uid 130 | } 131 | } 132 | 133 | return &process 134 | } 135 | 136 | // setOutputFunc set a function to receive process output 137 | func (process *Process) setOutputFunc(f OutputHandler) *Process { 138 | process.outputHandler = f 139 | 140 | return process 141 | } 142 | 143 | // Start start the process 144 | func (process *Process) start() <-chan *Process { 145 | go func() { 146 | startTime := time.Now() 147 | 148 | defer func() { 149 | process.SetPID(0) 150 | process.lastAliveTime = time.Now().Sub(startTime) 151 | 152 | log.Warningf("process %s finished", process.name) 153 | process.stat <- process 154 | }() 155 | 156 | cmd := process.createCmd() 157 | 158 | if process.outputHandler != nil { 159 | stdoutPipe, _ := cmd.StdoutPipe() 160 | stderrPipe, _ := cmd.StderrPipe() 161 | 162 | go process.consoleLog(LogTypeStdout, &stdoutPipe) 163 | go process.consoleLog(LogTypeStderr, &stderrPipe) 164 | } 165 | 166 | if err := cmd.Start(); err != nil { 167 | log.Errorf("process %s start failed: %s", process.name, err.Error()) 168 | process.SetLastErrorMessage(err.Error()) 169 | return 170 | } 171 | 172 | process.SetPID(cmd.Process.Pid) 173 | 174 | if err := cmd.Wait(); err != nil { 175 | log.Warningf("process %s stopped with error : %s", process.name, err.Error()) 176 | process.SetLastErrorMessage(err.Error()) 177 | } 178 | }() 179 | 180 | return process.stat 181 | } 182 | 183 | // retryDelayTime get the next retry delay 184 | func (process *Process) retryDelayTime() time.Duration { 185 | if process.lastAliveTime.Seconds() < 5 { 186 | return 5 * time.Second 187 | } 188 | 189 | return 0 190 | } 191 | 192 | // RemoveTimer remove the timer 193 | func (process *Process) removeTimer() { 194 | process.lock.Lock() 195 | defer process.lock.Unlock() 196 | 197 | if process.timer != nil { 198 | process.timer.Stop() 199 | process.timer = nil 200 | } 201 | } 202 | 203 | func (process *Process) stop(timeout time.Duration) { 204 | process.removeTimer() 205 | 206 | pid := process.GetPID() 207 | name := process.GetName() 208 | 209 | process.lock.Lock() 210 | defer process.lock.Unlock() 211 | 212 | if pid <= 0 { 213 | log.Warningf("process %s with pid=%d is invalid", name, pid) 214 | return 215 | } 216 | 217 | proc, err := os.FindProcess(pid) 218 | if err != nil { 219 | log.Warningf("process %s with pid=%d doesn't exist", name, pid) 220 | return 221 | } 222 | 223 | stopped := make(chan interface{}) 224 | go func() { 225 | proc.Signal(syscall.SIGTERM) 226 | close(stopped) 227 | 228 | log.Debugf("process %s gracefully stopped", name) 229 | }() 230 | 231 | select { 232 | case <-stopped: 233 | return 234 | case <-time.After(timeout): 235 | proc.Signal(syscall.SIGKILL) 236 | log.Debugf("process %s forced stopped", name) 237 | } 238 | } 239 | 240 | func (process *Process) consoleLog(logType OutputType, input *io.ReadCloser) error { 241 | reader := bufio.NewReader(*input) 242 | for { 243 | line, err := reader.ReadString('\n') 244 | if err != nil || io.EOF == err { 245 | if err != io.EOF { 246 | return fmt.Errorf("process %s output failed: %s", process.GetName(), err.Error()) 247 | } 248 | break 249 | } 250 | 251 | if process.outputHandler != nil { 252 | process.outputHandler(logType, strings.Trim(line, "\n"), process) 253 | } 254 | } 255 | 256 | return nil 257 | } 258 | -------------------------------------------------------------------------------- /process/process_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package process 4 | 5 | import ( 6 | "os/exec" 7 | "strconv" 8 | "syscall" 9 | ) 10 | 11 | func (process *Process) createCmd() *exec.Cmd { 12 | cmd := exec.Command(process.GetCommand(), process.GetArgs()...) 13 | // 这里就不重新创建进程组了，创建新的进程组，在当前程序意外退出后，启动的外部进程时无法自动关闭的 14 | // cmd.SysProcAttr = &syscall.SysProcAttr{ 15 | // Setpgid: true, 16 | // } 17 | 18 | if process.uid != "" { 19 | cmd.SysProcAttr.Credential = createCredential(process.uid) 20 | } 21 | 22 | return cmd 23 | } 24 | 25 | func createCredential(uid string) *syscall.Credential { 26 | credential := syscall.Credential{} 27 | if uid != "" { 28 | uidVal, _ := strconv.Atoi(uid) 29 | credential.Uid = uint32(uidVal) 30 | } 31 | 32 | return &credential 33 | } 34 | -------------------------------------------------------------------------------- /process/process_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package process 4 | 5 | import ( 6 | "os/exec" 7 | ) 8 | 9 | func (process *Process) createCmd() *exec.Cmd { 10 | cmd := exec.Command(process.GetCommand(), process.GetArgs()...) 11 | return cmd 12 | } 13 | -------------------------------------------------------------------------------- /process/program.go: -------------------------------------------------------------------------------- 1 | package process 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // Program is the program we want to execute 9 | type Program struct { 10 | Name string `json:"name,omitempty"` 11 | Command string `json:"command,omitempty"` 12 | User string `json:"user,omitempty"` 13 | ProcNum int `json:"proc_num,omitempty"` 14 | processes []*Process 15 | } 16 | 17 | // NewProgram create a new Program 18 | func NewProgram(name, command, username string, procNum int) *Program { 19 | return &Program{ 20 | Name: name, 21 | Command: command, 22 | User: username, 23 | ProcNum: procNum, 24 | processes: make([]*Process, 0), 25 | } 26 | } 27 | 28 | func (program *Program) initProcesses(outputFunc OutputHandler) *Program { 29 | snips := strings.Split(program.Command, " ") 30 | command, args := snips[0], snips[1:] 31 | 32 | for i := 0; i < program.ProcNum; i++ { 33 | program.processes = append(program.processes, NewProcess( 34 | fmt.Sprintf("%s/%d", program.Name, i), 35 | command, 36 | args, 37 | program.User, 38 | ).setOutputFunc(outputFunc)) 39 | } 40 | 41 | return program 42 | } 43 | 44 | // Processes get all processes for the program 45 | func (program *Program) Processes() []*Process { 46 | return program.processes 47 | } 48 | -------------------------------------------------------------------------------- /sql/extracter/sql.go: -------------------------------------------------------------------------------- 1 | package extracter 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strconv" 7 | 8 | "github.com/mylxsw/coll" 9 | ) 10 | 11 | // Column is a sql column info 12 | type Column struct { 13 | Name string `json:"name"` 14 | Type string `json:"type"` 15 | } 16 | 17 | // Rows sql rows object 18 | type Rows struct { 19 | Columns []Column `json:"columns"` 20 | DataSets [][]interface{} `json:"data_sets"` 21 | } 22 | 23 | // Extract export sql rows to Rows object 24 | func Extract(rows *sql.Rows) (*Rows, error) { 25 | types, err := rows.ColumnTypes() 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | var columns []Column 31 | if err := coll.MustNew(types).Map(func(t *sql.ColumnType) Column { 32 | return Column{ 33 | Name: t.Name(), 34 | Type: t.DatabaseTypeName(), 35 | } 36 | }).All(&columns); err != nil { 37 | return nil, err 38 | } 39 | 40 | dataSets := make([][]interface{}, 0) 41 | 42 | for rows.Next() { 43 | var data = coll.MustNew(types). 44 | Map(func(t *sql.ColumnType) interface{} { 45 | var tt interface{} 46 | return &tt 47 | }).Items().([]interface{}) 48 | 49 | if err := rows.Scan(data...); err != nil { 50 | return nil, err 51 | } 52 | 53 | dataSets = append(dataSets, coll.MustNew(data).Map(func(k *interface{}, index int) interface{} { 54 | if k == nil || *k == nil { 55 | return nil 56 | } 57 | 58 | res := fmt.Sprintf("%s", *k) 59 | switch types[index].DatabaseTypeName() { 60 | case "INT", "TINYINT", "BIGINT", "MEDIUMINT", "SMALLINT", "DECIMAL": 61 | intRes, _ := strconv.Atoi(res) 62 | return intRes 63 | } 64 | 65 | return res 66 | }).Items().([]interface{})) 67 | } 68 | 69 | res := Rows{ 70 | Columns: columns, 71 | DataSets: dataSets, 72 | } 73 | 74 | return &res, nil 75 | } 76 | --------------------------------------------------------------------------------