├── .gitignore ├── go-proxyd ├── vendor ├── proto │ ├── phproxyd-config.proto │ ├── phproxyd-config.pb.go │ └── phproxyd.proto ├── go-proxyd.go ├── conf │ └── phproxyd.conf ├── sharded_cache.go ├── version.go ├── vproc │ └── vproc.go └── vhash │ └── vhach.go ├── vendor ├── github.com │ ├── go-sql-driver │ │ └── mysql │ │ │ ├── .travis.yml │ │ │ ├── appengine.go │ │ │ ├── result.go │ │ │ ├── transaction.go │ │ │ ├── errors_test.go │ │ │ ├── AUTHORS │ │ │ ├── CONTRIBUTING.md │ │ │ ├── rows.go │ │ │ ├── buffer.go │ │ │ ├── statement.go │ │ │ ├── const.go │ │ │ ├── CHANGELOG.md │ │ │ ├── errors.go │ │ │ ├── driver.go │ │ │ ├── infile.go │ │ │ └── benchmark_test.go │ ├── gogo │ │ └── protobuf │ │ │ └── proto │ │ │ ├── text_gogo.go │ │ │ ├── properties_gogo.go │ │ │ ├── pointer_unsafe_gogo.go │ │ │ └── decode_gogo.go │ └── gorilla │ │ └── handlers │ │ └── compress.go └── badoo │ └── _packages │ ├── log │ ├── terminal_bsd.go │ ├── terminal_linux.go │ ├── terminal_notwindows.go │ ├── hooks.go │ ├── badoo_formatter.go │ ├── formatter.go │ ├── hooks │ │ └── syslog │ │ │ ├── syslog.go │ │ │ └── syslog_nofroze.go │ ├── logrus.go │ ├── text_formatter.go │ └── exported.go │ ├── service │ ├── tcpinfo_darwin.go │ ├── config_file.go │ ├── tcpinfo_unix.go │ ├── pidfile.go │ ├── pinba.go │ ├── http.go │ ├── logfile.go │ ├── signals.go │ ├── stats.go │ └── stats │ │ └── stats.gpbrpc.go │ ├── util │ ├── slice.go │ ├── osutil │ │ └── executable.go │ ├── structs │ │ ├── struct_walk.go │ │ ├── badoo.go │ │ └── protobuf.go │ ├── config_file.go │ └── deepcopy │ │ └── deepcopy.go │ ├── gpbrpc │ ├── pcm.go │ ├── codec_json.go │ ├── gpbrpc.go │ └── codec_gpbs.go │ └── dns │ └── dns.go ├── logs-collector ├── common │ ├── lines.proto │ ├── protocol_test.go │ ├── protocol.go │ └── lines.pb.go ├── smoke-test.sh ├── main.go ├── README.md └── logsproc │ └── processor.go ├── jobgen ├── killer.go ├── common.go ├── settings.go ├── logwriter.go └── api.go ├── conf └── thunder.conf ├── version.go ├── LICENSE ├── proto ├── thunder-config.proto ├── thunder.proto ├── phproxyd │ └── phproxyd.proto └── thunder.gpbrpc.go ├── common └── common.go ├── thunder.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | thunder 2 | -------------------------------------------------------------------------------- /go-proxyd/vendor: -------------------------------------------------------------------------------- 1 | ../vendor -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | go: 4 | - 1.2 5 | - 1.3 6 | - 1.4 7 | - tip 8 | 9 | before_script: 10 | - mysql -e 'create database gotest;' 11 | -------------------------------------------------------------------------------- /logs-collector/common/lines.proto: -------------------------------------------------------------------------------- 1 | package common; 2 | 3 | message LogLines { 4 | required string FileName = 1; 5 | required int64 Offset = 2; 6 | required uint64 Inode = 3; 7 | repeated string Lines = 4; 8 | } 9 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/terminal_bsd.go: -------------------------------------------------------------------------------- 1 | // +build darwin freebsd openbsd netbsd dragonfly 2 | 3 | package log 4 | 5 | import "syscall" 6 | 7 | const ioctlReadTermios = syscall.TIOCGETA 8 | 9 | type Termios syscall.Termios 10 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/tcpinfo_darwin.go: -------------------------------------------------------------------------------- 1 | // +build darwin 2 | 3 | package service 4 | 5 | import "errors" 6 | 7 | func GetLqInfo(srv *Server) (unacked, sacked uint32, err error) { 8 | return 0, 0, errors.New("unsupported on darwin") 9 | } 10 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/terminal_linux.go: -------------------------------------------------------------------------------- 1 | // Based on ssh/terminal: 2 | // Copyright 2013 The Go Authors. All rights reserved. 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | package log 7 | 8 | import "syscall" 9 | 10 | const ioctlReadTermios = syscall.TCGETS 11 | 12 | type Termios syscall.Termios 13 | -------------------------------------------------------------------------------- /go-proxyd/proto/phproxyd-config.proto: -------------------------------------------------------------------------------- 1 | package badoo.phproxyd; 2 | 3 | message phproxyd_config { 4 | message scripts_t { 5 | required string php_bin = 1; 6 | required string php_scripts_dir = 2; 7 | required string file_stdout_template = 3; 8 | required string file_stderr_template = 4; 9 | required string mem_stdout_template = 5; 10 | required string mem_stderr_template = 6; 11 | } 12 | 13 | required scripts_t scripts = 2; 14 | } 15 | 16 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/config_file.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // XXX(antoxa): just leaving util wrappers here, to avoid changing services that depend on this code 4 | 5 | import ( 6 | "badoo/_packages/util" 7 | ) 8 | 9 | func ParseConfigFromFile(conf_path string, to_struct interface{}) error { 10 | return util.ParseConfigFromFile(conf_path, to_struct) 11 | } 12 | 13 | func ParseConfigToStruct(data []byte, to_struct interface{}) error { 14 | return util.ParseConfigToStruct(data, to_struct) 15 | } 16 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/appengine.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | // +build appengine 10 | 11 | package mysql 12 | 13 | import ( 14 | "appengine/cloudsql" 15 | ) 16 | 17 | func init() { 18 | RegisterDial("cloudsql", cloudsql.Dial) 19 | } 20 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/slice.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // given a slice of strings `haystack', look for string `needle' in it 4 | // position has no meaning if found == false 5 | func StrSliceFind(haystack []string, needle string) (found bool, position int) { 6 | for i, val := range haystack { 7 | if val == needle { 8 | return true, i 9 | } 10 | } 11 | return false, -1 12 | } 13 | 14 | func StrSliceEqual(lhs, rhs []string) bool { 15 | if len(lhs) != len(rhs) { 16 | return false 17 | } 18 | 19 | for _, val := range lhs { 20 | found, _ := StrSliceFind(rhs, val) 21 | if !found { 22 | return false 23 | } 24 | } 25 | return true 26 | } 27 | 28 | func test() {} 29 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/terminal_notwindows.go: -------------------------------------------------------------------------------- 1 | // Based on ssh/terminal: 2 | // Copyright 2011 The Go Authors. All rights reserved. 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | // +build linux darwin freebsd openbsd netbsd dragonfly 7 | 8 | package log 9 | 10 | import ( 11 | "syscall" 12 | "unsafe" 13 | ) 14 | 15 | // IsTerminal returns true if the given file descriptor is a terminal. 16 | func IsTerminal() bool { 17 | fd := syscall.Stdout 18 | var termios Termios 19 | _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) 20 | return err == 0 21 | } 22 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/result.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | type mysqlResult struct { 12 | affectedRows int64 13 | insertId int64 14 | } 15 | 16 | func (res *mysqlResult) LastInsertId() (int64, error) { 17 | return res.insertId, nil 18 | } 19 | 20 | func (res *mysqlResult) RowsAffected() (int64, error) { 21 | return res.affectedRows, nil 22 | } 23 | -------------------------------------------------------------------------------- /go-proxyd/go-proxyd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "badoo/_packages/service" 6 | "badoo/_packages/service/config" 7 | "github.com/badoo/thunder/go-proxyd/proto" 8 | "syscall" 9 | ) 10 | 11 | type FullConfig struct { 12 | badoo_config.ServiceConfig 13 | badoo_phproxyd.PhproxydConfig 14 | } 15 | 16 | var config = FullConfig{} 17 | 18 | var ports = []service.Port{ 19 | service.GpbPort("phproxyd-gpb", &gpb_ctx, badoo_phproxyd.Gpbrpc), 20 | service.JsonPort("phproxyd-gpb/json", &gpb_ctx, badoo_phproxyd.Gpbrpc), 21 | } 22 | 23 | func main() { 24 | 25 | service.Initialize("conf/phproxyd.conf", &config) 26 | 27 | q, w, e := syscall.Syscall(syscall.SYS_GETPRIORITY, 1, 0, 0) 28 | log.Debug("GetPriority: %v %v %v", q, w, e) 29 | 30 | sh = newShardedCache(32) 31 | 32 | service.EventLoop(ports) 33 | } 34 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/transaction.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | type mysqlTx struct { 12 | mc *mysqlConn 13 | } 14 | 15 | func (tx *mysqlTx) Commit() (err error) { 16 | if tx.mc == nil || tx.mc.netConn == nil { 17 | return ErrInvalidConn 18 | } 19 | err = tx.mc.exec("COMMIT") 20 | tx.mc = nil 21 | return 22 | } 23 | 24 | func (tx *mysqlTx) Rollback() (err error) { 25 | if tx.mc == nil || tx.mc.netConn == nil { 26 | return ErrInvalidConn 27 | } 28 | err = tx.mc.exec("ROLLBACK") 29 | tx.mc = nil 30 | return 31 | } 32 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/tcpinfo_unix.go: -------------------------------------------------------------------------------- 1 | // +build linux 2 | 3 | package service 4 | 5 | import ( 6 | "fmt" 7 | "net" 8 | "syscall" 9 | "unsafe" 10 | ) 11 | 12 | func getsockoptTCPInfo(fd uintptr) (*syscall.TCPInfo, error) { 13 | var tcpinfo syscall.TCPInfo 14 | var len = syscall.SizeofTCPInfo 15 | _, _, err := syscall.Syscall6( 16 | syscall.SYS_GETSOCKOPT, 17 | fd, 18 | syscall.IPPROTO_TCP, 19 | syscall.TCP_INFO, 20 | uintptr(unsafe.Pointer(&tcpinfo)), 21 | uintptr(unsafe.Pointer(&len)), 22 | 0) 23 | 24 | if 0 != err { 25 | return nil, error(err) 26 | } 27 | 28 | return &tcpinfo, nil 29 | } 30 | 31 | func GetLqInfo(srv *Server) (unacked, sacked uint32, err error) { 32 | l, ok := srv.Server.Listener.(*net.TCPListener) 33 | if !ok { 34 | return 0, 0, fmt.Errorf("undefined for non-tcp listeners") 35 | } 36 | 37 | tcpinfo, err := getsockoptTCPInfo(getRealFdFromTCPListener(l)) 38 | if err != nil { 39 | return 40 | } 41 | 42 | unacked = tcpinfo.Unacked 43 | sacked = tcpinfo.Sacked 44 | 45 | return 46 | } 47 | -------------------------------------------------------------------------------- /logs-collector/smoke-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -x 2 | go build -o thunder-test 3 | 4 | rm -rf tmp-source tmp-target offsets.db offsets.db.tmp 5 | 6 | mkdir -p tmp-source tmp-target 7 | ./thunder-test logs-processor --listen-address='127.0.0.1:7357' --target-dir='tmp-target' & 8 | sleep 0.3 9 | ./thunder-test logs-collector --offsets-db='offsets.db' --target-address='127.0.0.1:7357' --source-dir='tmp-source' & 10 | sleep 0.3 11 | 12 | echo Test1 >> tmp-source/test.log 13 | echo Test2 >> tmp-source/test.log 14 | 15 | sleep 0.3 16 | 17 | SUCCESS=1 18 | 19 | if ! test "`cat tmp-source/test.log`" = "`cat tmp-target/test.log`"; then 20 | echo 'Test failed! Expected contents do not match source' 21 | SUCCESS=0 22 | fi 23 | 24 | killall thunder-test 25 | 26 | sleep 0.3 27 | 28 | rm -rf tmp-source tmp-target offsets.db offsets.db.tmp thunder-test 29 | 30 | set +x 31 | 32 | echo 33 | echo 34 | 35 | if test "$SUCCESS" -eq 0; then 36 | echo 'Test did not pass! Something went wrong, see errors above' 37 | exit 1 38 | else 39 | echo Smoke test passed 40 | fi 41 | -------------------------------------------------------------------------------- /logs-collector/common/protocol_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/golang/protobuf/proto" 10 | ) 11 | 12 | func TestTransport(t *testing.T) { 13 | var b bytes.Buffer 14 | 15 | testLines := &LogLines{ 16 | FileName: proto.String("test filename"), 17 | Inode: proto.Uint64(100), 18 | Offset: proto.Int64(3), 19 | Lines: []string{ 20 | "Test line 1\n", 21 | "Test line 2\n", 22 | "Test line 3\n", 23 | strings.Repeat("Supertest", 10000), 24 | }, 25 | } 26 | 27 | err := SendMessage(&b, testLines) 28 | if err != nil { 29 | t.Errorf("Unexpected error when sending message: %s", err.Error()) 30 | return 31 | } 32 | 33 | decodedLines, err := ReceiveMessage(&b) 34 | if err != nil { 35 | t.Errorf("Unexpected error when receiving message: %s", err.Error()) 36 | return 37 | } 38 | 39 | if !reflect.DeepEqual(testLines, decodedLines) { 40 | t.Errorf("Structures are not equal:\nExpected %+v,\nGot %+v", testLines, decodedLines) 41 | return 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /go-proxyd/conf/phproxyd.conf: -------------------------------------------------------------------------------- 1 | { 2 | "daemon_config": { 3 | "listen": [ 4 | {"proto": "phproxyd-gpb", "address": "0.0.0.0:11100" }, 5 | {"proto": "phproxyd-gpb/json", "address": "0.0.0.0:11101" }, 6 | {"proto": "service-stats-gpb", "address": "0.0.0.0:11102" }, 7 | {"proto": "service-stats-gpb/json", "address": "0.0.0.0:11103" }, 8 | ], 9 | "user": "wwwrun", 10 | "pid_file": "var/phproxyd.pid", 11 | "log_file": "var/phproxyd.log", 12 | "chdir": "/local/tmp", 13 | "service_name": "proxyd", 14 | "service_instance_name": "localhost", 15 | }, 16 | "scripts": { 17 | "php_bin" : "/local/php/bin/php", 18 | "php_scripts_dir": "/local/www/", 19 | "file_stdout_template": "/local/logs/phproxyd.%d.out.log", 20 | "file_stderr_template": "/local/logs/phproxyd.%d.err.log", 21 | "mem_stdout_template": "/local/logs/mem/phproxyd.%d.err.log", 22 | "mem_stderr_template": "/local/logs/mem/phproxyd.%d.err.log", 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/errors_test.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import ( 12 | "bytes" 13 | "log" 14 | "testing" 15 | ) 16 | 17 | func TestErrorsSetLogger(t *testing.T) { 18 | previous := errLog 19 | defer func() { 20 | errLog = previous 21 | }() 22 | 23 | // set up logger 24 | const expected = "prefix: test\n" 25 | buffer := bytes.NewBuffer(make([]byte, 0, 64)) 26 | logger := log.New(buffer, "prefix: ", 0) 27 | 28 | // print 29 | SetLogger(logger) 30 | errLog.Print("test") 31 | 32 | // check result 33 | if actual := buffer.String(); actual != expected { 34 | t.Errorf("expected %q, got %q", expected, actual) 35 | } 36 | } 37 | 38 | func TestErrorsStrictIgnoreNotes(t *testing.T) { 39 | runTests(t, dsn+"&sql_notes=false", func(dbt *DBTest) { 40 | dbt.mustExec("DROP TABLE IF EXISTS does_not_exist") 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/hooks.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | // A hook to be fired when logging on the logging levels returned from 4 | // `Levels()` on your implementation of the interface. Note that this is not 5 | // fired in a goroutine or a channel with workers, you should handle such 6 | // functionality yourself if your call is non-blocking and you don't wish for 7 | // the logging calls for levels returned from `Levels()` to block. 8 | type Hook interface { 9 | Levels() []Level 10 | Fire(*Entry) error 11 | } 12 | 13 | // Internal type for storing the hooks on a logger instance. 14 | type LevelHooks map[Level][]Hook 15 | 16 | // Add a hook to an instance of logger. This is called with 17 | // `log.Hooks.Add(new(MyHook))` where `MyHook` implements the `Hook` interface. 18 | func (hooks LevelHooks) Add(hook Hook) { 19 | for _, level := range hook.Levels() { 20 | hooks[level] = append(hooks[level], hook) 21 | } 22 | } 23 | 24 | // Fire all the hooks for the passed level. Used by `entry.log` to fire 25 | // appropriate hooks for a log entry. 26 | func (hooks LevelHooks) Fire(level Level, entry *Entry) error { 27 | for _, hook := range hooks[level] { 28 | if err := hook.Fire(entry); err != nil { 29 | return err 30 | } 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/badoo_formatter.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "sort" 8 | ) 9 | 10 | type BadooFormatter struct { 11 | TextFormatter 12 | } 13 | 14 | func (f *BadooFormatter) Format(entry *Entry) ([]byte, error) { 15 | const layout = "Jan _2 15:04:05.000000" 16 | 17 | b := &bytes.Buffer{} 18 | fmt.Fprintf(b, "%s %s <%d> %s", entry.Time.Format(layout), getLevelStr(entry.Level), os.Getpid(), entry.Message) 19 | 20 | if len(entry.Data) > 0 { 21 | 22 | // FIXME(antoxa): disabling this sorting thing speeds this one up by ~35% 23 | var keys []string 24 | for k := range entry.Data { 25 | keys = append(keys, k) 26 | } 27 | sort.Strings(keys) 28 | 29 | b.WriteByte(' ') 30 | 31 | for _, key := range keys { 32 | f.appendKeyValue(b, key, entry.Data[key]) 33 | } 34 | } 35 | 36 | b.WriteByte('\n') 37 | 38 | return b.Bytes(), nil 39 | } 40 | 41 | func getLevelStr(l Level) string { 42 | switch l { 43 | case DebugLevel: 44 | return "[DEBUG]" 45 | case InfoLevel: 46 | return "[INFO] " 47 | case WarnLevel: 48 | return "[WARN] " 49 | case ErrorLevel: 50 | return "[ERROR]" 51 | case FatalLevel: 52 | return "[FATAL]" 53 | case PanicLevel: 54 | return "[PANIC]" 55 | default: 56 | return "[UNKNOWN]" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /go-proxyd/sharded_cache.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/badoo/thunder/go-proxyd/vhash" 5 | ) 6 | 7 | type shardedCache struct { 8 | caches []*vhash.Vhash 9 | } 10 | 11 | var sh *shardedCache 12 | 13 | func newShardedCache(shards int) *shardedCache { 14 | scache := shardedCache{ 15 | caches: make([]*vhash.Vhash, shards), 16 | } 17 | 18 | for i := 0; i < len(scache.caches); i++ { 19 | scache.caches[i] = vhash.NewVhash() 20 | } 21 | 22 | return &scache 23 | } 24 | 25 | func (scache *shardedCache) getCache(key uint64) *vhash.Vhash { 26 | return scache.caches[key%uint64(len(scache.caches))] 27 | } 28 | 29 | func (scache *shardedCache) Get(key uint64) (v vhash.Value, ok bool) { 30 | shard := scache.getCache(key) 31 | v, ok = shard.Get(key) 32 | return 33 | } 34 | 35 | func (scache *shardedCache) Set(key uint64, value vhash.Value) { 36 | shard := scache.getCache(key) 37 | shard.Set(key, value) 38 | } 39 | 40 | func (scache *shardedCache) Delete(key uint64) bool { 41 | shard := scache.getCache(key) 42 | return shard.Delete(key) 43 | } 44 | 45 | func (scache *shardedCache) Lock(key uint64) { 46 | shard := scache.getCache(key) 47 | shard.Lock() 48 | } 49 | 50 | func (scache *shardedCache) Unlock(key uint64) { 51 | shard := scache.getCache(key) 52 | shard.Unlock() 53 | } 54 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/osutil/executable.go: -------------------------------------------------------------------------------- 1 | // os utilities 2 | // getCurrentBinary code - ninja'd from https://github.com/kardianos/osext 3 | package osutil 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "runtime" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | currentBinary string // cached 14 | ) 15 | 16 | func GetCurrentBinary() (string, error) { 17 | var err error 18 | 19 | if currentBinary != "" { 20 | return currentBinary, nil 21 | } 22 | 23 | currentBinary, err = getCurrentBinary() 24 | return currentBinary, err 25 | } 26 | 27 | func getCurrentBinary() (string, error) { 28 | switch runtime.GOOS { 29 | case "linux": 30 | const deletedTag = " (deleted)" 31 | execpath, err := os.Readlink("/proc/self/exe") 32 | if err != nil { 33 | return execpath, err 34 | } 35 | execpath = strings.TrimSuffix(execpath, deletedTag) 36 | execpath = strings.TrimPrefix(execpath, deletedTag) 37 | return execpath, nil 38 | case "netbsd": 39 | return os.Readlink("/proc/curproc/exe") 40 | case "dragonfly": 41 | return os.Readlink("/proc/curproc/file") 42 | case "solaris": 43 | return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid())) 44 | case "darwin": 45 | return "", fmt.Errorf("no /proc on darwin") 46 | } 47 | return "", fmt.Errorf("osutil.GetCurrentBinary not implemented for %s", runtime.GOOS) 48 | } 49 | -------------------------------------------------------------------------------- /jobgen/killer.go: -------------------------------------------------------------------------------- 1 | package jobgen 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "sync" 6 | ) 7 | 8 | var ( 9 | killerThreads struct { 10 | v map[string]map[string]bool // class_name => map(dispatcher_location => true) 11 | sync.Mutex 12 | } 13 | ) 14 | 15 | func startKilling(className string) { 16 | killerThreads.Lock() 17 | defer killerThreads.Unlock() 18 | 19 | dispatchThreads.Lock() 20 | defer dispatchThreads.Unlock() 21 | 22 | locs, ok := dispatchThreads.v[className] 23 | if !ok { 24 | return 25 | } 26 | 27 | killMap := killerThreads.v[className] 28 | if killMap == nil { 29 | killMap = make(map[string]bool) 30 | killerThreads.v[className] = killMap 31 | } 32 | 33 | for loc, dt := range locs { 34 | if killMap[loc] { 35 | continue 36 | } 37 | 38 | req := &KillRequest{ 39 | ResCh: make(chan error, 1), 40 | } 41 | 42 | log.Printf("Sending kill request to class=%s, location=%s", className, loc) 43 | select { 44 | case dt.killRequestCh <- req: 45 | killMap[loc] = true 46 | default: 47 | log.Warnf("Could not send kill request to class=%s, location=%s, kill channel was busy", className, loc) 48 | } 49 | } 50 | } 51 | 52 | func continueDispatchAfterKill(className string) { 53 | killerThreads.Lock() 54 | defer killerThreads.Unlock() 55 | 56 | delete(killerThreads.v, className) 57 | } 58 | -------------------------------------------------------------------------------- /conf/thunder.conf: -------------------------------------------------------------------------------- 1 | { 2 | "daemon_config": { 3 | "listen": [ 4 | { "proto": "thunder-gpb", "address": "0.0.0.0:6557" }, 5 | { "proto": "thunder-gpb/json", "address": "0.0.0.0:6558" }, 6 | { "proto": "service-stats-gpb", "address": "0.0.0.0:6559" }, 7 | { "proto": "service-stats-gpb/json", "address": "0.0.0.0:6560" } 8 | ], 9 | 10 | "service_name": "thunder", 11 | "service_instance_name": "dbcloud:6557", 12 | 13 | "max_cpus": 0, 14 | "http_pprof_addr": "0.0.0.0:6561", 15 | 16 | "log_level": "NOTICE", 17 | "user": "nobody", 18 | "pid_file": "/tmp/thunder.pid", 19 | "log_file": "-" 20 | }, 21 | 22 | "mysql": { 23 | "dsn": "login:password@tcp(dbcloud:3306)/Thunder?strict=true", 24 | "max_connections": 8 25 | }, 26 | 27 | "default": { 28 | "parasite_memory": 209715200, 29 | "min_memory": 134217728, 30 | "min_memory_ratio": 0.01, 31 | "min_idle_cpu": -0.15 32 | }, 33 | 34 | "launcher": { 35 | "host_suffix": ":11100", 36 | "base_path": "/var/code/run.py", 37 | "developer_path": "/home/%s/code/run.php", 38 | "rqh_log_file": "/var/lsd/thunder_fail.log" 39 | }, 40 | 41 | "exit_with_parent": false, 42 | "delay": 0, 43 | "class_workers": 2, 44 | "is_devel": true 45 | } 46 | -------------------------------------------------------------------------------- /go-proxyd/version.go: -------------------------------------------------------------------------------- 1 | // generated by: generate_version_info.sh "1.0.0" "vvv@corp.badoo.com" "" "/Applications/Xcode.app/Contents/Developer/usr/bin/make - " 2 | // DO NOT EDIT! 3 | 4 | package main 5 | 6 | import ( 7 | "badoo/_packages/service" 8 | "badoo/_packages/service/stats" 9 | "github.com/gogo/protobuf/proto" 10 | ) 11 | 12 | func init() { 13 | service.VersionInfo = badoo_service.ResponseVersion{ 14 | Version: proto.String("1.0.0"), 15 | Maintainer: proto.String("vvv@corp.badoo.com"), 16 | AutoBuildTag: proto.String(""), 17 | BuildDate: proto.String("Thu Sep 15 10:26:36 UTC 2016"), 18 | BuildHost: proto.String("nasretdinov.local"), 19 | BuildGoVersion: proto.String("go version go1.7 darwin/amd64"), 20 | BuildCommand: proto.String("/Applications/Xcode.app/Contents/Developer/usr/bin/make - "), 21 | VcsType: proto.String("git"), 22 | VcsBasename: proto.String("go-proxyd"), 23 | VcsNum: proto.String("31"), 24 | VcsDate: proto.String("2014-02-17T16:03:45+0400"), 25 | VcsBranch: proto.String("master"), 26 | VcsTag: proto.String(""), 27 | VcsTick: proto.String("31"), 28 | VcsFullHash: proto.String("a10437837ce1c22015dbcaef9796644cadb8f0b1"), 29 | VcsShortHash: proto.String("a104378"), 30 | VcsWcModified: proto.String("1"), 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | // generated by: generate_version_info.sh "1.2.0" "y.nasretdinov@corp.badoo.com" "" "/Applications/Xcode.app/Contents/Developer/usr/bin/make - " 2 | // DO NOT EDIT! 3 | 4 | package main 5 | 6 | import ( 7 | "badoo/_packages/service" 8 | "badoo/_packages/service/stats" 9 | "github.com/gogo/protobuf/proto" 10 | ) 11 | 12 | func init() { 13 | service.VersionInfo = badoo_service.ResponseVersion{ 14 | Version: proto.String("1.2.0"), 15 | Maintainer: proto.String("y.nasretdinov@corp.badoo.com"), 16 | AutoBuildTag: proto.String(""), 17 | BuildDate: proto.String("Tue Aug 30 13:58:14 UTC 2016"), 18 | BuildHost: proto.String("nasretdinov.local"), 19 | BuildGoVersion: proto.String("go version go1.7 darwin/amd64"), 20 | BuildCommand: proto.String("/Applications/Xcode.app/Contents/Developer/usr/bin/make - "), 21 | VcsType: proto.String("git"), 22 | VcsBasename: proto.String("thunder"), 23 | VcsNum: proto.String("51"), 24 | VcsDate: proto.String("2016-06-22T10:03:37+0000"), 25 | VcsBranch: proto.String("CT-2552_kill_right_from_init"), 26 | VcsTag: proto.String(""), 27 | VcsTick: proto.String("51"), 28 | VcsFullHash: proto.String("9e79ceebe05bd5d64a7af2b547a55dc55f0dc449"), 29 | VcsShortHash: proto.String("9e79cee"), 30 | VcsWcModified: proto.String("1"), 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /logs-collector/common/protocol.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | 7 | "github.com/golang/protobuf/proto" 8 | ) 9 | 10 | func SendMessage(c io.Writer, ln *LogLines) (err error) { 11 | var data []byte 12 | 13 | if data, err = proto.Marshal(ln); err != nil { 14 | return 15 | } 16 | 17 | l := uint32(len(data) + 4) 18 | 19 | _, err = c.Write([]byte{ 20 | byte(l >> 24), // message length 21 | byte(l >> 16), 22 | byte(l >> 8), 23 | byte(l), 24 | 0, // message id, currently always equal to zero 25 | 0, 26 | 0, 27 | 0, 28 | }) 29 | 30 | if err != nil { 31 | return 32 | } 33 | 34 | _, err = c.Write(data) 35 | return 36 | } 37 | 38 | func ReceiveMessage(c io.Reader) (ln *LogLines, err error) { 39 | var lenbuf = make([]byte, 4) 40 | if _, err = io.ReadFull(c, lenbuf); err != nil { 41 | return 42 | } 43 | 44 | l := uint32(uint32(lenbuf[0])<<24 + uint32(lenbuf[1])<<16 + uint32(lenbuf[2])<<8 + uint32(lenbuf[3])) 45 | if l <= 4 { 46 | err = errors.New("Message length must be greater than 4 bytes") 47 | return 48 | } 49 | 50 | buf := make([]byte, l) 51 | if _, err = io.ReadFull(c, buf); err != nil { 52 | return 53 | } 54 | 55 | // we ignore first 4 bytes as they are message id which is always equal to zero in current implementation 56 | ln = new(LogLines) 57 | err = proto.Unmarshal(buf[4:], ln) 58 | return 59 | } 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Badoo Development 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | -------------------------------------------------------------------------------- /go-proxyd/vproc/vproc.go: -------------------------------------------------------------------------------- 1 | package vproc 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "os" 6 | "os/exec" 7 | ) 8 | 9 | type Vproc struct { 10 | Cmd *exec.Cmd 11 | Ps *os.ProcessState 12 | Id uint64 13 | SendResponse bool 14 | } 15 | 16 | func NewProcess(id uint64, bin string, cmd string, args []string, stdout_file, stderr_file string) (*Vproc, error) { 17 | p := new(Vproc) 18 | p.Id = id 19 | p.Cmd = exec.Command(bin, cmd) 20 | var a = []string{bin, cmd} 21 | a = append(a, args...) 22 | 23 | p.Cmd.Args = a 24 | p.Cmd.Env = os.Environ() 25 | 26 | log.Debug("%v\n", a) 27 | 28 | stdout, err := os.OpenFile(stdout_file, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666) 29 | if err != nil { 30 | return nil, err 31 | } 32 | p.Cmd.Stdout = stdout 33 | 34 | stderr, err := os.OpenFile(stderr_file, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666) 35 | if err != nil { 36 | return nil, err 37 | } 38 | p.Cmd.Stderr = stderr 39 | 40 | return p, nil 41 | } 42 | 43 | func (vp *Vproc) Start() error { 44 | err := vp.Cmd.Start() 45 | if err != nil { 46 | log.Errorf("%v\n", err) 47 | } 48 | 49 | return err 50 | } 51 | 52 | func (vp *Vproc) Wait() (*os.ProcessState, error) { 53 | ps, err := vp.Cmd.Process.Wait() 54 | if err != nil { 55 | log.Errorf("%v\n", err) 56 | return ps, err 57 | } 58 | return ps, err 59 | } 60 | 61 | func (vp *Vproc) Kill() error { 62 | return vp.Cmd.Process.Kill() 63 | } 64 | 65 | func (vp *Vproc) Size() int { 66 | return 1 67 | } 68 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/structs/struct_walk.go: -------------------------------------------------------------------------------- 1 | // reflection based tools for checking/processing config (and maybe not just config) data 2 | package structs 3 | 4 | import ( 5 | "reflect" 6 | ) 7 | 8 | // walks the gpb message and calls function `f` for each struct field 9 | // handles pointer/slice indirections 10 | // TODO(antoxa): move this to badoo/_packages/util somewhere 11 | func WalkStructFieldsRecursive(rv reflect.Value, f func(parent reflect.Value, fieldNo int) error) (err error) { 12 | 13 | rt := rv.Type() 14 | 15 | switch rt.Kind() { 16 | case reflect.Struct: 17 | 18 | // handle all struct fields recursively, depth first 19 | for i := 0; i < rt.NumField(); i++ { 20 | 21 | // user function first 22 | err := f(rv, i) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | // recurse second 28 | err = WalkStructFieldsRecursive(rv.Field(i), f) 29 | if nil != err { 30 | return err 31 | } 32 | } 33 | 34 | case reflect.Slice: 35 | // handle all slice values recursively, depth first 36 | for i := 0; i < rv.Len(); i++ { 37 | err = WalkStructFieldsRecursive(rv.Index(i), f) 38 | if nil != err { 39 | return err 40 | } 41 | } 42 | 43 | case reflect.Ptr: 44 | // just recurse into pointers, this is required for struct pointers 45 | // recursion will terminate inside this call if it's just a pointer struct field 46 | if !rv.IsNil() { 47 | return WalkStructFieldsRecursive(reflect.Indirect(rv), f) 48 | } 49 | } 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /logs-collector/main.go: -------------------------------------------------------------------------------- 1 | //go:generate protoc --go_out=. common/lines.proto 2 | 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "fmt" 8 | "os" 9 | 10 | "github.com/badoo/thunder/logscol" 11 | "github.com/badoo/thunder/logsproc" 12 | ) 13 | 14 | const ( 15 | ACTION_LOGS_COLLECTOR = "logs-collector" 16 | ACTION_LOGS_PROCESSOR = "logs-processor" 17 | ) 18 | 19 | func dieWithUsage() { 20 | fmt.Fprintln(os.Stderr, "Usage: thunder []") 21 | fmt.Fprintln(os.Stderr, " can be one of the following:") 22 | fmt.Fprintln(os.Stderr, " "+ACTION_LOGS_COLLECTOR+" - logs collector") 23 | fmt.Fprintln(os.Stderr, " "+ACTION_LOGS_PROCESSOR+" - logs processor") 24 | os.Exit(1) 25 | } 26 | 27 | func main() { 28 | var action string 29 | 30 | // rewrite arguments so that we can support multiple "actions" in a single binary 31 | newArgs := make([]string, 0, len(os.Args)) 32 | for i, arg := range os.Args { 33 | if i > 0 && arg[0] != '-' && action == "" { 34 | action = arg 35 | continue 36 | } 37 | 38 | newArgs = append(newArgs, arg) 39 | } 40 | os.Args = newArgs 41 | 42 | if action == "" { 43 | fmt.Fprintln(os.Stderr, "Error: action was not specified") 44 | dieWithUsage() 45 | } 46 | 47 | if action == ACTION_LOGS_COLLECTOR { 48 | logscol.InitFlags() 49 | flag.Parse() 50 | logscol.Run() 51 | return 52 | } else if action == ACTION_LOGS_PROCESSOR { 53 | logsproc.InitFlags() 54 | flag.Parse() 55 | logsproc.Run() 56 | return 57 | } 58 | 59 | dieWithUsage() 60 | } 61 | -------------------------------------------------------------------------------- /go-proxyd/vhash/vhach.go: -------------------------------------------------------------------------------- 1 | package vhash 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type Vhash struct { 8 | mu sync.Mutex 9 | table map[uint64]*Item 10 | } 11 | 12 | // Values that go into Vhash need to satisfy this interface. 13 | type Value interface { 14 | } 15 | 16 | type Item struct { 17 | Key uint64 18 | Value Value 19 | } 20 | 21 | type entry struct { 22 | key uint64 23 | value Value 24 | } 25 | 26 | func NewVhash() *Vhash { 27 | return &Vhash{ 28 | table: make(map[uint64]*Item), 29 | } 30 | } 31 | 32 | func (vh *Vhash) Lock() { 33 | vh.mu.Lock() 34 | } 35 | 36 | func (vh *Vhash) Unlock() { 37 | vh.mu.Unlock() 38 | } 39 | 40 | func (vh *Vhash) Get(key uint64) (v Value, ok bool) { 41 | element := vh.table[key] 42 | if element == nil { 43 | return nil, false 44 | } 45 | return element.Value, true 46 | } 47 | 48 | func (vh *Vhash) Set(key uint64, value Value) { 49 | if element := vh.table[key]; element != nil { 50 | vh.updateInplace(element, value) 51 | } else { 52 | vh.addNew(key, value) 53 | } 54 | } 55 | 56 | func (vh *Vhash) Delete(key uint64) bool { 57 | element := vh.table[key] 58 | if element == nil { 59 | return false 60 | } 61 | 62 | delete(vh.table, key) 63 | 64 | return true 65 | } 66 | 67 | func (vh *Vhash) Clear() { 68 | vh.Lock() 69 | defer vh.Unlock() 70 | 71 | vh.table = make(map[uint64]*Item) 72 | } 73 | 74 | func (vh *Vhash) updateInplace(element *Item, value Value) { 75 | element.Value = value 76 | } 77 | 78 | func (vh *Vhash) addNew(key uint64, value Value) { 79 | newEntry := &Item{key, value} 80 | vh.table[key] = newEntry 81 | } 82 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/formatter.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import "time" 4 | 5 | const DefaultTimestampFormat = time.RFC3339 6 | 7 | // The Formatter interface is used to implement a custom Formatter. It takes an 8 | // `Entry`. It exposes all the fields, including the default ones: 9 | // 10 | // * `entry.Data["msg"]`. The message passed from Info, Warn, Error .. 11 | // * `entry.Data["time"]`. The timestamp. 12 | // * `entry.Data["level"]. The level the entry was logged at. 13 | // 14 | // Any additional fields added with `WithField` or `WithFields` are also in 15 | // `entry.Data`. Format is expected to return an array of bytes which are then 16 | // logged to `logger.Out`. 17 | type Formatter interface { 18 | Format(*Entry) ([]byte, error) 19 | } 20 | 21 | // This is to not silently overwrite `time`, `msg` and `level` fields when 22 | // dumping it. If this code wasn't there doing: 23 | // 24 | // logrus.WithField("level", 1).Info("hello") 25 | // 26 | // Would just silently drop the user provided level. Instead with this code 27 | // it'll logged as: 28 | // 29 | // {"level": "info", "fields.level": 1, "msg": "hello", "time": "..."} 30 | // 31 | // It's not exported because it's still using Data in an opinionated way. It's to 32 | // avoid code duplication between the two default formatters. 33 | func prefixFieldClashes(data Fields) { 34 | _, ok := data["time"] 35 | if ok { 36 | data["fields.time"] = data["time"] 37 | } 38 | 39 | _, ok = data["msg"] 40 | if ok { 41 | data["fields.msg"] = data["msg"] 42 | } 43 | 44 | _, ok = data["level"] 45 | if ok { 46 | data["fields.level"] = data["level"] 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/hooks/syslog/syslog.go: -------------------------------------------------------------------------------- 1 | package log_syslog 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "fmt" 6 | "log/syslog" 7 | "os" 8 | ) 9 | 10 | // SyslogHook to send logs via syslog. 11 | type SyslogHook struct { 12 | Writer *syslog.Writer 13 | SyslogNetwork string 14 | SyslogRaddr string 15 | } 16 | 17 | // Creates a hook to be added to an instance of logger. This is called with 18 | // `hook, err := NewSyslogHook("udp", "localhost:514", syslog.LOG_DEBUG, "")` 19 | // `if err == nil { log.Hooks.Add(hook) }` 20 | func NewSyslogHook(network, raddr string, priority syslog.Priority, tag string) (*SyslogHook, error) { 21 | w, err := syslog.Dial(network, raddr, priority, tag) 22 | return &SyslogHook{w, network, raddr}, err 23 | } 24 | 25 | func (hook *SyslogHook) Fire(entry *log.Entry) error { 26 | line, err := entry.String() 27 | if err != nil { 28 | fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) 29 | return err 30 | } 31 | 32 | switch entry.Level { 33 | case log.PanicLevel: 34 | return hook.Writer.Crit(line) 35 | case log.FatalLevel: 36 | return hook.Writer.Crit(line) 37 | case log.ErrorLevel: 38 | return hook.Writer.Err(line) 39 | case log.WarnLevel: 40 | return hook.Writer.Warning(line) 41 | case log.InfoLevel: 42 | return hook.Writer.Info(line) 43 | case log.DebugLevel: 44 | return hook.Writer.Debug(line) 45 | default: 46 | return nil 47 | } 48 | } 49 | 50 | func (hook *SyslogHook) Levels() []log.Level { 51 | return []log.Level{ 52 | log.PanicLevel, 53 | log.FatalLevel, 54 | log.ErrorLevel, 55 | log.WarnLevel, 56 | log.InfoLevel, 57 | log.DebugLevel, 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /logs-collector/common/lines.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: common/lines.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package common is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | common/lines.proto 10 | 11 | It has these top-level messages: 12 | LogLines 13 | */ 14 | package common 15 | 16 | import proto "github.com/golang/protobuf/proto" 17 | import math "math" 18 | 19 | // Reference imports to suppress errors if they are not otherwise used. 20 | var _ = proto.Marshal 21 | var _ = math.Inf 22 | 23 | type LogLines struct { 24 | FileName *string `protobuf:"bytes,1,req" json:"FileName,omitempty"` 25 | Offset *int64 `protobuf:"varint,2,req" json:"Offset,omitempty"` 26 | Inode *uint64 `protobuf:"varint,3,req" json:"Inode,omitempty"` 27 | Lines []string `protobuf:"bytes,4,rep" json:"Lines,omitempty"` 28 | XXX_unrecognized []byte `json:"-"` 29 | } 30 | 31 | func (m *LogLines) Reset() { *m = LogLines{} } 32 | func (m *LogLines) String() string { return proto.CompactTextString(m) } 33 | func (*LogLines) ProtoMessage() {} 34 | 35 | func (m *LogLines) GetFileName() string { 36 | if m != nil && m.FileName != nil { 37 | return *m.FileName 38 | } 39 | return "" 40 | } 41 | 42 | func (m *LogLines) GetOffset() int64 { 43 | if m != nil && m.Offset != nil { 44 | return *m.Offset 45 | } 46 | return 0 47 | } 48 | 49 | func (m *LogLines) GetInode() uint64 { 50 | if m != nil && m.Inode != nil { 51 | return *m.Inode 52 | } 53 | return 0 54 | } 55 | 56 | func (m *LogLines) GetLines() []string { 57 | if m != nil { 58 | return m.Lines 59 | } 60 | return nil 61 | } 62 | 63 | func init() { 64 | } 65 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Go-MySQL-Driver authors for copyright purposes. 2 | 3 | # If you are submitting a patch, please add your name or the name of the 4 | # organization which holds the copyright to this list in alphabetical order. 5 | 6 | # Names should be added to this file as 7 | # Name 8 | # The email address is not required for organizations. 9 | # Please keep the list sorted. 10 | 11 | 12 | # Individual Persons 13 | 14 | Aaron Hopkins 15 | Arne Hormann 16 | Carlos Nieto 17 | Chris Moos 18 | DisposaBoy 19 | Frederick Mayle 20 | Gustavo Kristic 21 | Hanno Braun 22 | Henri Yandell 23 | Hirotaka Yamamoto 24 | INADA Naoki 25 | James Harr 26 | Jian Zhen 27 | Joshua Prunier 28 | Julien Schmidt 29 | Kamil Dziedzic 30 | Leonardo YongUk Kim 31 | Lucas Liu 32 | Luke Scott 33 | Michael Woolnough 34 | Nicola Peduzzi 35 | Runrioter Wung 36 | Soroush Pour 37 | Stan Putrya 38 | Xiaobing Jiang 39 | Xiuming Chen 40 | Julien Lefevre 41 | 42 | # Organizations 43 | 44 | Barracuda Networks, Inc. 45 | Google Inc. 46 | Stripe Inc. 47 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | ## Reporting Issues 4 | 5 | Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). 6 | 7 | Please provide the following minimum information: 8 | * Your Go-MySQL-Driver version (or git SHA) 9 | * Your Go version (run `go version` in your console) 10 | * A detailed issue description 11 | * Error Log if present 12 | * If possible, a short example 13 | 14 | 15 | ## Contributing Code 16 | 17 | By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. 18 | Don't forget to add yourself to the AUTHORS file. 19 | 20 | ### Pull Requests Checklist 21 | 22 | Please check the following points before submitting your pull request: 23 | - [x] Code compiles correctly 24 | - [x] Created tests, if possible 25 | - [x] All tests pass 26 | - [x] Extended the README / documentation, if necessary 27 | - [x] Added yourself to the AUTHORS file 28 | 29 | ### Code Review 30 | 31 | Everyone is invited to review and comment on pull requests. 32 | If it looks fine to you, comment with "LGTM" (Looks good to me). 33 | 34 | If changes are required, notice the reviewers with "PTAL" (Please take another look) after committing the fixes. 35 | 36 | Before merging the Pull Request, at least one [team member](https://github.com/go-sql-driver?tab=members) must have commented with "LGTM". 37 | 38 | ## Development Ideas 39 | 40 | If you are looking for ideas for code contributions, please check our [Development Ideas](https://github.com/go-sql-driver/mysql/wiki/Development-Ideas) Wiki page. 41 | -------------------------------------------------------------------------------- /proto/thunder-config.proto: -------------------------------------------------------------------------------- 1 | package thunder; 2 | 3 | message config_t { 4 | message mysql_t { 5 | required string dsn = 1; 6 | required uint32 max_connections = 2; 7 | }; 8 | 9 | message default_t { 10 | // average parasite memory on devel and production servers if we were to guess 11 | optional int64 parasite_memory = 1 [default = 838860800]; 12 | 13 | // minimum requirements for hosts to be considered alive 14 | optional int64 min_memory = 2 [default = 1073741824]; 15 | optional double min_memory_ratio = 3 [default = 0.01]; 16 | 17 | optional double min_idle_cpu = 4 [default = -0.15]; 18 | 19 | // average script memory usage in bytes 20 | optional int64 max_memory = 5 [default = 52428800]; 21 | optional double rusage = 6 [default = 0.02]; 22 | }; 23 | 24 | required mysql_t mysql = 1; 25 | optional bool exit_with_parent = 2 [default = false]; 26 | optional uint32 delay = 3 [default = 0]; 27 | required default_t default = 4; 28 | optional int64 cycle_ms = 5 [default = 1000]; // how long to wait between job generation cycles in ms 29 | optional int64 full_timetable_reload_interval_ms = 6 [default = 1000]; // how often to fully reload timetable 30 | 31 | message launcher_t { 32 | required string host_suffix = 1; // should be ":", e.g. ":11100" for devel and ":6443" for production 33 | required string base_path = 2; // should be "/local/www/_scripts/run.php" or "/home/wwwrun/badoo/_scripts/run.php" 34 | optional string developer_path = 3; // if developer dirs are supported, specify it as "/home/%s/badoo/_scripts/run.php" 35 | required string rqh_log_file = 4; // path to rqh.out.log 36 | } 37 | 38 | required launcher_t launcher = 7; // settings for launcher 39 | required bool is_devel = 8; 40 | } 41 | -------------------------------------------------------------------------------- /logs-collector/README.md: -------------------------------------------------------------------------------- 1 | # thunder 2 | Our cloud system, currently containing only logs streamer, more to come in future. 3 | 4 | # Logs 5 | Logs transport system: 6 | 7 | ## Logs processor 8 | 9 | The component to be installed on a central logs receiver that writes incoming logs into target dir that you specified. 10 | Launched as `thunder logs-processor --target-dir=/var/logs/thunder/target/ --listen-address='0.0.0.0:1065'` 11 | 12 | Daemon will write logs that were received from other servers with the same names that were present at source servers. 13 | 14 | **Note:** when you rotate your logs using logrotate or any other mechanism, you need to send SIGUSR1 or SIGHUP to thunder logs processor in order for it to re-open log files. The logs-processor itself does not rotate anything 15 | 16 | ## Logs collector 17 | 18 | The component to be installed everywhere you want to collect logs from. 19 | Launched as `thunder logs-collector --offsets-db='/tmp/thunder-offsets.db' --target-address=':7357' --source-dir='/var/logs/thunder/source/'` 20 | 21 | The specified command collects all files from specified directory and streams them to the specified location in target-address. Daemon also does rotate log files every 10 MiB. Streaming is done in real time using inotify or similar mechanisms in your OS. There are two options to write into files to prevent data loss during rotate event: 22 | 23 | 1. open(..., O_APPEND), write(...), close(...) - do small writes files, opening and closing them every time 24 | 2. open(..., O_APPEND), flock(..., LOCK_SH), write(...), ..., write(...), close(...) - open log files once and then set shared lock, so that old files are not deleted while you can write there 25 | 26 | This daemon does not handle connection failures and exits. So you need to have a separate watcher for this daemon that would relaunch it if it exits because of connection problems. 27 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/hooks/syslog/syslog_nofroze.go: -------------------------------------------------------------------------------- 1 | package log_syslog 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "log/syslog" 6 | "sync/atomic" 7 | ) 8 | 9 | // SyslogHook to send logs via syslog. 10 | type SyslogHookNoFrozen struct { 11 | syslogHook *SyslogHook 12 | logCh chan *log.Entry 13 | exitCh chan struct{} 14 | skipped uint64 15 | } 16 | 17 | // Creates a hook to be added to an instance of logger. This is called with 18 | // `hook, err := NewSyslogHookNoFrozen("udp", "localhost:514", syslog.LOG_DEBUG, "")` 19 | func NewSyslogHookNoFrozen(network, raddr string, priority syslog.Priority, tag string) (*SyslogHookNoFrozen, error) { 20 | syslogHook, err := NewSyslogHook(network, raddr, priority, tag) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | hook := &SyslogHookNoFrozen{ 26 | syslogHook: syslogHook, 27 | logCh: make(chan *log.Entry, 1000), 28 | exitCh: make(chan struct{}), 29 | } 30 | go hook.serve() 31 | return hook, err 32 | } 33 | 34 | func (hook *SyslogHookNoFrozen) Fire(entry *log.Entry) error { 35 | select { 36 | case hook.logCh <- entry: 37 | default: 38 | skippedNew := atomic.AddUint64(&hook.skipped, uint64(1)) 39 | log.WithoutHooks().Errorf("syslog entry %v skipped (total %d)", entry, skippedNew) 40 | } 41 | return nil 42 | } 43 | 44 | func (hook *SyslogHookNoFrozen) Levels() []log.Level { 45 | return hook.syslogHook.Levels() 46 | } 47 | 48 | func (hook *SyslogHookNoFrozen) Exit() { 49 | hook.exitCh <- struct{}{} 50 | } 51 | 52 | func (hook *SyslogHookNoFrozen) GetSkippedCount() uint64 { 53 | return atomic.LoadUint64(&hook.skipped) 54 | } 55 | 56 | func (hook *SyslogHookNoFrozen) serve() { 57 | for { 58 | select { 59 | case entry := <-hook.logCh: 60 | err := hook.syslogHook.Fire(entry) 61 | if err != nil { 62 | log.WithoutHooks().Errorf("syslog error: %v", err) 63 | } 64 | case <-hook.exitCh: 65 | return 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /jobgen/common.go: -------------------------------------------------------------------------------- 1 | package jobgen 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "math/rand" 7 | "time" 8 | 9 | "badoo/_packages/log" 10 | ) 11 | 12 | var defaultParasiteMemory int64 13 | 14 | func getParasiteMemory(info *ServerInfo) int64 { 15 | if info.mem_parasite.Valid { 16 | return info.mem_parasite.Int64 17 | } 18 | 19 | return defaultParasiteMemory 20 | } 21 | 22 | func getFreeMem(info *ServerInfo) int64 { 23 | return info.mem_free.Int64 + info.mem_cached.Int64 - info.swap_used.Int64 24 | } 25 | 26 | func getNextJobGenerateTs(className string, firstRun bool, prevTs uint64, settings *ScriptSettings) (nextTs uint64) { 27 | if !settings.have_next_ts_callback { 28 | if firstRun { 29 | nextTs = uint64(time.Now().Unix()) 30 | } else { 31 | nextTs = prevTs + uint64(settings.repeat) 32 | } 33 | } else if settings.next_ts_callback.Callback == "cron" { 34 | if _, ok := settings.next_ts_callback.Settings["cron"]; !ok { 35 | log.Warning("'cron' section is absent for next ts callback for " + className) 36 | return 37 | } 38 | 39 | now := uint64(time.Now().Unix()) 40 | 41 | if firstRun { 42 | prevTs = now 43 | } 44 | 45 | nextTs = getNextLaunchTsForCron(prevTs, now, rand.Intn(60), settings.next_ts_callback.Settings["cron"]) 46 | } else { 47 | log.Warning("Callbacks other than cron are not supported for " + className) 48 | } 49 | 50 | return 51 | } 52 | 53 | func getLocationIdx(locationType string, location string) (res string, err error) { 54 | if locationType == LOCATION_TYPE_EACH { 55 | res = location 56 | } else if locationType == LOCATION_TYPE_ANY { 57 | res = DEFAULT_LOCATION_IDX 58 | } else { 59 | err = errors.New("Incorrect location type: " + locationType) 60 | } 61 | 62 | return 63 | } 64 | 65 | func parseJobs(jobsStr string) (jobs Jobs, err error) { 66 | err = json.Unmarshal([]byte(jobsStr), &jobs) 67 | return 68 | } 69 | 70 | func parseNextTsCallback(nextTsCallbackStr string) (callback NextTsCallback, err error) { 71 | err = json.Unmarshal([]byte(nextTsCallbackStr), &callback) 72 | return 73 | } 74 | -------------------------------------------------------------------------------- /common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "time" 8 | 9 | "badoo/_packages/service/config" 10 | "github.com/badoo/thunder/proto" 11 | "database/sql" 12 | ) 13 | 14 | type ( 15 | FullConfig struct { 16 | badoo_config.ServiceConfig 17 | thunder.ConfigT 18 | } 19 | 20 | Jobs struct { 21 | Type string 22 | Min int 23 | Max int 24 | Have_finish_jobs bool 25 | Temporary bool 26 | } 27 | 28 | NextTsCallback struct { 29 | Callback string 30 | Settings map[string]string 31 | } 32 | 33 | ScriptSettings struct { 34 | id uint64 35 | class_name string 36 | instance_count int 37 | max_time int 38 | jobs Jobs 39 | have_next_ts_callback bool 40 | next_ts_callback NextTsCallback 41 | repeat uint32 42 | retry sql.NullInt64 43 | ttl uint32 44 | repeat_job sql.NullInt64 45 | retry_job uint32 46 | location string 47 | location_type string 48 | developer sql.NullString 49 | max_retries uint32 50 | profiling_enabled int 51 | created int64 52 | } 53 | ) 54 | 55 | var ( 56 | // TODO: implement setting of this value 57 | exitWithParent bool 58 | startDelaySec int 59 | ) 60 | 61 | func init() { 62 | rand.Seed(time.Now().UnixNano()) 63 | } 64 | 65 | type UInt64Slice []uint64 66 | 67 | func (p UInt64Slice) Len() int { return len(p) } 68 | func (p UInt64Slice) Less(i, j int) bool { return p[i] < p[j] } 69 | func (p UInt64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 70 | 71 | func Setup() { 72 | if exitWithParent { 73 | go func() { 74 | buf := make([]byte, 10) 75 | for { 76 | _, err := os.Stdin.Read(buf) 77 | if err != nil { 78 | fmt.Println("Cannot read from stdin (" + err.Error() + "), exiting") 79 | os.Exit(0) 80 | } 81 | } 82 | }() 83 | } 84 | 85 | if startDelaySec > 0 { 86 | time.Sleep(time.Duration(startDelaySec) * time.Second) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /vendor/github.com/gogo/protobuf/proto/text_gogo.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 2 | // http://github.com/gogo/protobuf/gogoproto 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | package proto 28 | 29 | import ( 30 | "fmt" 31 | "reflect" 32 | ) 33 | 34 | func writeEnum(w *textWriter, v reflect.Value, props *Properties) error { 35 | m, ok := enumStringMaps[props.Enum] 36 | if !ok { 37 | if err := writeAny(w, v, props); err != nil { 38 | return err 39 | } 40 | } 41 | key := int32(0) 42 | if v.Kind() == reflect.Ptr { 43 | key = int32(v.Elem().Int()) 44 | } else { 45 | key = int32(v.Int()) 46 | } 47 | s, ok := m[key] 48 | if !ok { 49 | if err := writeAny(w, v, props); err != nil { 50 | return err 51 | } 52 | } 53 | _, err := fmt.Fprint(w, s) 54 | return err 55 | } 56 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/gpbrpc/pcm.go: -------------------------------------------------------------------------------- 1 | package gpbrpc 2 | 3 | type pcmRequest struct { 4 | address string 5 | reply chan *Client 6 | } 7 | 8 | type poolType struct { 9 | head, tail *Client 10 | } 11 | 12 | type pcmType struct { 13 | reused chan *Client 14 | getconn chan *pcmRequest 15 | conns map[string]*poolType 16 | } 17 | 18 | type PcmType interface { 19 | GetClient(address string) *Client 20 | Reuse(client *Client) 21 | StartServing() 22 | } 23 | 24 | var Pcm PcmType 25 | 26 | func NewPcm() *pcmType { 27 | // singleton, hah? 28 | return &pcmType{ 29 | reused: make(chan *Client, 16), 30 | getconn: make(chan *pcmRequest, 16), 31 | conns: make(map[string]*poolType), 32 | } 33 | } 34 | 35 | func (pcm *pcmType) GetClient(address string) *Client { 36 | reply := make(chan *Client) 37 | /* RPC within one object between different goroutines */ 38 | pcm.getconn <- &pcmRequest{address: address, reply: reply} 39 | return <-reply 40 | } 41 | 42 | func (pcm *pcmType) Reuse(client *Client) { 43 | pcm.reused <- client 44 | } 45 | 46 | func (pcm *pcmType) fetch(address string) *Client { 47 | if p, ok := pcm.conns[address]; ok { 48 | ret := p.head 49 | if ret == nil { 50 | return nil 51 | } 52 | p.head = ret.next 53 | if ret.next != nil { 54 | ret.next.prev = nil 55 | } else { 56 | p.tail = nil 57 | } 58 | ret.prev = nil 59 | ret.next = nil 60 | return ret 61 | } 62 | return nil 63 | } 64 | 65 | func (pcm *pcmType) store(client *Client) { 66 | if _, ok := pcm.conns[client.address]; !ok { 67 | pcm.conns[client.address] = new(poolType) 68 | } 69 | p := pcm.conns[client.address] 70 | client.prev = nil 71 | client.next = p.head 72 | if p.head != nil { 73 | p.head.prev = client 74 | } else { 75 | p.tail = client 76 | } 77 | p.head = client 78 | } 79 | 80 | func (pcm *pcmType) StartServing() { 81 | for { 82 | select { 83 | case r := <-pcm.reused: 84 | pcm.store(r) 85 | case req := <-pcm.getconn: 86 | req.reply <- pcm.fetch(req.address) 87 | } 88 | } 89 | } 90 | 91 | func init() { 92 | /* singleton, because it is a global cache by definition */ 93 | Pcm = NewPcm() 94 | go Pcm.StartServing() 95 | } 96 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/structs/badoo.go: -------------------------------------------------------------------------------- 1 | package structs 2 | 3 | import ( 4 | "badoo/_packages/util/deepcopy" 5 | 6 | "fmt" 7 | "reflect" 8 | "strings" 9 | ) 10 | 11 | type BadooTags struct { 12 | IsSecret bool 13 | SecretVal string 14 | } 15 | 16 | func BadooTagsFromString(tag string) *BadooTags { 17 | result := &BadooTags{} 18 | 19 | if tag == "" { 20 | return nil 21 | } 22 | 23 | v := strings.Split(tag, ",") 24 | if len(v) < 1 { 25 | return nil 26 | } 27 | 28 | vv := strings.Split(v[0], "=") 29 | if vv[0] == "secret" { 30 | result.IsSecret = true 31 | } 32 | if len(vv) == 2 { 33 | result.SecretVal = vv[1] 34 | } else { 35 | result.SecretVal = "*******" 36 | } 37 | 38 | return result 39 | } 40 | 41 | // walks the struct and zeroes/resets all fields that are marked with `badoo:"secret"` tags 42 | func BadooStripSecretFields(st interface{}) (interface{}, error) { 43 | 44 | v := deepcopy.Iface(st) // copy the value to avoid wrecking the passed one 45 | 46 | err := WalkStructFieldsRecursive(reflect.ValueOf(v), 47 | func(parent reflect.Value, fieldNo int) error { 48 | pt := parent.Type() 49 | t := pt.Field(fieldNo) 50 | v := parent.Field(fieldNo) 51 | 52 | tags := BadooTagsFromString(t.Tag.Get("badoo")) 53 | if tags == nil || !tags.IsSecret { 54 | return nil 55 | } 56 | 57 | // field is empty = no need to reset 58 | if v.Interface() == reflect.Zero(t.Type).Interface() { 59 | return nil 60 | } 61 | 62 | // we MUST set the field, if we can't -> complain 63 | if !v.CanSet() { 64 | return fmt.Errorf("field %s::%s.%s is not settable but must be removed", t.PkgPath, pt.Name(), t.Name) 65 | } 66 | 67 | // now set the secret value 68 | // we handle setting to plain strings and string pointers, otherwise - set zero of whatever type the value is 69 | // XXX: if you redo it as a switch - don't forget to handle the case when it's a pointer to non-string type 70 | sv := reflect.ValueOf(tags.SecretVal) 71 | 72 | if v.Kind() == reflect.String { 73 | v.Set(sv) 74 | } else if v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.String { 75 | v.Elem().Set(sv) 76 | } else { 77 | v.Set(reflect.Zero(t.Type)) 78 | } 79 | 80 | return nil 81 | }) 82 | 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | return v, nil 88 | } 89 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/pidfile.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "syscall" 7 | ) 8 | 9 | type Pidfile struct { 10 | f *os.File 11 | path string 12 | } 13 | 14 | func pidfileOpenFile(path string) (*os.File, error) { 15 | // just don't create pidfile in case path is not set 16 | if "" == path { 17 | return nil, nil 18 | } 19 | 20 | return os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) 21 | } 22 | 23 | // PidfileTest just opens the file and closes it again, used by testconf mode only 24 | func PidfileTest(path string) error { 25 | pf, err := pidfileOpenFile(path) 26 | pf.Close() 27 | return err 28 | } 29 | 30 | // PidfileOpen opens new pidfile, locks it, truncates and writes current pid to it 31 | func PidfileOpen(path string) (*Pidfile, error) { 32 | 33 | pf, err := pidfileOpenFile(path) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | defer func() { 39 | if err != nil { 40 | pf.Close() 41 | } 42 | }() 43 | 44 | err = syscall.Flock(int(pf.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) 45 | if err != nil { 46 | return nil, fmt.Errorf("flock failed (other process running?): %s, %v", path, err) 47 | } 48 | 49 | err = pf.Truncate(0) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | _, err = fmt.Fprintf(pf, "%d\n", os.Getpid()) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | // everything is ok 60 | return &Pidfile{ 61 | f: pf, 62 | path: path, 63 | }, nil 64 | } 65 | 66 | func (p *Pidfile) Close() (err error) { 67 | if p.f != nil { 68 | err = p.f.Close() 69 | p.f = nil 70 | } 71 | return 72 | } 73 | 74 | func (p *Pidfile) MoveTo(new_path string) (*Pidfile, error) { 75 | if p.f == nil { 76 | return nil, fmt.Errorf("no pidfile is open") 77 | } 78 | 79 | if new_path == "" { 80 | return nil, fmt.Errorf("need path for new pidfile") 81 | } 82 | 83 | if err := os.Rename(p.path, new_path); err != nil { 84 | return nil, err 85 | } 86 | 87 | new_p := &Pidfile{ 88 | f: p.f, 89 | path: new_path, 90 | } 91 | 92 | p.f = nil 93 | 94 | return new_p, nil 95 | } 96 | 97 | func (p *Pidfile) CloseAndRemove() error { 98 | if p.f != nil { 99 | p.Close() // screw error here 100 | return os.Remove(p.path) 101 | } 102 | return nil 103 | } 104 | -------------------------------------------------------------------------------- /vendor/github.com/gorilla/handlers/compress.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Gorilla Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package handlers 6 | 7 | import ( 8 | "compress/flate" 9 | "compress/gzip" 10 | "io" 11 | "net/http" 12 | "strings" 13 | ) 14 | 15 | type compressResponseWriter struct { 16 | io.Writer 17 | http.ResponseWriter 18 | http.Hijacker 19 | } 20 | 21 | func (w *compressResponseWriter) WriteHeader(c int) { 22 | w.ResponseWriter.Header().Del("Content-Length") 23 | w.ResponseWriter.WriteHeader(c) 24 | } 25 | 26 | func (w *compressResponseWriter) Header() http.Header { 27 | return w.ResponseWriter.Header() 28 | } 29 | 30 | func (w *compressResponseWriter) Write(b []byte) (int, error) { 31 | h := w.ResponseWriter.Header() 32 | if h.Get("Content-Type") == "" { 33 | h.Set("Content-Type", http.DetectContentType(b)) 34 | } 35 | h.Del("Content-Length") 36 | 37 | return w.Writer.Write(b) 38 | } 39 | 40 | // CompressHandler gzip compresses HTTP responses for clients that support it 41 | // via the 'Accept-Encoding' header. 42 | func CompressHandler(h http.Handler) http.Handler { 43 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 44 | L: 45 | for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") { 46 | switch strings.TrimSpace(enc) { 47 | case "gzip": 48 | w.Header().Set("Content-Encoding", "gzip") 49 | w.Header().Add("Vary", "Accept-Encoding") 50 | 51 | gw := gzip.NewWriter(w) 52 | defer gw.Close() 53 | 54 | h, hok := w.(http.Hijacker) 55 | if !hok { /* w is not Hijacker... oh well... */ 56 | h = nil 57 | } 58 | 59 | w = &compressResponseWriter{ 60 | Writer: gw, 61 | ResponseWriter: w, 62 | Hijacker: h, 63 | } 64 | 65 | break L 66 | case "deflate": 67 | w.Header().Set("Content-Encoding", "deflate") 68 | w.Header().Add("Vary", "Accept-Encoding") 69 | 70 | fw, _ := flate.NewWriter(w, flate.DefaultCompression) 71 | defer fw.Close() 72 | 73 | h, hok := w.(http.Hijacker) 74 | if !hok { /* w is not Hijacker... oh well... */ 75 | h = nil 76 | } 77 | 78 | w = &compressResponseWriter{ 79 | Writer: fw, 80 | ResponseWriter: w, 81 | Hijacker: h, 82 | } 83 | 84 | break L 85 | } 86 | } 87 | 88 | h.ServeHTTP(w, r) 89 | }) 90 | } 91 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/rows.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import ( 12 | "database/sql/driver" 13 | "io" 14 | ) 15 | 16 | type mysqlField struct { 17 | tableName string 18 | name string 19 | flags fieldFlag 20 | fieldType byte 21 | decimals byte 22 | } 23 | 24 | type mysqlRows struct { 25 | mc *mysqlConn 26 | columns []mysqlField 27 | } 28 | 29 | type binaryRows struct { 30 | mysqlRows 31 | } 32 | 33 | type textRows struct { 34 | mysqlRows 35 | } 36 | 37 | type emptyRows struct{} 38 | 39 | func (rows *mysqlRows) Columns() []string { 40 | columns := make([]string, len(rows.columns)) 41 | if rows.mc.cfg.columnsWithAlias { 42 | for i := range columns { 43 | if tableName := rows.columns[i].tableName; len(tableName) > 0 { 44 | columns[i] = tableName + "." + rows.columns[i].name 45 | } else { 46 | columns[i] = rows.columns[i].name 47 | } 48 | } 49 | } else { 50 | for i := range columns { 51 | columns[i] = rows.columns[i].name 52 | } 53 | } 54 | return columns 55 | } 56 | 57 | func (rows *mysqlRows) Close() error { 58 | mc := rows.mc 59 | if mc == nil { 60 | return nil 61 | } 62 | if mc.netConn == nil { 63 | return ErrInvalidConn 64 | } 65 | 66 | // Remove unread packets from stream 67 | err := mc.readUntilEOF() 68 | rows.mc = nil 69 | return err 70 | } 71 | 72 | func (rows *binaryRows) Next(dest []driver.Value) error { 73 | if mc := rows.mc; mc != nil { 74 | if mc.netConn == nil { 75 | return ErrInvalidConn 76 | } 77 | 78 | // Fetch next row from stream 79 | return rows.readRow(dest) 80 | } 81 | return io.EOF 82 | } 83 | 84 | func (rows *textRows) Next(dest []driver.Value) error { 85 | if mc := rows.mc; mc != nil { 86 | if mc.netConn == nil { 87 | return ErrInvalidConn 88 | } 89 | 90 | // Fetch next row from stream 91 | return rows.readRow(dest) 92 | } 93 | return io.EOF 94 | } 95 | 96 | func (rows emptyRows) Columns() []string { 97 | return nil 98 | } 99 | 100 | func (rows emptyRows) Close() error { 101 | return nil 102 | } 103 | 104 | func (rows emptyRows) Next(dest []driver.Value) error { 105 | return io.EOF 106 | } 107 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/structs/protobuf.go: -------------------------------------------------------------------------------- 1 | package structs 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | // TODO: this needs to be moved to it's own package i guess ? 11 | // along with utilities for stripping comments and trailing commas from json 12 | type PBTags struct { 13 | Id int 14 | Type string 15 | Name string 16 | IsRequired bool 17 | DefValue string 18 | } 19 | 20 | func (p *PBTags) IsValid() bool { 21 | return 0 != p.Id 22 | } 23 | 24 | func PBTagsFromString(tags string) (p *PBTags) { 25 | p = &PBTags{} 26 | 27 | if "" == tags { 28 | return 29 | } 30 | 31 | v := strings.Split(tags, ",") 32 | if len(v) < 3 { 33 | return 34 | } 35 | 36 | p.Type = v[0] 37 | p.Id, _ = strconv.Atoi(v[1]) 38 | p.IsRequired = (v[2] == "req") 39 | 40 | for i := 3; i < len(v); i++ { 41 | kv := strings.SplitN(v[i], "=", 2) 42 | 43 | switch kv[0] { 44 | case "name": 45 | p.Name = kv[1] 46 | case "def": 47 | p.DefValue = kv[1] 48 | } 49 | } 50 | 51 | return p 52 | } 53 | 54 | // walks all struct fields and checks `protobuf` struct tags for required fields 55 | // if any field is marked as required by `protobuf` tag and is not set - we complain 56 | // NOTE: since with gogoprotobuf some fields can be marked with "(gogoproto.nullable) = false" 57 | // it's impossible to check if those were not set in config or set to zero value, this function only handles pointer fields 58 | func PBCheckRequiredFields(value interface{}) (err error) { 59 | return WalkStructFieldsRecursive(reflect.ValueOf(value), 60 | func(parent reflect.Value, fieldNo int) error { 61 | pt := parent.Type() 62 | t := pt.Field(fieldNo) 63 | v := parent.Field(fieldNo) 64 | 65 | tags := PBTagsFromString(t.Tag.Get("protobuf")) 66 | if tags == nil || !tags.IsRequired { 67 | return nil 68 | } 69 | 70 | // field is set = all good, skip it 71 | if !v.IsNil() { 72 | return nil 73 | } 74 | 75 | // required field is nil here = we need to return an error 76 | 77 | // try construct the best error message we can (take json field name and path) 78 | jsonName := func() string { 79 | jtag := t.Tag.Get("json") 80 | jname := strings.Split(jtag, ",")[0] // json name for that field 81 | 82 | if jname == "" || jname == "-" { 83 | return t.Name 84 | } 85 | return jname 86 | }() 87 | 88 | // TODO(antoxa): we can do better than reporting just the parent type name here (i.e. reconstruct 'full path' in json) 89 | return fmt.Errorf("required field \"%s\" missing in %s", jsonName, pt.Name()) 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /vendor/github.com/gogo/protobuf/proto/properties_gogo.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 2 | // http://github.com/gogo/protobuf/gogoproto 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | package proto 28 | 29 | import ( 30 | "fmt" 31 | "os" 32 | "reflect" 33 | ) 34 | 35 | func (p *Properties) setCustomEncAndDec(typ reflect.Type) { 36 | p.ctype = typ 37 | if p.Repeated { 38 | p.enc = (*Buffer).enc_custom_slice_bytes 39 | p.dec = (*Buffer).dec_custom_slice_bytes 40 | p.size = size_custom_slice_bytes 41 | } else if typ.Kind() == reflect.Ptr { 42 | p.enc = (*Buffer).enc_custom_bytes 43 | p.dec = (*Buffer).dec_custom_bytes 44 | p.size = size_custom_bytes 45 | } else { 46 | p.enc = (*Buffer).enc_custom_ref_bytes 47 | p.dec = (*Buffer).dec_custom_ref_bytes 48 | p.size = size_custom_ref_bytes 49 | } 50 | } 51 | 52 | func (p *Properties) setSliceOfNonPointerStructs(typ reflect.Type) { 53 | t2 := typ.Elem() 54 | p.sstype = typ 55 | p.stype = t2 56 | p.isMarshaler = isMarshaler(t2) 57 | p.isUnmarshaler = isUnmarshaler(t2) 58 | p.enc = (*Buffer).enc_slice_ref_struct_message 59 | p.dec = (*Buffer).dec_slice_ref_struct_message 60 | p.size = size_slice_ref_struct_message 61 | if p.Wire != "bytes" { 62 | fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T \n", typ, t2) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/pinba.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "badoo/_packages/pinba" 5 | 6 | "fmt" 7 | "sync" 8 | ) 9 | 10 | var ( 11 | pinbaSender *PinbaSender 12 | ) 13 | 14 | type PinbaInfo struct { 15 | Address string 16 | ServiceName string 17 | InstanceName string 18 | } 19 | 20 | type PinbaSender struct { 21 | *PinbaInfo 22 | clientPool sync.Pool 23 | } 24 | 25 | func NewPinbaSender(pi *PinbaInfo) *PinbaSender { 26 | return &PinbaSender{ 27 | PinbaInfo: pi, 28 | clientPool: sync.Pool{ 29 | New: func() interface{} { 30 | // ignoring error here, sucks 31 | // but there is a check when creating configuration for better error reporting 32 | c, _ := pinba.NewClient(pi.Address) 33 | return c 34 | }, 35 | }, 36 | } 37 | } 38 | 39 | func (ps *PinbaSender) Send(req *pinba.Request) error { 40 | cli := ps.GetClient() 41 | if cli == nil { 42 | return nil 43 | } 44 | 45 | req.Hostname = ps.ServiceName 46 | req.ServerName = ps.InstanceName 47 | 48 | err := cli.SendRequest(req) 49 | 50 | ps.clientPool.Put(cli) // this might be a frequent operation, save on defer 51 | 52 | return err 53 | } 54 | 55 | func (ps *PinbaSender) GetClient() *pinba.Client { 56 | return ps.clientPool.Get().(*pinba.Client) 57 | } 58 | 59 | // process pinba configuration and return internal configuration 60 | func PinbaInfoFromConfig(config Config) (*PinbaInfo, error) { 61 | daemonConfig := config.GetDaemonConfig() 62 | 63 | pi := &PinbaInfo{ 64 | Address: daemonConfig.GetPinbaAddress(), 65 | ServiceName: daemonConfig.GetServiceName(), 66 | InstanceName: daemonConfig.GetServiceInstanceName(), 67 | } 68 | 69 | if pi.Address == "" { 70 | return nil, fmt.Errorf("pinba_address not set or empty") 71 | } 72 | 73 | // check that client can be created, for better error reporting 74 | _, err := pinba.NewClient(pi.Address) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | return pi, nil 80 | } 81 | 82 | func IsPinbaConfigured() bool { 83 | return pinbaSender != nil 84 | } 85 | 86 | func GetPinbaSender() *PinbaSender { 87 | return pinbaSender 88 | } 89 | 90 | // send a request to pinba 91 | // will silently drop requests if pinba hs not been configured 92 | func SendToPinba(req *pinba.Request) error { 93 | if pinbaSender == nil { 94 | return nil 95 | } 96 | 97 | return pinbaSender.Send(req) 98 | } 99 | 100 | // grab a pinba Client for your own use 101 | // NOTE: client is NOT safe for use from multiple goroutines 102 | func GetPinbaClient() *pinba.Client { 103 | if pinbaSender == nil { 104 | return nil 105 | } 106 | 107 | return pinbaSender.GetClient() 108 | } 109 | -------------------------------------------------------------------------------- /jobgen/settings.go: -------------------------------------------------------------------------------- 1 | package jobgen 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "database/sql" 6 | "errors" 7 | "time" 8 | ) 9 | 10 | type ( 11 | Jobs struct { 12 | Type string 13 | Min int 14 | Max int 15 | Have_finish_jobs bool 16 | Temporary bool 17 | } 18 | 19 | NextTsCallback struct { 20 | Callback string 21 | Settings map[string]string 22 | } 23 | 24 | ScriptSettings struct { 25 | id uint64 26 | class_name string 27 | instance_count int 28 | max_time int 29 | jobs Jobs 30 | have_next_ts_callback bool 31 | next_ts_callback NextTsCallback 32 | repeat uint32 33 | retry sql.NullInt64 34 | ttl uint32 35 | repeat_job sql.NullInt64 36 | retry_job uint32 37 | location string 38 | location_type string 39 | developer sql.NullString 40 | max_retries uint32 41 | profiling_enabled int 42 | debug_enabled int 43 | named_params sql.NullString 44 | created int64 45 | } 46 | ) 47 | 48 | func loadSettingsFromRows(jiRows map[string]map[string]*JobInfoEntry, scripts map[string]*ScriptEntry) error { 49 | newIds := make(map[uint64]bool) 50 | jiRowsCnt := 0 51 | 52 | func() { 53 | allSettingsMutex.Lock() 54 | defer allSettingsMutex.Unlock() 55 | 56 | for _, row := range scripts { 57 | if _, ok := allSettings[row.settings_id]; !ok { 58 | newIds[row.settings_id] = true 59 | } 60 | } 61 | 62 | for _, rows := range jiRows { 63 | for _, row := range rows { 64 | jiRowsCnt++ 65 | 66 | if _, ok := allSettings[row.settings_id]; !ok { 67 | newIds[row.settings_id] = true 68 | } 69 | } 70 | } 71 | }() 72 | 73 | if len(newIds) > 0 { 74 | loadNewIdsTs := time.Now().UnixNano() 75 | err := loadNewIds(newIds) 76 | if err != nil { 77 | return err 78 | } 79 | log.Printf("Loaded %d new ids for %.5f sec", len(newIds), float64(time.Now().UnixNano()-loadNewIdsTs)/1e9) 80 | } 81 | 82 | log.Debugln(" Selected", len(scripts), "rows from scripts") 83 | log.Debugln(" Selected", jiRowsCnt, "rows from job info") 84 | return nil 85 | } 86 | 87 | func getScriptSettings(className string) (*ScriptSettings, error) { 88 | scriptsMap.Lock() 89 | defer scriptsMap.Unlock() 90 | 91 | script := scriptsMap.v[className] 92 | if script == nil { 93 | return nil, errors.New("Unknown class: " + className) 94 | } 95 | 96 | settings := script.settings 97 | if settings == nil { 98 | return nil, errors.New("Damaged script entry for class " + className + ": there are no settings") 99 | } 100 | 101 | return settings, nil 102 | } 103 | -------------------------------------------------------------------------------- /proto/thunder.proto: -------------------------------------------------------------------------------- 1 | package thunder; 2 | 3 | enum request_msgid { 4 | REQUEST_ADD_JOBS = 1; 5 | REQUEST_UPDATE_STATUS = 2; 6 | REQUEST_LOG_FINISH = 3; 7 | } 8 | 9 | enum response_msgid { 10 | RESPONSE_GENERIC = 1; 11 | RESPONSE_JOB_IDS = 2; 12 | RESPONSE_RUN_INFO = 3; 13 | } 14 | 15 | enum errno { 16 | ERRNO_GENERIC = 1; 17 | } 18 | 19 | message response_generic { 20 | required sint32 error_code = 1; 21 | optional string error_text = 2; 22 | } 23 | 24 | message request_heartbeat { 25 | required string hostname = 1; 26 | required string hostgroup = 2; 27 | 28 | message cpu_t { 29 | required float user = 1; 30 | required float sys = 2; 31 | required float nice = 3; 32 | required float iowait = 4; 33 | required float steal = 5; 34 | required float idle = 6; 35 | required float parasite = 7; 36 | } 37 | 38 | optional uint32 cpu_parrots_per_core = 3; 39 | required uint32 cpu_cores = 4; 40 | 41 | message memory_t { 42 | required int64 total = 1; 43 | required int64 free = 2; 44 | required int64 cached = 3; 45 | required int64 buffers = 4; 46 | required int64 parasite = 5; 47 | required int64 swap_total = 6; 48 | required int64 swap_used = 7; 49 | } 50 | 51 | optional cpu_t cpu = 5; 52 | optional memory_t memory = 6; 53 | } 54 | 55 | message request_phproxyd_heartbeat { 56 | required string hostname = 1; 57 | } 58 | 59 | message request_add_jobs { 60 | message job_t { 61 | optional int64 generation_id = 1; 62 | required string class_name = 2; 63 | required string job_data = 3; 64 | optional string method = 4; 65 | optional string location = 5; 66 | optional int64 next_launch_ts = 6; 67 | message repeat_t { 68 | optional int32 value = 1; 69 | } 70 | optional repeat_t repeat = 7; // if repeat field is set, but repeat.value is not, then it means NULL in DB 71 | optional uint64 settings_id = 8; 72 | } 73 | 74 | repeated job_t jobs = 1; 75 | } 76 | 77 | message response_job_ids { 78 | repeated uint64 job_id = 1; 79 | } 80 | 81 | message request_update_status { 82 | required string hostname = 1; // hostname where script is run (required because thunder has no global map of run ids) 83 | required uint64 run_id = 2; 84 | required string prev_status = 3; // previous run_status that is expected 85 | required string status = 4; // new run_status 86 | } 87 | 88 | message request_log_finish { 89 | required string hostname = 1; 90 | required uint64 run_id = 2; 91 | required string prev_status = 3; 92 | required bool success = 4; 93 | } 94 | 95 | message response_run_info { 96 | required string info = 1; // json_encode'd row from database table RunQueue 97 | } 98 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/logrus.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | // Fields type, used to pass to `WithFields`. 9 | type Fields map[string]interface{} 10 | 11 | // Level type 12 | type Level uint8 13 | 14 | // Convert the Level to a string. E.g. PanicLevel becomes "panic". 15 | func (level Level) String() string { 16 | switch level { 17 | case DebugLevel: 18 | return "debug" 19 | case InfoLevel: 20 | return "info" 21 | case WarnLevel: 22 | return "warning" 23 | case ErrorLevel: 24 | return "error" 25 | case FatalLevel: 26 | return "fatal" 27 | case PanicLevel: 28 | return "panic" 29 | } 30 | 31 | return "unknown" 32 | } 33 | 34 | // ParseLevel takes a string level and returns the Logrus log level constant. 35 | func ParseLevel(lvl string) (Level, error) { 36 | switch lvl { 37 | case "panic": 38 | return PanicLevel, nil 39 | case "fatal": 40 | return FatalLevel, nil 41 | case "error": 42 | return ErrorLevel, nil 43 | case "warn", "warning": 44 | return WarnLevel, nil 45 | case "info": 46 | return InfoLevel, nil 47 | case "debug": 48 | return DebugLevel, nil 49 | } 50 | 51 | var l Level 52 | return l, fmt.Errorf("not a valid logrus Level: %q", lvl) 53 | } 54 | 55 | // These are the different logging levels. You can set the logging level to log 56 | // on your instance of logger, obtained with `logrus.New()`. 57 | const ( 58 | // PanicLevel level, highest level of severity. Logs and then calls panic with the 59 | // message passed to Debug, Info, ... 60 | PanicLevel Level = iota 61 | // FatalLevel level. Logs and then calls `os.Exit(1)`. It will exit even if the 62 | // logging level is set to Panic. 63 | FatalLevel 64 | // ErrorLevel level. Logs. Used for errors that should definitely be noted. 65 | // Commonly used for hooks to send errors to an error tracking service. 66 | ErrorLevel 67 | // WarnLevel level. Non-critical entries that deserve eyes. 68 | WarnLevel 69 | // InfoLevel level. General operational entries about what's going on inside the 70 | // application. 71 | InfoLevel 72 | // DebugLevel level. Usually only enabled when debugging. Very verbose logging. 73 | DebugLevel 74 | ) 75 | 76 | // Won't compile if StdLogger can't be realized by a log.Logger 77 | var ( 78 | _ StdLogger = &log.Logger{} 79 | _ StdLogger = &Entry{} 80 | _ StdLogger = &Logger{} 81 | ) 82 | 83 | // StdLogger is what your logrus-enabled library should take, that way 84 | // it'll accept a stdlib logger and a logrus logger. There's no standard 85 | // interface, this is the closest we get, unfortunately. 86 | type StdLogger interface { 87 | Print(...interface{}) 88 | Printf(string, ...interface{}) 89 | Println(...interface{}) 90 | 91 | Fatal(...interface{}) 92 | Fatalf(string, ...interface{}) 93 | Fatalln(...interface{}) 94 | 95 | Panic(...interface{}) 96 | Panicf(string, ...interface{}) 97 | Panicln(...interface{}) 98 | } 99 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/http.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | 6 | "fmt" 7 | stdlog "log" 8 | "net" 9 | "os" 10 | "strings" 11 | "time" 12 | 13 | "net/http" 14 | _ "net/http/pprof" 15 | 16 | "github.com/gorilla/handlers" 17 | ) 18 | 19 | type httpLoggerWrapper struct { 20 | } 21 | 22 | func (l *httpLoggerWrapper) Write(data []byte) (int, error) { 23 | log.Errorf("[http pprof] %s", strings.Trim(string(data), " \t\r\n")) 24 | return len(data), nil 25 | } 26 | 27 | type httpServer struct { 28 | Addr string 29 | Listener net.Listener 30 | 31 | server *http.Server 32 | } 33 | 34 | func (s *httpServer) Serve() { 35 | s.server = &http.Server{ 36 | Addr: s.Addr, 37 | Handler: handlers.CompressHandler(http.DefaultServeMux), 38 | ReadTimeout: 10 * time.Second, 39 | WriteTimeout: 10 * time.Second, 40 | MaxHeaderBytes: 1 << 20, 41 | ErrorLog: stdlog.New(&httpLoggerWrapper{}, "", 0), 42 | } 43 | s.server.Serve(s.Listener) 44 | } 45 | 46 | func getHttpRestartSocket(rcd *RestartChildData, addr string) (*RestartSocket, *os.File) { 47 | if rcd == nil { 48 | return nil, nil 49 | } 50 | 51 | restartSocket := rcd.HttpPProfSocket 52 | if restartSocket.Address == "" { 53 | // parent doesn't have http_pprof_address 54 | return nil, nil 55 | } 56 | 57 | restartFile := os.NewFile(restartSocket.Fd, "") 58 | 59 | if restartSocket.Address != addr { 60 | return nil, restartFile 61 | } 62 | 63 | return &restartSocket, restartFile 64 | } 65 | 66 | // start http/pprof and expvar server 67 | // FIXME: refactor graceful restart copy-paste stuff here 68 | func newHttpServer(config Config, rcd *RestartChildData) (*httpServer, error) { 69 | var err error 70 | 71 | daemonConfig := config.GetDaemonConfig() 72 | 73 | addr := daemonConfig.GetHttpPprofAddr() 74 | if addr == "" { 75 | return nil, nil 76 | } 77 | 78 | restartSocket, restartFile := getHttpRestartSocket(rcd, addr) 79 | defer func() { 80 | if restartFile != nil { 81 | restartFile.Close() 82 | } 83 | }() 84 | 85 | var listener net.Listener 86 | if restartSocket == nil { 87 | listener, err = net.Listen("tcp", addr) 88 | if err != nil { 89 | return nil, fmt.Errorf("listen failed for server http_pprof at %s: %s", addr, err) 90 | } 91 | 92 | log.Infof("port http_pprof_addr bound to address %s", addr) 93 | } else { 94 | listener, err = net.FileListener(restartFile) // this dup()-s 95 | if err != nil { 96 | return nil, fmt.Errorf("failed to grab parent fd %d for http_pprof_addr at %s: %s", restartSocket.Fd, addr, err) 97 | } 98 | log.Infof("http_pprof_addr bound to address %s (parent fd: %d)", listener.Addr(), restartSocket.Fd) 99 | } 100 | 101 | server := &httpServer{ 102 | Addr: addr, 103 | Listener: listener, 104 | } 105 | 106 | return server, nil 107 | } 108 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/config_file.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // XXX(antoxa): for the lack of a better spot to put this code - let it rot here 4 | // the point being - moved the code here from service package, since some utils need it 5 | // but utils can't easily use service package itself, due to a ton of init machinery there 6 | 7 | import ( 8 | "badoo/_packages/util/structs" 9 | 10 | "encoding/json" 11 | "io/ioutil" 12 | ) 13 | 14 | func ParseConfigFromFile(conf_path string, to_struct interface{}) error { 15 | data, err := ioutil.ReadFile(conf_path) 16 | if err != nil { 17 | return err 18 | } 19 | 20 | data = PrepareJsonConfigData(data) 21 | 22 | err = ParseConfigToStruct(data, to_struct) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | return nil 28 | } 29 | 30 | func ParseConfigToStruct(data []byte, to_struct interface{}) error { 31 | 32 | err := json.Unmarshal(data, to_struct) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | err = structs.PBCheckRequiredFields(to_struct) 38 | if nil != err { 39 | return err 40 | } 41 | 42 | return nil 43 | } 44 | 45 | func PrepareJsonConfigData(d []byte) []byte { 46 | data := make([]byte, len(d)) 47 | copy(data, d) 48 | 49 | l := len(data) 50 | 51 | // first pass, remove comments 52 | for i := 0; i < l; { 53 | switch data[i] { 54 | default: 55 | i++ 56 | case '"': // no substitutions inside quoted strings 57 | i++ 58 | for ; i < l; i++ { 59 | if '"' == data[i] && '\\' != data[i-1] { 60 | i++ 61 | break 62 | } 63 | } 64 | case '/': 65 | i = jsonDataConsumeComments(data, i+1) 66 | } 67 | } 68 | 69 | // second pass, remove trailing commas 70 | for i := 0; i < l; i++ { 71 | if ']' == data[i] || '}' == data[i] { 72 | j := i - 1 73 | 74 | for j >= 0 && jsonIsWhitespace(data[j]) { 75 | j-- 76 | } 77 | 78 | for ; ',' == data[j]; j-- { 79 | data[j] = ' ' 80 | } 81 | } 82 | } 83 | 84 | return data 85 | } 86 | 87 | // ---------------------------------------------------------------------------------------------------------------------------------- 88 | // private json stuff 89 | 90 | func jsonDataConsumeComments(data []byte, i int) int { 91 | l := len(data) 92 | if i >= l { 93 | return i 94 | } 95 | 96 | if '/' == data[i] { // line comment 97 | data[i-1] = ' ' 98 | for ; (i < l) && ('\n' != data[i]); i++ { 99 | data[i] = ' ' 100 | } 101 | } else if '*' == data[i] { // block comment 102 | data[i-1] = ' ' 103 | data[i] = ' ' 104 | 105 | for i++; i < l; i++ { 106 | if '*' == data[i] { 107 | if ((i + 1) < l) && '/' == data[i+1] { 108 | data[i] = ' ' 109 | data[i+1] = ' ' 110 | i += 2 111 | break 112 | } 113 | } 114 | data[i] = ' ' 115 | } 116 | } 117 | return i 118 | } 119 | 120 | func jsonIsWhitespace(c byte) bool { 121 | return ' ' == c || '\t' == c || '\r' == c || '\n' == c 122 | } 123 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/logfile.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | syslog_hook "badoo/_packages/log/hooks/syslog" 6 | "fmt" 7 | "log/syslog" 8 | "os" 9 | "regexp" 10 | ) 11 | 12 | var ( 13 | logPath = "-" 14 | logLevel = log.DebugLevel 15 | syslogHookInited = false 16 | 17 | stderrLogger *log.Logger // for when you need a console unconditionally (such as config test errors) 18 | ) 19 | 20 | func parseSyslogIdentity(str string) string { 21 | if str == "" { 22 | return "" 23 | } 24 | 25 | type replaceTable struct { 26 | regExp string 27 | replace string 28 | } 29 | 30 | table := []replaceTable{ 31 | {"\\$\\{daemon\\}", VersionInfo.GetVcsBasename()}, 32 | {"\\$\\{version\\}", VersionInfo.GetVersion()}, 33 | {"\\$\\{service_name\\}", DaemonConfig().GetServiceName()}, 34 | {"\\$\\{service_instance_name\\}", DaemonConfig().GetServiceInstanceName()}, 35 | } 36 | 37 | for _, elem := range table { 38 | re := regexp.MustCompile(elem.regExp) 39 | str = re.ReplaceAllString(str, elem.replace) 40 | } 41 | 42 | return str 43 | } 44 | 45 | func initSyslogHook() error { 46 | if syslogHookInited { 47 | return nil 48 | } 49 | 50 | if DaemonConfig().SyslogIdentity != nil { 51 | syslogIdentity := parseSyslogIdentity(DaemonConfig().GetSyslogIdentity()) 52 | log.Debugf("syslog_identity: <%s>", syslogIdentity) 53 | 54 | syslogAddr := fmt.Sprintf("%s:%d", DaemonConfig().GetSyslogIp(), DaemonConfig().GetSyslogPort()) 55 | 56 | network := func() string { 57 | 58 | if DaemonConfig().GetSyslogIp() != "" && DaemonConfig().GetSyslogPort() != 0 { 59 | return "udp" 60 | } 61 | 62 | return "" 63 | }() 64 | 65 | hook, err := syslog_hook.NewSyslogHookNoFrozen(network, syslogAddr, syslog.LOG_LOCAL6, syslogIdentity) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | log.AddHook(hook) 71 | 72 | if DaemonConfig().GetSyslogSendBufferSize() != 0 { 73 | log.Warn("unsupported config option syslog_send_buffer_size, ignored") 74 | } 75 | } 76 | 77 | syslogHookInited = true 78 | 79 | return nil 80 | } 81 | 82 | func init() { 83 | log.SetLevel(log.DebugLevel) 84 | log.SetFormatter(&log.BadooFormatter{}) 85 | 86 | stderrLogger = &log.Logger{ 87 | Out: os.Stderr, 88 | Formatter: &log.BadooFormatter{}, 89 | Level: log.DebugLevel, 90 | } 91 | 92 | } 93 | 94 | func reopenLogfile(path string, level log.Level) (err error) { 95 | if err := initSyslogHook(); err != nil { 96 | return err 97 | } 98 | 99 | log.SetLevel(level) 100 | 101 | if "" == path || "-" == path { 102 | log.SetOutput(os.Stderr) 103 | } else { 104 | logfile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) 105 | if nil != err { 106 | return err 107 | } 108 | log.SetOutput(logfile) 109 | } 110 | 111 | logPath = path 112 | logLevel = level 113 | 114 | return nil 115 | } 116 | 117 | func GetLogLevel() log.Level { 118 | return logLevel 119 | } 120 | -------------------------------------------------------------------------------- /thunder.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "badoo/_packages/gpbrpc" 5 | "badoo/_packages/log" 6 | "badoo/_packages/service" 7 | "github.com/badoo/thunder/common" 8 | "github.com/badoo/thunder/db" 9 | "github.com/badoo/thunder/jobgen" 10 | "github.com/badoo/thunder/proto" 11 | "net/http" 12 | 13 | "github.com/gogo/protobuf/proto" 14 | ) 15 | 16 | var config = common.FullConfig{} 17 | 18 | type GPBContext struct { 19 | } 20 | 21 | func (ctx *GPBContext) RequestAddJobs(rctx gpbrpc.RequestT, request *thunder.RequestAddJobs) gpbrpc.ResultT { 22 | ids, err := jobgen.APIAcceptTTJobs(request.Jobs) 23 | 24 | if err != nil { 25 | return thunder.Gpbrpc.ErrorGeneric(err.Error()) 26 | } 27 | 28 | return gpbrpc.Result(&thunder.ResponseJobIds{JobId: ids}) 29 | } 30 | 31 | func (ctx *GPBContext) RequestUpdateStatus(rctx gpbrpc.RequestT, request *thunder.RequestUpdateStatus) gpbrpc.ResultT { 32 | err := jobgen.APIUpdateRunStatus(request.GetHostname(), request.GetRunId(), request.GetPrevStatus(), request.GetStatus()) 33 | if err != nil { 34 | return thunder.Gpbrpc.ErrorGeneric(err.Error()) 35 | } 36 | 37 | return thunder.Gpbrpc.OK() 38 | } 39 | 40 | func (ctx *GPBContext) RequestLogFinish(rctx gpbrpc.RequestT, request *thunder.RequestLogFinish) gpbrpc.ResultT { 41 | info, err := jobgen.APILogFinish(request.GetHostname(), request.GetRunId(), request.GetPrevStatus(), request.GetSuccess()) 42 | if err != nil { 43 | return thunder.Gpbrpc.ErrorGeneric(err.Error()) 44 | } 45 | 46 | return gpbrpc.Result(&thunder.ResponseRunInfo{Info: proto.String(info)}) 47 | } 48 | 49 | func checkSelect42() { 50 | rows, err := db.Query("SELECT 42") 51 | if err != nil { 52 | log.Fatal("Could not SELECT 42 from DB: " + err.Error()) 53 | } 54 | 55 | defer rows.Close() 56 | 57 | var intRes int 58 | 59 | for rows.Next() { 60 | if err := rows.Scan(&intRes); err != nil { 61 | log.Fatal("Could not fetch results of SELECT 42: " + err.Error()) 62 | } 63 | 64 | if intRes != 42 { 65 | log.Fatal("SELECT 42 did not return '42', perhaps something is broken in either client or server") 66 | } 67 | 68 | // TODO: maybe check that only one exists? 69 | } 70 | } 71 | 72 | func setupAndCheckMysql() { 73 | mysqlConf := config.GetMysql() 74 | 75 | log.Info("Connecting to MySQL") 76 | 77 | if err := db.Setup(mysqlConf.GetDsn(), mysqlConf.GetMaxConnections()); err != nil { 78 | log.Fatal("Could not estabilish db connection: " + err.Error()) 79 | } 80 | 81 | checkSelect42() 82 | 83 | log.Info("Connection successfull") 84 | } 85 | 86 | func main() { 87 | var gpb = GPBContext{} 88 | 89 | var ports = []service.Port{ 90 | service.GpbPort("thunder-gpb", &gpb, thunder.Gpbrpc), 91 | service.JsonPort("thunder-gpb/json", &gpb, thunder.Gpbrpc), 92 | } 93 | 94 | service.Initialize("conf/thunder.conf", &config) 95 | 96 | setupAndCheckMysql() 97 | 98 | http.HandleFunc("/debug/thunder/", debugPage) 99 | http.HandleFunc("/debug/thunder/jobs", jobsDebugPage) 100 | 101 | common.Setup() 102 | jobgen.Setup(config) 103 | go jobgen.WriteLogsThread(config.GetLauncher().GetRqhLogFile()) 104 | go jobgen.GenerateJobsCycle() 105 | 106 | service.EventLoop(ports) 107 | } 108 | -------------------------------------------------------------------------------- /go-proxyd/proto/phproxyd-config.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gogo. 2 | // source: phproxyd-config.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package badoo_phproxyd is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | phproxyd-config.proto 10 | 11 | It has these top-level messages: 12 | PhproxydConfig 13 | */ 14 | package badoo_phproxyd 15 | 16 | import proto "github.com/gogo/protobuf/proto" 17 | import fmt "fmt" 18 | import math "math" 19 | 20 | // Reference imports to suppress errors if they are not otherwise used. 21 | var _ = proto.Marshal 22 | var _ = fmt.Errorf 23 | var _ = math.Inf 24 | 25 | type PhproxydConfig struct { 26 | Scripts *PhproxydConfigScriptsT `protobuf:"bytes,2,req,name=scripts" json:"scripts,omitempty"` 27 | XXX_unrecognized []byte `json:"-"` 28 | } 29 | 30 | func (m *PhproxydConfig) Reset() { *m = PhproxydConfig{} } 31 | func (m *PhproxydConfig) String() string { return proto.CompactTextString(m) } 32 | func (*PhproxydConfig) ProtoMessage() {} 33 | 34 | func (m *PhproxydConfig) GetScripts() *PhproxydConfigScriptsT { 35 | if m != nil { 36 | return m.Scripts 37 | } 38 | return nil 39 | } 40 | 41 | type PhproxydConfigScriptsT struct { 42 | PhpBin *string `protobuf:"bytes,1,req,name=php_bin" json:"php_bin,omitempty"` 43 | PhpScriptsDir *string `protobuf:"bytes,2,req,name=php_scripts_dir" json:"php_scripts_dir,omitempty"` 44 | FileStdoutTemplate *string `protobuf:"bytes,3,req,name=file_stdout_template" json:"file_stdout_template,omitempty"` 45 | FileStderrTemplate *string `protobuf:"bytes,4,req,name=file_stderr_template" json:"file_stderr_template,omitempty"` 46 | MemStdoutTemplate *string `protobuf:"bytes,5,req,name=mem_stdout_template" json:"mem_stdout_template,omitempty"` 47 | MemStderrTemplate *string `protobuf:"bytes,6,req,name=mem_stderr_template" json:"mem_stderr_template,omitempty"` 48 | XXX_unrecognized []byte `json:"-"` 49 | } 50 | 51 | func (m *PhproxydConfigScriptsT) Reset() { *m = PhproxydConfigScriptsT{} } 52 | func (m *PhproxydConfigScriptsT) String() string { return proto.CompactTextString(m) } 53 | func (*PhproxydConfigScriptsT) ProtoMessage() {} 54 | 55 | func (m *PhproxydConfigScriptsT) GetPhpBin() string { 56 | if m != nil && m.PhpBin != nil { 57 | return *m.PhpBin 58 | } 59 | return "" 60 | } 61 | 62 | func (m *PhproxydConfigScriptsT) GetPhpScriptsDir() string { 63 | if m != nil && m.PhpScriptsDir != nil { 64 | return *m.PhpScriptsDir 65 | } 66 | return "" 67 | } 68 | 69 | func (m *PhproxydConfigScriptsT) GetFileStdoutTemplate() string { 70 | if m != nil && m.FileStdoutTemplate != nil { 71 | return *m.FileStdoutTemplate 72 | } 73 | return "" 74 | } 75 | 76 | func (m *PhproxydConfigScriptsT) GetFileStderrTemplate() string { 77 | if m != nil && m.FileStderrTemplate != nil { 78 | return *m.FileStderrTemplate 79 | } 80 | return "" 81 | } 82 | 83 | func (m *PhproxydConfigScriptsT) GetMemStdoutTemplate() string { 84 | if m != nil && m.MemStdoutTemplate != nil { 85 | return *m.MemStdoutTemplate 86 | } 87 | return "" 88 | } 89 | 90 | func (m *PhproxydConfigScriptsT) GetMemStderrTemplate() string { 91 | if m != nil && m.MemStderrTemplate != nil { 92 | return *m.MemStderrTemplate 93 | } 94 | return "" 95 | } 96 | -------------------------------------------------------------------------------- /proto/phproxyd/phproxyd.proto: -------------------------------------------------------------------------------- 1 | 2 | package badoo.phproxyd; 3 | 4 | enum request_msgid { 5 | REQUEST_RRD = 1; 6 | REQUEST_PING = 2; 7 | REQUEST_STAT = 3; 8 | REQUEST_RUN = 4; 9 | REQUEST_CHECK = 5; 10 | REQUEST_FREE = 6; 11 | REQUEST_TERMINATE = 7; 12 | } 13 | 14 | enum response_msgid { 15 | RESPONSE_GENERIC = 1; 16 | RESPONSE_PING = 2; 17 | RESPONSE_RRD = 3; 18 | RESPONSE_STAT = 4; 19 | RESPONSE_CHECK = 5; 20 | } 21 | 22 | enum errno { 23 | ERRNO_GENERIC = 1; 24 | ERRNO_RUN_FAILED = 2; 25 | ERRNO_ALREADY_RUNNING = 3; 26 | ERRNO_NOT_FOUND = 4; 27 | ERRNO_WORKING = 5; 28 | ERRNO_NO_MEMORY = 6; 29 | ERRNO_SUCCESS_FINISHED = 7; 30 | ERRNO_FAILED_FINISHED = 8; 31 | ERRNO_WAIT_FOR_FREE = 9; 32 | ERRNO_START_CHILD_FAILED = 10; 33 | ERRNO_TOO_BUSY = 11; 34 | } 35 | 36 | enum phproxyd_status_t { 37 | STATUS_OK = 1; 38 | STATUS_FAILED = 2; 39 | } 40 | 41 | enum store_t { 42 | FILES = 1; 43 | MEMORY = 2; 44 | } 45 | 46 | message tag_stat_t { 47 | required string tag = 1; 48 | required uint32 success = 2; 49 | required uint32 failed = 3; 50 | } 51 | 52 | message tag_rrd_t { 53 | required string tag = 1; 54 | required uint64 tm_clean = 2; 55 | required uint32 success = 3; 56 | required uint32 failed = 4; 57 | } 58 | 59 | message response_generic { 60 | required sint32 error_code = 1; 61 | optional string error_text = 2; 62 | } 63 | 64 | message request_ping { 65 | } 66 | 67 | message request_rrd { 68 | repeated string tags = 1; 69 | required uint32 clean = 2; 70 | } 71 | 72 | message request_stat { 73 | } 74 | 75 | message response_ping { 76 | required phproxyd_status_t status = 1; 77 | } 78 | 79 | message response_rrd { 80 | required string vmsize = 1; 81 | required uint64 utime_tv_sec = 2; 82 | required uint64 utime_tv_usec = 3; 83 | required uint64 stime_tv_sec = 4; 84 | required uint64 stime_tv_usec = 5; 85 | required uint64 commands_cnt = 6; 86 | required uint64 failed_commands_cnt = 7; 87 | 88 | repeated tag_rrd_t rrd = 8; 89 | 90 | optional uint64 too_busy_cnt = 9; 91 | } 92 | 93 | message response_stat { 94 | required string vmsize = 1; 95 | required uint64 utime_tv_sec = 2; 96 | required uint64 utime_tv_usec = 3; 97 | required uint64 stime_tv_sec = 4; 98 | required uint64 stime_tv_usec = 5; 99 | required uint64 commands_cnt = 6; 100 | required uint64 failed_commands_cnt = 7; 101 | 102 | repeated tag_stat_t stats = 8; 103 | optional uint64 hash_size = 9; 104 | optional uint64 too_busy_cnt = 10; 105 | } 106 | 107 | message request_run { 108 | required string script = 1; 109 | required uint64 hash = 2; 110 | required string tag = 3; 111 | optional sint32 force = 4; 112 | repeated string params = 5; 113 | optional store_t store = 6 [default = MEMORY]; 114 | optional bool free_after_run = 7 [default = true]; 115 | } 116 | 117 | message request_check { 118 | required uint64 hash = 2; 119 | } 120 | 121 | message request_free { 122 | required uint64 hash = 2; 123 | } 124 | 125 | message response_check { 126 | required sint32 error_code = 1; 127 | optional string error_text = 2; 128 | optional string response = 3; 129 | optional uint64 utime_tv_sec = 4; 130 | optional uint64 utime_tv_usec = 5; 131 | optional uint64 stime_tv_sec = 6; 132 | optional uint64 stime_tv_usec = 7; 133 | optional sint32 retcode = 8; 134 | optional sint32 maxrss = 9; 135 | } 136 | 137 | message request_terminate { 138 | required uint64 hash = 1; 139 | } 140 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/gpbrpc/codec_json.go: -------------------------------------------------------------------------------- 1 | package gpbrpc 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "net" 8 | "strings" 9 | 10 | "github.com/gogo/protobuf/proto" 11 | ) 12 | 13 | const ( 14 | JSON_MaxMessageLength = 1024 * 1024 // TODO: make this configurable 15 | ) 16 | 17 | var ( 18 | JsonCodec = jsonCodec{} 19 | ) 20 | 21 | type jsonCodec struct { 22 | } 23 | 24 | func (c *jsonCodec) ReadRequest(p Protocol, conn net.Conn) (uint32, proto.Message, int, ConnStatus, error) { 25 | _, line, readln, status, err := ReadJsonPacket(conn) 26 | if err != nil { 27 | return 0, nil, readln, status, err 28 | } 29 | 30 | r := bytes.IndexAny(line, " ") 31 | if r == -1 { 32 | return 0, nil, readln, ConnOK, fmt.Errorf("parse error: need message name") 33 | } 34 | 35 | message_name := "request_" + strings.ToLower(string(line[:r])) 36 | 37 | msgid := MapRequestNameToId(p, message_name) 38 | msg := p.GetRequestMsg(msgid) 39 | if msg == nil { 40 | return 0, nil, readln, ConnOK, fmt.Errorf("parse error: unknown message id = %d", msgid) 41 | } 42 | err = json.Unmarshal(line[r+1:], msg) 43 | if err != nil { 44 | return 0, nil, readln, ConnOK, fmt.Errorf("parse error: message %s, error: %s", message_name, err) 45 | } 46 | return msgid, msg, readln, status, nil 47 | } 48 | 49 | func (c *jsonCodec) WriteResponse(p Protocol, conn net.Conn, msg proto.Message) (int, error) { 50 | msg_name := JsonStripMsgname(MapResponseIdToName(p, p.GetResponseMsgid(msg))) 51 | return WriteJsonPacket(conn, msg_name, msg) 52 | } 53 | 54 | // --------------------------------------------------------------------------------------------------------------- 55 | // helper functions 56 | 57 | func ReadJsonPacket(conn net.Conn) (uint32, []byte, int, ConnStatus, error) { 58 | line := make([]byte, JSON_MaxMessageLength) 59 | 60 | read_bytes, err := readLine(conn, line) 61 | if err != nil { 62 | if read_bytes == 0 { 63 | return 0, nil, read_bytes, ConnEOF, fmt.Errorf("read error: %s", err) 64 | } 65 | if read_bytes == len(line) { // FIXME: this one is not really needed, readLine() check this cond 66 | return 0, nil, read_bytes, ConnIOError, fmt.Errorf("read error: %s", err) 67 | } 68 | return 0, nil, read_bytes, ConnIOError, fmt.Errorf("read error: %s", err) 69 | } 70 | 71 | return 0, line[:read_bytes], read_bytes, ConnOK, nil 72 | } 73 | 74 | func WriteJsonPacket(conn net.Conn, msgname string, msg proto.Message) (int, error) { 75 | var body []byte 76 | var err error 77 | 78 | if msg != nil { 79 | body, err = json.Marshal(msg) 80 | if err != nil { 81 | return 0, fmt.Errorf("encode error: %s", err) 82 | } 83 | } 84 | 85 | packet := bytes.NewBufferString(msgname) 86 | 87 | packet.Write([]byte(" ")) 88 | packet.Write(body) 89 | packet.Write([]byte("\r\n")) 90 | 91 | written_bytes, err := writeAll(conn, packet.Bytes()) 92 | if err != nil || written_bytes != len(packet.Bytes()) { 93 | if err != nil { 94 | return written_bytes, fmt.Errorf("write error: %s", err) 95 | } 96 | return written_bytes, fmt.Errorf("write error: connection unexpectedly closed, err: %s", err) 97 | } 98 | 99 | return written_bytes, nil 100 | } 101 | 102 | // used by json and Client, need to move to proper spot or generate it 103 | func JsonStripMsgname(name string) string { 104 | if name[:len("response_")] == "response_" { 105 | return name[len("response_"):] 106 | } else if name[:len("request_")] == "request_" { 107 | return name[len("request_"):] 108 | } else { 109 | panic("oops, " + name) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/buffer.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import "io" 12 | 13 | const defaultBufSize = 4096 14 | 15 | // A buffer which is used for both reading and writing. 16 | // This is possible since communication on each connection is synchronous. 17 | // In other words, we can't write and read simultaneously on the same connection. 18 | // The buffer is similar to bufio.Reader / Writer but zero-copy-ish 19 | // Also highly optimized for this particular use case. 20 | type buffer struct { 21 | buf []byte 22 | rd io.Reader 23 | idx int 24 | length int 25 | } 26 | 27 | func newBuffer(rd io.Reader) buffer { 28 | var b [defaultBufSize]byte 29 | return buffer{ 30 | buf: b[:], 31 | rd: rd, 32 | } 33 | } 34 | 35 | // fill reads into the buffer until at least _need_ bytes are in it 36 | func (b *buffer) fill(need int) error { 37 | n := b.length 38 | 39 | // move existing data to the beginning 40 | if n > 0 && b.idx > 0 { 41 | copy(b.buf[0:n], b.buf[b.idx:]) 42 | } 43 | 44 | // grow buffer if necessary 45 | // TODO: let the buffer shrink again at some point 46 | // Maybe keep the org buf slice and swap back? 47 | if need > len(b.buf) { 48 | // Round up to the next multiple of the default size 49 | newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) 50 | copy(newBuf, b.buf) 51 | b.buf = newBuf 52 | } 53 | 54 | b.idx = 0 55 | 56 | for { 57 | nn, err := b.rd.Read(b.buf[n:]) 58 | n += nn 59 | 60 | switch err { 61 | case nil: 62 | if n < need { 63 | continue 64 | } 65 | b.length = n 66 | return nil 67 | 68 | case io.EOF: 69 | if n >= need { 70 | b.length = n 71 | return nil 72 | } 73 | return io.ErrUnexpectedEOF 74 | 75 | default: 76 | return err 77 | } 78 | } 79 | } 80 | 81 | // returns next N bytes from buffer. 82 | // The returned slice is only guaranteed to be valid until the next read 83 | func (b *buffer) readNext(need int) ([]byte, error) { 84 | if b.length < need { 85 | // refill 86 | if err := b.fill(need); err != nil { 87 | return nil, err 88 | } 89 | } 90 | 91 | offset := b.idx 92 | b.idx += need 93 | b.length -= need 94 | return b.buf[offset:b.idx], nil 95 | } 96 | 97 | // returns a buffer with the requested size. 98 | // If possible, a slice from the existing buffer is returned. 99 | // Otherwise a bigger buffer is made. 100 | // Only one buffer (total) can be used at a time. 101 | func (b *buffer) takeBuffer(length int) []byte { 102 | if b.length > 0 { 103 | return nil 104 | } 105 | 106 | // test (cheap) general case first 107 | if length <= defaultBufSize || length <= cap(b.buf) { 108 | return b.buf[:length] 109 | } 110 | 111 | if length < maxPacketSize { 112 | b.buf = make([]byte, length) 113 | return b.buf 114 | } 115 | return make([]byte, length) 116 | } 117 | 118 | // shortcut which can be used if the requested buffer is guaranteed to be 119 | // smaller than defaultBufSize 120 | // Only one buffer (total) can be used at a time. 121 | func (b *buffer) takeSmallBuffer(length int) []byte { 122 | if b.length == 0 { 123 | return b.buf[:length] 124 | } 125 | return nil 126 | } 127 | 128 | // takeCompleteBuffer returns the complete existing buffer. 129 | // This can be used if the necessary buffer size is unknown. 130 | // Only one buffer (total) can be used at a time. 131 | func (b *buffer) takeCompleteBuffer() []byte { 132 | if b.length == 0 { 133 | return b.buf 134 | } 135 | return nil 136 | } 137 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/statement.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import ( 12 | "database/sql/driver" 13 | "fmt" 14 | "reflect" 15 | "strconv" 16 | ) 17 | 18 | type mysqlStmt struct { 19 | mc *mysqlConn 20 | id uint32 21 | paramCount int 22 | columns []mysqlField // cached from the first query 23 | } 24 | 25 | func (stmt *mysqlStmt) Close() error { 26 | if stmt.mc == nil || stmt.mc.netConn == nil { 27 | errLog.Print(ErrInvalidConn) 28 | return driver.ErrBadConn 29 | } 30 | 31 | err := stmt.mc.writeCommandPacketUint32(comStmtClose, stmt.id) 32 | stmt.mc = nil 33 | return err 34 | } 35 | 36 | func (stmt *mysqlStmt) NumInput() int { 37 | return stmt.paramCount 38 | } 39 | 40 | func (stmt *mysqlStmt) ColumnConverter(idx int) driver.ValueConverter { 41 | return converter{} 42 | } 43 | 44 | func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { 45 | if stmt.mc.netConn == nil { 46 | errLog.Print(ErrInvalidConn) 47 | return nil, driver.ErrBadConn 48 | } 49 | // Send command 50 | err := stmt.writeExecutePacket(args) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | mc := stmt.mc 56 | 57 | mc.affectedRows = 0 58 | mc.insertId = 0 59 | 60 | // Read Result 61 | resLen, err := mc.readResultSetHeaderPacket() 62 | if err == nil { 63 | if resLen > 0 { 64 | // Columns 65 | err = mc.readUntilEOF() 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | // Rows 71 | err = mc.readUntilEOF() 72 | } 73 | if err == nil { 74 | return &mysqlResult{ 75 | affectedRows: int64(mc.affectedRows), 76 | insertId: int64(mc.insertId), 77 | }, nil 78 | } 79 | } 80 | 81 | return nil, err 82 | } 83 | 84 | func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { 85 | if stmt.mc.netConn == nil { 86 | errLog.Print(ErrInvalidConn) 87 | return nil, driver.ErrBadConn 88 | } 89 | // Send command 90 | err := stmt.writeExecutePacket(args) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | mc := stmt.mc 96 | 97 | // Read Result 98 | resLen, err := mc.readResultSetHeaderPacket() 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | rows := new(binaryRows) 104 | rows.mc = mc 105 | 106 | if resLen > 0 { 107 | // Columns 108 | // If not cached, read them and cache them 109 | if stmt.columns == nil { 110 | rows.columns, err = mc.readColumns(resLen) 111 | stmt.columns = rows.columns 112 | } else { 113 | rows.columns = stmt.columns 114 | err = mc.readUntilEOF() 115 | } 116 | } 117 | 118 | return rows, err 119 | } 120 | 121 | type converter struct{} 122 | 123 | func (c converter) ConvertValue(v interface{}) (driver.Value, error) { 124 | if driver.IsValue(v) { 125 | return v, nil 126 | } 127 | 128 | rv := reflect.ValueOf(v) 129 | switch rv.Kind() { 130 | case reflect.Ptr: 131 | // indirect pointers 132 | if rv.IsNil() { 133 | return nil, nil 134 | } 135 | return c.ConvertValue(rv.Elem().Interface()) 136 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 137 | return rv.Int(), nil 138 | case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: 139 | return int64(rv.Uint()), nil 140 | case reflect.Uint64: 141 | u64 := rv.Uint() 142 | if u64 >= 1<<63 { 143 | return strconv.FormatUint(u64, 10), nil 144 | } 145 | return int64(u64), nil 146 | case reflect.Float32, reflect.Float64: 147 | return rv.Float(), nil 148 | } 149 | return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind()) 150 | } 151 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/const.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | const ( 12 | minProtocolVersion byte = 10 13 | maxPacketSize = 1<<24 - 1 14 | timeFormat = "2006-01-02 15:04:05.999999" 15 | ) 16 | 17 | // MySQL constants documentation: 18 | // http://dev.mysql.com/doc/internals/en/client-server-protocol.html 19 | 20 | const ( 21 | iOK byte = 0x00 22 | iLocalInFile byte = 0xfb 23 | iEOF byte = 0xfe 24 | iERR byte = 0xff 25 | ) 26 | 27 | // https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags 28 | type clientFlag uint32 29 | 30 | const ( 31 | clientLongPassword clientFlag = 1 << iota 32 | clientFoundRows 33 | clientLongFlag 34 | clientConnectWithDB 35 | clientNoSchema 36 | clientCompress 37 | clientODBC 38 | clientLocalFiles 39 | clientIgnoreSpace 40 | clientProtocol41 41 | clientInteractive 42 | clientSSL 43 | clientIgnoreSIGPIPE 44 | clientTransactions 45 | clientReserved 46 | clientSecureConn 47 | clientMultiStatements 48 | clientMultiResults 49 | clientPSMultiResults 50 | clientPluginAuth 51 | clientConnectAttrs 52 | clientPluginAuthLenEncClientData 53 | clientCanHandleExpiredPasswords 54 | clientSessionTrack 55 | clientDeprecateEOF 56 | ) 57 | 58 | const ( 59 | comQuit byte = iota + 1 60 | comInitDB 61 | comQuery 62 | comFieldList 63 | comCreateDB 64 | comDropDB 65 | comRefresh 66 | comShutdown 67 | comStatistics 68 | comProcessInfo 69 | comConnect 70 | comProcessKill 71 | comDebug 72 | comPing 73 | comTime 74 | comDelayedInsert 75 | comChangeUser 76 | comBinlogDump 77 | comTableDump 78 | comConnectOut 79 | comRegisterSlave 80 | comStmtPrepare 81 | comStmtExecute 82 | comStmtSendLongData 83 | comStmtClose 84 | comStmtReset 85 | comSetOption 86 | comStmtFetch 87 | ) 88 | 89 | // https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnType 90 | const ( 91 | fieldTypeDecimal byte = iota 92 | fieldTypeTiny 93 | fieldTypeShort 94 | fieldTypeLong 95 | fieldTypeFloat 96 | fieldTypeDouble 97 | fieldTypeNULL 98 | fieldTypeTimestamp 99 | fieldTypeLongLong 100 | fieldTypeInt24 101 | fieldTypeDate 102 | fieldTypeTime 103 | fieldTypeDateTime 104 | fieldTypeYear 105 | fieldTypeNewDate 106 | fieldTypeVarChar 107 | fieldTypeBit 108 | ) 109 | const ( 110 | fieldTypeNewDecimal byte = iota + 0xf6 111 | fieldTypeEnum 112 | fieldTypeSet 113 | fieldTypeTinyBLOB 114 | fieldTypeMediumBLOB 115 | fieldTypeLongBLOB 116 | fieldTypeBLOB 117 | fieldTypeVarString 118 | fieldTypeString 119 | fieldTypeGeometry 120 | ) 121 | 122 | type fieldFlag uint16 123 | 124 | const ( 125 | flagNotNULL fieldFlag = 1 << iota 126 | flagPriKey 127 | flagUniqueKey 128 | flagMultipleKey 129 | flagBLOB 130 | flagUnsigned 131 | flagZeroFill 132 | flagBinary 133 | flagEnum 134 | flagAutoIncrement 135 | flagTimestamp 136 | flagSet 137 | flagUnknown1 138 | flagUnknown2 139 | flagUnknown3 140 | flagUnknown4 141 | ) 142 | 143 | // http://dev.mysql.com/doc/internals/en/status-flags.html 144 | type statusFlag uint16 145 | 146 | const ( 147 | statusInTrans statusFlag = 1 << iota 148 | statusInAutocommit 149 | statusReserved // Not in documentation 150 | statusMoreResultsExists 151 | statusNoGoodIndexUsed 152 | statusNoIndexUsed 153 | statusCursorExists 154 | statusLastRowSent 155 | statusDbDropped 156 | statusNoBackslashEscapes 157 | statusMetadataChanged 158 | statusQueryWasSlow 159 | statusPsOutParams 160 | statusInTransReadonly 161 | statusSessionStateChanged 162 | ) 163 | -------------------------------------------------------------------------------- /jobgen/logwriter.go: -------------------------------------------------------------------------------- 1 | package jobgen 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "time" 7 | 8 | "badoo/_packages/log" 9 | "encoding/json" 10 | ) 11 | 12 | type ( 13 | FinishResultRusage struct { 14 | UserTime float64 `json:"user_time"` 15 | SysTime float64 `json:"sys_time"` 16 | RealTime float64 `json:"real_time"` 17 | MaxMemory uint64 `json:"max_memory"` 18 | } 19 | 20 | FinishResultRunInfo struct { 21 | Id uint64 `json:"id"` 22 | TimetableId uint64 `json:"timetable_id"` 23 | GenerationId uint64 `json:"generation_id"` 24 | Hostname string `json:"hostname"` 25 | HostnameIdx uint32 `json:"hostname_idx"` 26 | ClassName string `json:"class_name"` 27 | JobData string `json:"job_data"` 28 | Method string `json:"method"` 29 | RunStatus string `json:"run_status"` 30 | Created interface{} `json:"created"` 31 | WaitingTs interface{} `json:"waiting_ts"` 32 | ShouldInitTs interface{} `json:"should_init_ts"` 33 | InitAttempts uint32 `json:"init_attempts"` 34 | InitTs interface{} `json:"init_ts"` 35 | RunningTs interface{} `json:"running_ts"` 36 | MaxFinishedTs interface{} `json:"max_finished_ts"` 37 | FinishedTs interface{} `json:"finished_ts"` 38 | StoppedEmployeeId int64 `json:"stopped_employee_id"` 39 | Token string `json:"token"` 40 | RetryAttempt uint32 `json:"retry_attempt"` 41 | SettingsId uint64 `json:"settings_id"` 42 | } 43 | 44 | FinishResult struct { 45 | Id uint64 `json:"id"` 46 | TimetableId uint64 `json:"timetable_id"` 47 | ClassName string `json:"class_name"` 48 | Hostname string `json:"hostname"` 49 | Success bool `json:"success"` 50 | PrevStatus string `json:"prev_status"` 51 | Rusage FinishResultRusage `json:"rusage"` 52 | ProfilingUrl string `json:"profiling_url"` 53 | Initial bool `json:"initial"` 54 | Timestamp uint64 `json:"timestamp"` 55 | RunInfo FinishResultRunInfo `json:"run_info"` 56 | } 57 | ) 58 | 59 | var ( 60 | rqhLog = make(chan *FinishResult, 100) 61 | ) 62 | 63 | func WriteLogsThread(filename string) { 64 | log.Infof("Started write logs thread to file=%s", filename) 65 | 66 | reopenTick := time.Tick(time.Second * 10) 67 | 68 | fp, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 69 | var wr *bufio.Writer 70 | 71 | if err != nil { 72 | log.Errorf("Could not open %s: %s", filename, err.Error()) 73 | } else { 74 | wr = bufio.NewWriterSize(fp, 65536) 75 | } 76 | 77 | for { 78 | select { 79 | case <-reopenTick: 80 | if fp != nil { 81 | fp.Close() 82 | } 83 | 84 | fp, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 85 | if err != nil { 86 | log.Warnf("Could not reopen %s: %s", err.Error()) 87 | wr = nil 88 | fp = nil 89 | } else { 90 | wr = bufio.NewWriterSize(fp, 65536) 91 | } 92 | case ev := <-rqhLog: 93 | l := len(rqhLog) 94 | evs := make([]*FinishResult, 0, l+1) 95 | evs = append(evs, ev) 96 | 97 | for i := 0; i < l; i++ { 98 | evs = append(evs, <-rqhLog) 99 | } 100 | 101 | if wr != nil { 102 | encoder := json.NewEncoder(wr) 103 | 104 | for _, e := range evs { 105 | if err = encoder.Encode(e); err != nil { 106 | log.Errorf("Could not write to %s: %s", filename, err.Error()) 107 | } 108 | } 109 | 110 | if err = wr.Flush(); err != nil { 111 | log.Errorf("Could not flush contents to %s: %s", filename, err.Error()) 112 | } 113 | } else { 114 | log.Errorf("Failed to write %d events to rqh log because file %s could not be opened", len(evs), filename) 115 | } 116 | } 117 | 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /logs-collector/logsproc/processor.go: -------------------------------------------------------------------------------- 1 | package logsproc 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "io" 7 | "log" 8 | "net" 9 | "os" 10 | "os/signal" 11 | "path/filepath" 12 | "sync" 13 | "syscall" 14 | 15 | "github.com/badoo/thunder/common" 16 | ) 17 | 18 | type ( 19 | msg struct { 20 | lines []string 21 | resp chan error 22 | } 23 | ) 24 | 25 | var ( 26 | targetDir string 27 | listenAddress string 28 | 29 | fileChans = make(map[string]chan *msg) // nil message means that file reopen is required 30 | fileChansMutex sync.Mutex 31 | ) 32 | 33 | func InitFlags() { 34 | flag.StringVar(&targetDir, "target-dir", "/local/logs/sf", "Target directory where to put logs") 35 | flag.StringVar(&listenAddress, "listen-address", "0.0.0.0:1065", "Listen address") 36 | } 37 | 38 | func Run() { 39 | c := make(chan os.Signal, 1) 40 | signal.Notify(c, syscall.SIGHUP, syscall.SIGUSR1) 41 | go func() { 42 | for { 43 | <-c 44 | log.Println("Forcing re-open for log files") 45 | 46 | fileChansMutex.Lock() 47 | for _, ch := range fileChans { 48 | ch <- nil 49 | } 50 | fileChansMutex.Unlock() 51 | } 52 | }() 53 | 54 | ln, err := net.Listen("tcp", listenAddress) 55 | if err != nil { 56 | log.Fatalln("Could not listen supplied port: " + err.Error()) 57 | } 58 | 59 | log.Println("Listening on", listenAddress) 60 | 61 | for { 62 | conn, err := ln.Accept() 63 | if err != nil { 64 | log.Fatalln("Could not accept connection: " + err.Error()) 65 | } 66 | 67 | go handleConnection(conn) 68 | } 69 | } 70 | 71 | func handleConnection(conn net.Conn) { 72 | defer conn.Close() 73 | 74 | for { 75 | ln, err := common.ReceiveMessage(conn) 76 | if err != nil { 77 | if err != io.EOF { 78 | log.Println("Got error whe receiving message: ", err.Error()) 79 | } 80 | 81 | return 82 | } 83 | 84 | writeLines(ln) 85 | 86 | // write response about file and offset 87 | if err = common.SendMessage(conn, &common.LogLines{FileName: ln.FileName, Inode: ln.Inode, Offset: ln.Offset}); err != nil { 88 | log.Println("Could not send message:", err.Error()) 89 | return 90 | } 91 | } 92 | } 93 | 94 | func openFile(filename string) *os.File { 95 | targetPath := targetDir + "/" + filepath.Clean(filename) 96 | 97 | fp, err := os.OpenFile(targetPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) 98 | if err != nil { 99 | log.Fatalln("Could not open " + filename + " for writing: " + err.Error()) 100 | } 101 | 102 | return fp 103 | } 104 | 105 | func writeThread(filename string, ch chan *msg) { 106 | var err error 107 | fp := openFile(filename) 108 | w := bufio.NewWriterSize(fp, 65536) 109 | 110 | for { 111 | messages := make([]*msg, 0) 112 | messages = append(messages, <-ch) 113 | if l := len(ch); l > 0 { 114 | for l > 0 { 115 | messages = append(messages, <-ch) 116 | l-- 117 | } 118 | } 119 | 120 | for _, m := range messages { 121 | if m == nil { 122 | if err = w.Flush(); err != nil { 123 | log.Fatalln("Could not flush write: " + err.Error()) 124 | } 125 | fp.Close() 126 | fp = openFile(filename) 127 | w = bufio.NewWriter(fp) 128 | continue 129 | } 130 | 131 | for _, s := range m.lines { 132 | if _, err = w.WriteString(s); err != nil { 133 | log.Fatalln("Could not write logs: " + err.Error()) 134 | } 135 | } 136 | } 137 | 138 | if err = w.Flush(); err != nil { 139 | log.Fatalln("Could not flush writes: " + err.Error()) 140 | } 141 | 142 | for _, m := range messages { 143 | if m != nil { 144 | m.resp <- nil 145 | } 146 | } 147 | } 148 | } 149 | 150 | func writeLines(ln *common.LogLines) (err error) { 151 | fileChansMutex.Lock() 152 | ch, ok := fileChans[*ln.FileName] 153 | if !ok { 154 | ch = make(chan *msg, 500) 155 | go writeThread(*ln.FileName, ch) 156 | fileChans[*ln.FileName] = ch 157 | } 158 | fileChansMutex.Unlock() 159 | 160 | respCh := make(chan error, 1) 161 | ch <- &msg{lines: ln.Lines, resp: respCh} 162 | err = <-respCh 163 | 164 | return 165 | } 166 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/signals.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | ) 10 | 11 | // wait_for_signals tells the caller how to exit 12 | type ExitMethod int 13 | 14 | const ( 15 | EXIT_IMMEDIATELY = iota 16 | EXIT_GRACEFULLY 17 | ) 18 | 19 | func SignalName(sig os.Signal) string { 20 | switch sig { 21 | case syscall.SIGABRT: 22 | return "SIGABRT" 23 | case syscall.SIGALRM: 24 | return "SIGALRM" 25 | case syscall.SIGBUS: 26 | return "SIGBUS" 27 | case syscall.SIGCHLD: 28 | return "SIGCHLD" 29 | case syscall.SIGCONT: 30 | return "SIGCONT" 31 | case syscall.SIGFPE: 32 | return "SIGFPE" 33 | case syscall.SIGHUP: 34 | return "SIGHUP" 35 | case syscall.SIGILL: 36 | return "SIGILL" 37 | case syscall.SIGINT: 38 | return "SIGINT" 39 | case syscall.SIGIO: 40 | return "SIGIO" 41 | case syscall.SIGKILL: 42 | return "SIGKILL" 43 | case syscall.SIGPIPE: 44 | return "SIGPIPE" 45 | case syscall.SIGPROF: 46 | return "SIGPROF" 47 | case syscall.SIGQUIT: 48 | return "SIGQUIT" 49 | case syscall.SIGSEGV: 50 | return "SIGSEGV" 51 | case syscall.SIGSTOP: 52 | return "SIGSTOP" 53 | case syscall.SIGSYS: 54 | return "SIGSYS" 55 | case syscall.SIGTERM: 56 | return "SIGTERM" 57 | case syscall.SIGTRAP: 58 | return "SIGTRAP" 59 | case syscall.SIGTSTP: 60 | return "SIGTSTP" 61 | case syscall.SIGTTIN: 62 | return "SIGTTIN" 63 | case syscall.SIGTTOU: 64 | return "SIGTTOU" 65 | case syscall.SIGURG: 66 | return "SIGURG" 67 | case syscall.SIGUSR1: 68 | return "SIGUSR1" 69 | case syscall.SIGUSR2: 70 | return "SIGUSR2" 71 | case syscall.SIGVTALRM: 72 | return "SIGVTALRM" 73 | case syscall.SIGWINCH: 74 | return "SIGWINCH" 75 | case syscall.SIGXCPU: 76 | return "SIGXCPU" 77 | case syscall.SIGXFSZ: 78 | return "SIGXFSZ" 79 | default: 80 | return fmt.Sprintf("SIGNAL (%s)", sig) 81 | } 82 | } 83 | 84 | func sigaction__graceful_shutdown(sig os.Signal) { 85 | log.Infof("got %s, shutting down", SignalName(sig)) 86 | } 87 | 88 | func sigaction__reopen_logs(sig os.Signal) { 89 | log.Infof("got %s, reopening logfile: %s", SignalName(sig), logPath) 90 | 91 | if err := reopenLogfile(logPath, logLevel); err != nil { 92 | log.Errorf("can't reopen log file: %s", err) 93 | } 94 | 95 | log.Infof("sigaction__reopen_logs: new log opened: %s", logPath) 96 | } 97 | 98 | func sigaction__graceful_restart(sig os.Signal) { 99 | log.Infof("got %s, restarting gracefully", SignalName(sig)) 100 | 101 | if err := InitiateRestart(); err != nil { 102 | log.Errorf("can't initiate restart: %s", err) 103 | } 104 | } 105 | 106 | func wait_for_signals() ExitMethod { 107 | c := make(chan os.Signal, 5) 108 | signal.Notify(c, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGINT, syscall.SIGUSR2) 109 | 110 | for { 111 | select { 112 | case sig := <-c: 113 | 114 | switch sig { 115 | case syscall.SIGTERM, syscall.SIGINT: 116 | sigaction__graceful_shutdown(sig) 117 | return EXIT_IMMEDIATELY 118 | 119 | case syscall.SIGUSR1: 120 | if RestartInprogress() == false { // FIXME: why ingore log reopen ? 121 | sigaction__reopen_logs(sig) 122 | } else { 123 | log.Infof("service is restarting. ignoring %s", SignalName(sig)) 124 | } 125 | 126 | case syscall.SIGUSR2: 127 | if RestartInprogress() == false { 128 | sigaction__graceful_restart(sig) 129 | } else { 130 | log.Infof("service is restarting. ignoring %s", SignalName(sig)) 131 | } 132 | 133 | case syscall.SIGQUIT: 134 | if RestartInprogress() == false { 135 | // not a restart sequence, someone's just sent us SIGQUIT 136 | sigaction__graceful_shutdown(sig) 137 | return EXIT_IMMEDIATELY 138 | } 139 | 140 | FinalizeRestartWithSuccess() 141 | return EXIT_GRACEFULLY 142 | } 143 | 144 | case wp := <-RestartChildWaitChan(): 145 | FinalizeRestartWithError(wp) 146 | 147 | case <-RestartChildTimeoutChan(): 148 | FinalizeRestartWithTimeout() 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /go-proxyd/proto/phproxyd.proto: -------------------------------------------------------------------------------- 1 | 2 | package badoo.phproxyd; 3 | 4 | enum request_msgid { 5 | REQUEST_RRD = 1; 6 | REQUEST_PING = 2; 7 | REQUEST_STATS = 3; 8 | REQUEST_RUN = 4; 9 | REQUEST_CHECK = 5; 10 | REQUEST_FREE = 6; 11 | REQUEST_TERMINATE = 7; 12 | REQUEST_SOFT_RUN = 8; 13 | REQUEST_SOFT_CHECK = 9; 14 | REQUEST_SOFT_FREE = 10; 15 | } 16 | 17 | enum response_msgid { 18 | RESPONSE_GENERIC = 1; 19 | RESPONSE_PING = 2; 20 | RESPONSE_RRD = 3; 21 | RESPONSE_STATS = 4; 22 | RESPONSE_CHECK = 5; 23 | RESPONSE_SOFT_CHECK = 6; 24 | RESPONSE_SOFT_RUN = 7; 25 | } 26 | 27 | enum errno { 28 | ERRNO_GENERIC = 1; 29 | ERRNO_RUN_FAILED = 2; 30 | ERRNO_ALREADY_RUNNING = 3; 31 | ERRNO_NOT_FOUND = 4; 32 | ERRNO_WORKING = 5; 33 | ERRNO_NO_MEMORY = 6; 34 | ERRNO_SUCCESS_FINISHED = 7; 35 | ERRNO_FAILED_FINISHED = 8; 36 | ERRNO_WAIT_FOR_FREE = 9; 37 | ERRNO_START_CHILD_FAILED = 10; 38 | } 39 | 40 | enum phproxyd_status_t { 41 | STATUS_OK = 1; 42 | STATUS_FAILED = 2; 43 | } 44 | 45 | enum store_t { 46 | FILES = 1; 47 | MEMORY = 2; 48 | } 49 | 50 | message tag_stat_t { 51 | required string tag = 1; 52 | required uint32 success = 2; 53 | required uint32 failed = 3; 54 | } 55 | 56 | message tag_rrd_t { 57 | required string tag = 1; 58 | required uint64 tm_clean = 2; 59 | required uint32 success = 3; 60 | required uint32 failed = 4; 61 | } 62 | 63 | message response_generic { 64 | required sint32 error_code = 1; 65 | optional string error_text = 2; 66 | } 67 | 68 | message request_ping { 69 | } 70 | 71 | message request_rrd { 72 | repeated string tags = 1; 73 | required uint32 clean = 2; 74 | } 75 | 76 | message request_stats { 77 | } 78 | 79 | message response_ping { 80 | required phproxyd_status_t status = 1; 81 | } 82 | 83 | message response_rrd { 84 | required string vmsize = 1; 85 | required uint64 utime_tv_sec = 2; 86 | required uint64 utime_tv_usec = 3; 87 | required uint64 stime_tv_sec = 4; 88 | required uint64 stime_tv_usec = 5; 89 | required uint64 commands_cnt = 6; 90 | required uint64 failed_commands_cnt = 7; 91 | 92 | repeated tag_rrd_t rrd = 8; 93 | } 94 | 95 | message response_stats { 96 | required string vmsize = 1; 97 | required uint64 utime_tv_sec = 2; 98 | required uint64 utime_tv_usec = 3; 99 | required uint64 stime_tv_sec = 4; 100 | required uint64 stime_tv_usec = 5; 101 | required uint64 commands_cnt = 6; 102 | required uint64 failed_commands_cnt = 7; 103 | 104 | repeated tag_stat_t stats = 8; 105 | } 106 | 107 | message request_run { 108 | required string script = 1; 109 | required uint64 hash = 2; 110 | required string tag = 3; 111 | optional sint32 force = 4; 112 | repeated string params = 5; 113 | optional store_t store = 6 [default = MEMORY]; 114 | optional bool free_after_run = 7 [default = true]; 115 | } 116 | 117 | message request_soft_run { 118 | required string script = 1; 119 | optional uint64 hash = 2; 120 | required string data = 3; 121 | } 122 | 123 | message response_soft_run { 124 | optional uint64 hash = 1; 125 | } 126 | 127 | message request_check { 128 | required uint64 hash = 2; 129 | } 130 | 131 | message request_soft_check { 132 | repeated uint64 hash = 1; 133 | } 134 | 135 | message request_free { 136 | required uint64 hash = 2; 137 | } 138 | 139 | message request_soft_free { 140 | repeated uint64 hash = 1; 141 | } 142 | 143 | message response_check { 144 | required sint32 error_code = 1; 145 | optional string error_text = 2; 146 | optional string response = 3; 147 | optional uint64 utime_tv_sec = 4; 148 | optional uint64 utime_tv_usec = 5; 149 | optional uint64 stime_tv_sec = 6; 150 | optional uint64 stime_tv_usec = 7; 151 | optional sint32 retcode = 8; 152 | } 153 | 154 | message soft_check_t { 155 | required uint64 hash = 1; 156 | required string response = 2; 157 | } 158 | 159 | message response_soft_check { 160 | repeated soft_check_t stats = 1; 161 | } 162 | 163 | message request_terminate { 164 | required uint64 hash = 1; 165 | } 166 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/text_formatter.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "runtime" 7 | "sort" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | const ( 13 | nocolor = 0 14 | red = 31 15 | green = 32 16 | yellow = 33 17 | blue = 34 18 | gray = 37 19 | ) 20 | 21 | var ( 22 | baseTimestamp time.Time 23 | isTerminal bool 24 | ) 25 | 26 | func init() { 27 | baseTimestamp = time.Now() 28 | isTerminal = IsTerminal() 29 | } 30 | 31 | func miniTS() int { 32 | return int(time.Since(baseTimestamp) / time.Second) 33 | } 34 | 35 | type TextFormatter struct { 36 | // Set to true to bypass checking for a TTY before outputting colors. 37 | ForceColors bool 38 | 39 | // Force disabling colors. 40 | DisableColors bool 41 | 42 | // Disable timestamp logging. useful when output is redirected to logging 43 | // system that already adds timestamps. 44 | DisableTimestamp bool 45 | 46 | // Enable logging the full timestamp when a TTY is attached instead of just 47 | // the time passed since beginning of execution. 48 | FullTimestamp bool 49 | 50 | // TimestampFormat to use for display when a full timestamp is printed 51 | TimestampFormat string 52 | 53 | // The fields are sorted by default for a consistent output. For applications 54 | // that log extremely frequently and don't use the JSON formatter this may not 55 | // be desired. 56 | DisableSorting bool 57 | } 58 | 59 | func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { 60 | var keys []string = make([]string, 0, len(entry.Data)) 61 | for k := range entry.Data { 62 | keys = append(keys, k) 63 | } 64 | 65 | if !f.DisableSorting { 66 | sort.Strings(keys) 67 | } 68 | 69 | b := &bytes.Buffer{} 70 | 71 | prefixFieldClashes(entry.Data) 72 | 73 | isColorTerminal := isTerminal && (runtime.GOOS != "windows") 74 | isColored := (f.ForceColors || isColorTerminal) && !f.DisableColors 75 | 76 | timestampFormat := f.TimestampFormat 77 | if timestampFormat == "" { 78 | timestampFormat = DefaultTimestampFormat 79 | } 80 | if isColored { 81 | f.printColored(b, entry, keys, timestampFormat) 82 | } else { 83 | if !f.DisableTimestamp { 84 | f.appendKeyValue(b, "time", entry.Time.Format(timestampFormat)) 85 | } 86 | f.appendKeyValue(b, "level", entry.Level.String()) 87 | if entry.Message != "" { 88 | f.appendKeyValue(b, "msg", entry.Message) 89 | } 90 | for _, key := range keys { 91 | f.appendKeyValue(b, key, entry.Data[key]) 92 | } 93 | } 94 | 95 | b.WriteByte('\n') 96 | return b.Bytes(), nil 97 | } 98 | 99 | func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []string, timestampFormat string) { 100 | var levelColor int 101 | switch entry.Level { 102 | case DebugLevel: 103 | levelColor = gray 104 | case WarnLevel: 105 | levelColor = yellow 106 | case ErrorLevel, FatalLevel, PanicLevel: 107 | levelColor = red 108 | default: 109 | levelColor = blue 110 | } 111 | 112 | levelText := strings.ToUpper(entry.Level.String())[0:4] 113 | 114 | if !f.FullTimestamp { 115 | fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, miniTS(), entry.Message) 116 | } else { 117 | fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(timestampFormat), entry.Message) 118 | } 119 | for _, k := range keys { 120 | v := entry.Data[k] 121 | fmt.Fprintf(b, " \x1b[%dm%s\x1b[0m=%+v", levelColor, k, v) 122 | } 123 | } 124 | 125 | func needsQuoting(text string) bool { 126 | for _, ch := range text { 127 | if !((ch >= 'a' && ch <= 'z') || 128 | (ch >= 'A' && ch <= 'Z') || 129 | (ch >= '0' && ch <= '9') || 130 | ch == '-' || ch == '.') { 131 | return false 132 | } 133 | } 134 | return true 135 | } 136 | 137 | func (f *TextFormatter) appendKeyValue(b *bytes.Buffer, key string, value interface{}) { 138 | 139 | b.WriteString(key) 140 | b.WriteByte('=') 141 | 142 | switch value := value.(type) { 143 | case string: 144 | if needsQuoting(value) { 145 | b.WriteString(value) 146 | } else { 147 | fmt.Fprintf(b, "%q", value) 148 | } 149 | case error: 150 | errmsg := value.Error() 151 | if needsQuoting(errmsg) { 152 | b.WriteString(errmsg) 153 | } else { 154 | fmt.Fprintf(b, "%q", value) 155 | } 156 | default: 157 | fmt.Fprint(b, value) 158 | } 159 | 160 | b.WriteByte(' ') 161 | } 162 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## HEAD 2 | 3 | Changes: 4 | 5 | - Go 1.1 is no longer supported 6 | - Use decimals field from MySQL to format time types (#249) 7 | - Buffer optimizations (#269) 8 | - TLS ServerName defaults to the host (#283) 9 | 10 | Bugfixes: 11 | 12 | - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) 13 | - Fixed handling of queries without columns and rows (#255) 14 | - Fixed a panic when SetKeepAlive() failed (#298) 15 | 16 | New Features: 17 | - Support for returning table alias on Columns() (#289) 18 | - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318) 19 | 20 | 21 | ## Version 1.2 (2014-06-03) 22 | 23 | Changes: 24 | 25 | - We switched back to a "rolling release". `go get` installs the current master branch again 26 | - Version v1 of the driver will not be maintained anymore. Go 1.0 is no longer supported by this driver 27 | - Exported errors to allow easy checking from application code 28 | - Enabled TCP Keepalives on TCP connections 29 | - Optimized INFILE handling (better buffer size calculation, lazy init, ...) 30 | - The DSN parser also checks for a missing separating slash 31 | - Faster binary date / datetime to string formatting 32 | - Also exported the MySQLWarning type 33 | - mysqlConn.Close returns the first error encountered instead of ignoring all errors 34 | - writePacket() automatically writes the packet size to the header 35 | - readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets 36 | 37 | New Features: 38 | 39 | - `RegisterDial` allows the usage of a custom dial function to establish the network connection 40 | - Setting the connection collation is possible with the `collation` DSN parameter. This parameter should be preferred over the `charset` parameter 41 | - Logging of critical errors is configurable with `SetLogger` 42 | - Google CloudSQL support 43 | 44 | Bugfixes: 45 | 46 | - Allow more than 32 parameters in prepared statements 47 | - Various old_password fixes 48 | - Fixed TestConcurrent test to pass Go's race detection 49 | - Fixed appendLengthEncodedInteger for large numbers 50 | - Renamed readLengthEnodedString to readLengthEncodedString and skipLengthEnodedString to skipLengthEncodedString (fixed typo) 51 | 52 | 53 | ## Version 1.1 (2013-11-02) 54 | 55 | Changes: 56 | 57 | - Go-MySQL-Driver now requires Go 1.1 58 | - Connections now use the collation `utf8_general_ci` by default. Adding `&charset=UTF8` to the DSN should not be necessary anymore 59 | - Made closing rows and connections error tolerant. This allows for example deferring rows.Close() without checking for errors 60 | - `[]byte(nil)` is now treated as a NULL value. Before, it was treated like an empty string / `[]byte("")` 61 | - DSN parameter values must now be url.QueryEscape'ed. This allows text values to contain special characters, such as '&'. 62 | - Use the IO buffer also for writing. This results in zero allocations (by the driver) for most queries 63 | - Optimized the buffer for reading 64 | - stmt.Query now caches column metadata 65 | - New Logo 66 | - Changed the copyright header to include all contributors 67 | - Improved the LOAD INFILE documentation 68 | - The driver struct is now exported to make the driver directly accessible 69 | - Refactored the driver tests 70 | - Added more benchmarks and moved all to a separate file 71 | - Other small refactoring 72 | 73 | New Features: 74 | 75 | - Added *old_passwords* support: Required in some cases, but must be enabled by adding `allowOldPasswords=true` to the DSN since it is insecure 76 | - Added a `clientFoundRows` parameter: Return the number of matching rows instead of the number of rows changed on UPDATEs 77 | - Added TLS/SSL support: Use a TLS/SSL encrypted connection to the server. Custom TLS configs can be registered and used 78 | 79 | Bugfixes: 80 | 81 | - Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification 82 | - Convert to DB timezone when inserting `time.Time` 83 | - Splitted packets (more than 16MB) are now merged correctly 84 | - Fixed false positive `io.EOF` errors when the data was fully read 85 | - Avoid panics on reuse of closed connections 86 | - Fixed empty string producing false nil values 87 | - Fixed sign byte for positive TIME fields 88 | 89 | 90 | ## Version 1.0 (2013-05-14) 91 | 92 | Initial Release 93 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/errors.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import ( 12 | "database/sql/driver" 13 | "errors" 14 | "fmt" 15 | "io" 16 | "log" 17 | "os" 18 | ) 19 | 20 | // Various errors the driver might return. Can change between driver versions. 21 | var ( 22 | ErrInvalidConn = errors.New("Invalid Connection") 23 | ErrMalformPkt = errors.New("Malformed Packet") 24 | ErrNoTLS = errors.New("TLS encryption requested but server does not support TLS") 25 | ErrOldPassword = errors.New("This user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") 26 | ErrCleartextPassword = errors.New("This user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN.") 27 | ErrUnknownPlugin = errors.New("The authentication plugin is not supported.") 28 | ErrOldProtocol = errors.New("MySQL-Server does not support required Protocol 41+") 29 | ErrPktSync = errors.New("Commands out of sync. You can't run this command now") 30 | ErrPktSyncMul = errors.New("Commands out of sync. Did you run multiple statements at once?") 31 | ErrPktTooLarge = errors.New("Packet for query is too large. You can change this value on the server by adjusting the 'max_allowed_packet' variable.") 32 | ErrBusyBuffer = errors.New("Busy buffer") 33 | ) 34 | 35 | var errLog Logger = log.New(os.Stderr, "[MySQL] ", log.Ldate|log.Ltime|log.Lshortfile) 36 | 37 | // Logger is used to log critical error messages. 38 | type Logger interface { 39 | Print(v ...interface{}) 40 | } 41 | 42 | // SetLogger is used to set the logger for critical errors. 43 | // The initial logger is os.Stderr. 44 | func SetLogger(logger Logger) error { 45 | if logger == nil { 46 | return errors.New("logger is nil") 47 | } 48 | errLog = logger 49 | return nil 50 | } 51 | 52 | // MySQLError is an error type which represents a single MySQL error 53 | type MySQLError struct { 54 | Number uint16 55 | Message string 56 | } 57 | 58 | func (me *MySQLError) Error() string { 59 | return fmt.Sprintf("Error %d: %s", me.Number, me.Message) 60 | } 61 | 62 | // MySQLWarnings is an error type which represents a group of one or more MySQL 63 | // warnings 64 | type MySQLWarnings []MySQLWarning 65 | 66 | func (mws MySQLWarnings) Error() string { 67 | var msg string 68 | for i, warning := range mws { 69 | if i > 0 { 70 | msg += "\r\n" 71 | } 72 | msg += fmt.Sprintf( 73 | "%s %s: %s", 74 | warning.Level, 75 | warning.Code, 76 | warning.Message, 77 | ) 78 | } 79 | return msg 80 | } 81 | 82 | // MySQLWarning is an error type which represents a single MySQL warning. 83 | // Warnings are returned in groups only. See MySQLWarnings 84 | type MySQLWarning struct { 85 | Level string 86 | Code string 87 | Message string 88 | } 89 | 90 | func (mc *mysqlConn) getWarnings() (err error) { 91 | rows, err := mc.Query("SHOW WARNINGS", nil) 92 | if err != nil { 93 | return 94 | } 95 | 96 | var warnings = MySQLWarnings{} 97 | var values = make([]driver.Value, 3) 98 | 99 | for { 100 | err = rows.Next(values) 101 | switch err { 102 | case nil: 103 | warning := MySQLWarning{} 104 | 105 | if raw, ok := values[0].([]byte); ok { 106 | warning.Level = string(raw) 107 | } else { 108 | warning.Level = fmt.Sprintf("%s", values[0]) 109 | } 110 | if raw, ok := values[1].([]byte); ok { 111 | warning.Code = string(raw) 112 | } else { 113 | warning.Code = fmt.Sprintf("%s", values[1]) 114 | } 115 | if raw, ok := values[2].([]byte); ok { 116 | warning.Message = string(raw) 117 | } else { 118 | warning.Message = fmt.Sprintf("%s", values[0]) 119 | } 120 | 121 | warnings = append(warnings, warning) 122 | 123 | case io.EOF: 124 | return warnings 125 | 126 | default: 127 | rows.Close() 128 | return 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/driver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. 2 | // 3 | // This Source Code Form is subject to the terms of the Mozilla Public 4 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 5 | // You can obtain one at http://mozilla.org/MPL/2.0/. 6 | 7 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 8 | // 9 | // The driver should be used via the database/sql package: 10 | // 11 | // import "database/sql" 12 | // import _ "github.com/go-sql-driver/mysql" 13 | // 14 | // db, err := sql.Open("mysql", "user:password@/dbname") 15 | // 16 | // See https://github.com/go-sql-driver/mysql#usage for details 17 | package mysql 18 | 19 | import ( 20 | "database/sql" 21 | "database/sql/driver" 22 | "net" 23 | ) 24 | 25 | // This struct is exported to make the driver directly accessible. 26 | // In general the driver is used via the database/sql package. 27 | type MySQLDriver struct{} 28 | 29 | // DialFunc is a function which can be used to establish the network connection. 30 | // Custom dial functions must be registered with RegisterDial 31 | type DialFunc func(addr string) (net.Conn, error) 32 | 33 | var dials map[string]DialFunc 34 | 35 | // RegisterDial registers a custom dial function. It can then be used by the 36 | // network address mynet(addr), where mynet is the registered new network. 37 | // addr is passed as a parameter to the dial function. 38 | func RegisterDial(net string, dial DialFunc) { 39 | if dials == nil { 40 | dials = make(map[string]DialFunc) 41 | } 42 | dials[net] = dial 43 | } 44 | 45 | // Open new Connection. 46 | // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how 47 | // the DSN string is formated 48 | func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { 49 | var err error 50 | 51 | // New mysqlConn 52 | mc := &mysqlConn{ 53 | maxPacketAllowed: maxPacketSize, 54 | maxWriteSize: maxPacketSize - 1, 55 | } 56 | mc.cfg, err = parseDSN(dsn) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | // Connect to Server 62 | if dial, ok := dials[mc.cfg.net]; ok { 63 | mc.netConn, err = dial(mc.cfg.addr) 64 | } else { 65 | nd := net.Dialer{Timeout: mc.cfg.timeout} 66 | mc.netConn, err = nd.Dial(mc.cfg.net, mc.cfg.addr) 67 | } 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | // Enable TCP Keepalives on TCP connections 73 | if tc, ok := mc.netConn.(*net.TCPConn); ok { 74 | if err := tc.SetKeepAlive(true); err != nil { 75 | // Don't send COM_QUIT before handshake. 76 | mc.netConn.Close() 77 | mc.netConn = nil 78 | return nil, err 79 | } 80 | } 81 | 82 | mc.buf = newBuffer(mc.netConn) 83 | 84 | // Reading Handshake Initialization Packet 85 | cipher, err := mc.readInitPacket() 86 | if err != nil { 87 | mc.Close() 88 | return nil, err 89 | } 90 | 91 | // Send Client Authentication Packet 92 | if err = mc.writeAuthPacket(cipher); err != nil { 93 | mc.Close() 94 | return nil, err 95 | } 96 | 97 | // Read Result Packet 98 | err = mc.readResultOK() 99 | if err != nil { 100 | // Retry with old authentication method, if allowed 101 | if mc.cfg != nil && mc.cfg.allowOldPasswords && err == ErrOldPassword { 102 | if err = mc.writeOldAuthPacket(cipher); err != nil { 103 | mc.Close() 104 | return nil, err 105 | } 106 | if err = mc.readResultOK(); err != nil { 107 | mc.Close() 108 | return nil, err 109 | } 110 | } else if mc.cfg != nil && mc.cfg.allowCleartextPasswords && err == ErrCleartextPassword { 111 | if err = mc.writeClearAuthPacket(); err != nil { 112 | mc.Close() 113 | return nil, err 114 | } 115 | if err = mc.readResultOK(); err != nil { 116 | mc.Close() 117 | return nil, err 118 | } 119 | } else { 120 | mc.Close() 121 | return nil, err 122 | } 123 | 124 | } 125 | 126 | // Get max allowed packet size 127 | maxap, err := mc.getSystemVar("max_allowed_packet") 128 | if err != nil { 129 | mc.Close() 130 | return nil, err 131 | } 132 | mc.maxPacketAllowed = stringToInt(maxap) - 1 133 | if mc.maxPacketAllowed < maxPacketSize { 134 | mc.maxWriteSize = mc.maxPacketAllowed 135 | } 136 | 137 | // Handle DSN Params 138 | err = mc.handleParams() 139 | if err != nil { 140 | mc.Close() 141 | return nil, err 142 | } 143 | 144 | return mc, nil 145 | } 146 | 147 | func init() { 148 | sql.Register("mysql", &MySQLDriver{}) 149 | } 150 | -------------------------------------------------------------------------------- /vendor/github.com/gogo/protobuf/proto/pointer_unsafe_gogo.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 2 | // http://github.com/gogo/protobuf/gogoproto 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | // +build !appengine 28 | 29 | // This file contains the implementation of the proto field accesses using package unsafe. 30 | 31 | package proto 32 | 33 | import ( 34 | "reflect" 35 | "unsafe" 36 | ) 37 | 38 | func structPointer_InterfaceAt(p structPointer, f field, t reflect.Type) interface{} { 39 | point := unsafe.Pointer(uintptr(p) + uintptr(f)) 40 | r := reflect.NewAt(t, point) 41 | return r.Interface() 42 | } 43 | 44 | func structPointer_InterfaceRef(p structPointer, f field, t reflect.Type) interface{} { 45 | point := unsafe.Pointer(uintptr(p) + uintptr(f)) 46 | r := reflect.NewAt(t, point) 47 | if r.Elem().IsNil() { 48 | return nil 49 | } 50 | return r.Elem().Interface() 51 | } 52 | 53 | func copyUintPtr(oldptr, newptr uintptr, size int) { 54 | oldbytes := make([]byte, 0) 55 | oldslice := (*reflect.SliceHeader)(unsafe.Pointer(&oldbytes)) 56 | oldslice.Data = oldptr 57 | oldslice.Len = size 58 | oldslice.Cap = size 59 | newbytes := make([]byte, 0) 60 | newslice := (*reflect.SliceHeader)(unsafe.Pointer(&newbytes)) 61 | newslice.Data = newptr 62 | newslice.Len = size 63 | newslice.Cap = size 64 | copy(newbytes, oldbytes) 65 | } 66 | 67 | func structPointer_Copy(oldptr structPointer, newptr structPointer, size int) { 68 | copyUintPtr(uintptr(oldptr), uintptr(newptr), size) 69 | } 70 | 71 | func appendStructPointer(base structPointer, f field, typ reflect.Type) structPointer { 72 | size := typ.Elem().Size() 73 | oldHeader := structPointer_GetSliceHeader(base, f) 74 | newLen := oldHeader.Len + 1 75 | slice := reflect.MakeSlice(typ, newLen, newLen) 76 | bas := toStructPointer(slice) 77 | for i := 0; i < oldHeader.Len; i++ { 78 | newElemptr := uintptr(bas) + uintptr(i)*size 79 | oldElemptr := oldHeader.Data + uintptr(i)*size 80 | copyUintPtr(oldElemptr, newElemptr, int(size)) 81 | } 82 | 83 | oldHeader.Data = uintptr(bas) 84 | oldHeader.Len = newLen 85 | oldHeader.Cap = newLen 86 | 87 | return structPointer(unsafe.Pointer(uintptr(unsafe.Pointer(bas)) + uintptr(uintptr(newLen-1)*size))) 88 | } 89 | 90 | func structPointer_FieldPointer(p structPointer, f field) structPointer { 91 | return structPointer(unsafe.Pointer(uintptr(p) + uintptr(f))) 92 | } 93 | 94 | func structPointer_GetRefStructPointer(p structPointer, f field) structPointer { 95 | return structPointer((*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f)))) 96 | } 97 | 98 | func structPointer_GetSliceHeader(p structPointer, f field) *reflect.SliceHeader { 99 | return (*reflect.SliceHeader)(unsafe.Pointer(uintptr(p) + uintptr(f))) 100 | } 101 | 102 | func structPointer_Add(p structPointer, size field) structPointer { 103 | return structPointer(unsafe.Pointer(uintptr(p) + uintptr(size))) 104 | } 105 | 106 | func structPointer_Len(p structPointer, f field) int { 107 | return len(*(*[]interface{})(unsafe.Pointer(structPointer_GetRefStructPointer(p, f)))) 108 | } 109 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/util/deepcopy/deepcopy.go: -------------------------------------------------------------------------------- 1 | // Package deepcopy deep copies maps, slices, structs, etc. A standard copy will copy 2 | // the pointers: deep copy copies the values pointed to. 3 | // 4 | // Only what is needed has been implemented. Could make more dynamic, at the 5 | // cost of reflection. Either adjust as needed or create a new function. 6 | // 7 | // Copyright (c)2014, Joel Scoble (github.com/mohae), all rights reserved. 8 | // License: MIT, for more details check the included LICENSE.txt. 9 | package deepcopy 10 | 11 | import ( 12 | "reflect" 13 | ) 14 | 15 | // InterfaceToSliceOfStrings takes an interface that is either a slice of 16 | // strings or a string and returns a deep copy of it as a slice of strings. 17 | func InterfaceToSliceOfStrings(v interface{}) []string { 18 | if v == nil { 19 | return nil 20 | } 21 | var sl []string 22 | switch reflect.TypeOf(v).Kind() { 23 | case reflect.Slice: 24 | s := reflect.ValueOf(v) 25 | sl = make([]string, s.Len(), s.Len()) 26 | for i := 0; i < s.Len(); i++ { 27 | sl[i] = s.Index(i).Interface().(string) 28 | } 29 | case reflect.String: 30 | sl = append(sl, reflect.ValueOf(v).Interface().(string)) 31 | default: 32 | return nil 33 | } 34 | return sl 35 | } 36 | 37 | // InterfaceToSliceOfInts takes an interface that is a slice of ints and returns 38 | // a deep copy of it as a slice of strings. An error is returned if the 39 | // interface is not a slice of strings. 40 | func InterfaceToSliceOfInts(v interface{}) []int { 41 | if v == nil { 42 | return nil 43 | } 44 | var sl []int 45 | switch reflect.TypeOf(v).Kind() { 46 | case reflect.Slice: 47 | s := reflect.ValueOf(v) 48 | sl = make([]int, s.Len(), s.Len()) 49 | for i := 0; i < s.Len(); i++ { 50 | sl[i] = s.Index(i).Interface().(int) 51 | } 52 | case reflect.Int: 53 | sl = append(sl, reflect.ValueOf(v).Interface().(int)) 54 | default: 55 | return nil 56 | } 57 | return sl 58 | } 59 | 60 | // Iface recursively deep copies an interface{} 61 | func Iface(iface interface{}) interface{} { 62 | if iface == nil { 63 | return nil 64 | } 65 | // Make the interface a reflect.Value 66 | original := reflect.ValueOf(iface) 67 | // Make a copy of the same type as the original. 68 | cpy := reflect.New(original.Type()).Elem() 69 | // Recursively copy the original. 70 | copyRecursive(original, cpy) 71 | // Return theb copy as an interface. 72 | return cpy.Interface() 73 | } 74 | 75 | // copyRecursive does the actual copying of the interface. It currently has 76 | // limited support for what it can handle. Add as needed. 77 | func copyRecursive(original, cpy reflect.Value) { 78 | // handle according to original's Kind 79 | switch original.Kind() { 80 | case reflect.Ptr: 81 | // Get the actual value being pointed to. 82 | originalValue := original.Elem() 83 | // if it isn't valid, return. 84 | if !originalValue.IsValid() { 85 | return 86 | } 87 | cpy.Set(reflect.New(originalValue.Type())) 88 | copyRecursive(originalValue, cpy.Elem()) 89 | case reflect.Interface: 90 | // Get the value for the interface, not the pointer. 91 | originalValue := original.Elem() 92 | if !originalValue.IsValid() { 93 | return 94 | } 95 | // Get the value by calling Elem(). 96 | copyValue := reflect.New(originalValue.Type()).Elem() 97 | copyRecursive(originalValue, copyValue) 98 | cpy.Set(copyValue) 99 | case reflect.Struct: 100 | // Go through each field of the struct and copy it. 101 | for i := 0; i < original.NumField(); i++ { 102 | if cpy.Field(i).CanSet() { 103 | copyRecursive(original.Field(i), cpy.Field(i)) 104 | } 105 | } 106 | case reflect.Slice: 107 | // Make a new slice and copy each element. 108 | cpy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap())) 109 | for i := 0; i < original.Len(); i++ { 110 | copyRecursive(original.Index(i), cpy.Index(i)) 111 | } 112 | case reflect.Map: 113 | cpy.Set(reflect.MakeMap(original.Type())) 114 | for _, key := range original.MapKeys() { 115 | originalValue := original.MapIndex(key) 116 | copyValue := reflect.New(originalValue.Type()).Elem() 117 | copyRecursive(originalValue, copyValue) 118 | cpy.SetMapIndex(key, copyValue) 119 | } 120 | // Set the actual values from here on. 121 | case reflect.String: 122 | cpy.SetString(original.Interface().(string)) 123 | case reflect.Int: 124 | cpy.SetInt(int64(original.Interface().(int))) 125 | case reflect.Bool: 126 | cpy.SetBool(original.Interface().(bool)) 127 | case reflect.Float64: 128 | cpy.SetFloat(original.Interface().(float64)) 129 | 130 | default: 131 | cpy.Set(original) 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/gpbrpc/gpbrpc.go: -------------------------------------------------------------------------------- 1 | package gpbrpc 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "net" 7 | "reflect" 8 | "strings" 9 | 10 | "github.com/gogo/protobuf/proto" 11 | ) 12 | 13 | type ConnStatus int32 14 | 15 | const ( 16 | ConnOK ConnStatus = 0 /* It's okay to use this connection */ 17 | ConnEOF ConnStatus = 1 /* EOF was detected up on read of the very first byte of a packet */ 18 | ConnIOError ConnStatus = 2 /* IO error */ 19 | ConnParseError ConnStatus = 3 /* Received unacceptable data from a peer */ 20 | // this one is unused for now 21 | // ConnBufferFull ConnStatus = 4 /* Buffer for io full */ 22 | ) 23 | 24 | // Knows about actual protocol structure, messages, their dispatch and specific protobuf structures implementing messages 25 | // Real implemtations are generated by protoc-gen-gpbrpc-go 26 | // in *.gpbrpc.go structure implementing this interface is called Gpbrpc for clarity :) 27 | // (clarity in that it resides in a different package, so naming is important) 28 | type Protocol interface { 29 | Dispatch(request RequestT, handler interface{}) ResultT 30 | ErrorGeneric(args ...interface{}) ResultT 31 | GetPackageName() string 32 | 33 | // request 34 | GetRequestMsgid(msg proto.Message) uint32 // panics if unknown message (validate messages before passing, have to do it sometime anyway) 35 | GetRequestMsg(msgid uint32) proto.Message // returns nil on unknown msgid 36 | GetRequestNameToIdMap() map[string]uint32 37 | GetRequestIdToNameMap() map[uint32]string 38 | 39 | // response 40 | GetResponseMsg(msgid uint32) proto.Message // panics if unknown message (validate messages before passing, have to do it sometime anyway) 41 | GetResponseMsgid(msg proto.Message) uint32 // returns nil on unknown msgid 42 | GetResponseNameToIdMap() map[string]uint32 43 | GetResponseIdToNameMap() map[uint32]string 44 | } 45 | 46 | // --------------------------------------------------------------------------------------------------------------- 47 | // Helpers 48 | 49 | func MapRequestNameToId(p Protocol, name string) uint32 { 50 | return p.GetRequestNameToIdMap()[name] 51 | } 52 | 53 | func MapRequestIdToName(p Protocol, msgid uint32) string { 54 | return p.GetRequestIdToNameMap()[msgid] 55 | } 56 | 57 | func MapResponseNameToId(p Protocol, name string) uint32 { 58 | return p.GetResponseNameToIdMap()[name] 59 | } 60 | 61 | func MapResponseIdToName(p Protocol, msgid uint32) string { 62 | return p.GetResponseIdToNameMap()[msgid] 63 | } 64 | 65 | func MakeRequestIdToPinbaNameMap(p Protocol) map[uint32]string { 66 | result := make(map[uint32]string) 67 | 68 | for id, name := range p.GetRequestIdToNameMap() { 69 | result[id] = p.GetPackageName() + "." + name 70 | } 71 | 72 | return result 73 | } 74 | 75 | // get message name, without any regard for Protocol it might belong to 76 | // it's heavywheight on GC, so don't use if possible to avoid, use helpers defined above! 77 | func Msgname(pb proto.Message) string { 78 | names := strings.Split(reflect.TypeOf(pb).String(), ".") 79 | return uncamelize(names[len(names)-1]) 80 | } 81 | 82 | // helper for Msgname() 83 | func uncamelize(s string) string { 84 | t := make([]byte, 0, 32) 85 | for i := 0; i < len(s); i++ { 86 | ch := s[i] 87 | if ch >= 'A' && ch <= 'Z' { 88 | if len(t) > 0 { 89 | t = append(t, '_') 90 | } 91 | } 92 | t = append(t, ch) 93 | } 94 | return strings.ToUpper(string(t)) 95 | } 96 | 97 | // --------------------------------------------------------------------------------------------------------------- 98 | // low level io helpers, need a separate package, do they ? 99 | 100 | func readAll(r io.Reader, buf []byte) (int, error) { 101 | buf_len := len(buf) 102 | read := 0 103 | 104 | for buf_len-read > 0 { 105 | read_oneshot, err := r.Read(buf) 106 | read += read_oneshot 107 | if err != nil { 108 | return read, err 109 | } 110 | buf = buf[read_oneshot:] 111 | } 112 | return read, nil 113 | } 114 | 115 | func writeAll(w io.Writer, buf []byte) (int, error) { 116 | buf_len := len(buf) 117 | written := 0 118 | 119 | for buf_len-written > 0 { 120 | written_oneshot, err := w.Write(buf) 121 | written += written_oneshot 122 | if err != nil { 123 | return written, err 124 | } 125 | buf = buf[written_oneshot:] 126 | } 127 | 128 | return written, nil 129 | } 130 | 131 | // FIXME: this does NOT work with multiple message in buffer (will handle just the first one) 132 | // and it's not trivially fixable, need to go up to codec and buffer data there 133 | func readLine(conn net.Conn, buf []byte) (read int, err error) { 134 | buf_len := len(buf) 135 | read = 0 136 | for buf_len-read > 0 { 137 | var read_oneshot int 138 | read_oneshot, err = conn.Read(buf[read:]) 139 | read += read_oneshot 140 | if err != nil { 141 | break 142 | } 143 | if buf[read-1] == '\n' { 144 | break 145 | } 146 | } 147 | 148 | if buf_len == read && buf[read-1] != '\n' { 149 | err = errors.New("read error: message too long") 150 | } 151 | 152 | return 153 | } 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Thunder, Badoo cloud system 2 | 3 | ## Описание 4 | 5 | Thunder представляет из себя реализацию управляющей логики для запуска заданий по расписанию, или добавляемых через API. 6 | Упрощенно можно рассматривать эту систему, как реализацию «облачного cron». Вы задаете расписание для запуска путем 7 | добавления записей в таблицу в MySQL и дальше система начинает запускать указанные задания и следить за их исполнением. 8 | 9 | ## Основные компоненты 10 | Для работы thunder нужны следующие компоненты: 11 | 12 | 1. Демон thunder 13 | 2. MySQL 14 | 3. Агент для запуска заданий (например go-proxyd) 15 | 4. LSD, опционально (github.com/badoo/lsd) 16 | 5. Код для уведомления об успешном запуске и завершении заданий* 17 | 6. Heartbeat* 18 | 19 | Компоненты со звездочкой должны быть написаны самостоятельно 20 | 21 | ## Сборка и конфигурация thunder 22 | Чтобы установить thunder, выполните команду ```go get github.com/badoo/thunder``` с помощью go версии 1.6+. 23 | 24 | Пример конфигурации находится в файле conf/thunder.conf. Секция daemon_config задает порты для общения с демоном, по 25 | умолчанию он слушает JSON-запросы на порту 6558. Дебаг-интерфейс доступен порту 6561 по адресу ```http://host:6561/debug/thunder```. 26 | 27 | ### Основные конфигурационные опции 28 | * ```mysql.dsn``` — параметры для доступа к MySQL 29 | * ```launcher.host_suffix``` — суффикс (с указанием порта), который будет добавляться к полю hostname в базе для соединения с агентом 30 | * ```launcher.base_path``` — путь до «скрипта» в поле script при запросах типа run к агенту 31 | * ```launcher.developer_path``` — путь до «скрипта» в поле script, если указано поле developer (%s заменяется на значение поля developer) 32 | * ```launcher.rqh_log_file``` — путь до лог-файла, в который будут записаны данные о неуспешных запусках 33 | * ```is_devel``` — если выставлено в false, то значение launcher.developer_path перестанет учитываться 34 | 35 | ## Назначение таблиц в MySQL в файле structure.sql 36 | * ```Script``` — список «скриптов» (типов заданий, которые можно запускать) 37 | * ```ScriptSettings``` — настройки для скриптов, вынесены отдельно, поскольку используются в RunQueue и ScriptTimetable 38 | * ```ScriptFlags``` — флаги, используемые для того, чтобы «убивать», останавливать и запускать скрипты 39 | * ```Server``` — список серверов с разбиением на группы, также содержит загрузку CPU, потребление памяти и др. 40 | * ```ScriptTimetable``` — очередь для запуска заданий и их статус 41 | * ```RunQueue``` — после того, как для записи в ScriptTimetable был выбран hostname, она попадает в RunQueue для непосредственного запуска 42 | * ```ScriptJobInfo``` — информация о поколениях заданий (о них рассказано ниже) 43 | 44 | ## Написание Heartbeat-скрипта 45 | Для того, чтобы thunder «узнал» о существовании хостов, нужно запускать Heartbeat-скрипт, например из cron. 46 | Heartbeat-скрипт должен собирать информацию о текущей загрузке процессора (в процентах) и по потреблению памяти (в байтах). 47 | Помимо этого, Heartbeat-скрипт должен определять имя кластера (группы), в который входит данный хост. 48 | Информация должна вставляться в таблицу Server раз в 10 секунд. 49 | 50 | Описание полей: 51 | 52 | * ```hostname``` — имя сервера, возвращаемое системным вызовом gethostname() 53 | * ```group``` — имя группы (кластера), в который входит этот сервер 54 | * ```cpu_user``` — user% загрузки процессора (значение от 0 до 100) 55 | * ```cpu_sys``` — system% 56 | * ```cpu_nice``` — nice% 57 | * ```cpu_iowait``` — iowait% 58 | * ```cpu_steal``` — steal% 59 | * ```cpu_idle``` — какой процент процессора свободен (значение от 0 до 100) 60 | * ```cpu_parasite``` — сколько процентов CPU потребляют программы, которые не запущены с помощью thunder («паразиты») 61 | * ```cpu_parrots_per_core``` — индекс производительности хоста в расчете на одно ядро («попугаи») 62 | * ```cpu_cores``` — количество ядер 63 | * ```mem_total``` — общее количество памяти на хосте в байтах 64 | * ```mem_free``` — количество свободной оперативной памяти в байтах (free) 65 | * ```mem_cached``` — количество памяти, занятое кешом файловой системы (cached) 66 | * ```mem_buffers``` — количество памяти, отведенное под дисковые и другие буферы (buffers) 67 | * ```mem_parasite``` — количество памяти, потребляемое «паразитами» (программы, запущенные не с помощью thunder) 68 | * ```swap_total``` — общий объем swap-раздела 69 | * ```swap_used``` — используемый объем swap-раздела 70 | * ```min_memory_ratio``` — опциональное поле, определяет минимальное соотношение свободной памяти к общему объему, при котором хост всё ещё считается «живым» (полезно выставлять в -1 для маленьких виртуальных машин) 71 | * ```updated``` — TIMESTAMP последнего времени обновления информации о сервере (выставляется автоматически в MySQL при выполнении UPDATE) 72 | * ```phproxyd_heartbeat_ts``` — TIMESTAMP последнего «хартбита», посчитанный через запуск задания через go-proxyd (об этом ниже) 73 | * ```disabled_ts``` — опциональное поле, которое временно отключает хост из балансировки, если выставлено в не-NULL значение 74 | * ```min_memory``` — опциональное поле, минимальное количество свободной памяти, при которой хост считается «живым» (полезно выставлять в -1045057536 для маленьких виртуальных машин) 75 | * ```min_parrots``` — опциональное поле, минимальное количество «попугаев», при которых хост всё ещё считается «живым» 76 | 77 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/dns/dns.go: -------------------------------------------------------------------------------- 1 | package dns 2 | 3 | import ( 4 | "badoo/_packages/log" 5 | "badoo/_packages/util" 6 | "fmt" 7 | "net" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | type Resolver interface { 13 | LookupHost(host string) ([]string, error) 14 | LookupHostEx(host string) (ips []string, cached bool, err error) 15 | LookupHostPort(hostport string) ([]string, error) // antoxa: temporary, for gpbrpc.Client 16 | Stop() 17 | } 18 | 19 | type cacheItem struct { 20 | host string 21 | ips []string 22 | err error 23 | } 24 | 25 | type resolveReq struct { 26 | host string 27 | responseC chan<- cacheItem 28 | } 29 | 30 | type resolver struct { 31 | resolveInterval time.Duration 32 | logger *log.Logger 33 | 34 | cacheLock sync.RWMutex 35 | cache map[string]cacheItem 36 | resolveC chan resolveReq 37 | quitC chan struct{} 38 | stopped bool 39 | } 40 | 41 | var ( 42 | std = NewResolver(30*time.Second, log.StandardLogger()) // default resolver, no logger 43 | 44 | ErrStopped = fmt.Errorf("Resolver is stopped") 45 | ) 46 | 47 | func (r *resolver) LookupHost(host string) ([]string, error) { 48 | ips, _, err := r.LookupHostEx(host) 49 | return ips, err 50 | } 51 | 52 | func (r *resolver) LookupHostEx(host string) (ips []string, cached bool, err error) { 53 | if r.stopped { 54 | return nil, false, ErrStopped 55 | } 56 | 57 | r.cacheLock.RLock() 58 | item, ok := r.cache[host] 59 | r.cacheLock.RUnlock() // unlock early! 60 | 61 | if ok { 62 | return item.ips, true, nil 63 | } 64 | 65 | // cache miss - slowpath 66 | 67 | result := func() cacheItem { 68 | resultC := make(chan cacheItem, 1) 69 | defer close(resultC) 70 | 71 | r.resolveC <- resolveReq{host, resultC} 72 | return <-resultC 73 | }() 74 | 75 | return result.ips, false, result.err 76 | } 77 | 78 | func (r *resolver) LookupHostPort(hostport string) ([]string, error) { 79 | if r.stopped { // detect stopped early 80 | return nil, ErrStopped 81 | } 82 | 83 | host, _, err := net.SplitHostPort(hostport) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return r.LookupHost(host) 89 | } 90 | 91 | func (r *resolver) Stop() { 92 | if !r.stopped { 93 | close(r.quitC) 94 | r.stopped = true 95 | } 96 | } 97 | 98 | func (r *resolver) resolveLoop() { 99 | for { 100 | select { 101 | case req := <-r.resolveC: 102 | 103 | // check cache again, in case we've just resolved that hostname 104 | // i.e. imagine service starts up, cache is empty and multiple goroutines all request to resolve the same hostname 105 | r.cacheLock.RLock() 106 | item, exists := r.cache[req.host] 107 | r.cacheLock.RUnlock() 108 | 109 | if exists { 110 | req.responseC <- item 111 | } else { 112 | item, _ := r.resolveAndUpdate(cacheItem{host: req.host}) 113 | req.responseC <- item 114 | } 115 | 116 | case <-r.quitC: 117 | return 118 | } 119 | } 120 | } 121 | 122 | func (r *resolver) updateLoop() { 123 | ticker := time.NewTicker(r.resolveInterval) 124 | defer ticker.Stop() 125 | 126 | copyCacheItems := func() []cacheItem { 127 | r.cacheLock.RLock() 128 | defer r.cacheLock.RUnlock() 129 | 130 | items := make([]cacheItem, 0, len(r.cache)) 131 | for _, item := range r.cache { 132 | items = append(items, item) 133 | } 134 | 135 | return items 136 | } 137 | 138 | for { 139 | select { 140 | case <-ticker.C: 141 | 142 | items := copyCacheItems() 143 | 144 | // TODO(antoxa): maybe do this in parallel 145 | for _, item := range items { 146 | r.resolveAndUpdate(item) 147 | } 148 | 149 | case <-r.quitC: 150 | return 151 | } 152 | } 153 | } 154 | 155 | func (r *resolver) resolveAndUpdate(item cacheItem) (cacheItem, error) { 156 | ips, err := net.LookupHost(item.host) 157 | if err != nil { 158 | if r.logger != nil { 159 | r.logger.Warnf("resolveAndUpdate %v: %v", item.host, err) 160 | } 161 | 162 | r.cacheLock.Lock() 163 | delete(r.cache, item.host) 164 | r.cacheLock.Unlock() 165 | 166 | return cacheItem{err: err}, err 167 | } 168 | 169 | if item.ips == nil || !util.StrSliceEqual(item.ips, ips) { 170 | if r.logger != nil { 171 | r.logger.Debugf("resolveAndUpdate: %s ips changed %v -> %v", item.host, item.ips, ips) 172 | } 173 | 174 | item.ips = ips 175 | item.err = err 176 | 177 | r.cacheLock.Lock() 178 | r.cache[item.host] = item 179 | r.cacheLock.Unlock() 180 | } 181 | 182 | return item, nil 183 | } 184 | 185 | func NewResolver(resolveInterval time.Duration, logger *log.Logger) Resolver { 186 | r := &resolver{ 187 | resolveInterval: resolveInterval, 188 | logger: logger, 189 | cache: make(map[string]cacheItem), 190 | resolveC: make(chan resolveReq, 100), // arbitrary buffer length 191 | quitC: make(chan struct{}), 192 | } 193 | go r.resolveLoop() 194 | go r.updateLoop() 195 | return r 196 | } 197 | 198 | func StdResolver() Resolver { 199 | return std 200 | } 201 | 202 | func LookupHost(host string) ([]string, error) { 203 | return std.LookupHost(host) 204 | } 205 | 206 | func LookupHostEx(host string) ([]string, bool, error) { 207 | return std.LookupHostEx(host) 208 | } 209 | 210 | func LookupHostPort(hostport string) ([]string, error) { 211 | return std.LookupHostPort(hostport) 212 | } 213 | -------------------------------------------------------------------------------- /proto/thunder.gpbrpc.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gpbrpc-go. 2 | // source: thunder.proto 3 | // DO NOT EDIT! 4 | 5 | package thunder 6 | 7 | import "github.com/gogo/protobuf/proto" 8 | import "badoo/_packages/gpbrpc" 9 | import "fmt" 10 | 11 | type GpbrpcType struct { 12 | } 13 | 14 | var Gpbrpc GpbrpcType 15 | 16 | var RequestMsgid_gpbrpc_name = map[uint32]string{ 17 | 1: "request_add_jobs", 18 | 2: "request_update_status", 19 | 3: "request_log_finish", 20 | } 21 | 22 | var RequestMsgid_gpbrpc_value = map[string]uint32{ 23 | "request_add_jobs": 1, 24 | "request_update_status": 2, 25 | "request_log_finish": 3, 26 | } 27 | 28 | var ResponseMsgid_gpbrpc_name = map[uint32]string{ 29 | 1: "response_generic", 30 | 2: "response_job_ids", 31 | 3: "response_run_info", 32 | } 33 | 34 | var ResponseMsgid_gpbrpc_value = map[string]uint32{ 35 | "response_generic": 1, 36 | "response_job_ids": 2, 37 | "response_run_info": 3, 38 | } 39 | 40 | func (GpbrpcType) GetRequestMsgid(msg proto.Message) uint32 { 41 | switch msg.(type) { 42 | case *RequestAddJobs: 43 | return uint32(RequestMsgid_REQUEST_ADD_JOBS) 44 | case *RequestUpdateStatus: 45 | return uint32(RequestMsgid_REQUEST_UPDATE_STATUS) 46 | case *RequestLogFinish: 47 | return uint32(RequestMsgid_REQUEST_LOG_FINISH) 48 | default: 49 | panic("you gave me the wrong message") 50 | } 51 | } 52 | 53 | func (GpbrpcType) GetRequestNameToIdMap() map[string]uint32 { 54 | return RequestMsgid_gpbrpc_value 55 | } 56 | 57 | func (GpbrpcType) GetRequestIdToNameMap() map[uint32]string { 58 | return RequestMsgid_gpbrpc_name 59 | } 60 | 61 | func (GpbrpcType) GetResponseNameToIdMap() map[string]uint32 { 62 | return ResponseMsgid_gpbrpc_value 63 | } 64 | 65 | func (GpbrpcType) GetResponseIdToNameMap() map[uint32]string { 66 | return ResponseMsgid_gpbrpc_name 67 | } 68 | 69 | func (GpbrpcType) GetResponseMsgid(msg proto.Message) uint32 { 70 | switch msg.(type) { 71 | case *ResponseGeneric: 72 | return uint32(ResponseMsgid_RESPONSE_GENERIC) 73 | case *ResponseJobIds: 74 | return uint32(ResponseMsgid_RESPONSE_JOB_IDS) 75 | case *ResponseRunInfo: 76 | return uint32(ResponseMsgid_RESPONSE_RUN_INFO) 77 | default: 78 | panic("you gave me the wrong message") 79 | } 80 | } 81 | 82 | func (GpbrpcType) GetPackageName() string { 83 | return "thunder" 84 | } 85 | 86 | func (GpbrpcType) GetRequestMsg(request_msgid uint32) proto.Message { 87 | switch RequestMsgid(request_msgid) { 88 | case RequestMsgid_REQUEST_ADD_JOBS: 89 | return &RequestAddJobs{} 90 | case RequestMsgid_REQUEST_UPDATE_STATUS: 91 | return &RequestUpdateStatus{} 92 | case RequestMsgid_REQUEST_LOG_FINISH: 93 | return &RequestLogFinish{} 94 | default: 95 | return nil 96 | } 97 | } 98 | 99 | func (GpbrpcType) GetResponseMsg(response_msgid uint32) proto.Message { 100 | switch ResponseMsgid(response_msgid) { 101 | case ResponseMsgid_RESPONSE_GENERIC: 102 | return &ResponseGeneric{} 103 | case ResponseMsgid_RESPONSE_JOB_IDS: 104 | return &ResponseJobIds{} 105 | case ResponseMsgid_RESPONSE_RUN_INFO: 106 | return &ResponseRunInfo{} 107 | default: 108 | return nil 109 | } 110 | } 111 | 112 | type GpbrpcInterface interface { 113 | RequestAddJobs(rctx gpbrpc.RequestT, request *RequestAddJobs) gpbrpc.ResultT 114 | RequestUpdateStatus(rctx gpbrpc.RequestT, request *RequestUpdateStatus) gpbrpc.ResultT 115 | RequestLogFinish(rctx gpbrpc.RequestT, request *RequestLogFinish) gpbrpc.ResultT 116 | } 117 | 118 | func (GpbrpcType) Dispatch(rctx gpbrpc.RequestT, abstract_service interface{}) gpbrpc.ResultT { 119 | 120 | service := abstract_service.(GpbrpcInterface) 121 | 122 | switch RequestMsgid(rctx.MessageId) { 123 | case RequestMsgid_REQUEST_ADD_JOBS: 124 | r := rctx.Message.(*RequestAddJobs) 125 | return service.RequestAddJobs(rctx, r) 126 | case RequestMsgid_REQUEST_UPDATE_STATUS: 127 | r := rctx.Message.(*RequestUpdateStatus) 128 | return service.RequestUpdateStatus(rctx, r) 129 | case RequestMsgid_REQUEST_LOG_FINISH: 130 | r := rctx.Message.(*RequestLogFinish) 131 | return service.RequestLogFinish(rctx, r) 132 | default: 133 | panic("screw you") 134 | } 135 | } 136 | 137 | var okResult = gpbrpc.Result(&ResponseGeneric{ErrorCode: proto.Int32(0)}) 138 | 139 | func (GpbrpcType) OK(args ...interface{}) gpbrpc.ResultT { 140 | if len(args) == 0 { 141 | return okResult 142 | } 143 | return gpbrpc.Result(&ResponseGeneric{ErrorCode: proto.Int32(0), 144 | ErrorText: proto.String(fmt.Sprint(args...))}) 145 | } 146 | 147 | func (GpbrpcType) ErrorGeneric(args ...interface{}) gpbrpc.ResultT { 148 | return gpbrpc.Result(&ResponseGeneric{ErrorCode: proto.Int32(-int32(Errno_ERRNO_GENERIC)), 149 | ErrorText: proto.String(fmt.Sprint(args...))}) 150 | } 151 | 152 | /* 153 | func ($receiver$) RequestAddJobs(rctx gpbrpc.RequestT, request *$proto$.RequestAddJobs) gpbrpc.ResultT { 154 | return $proto$.Gpbrpc.ErrorGeneric("not implemented") 155 | } 156 | 157 | func ($receiver$) RequestUpdateStatus(rctx gpbrpc.RequestT, request *$proto$.RequestUpdateStatus) gpbrpc.ResultT { 158 | return $proto$.Gpbrpc.ErrorGeneric("not implemented") 159 | } 160 | 161 | func ($receiver$) RequestLogFinish(rctx gpbrpc.RequestT, request *$proto$.RequestLogFinish) gpbrpc.ResultT { 162 | return $proto$.Gpbrpc.ErrorGeneric("not implemented") 163 | } 164 | 165 | */ 166 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/stats.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "badoo/_packages/gpbrpc" 5 | "badoo/_packages/service/stats" 6 | "encoding/json" 7 | "fmt" 8 | "net" 9 | "reflect" 10 | "sync/atomic" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/gogo/protobuf/proto" 15 | ) 16 | 17 | type StatsCtx struct { 18 | } 19 | 20 | var ( 21 | // public variable, is supposed to be set by user (for example in their package's init function) 22 | // have some defaults, in case user doesn't set it at all 23 | VersionInfo = badoo_service.ResponseVersion{ 24 | Version: proto.String("0.0.0"), 25 | Maintainer: proto.String("NO-USERNAME-SET@corp.badoo.com"), 26 | } 27 | 28 | stats_ctx = &StatsCtx{} 29 | ) 30 | 31 | // this function extracts underlying fd from TCPListener 32 | // CALLER MUST TREAT THIS FD AS READ-ONLY 33 | // the purpose here is to extract real non dup()'d fd for use in GetsockoptTCPInfo() 34 | func getRealFdFromTCPListener(l *net.TCPListener) uintptr { 35 | file := reflect.ValueOf(l).Elem().FieldByName("fd").Elem() 36 | return uintptr(file.FieldByName("sysfd").Int()) 37 | } 38 | 39 | // these functions should be moved somewhere to util package? ... someday when we need it 40 | func Getrusage(who int) (*syscall.Rusage, error) { 41 | rusage := &syscall.Rusage{} 42 | err := syscall.Getrusage(who, rusage) 43 | return rusage, err 44 | } 45 | 46 | func timevalToFloat32(tv *syscall.Timeval) float32 { 47 | return float32(tv.Sec) + float32(tv.Usec)/float32(1000*1000) 48 | } 49 | 50 | func GatherServiceStats() (*badoo_service.ResponseStats, error) { 51 | ru, err := Getrusage(syscall.RUSAGE_SELF) 52 | if nil != err { 53 | return nil, fmt.Errorf("getrusage: %v", err) 54 | } 55 | 56 | // ports stats first 57 | ports := make([]*badoo_service.ResponseStatsPortStats, len(StartedServers)) 58 | 59 | i := 0 60 | total_connections := uint32(0) 61 | for _, srv := range StartedServers { 62 | port_stats := &badoo_service.ResponseStatsPortStats{} 63 | 64 | stats := srv.Server.Stats 65 | 66 | // listen queue information 67 | unacked, sacked, err := GetLqInfo(srv) 68 | if err == nil { 69 | port_stats.LqCur = proto.Uint32(unacked) 70 | port_stats.LqMax = proto.Uint32(sacked) 71 | } 72 | 73 | port_connections := atomic.LoadUint64(&stats.ConnCur) 74 | total_connections += uint32(port_connections) 75 | 76 | // general stats 77 | port_stats.Proto = proto.String(srv.Name) 78 | port_stats.Address = proto.String(srv.Address) 79 | port_stats.ConnCur = proto.Uint64(port_connections) 80 | port_stats.ConnTotal = proto.Uint64(atomic.LoadUint64(&stats.ConnTotal)) 81 | port_stats.Requests = proto.Uint64(atomic.LoadUint64(&stats.Requests)) 82 | port_stats.BytesRead = proto.Uint64(atomic.LoadUint64(&stats.BytesRead)) 83 | port_stats.BytesWritten = proto.Uint64(atomic.LoadUint64(&stats.BytesWritten)) 84 | 85 | // per request stats 86 | port_stats.RequestStats = make([]*badoo_service.ResponseStatsPortStatsRequestStatsT, 0, len(badoo_service.RequestMsgid_name)) 87 | for msg_id, msg_name := range srv.Server.Proto.GetRequestIdToNameMap() { 88 | port_stats.RequestStats = append(port_stats.RequestStats, &badoo_service.ResponseStatsPortStatsRequestStatsT{ 89 | Name: proto.String(msg_name), 90 | Count: proto.Uint64(atomic.LoadUint64(&stats.RequestsIdStat[msg_id])), 91 | }) 92 | } 93 | 94 | ports[i] = port_stats 95 | i++ 96 | } 97 | 98 | r := &badoo_service.ResponseStats{ 99 | Uptime: proto.Uint32(uint32(time.Since(GetStartupTime()).Seconds())), 100 | RusageSelf: &badoo_service.ResponseStatsRusage{ 101 | RuUtime: proto.Float32(timevalToFloat32(&ru.Utime)), 102 | RuStime: proto.Float32(timevalToFloat32(&ru.Stime)), 103 | RuMaxrss: proto.Uint64(uint64(ru.Maxrss)), 104 | RuMinflt: proto.Uint64(uint64(ru.Minflt)), 105 | RuMajflt: proto.Uint64(uint64(ru.Majflt)), 106 | RuInblock: proto.Uint64(uint64(ru.Inblock)), 107 | RuOublock: proto.Uint64(uint64(ru.Oublock)), 108 | RuNvcsw: proto.Uint64(uint64(ru.Nvcsw)), 109 | RuNivcsw: proto.Uint64(uint64(ru.Nivcsw)), 110 | }, 111 | Ports: ports, 112 | Connections: proto.Uint32(total_connections), 113 | InitPhaseDuration: proto.Uint32(uint32(GetInitPhaseDuration().Seconds())), 114 | } 115 | 116 | return r, nil 117 | } 118 | 119 | func (s *StatsCtx) RequestStats(rctx gpbrpc.RequestT, request *badoo_service.RequestStats) gpbrpc.ResultT { 120 | stats, err := GatherServiceStats() 121 | if err != nil { 122 | return badoo_service.Gpbrpc.ErrorGeneric(err.Error()) 123 | } 124 | 125 | return gpbrpc.Result(stats) 126 | } 127 | 128 | func (s *StatsCtx) RequestVersion(rctx gpbrpc.RequestT, request *badoo_service.RequestVersion) gpbrpc.ResultT { 129 | return gpbrpc.Result(&VersionInfo) 130 | } 131 | 132 | func (s *StatsCtx) RequestConfigJson(rctx gpbrpc.RequestT, request *badoo_service.RequestConfigJson) gpbrpc.ResultT { 133 | buf, err := json.Marshal(config) 134 | if err != nil { 135 | return gpbrpc.Result(&badoo_service.ResponseGeneric{ 136 | ErrorCode: proto.Int32(-int32(badoo_service.Errno_ERRNO_GENERIC)), 137 | ErrorText: proto.String(fmt.Sprintf("error while marshalling config to json: %s", err)), 138 | }) 139 | } 140 | 141 | result := badoo_service.ResponseConfigJson{Json: proto.String(string(buf))} 142 | 143 | return gpbrpc.Result(&result) 144 | } 145 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/service/stats/stats.gpbrpc.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gpbrpc-go. 2 | // source: stats/stats.proto 3 | // DO NOT EDIT! 4 | 5 | package badoo_service 6 | 7 | import "github.com/gogo/protobuf/proto" 8 | import "badoo/_packages/gpbrpc" 9 | import "fmt" 10 | 11 | type GpbrpcType struct { 12 | } 13 | 14 | var Gpbrpc GpbrpcType 15 | 16 | var RequestMsgid_gpbrpc_name = map[uint32]string{ 17 | 1: "request_stats", 18 | 2: "request_version", 19 | 6: "request_config_json", 20 | } 21 | 22 | var RequestMsgid_gpbrpc_value = map[string]uint32{ 23 | "request_stats": 1, 24 | "request_version": 2, 25 | "request_config_json": 6, 26 | } 27 | 28 | var ResponseMsgid_gpbrpc_name = map[uint32]string{ 29 | 1: "response_generic", 30 | 2: "response_stats", 31 | 3: "response_version", 32 | 6: "response_config_json", 33 | } 34 | 35 | var ResponseMsgid_gpbrpc_value = map[string]uint32{ 36 | "response_generic": 1, 37 | "response_stats": 2, 38 | "response_version": 3, 39 | "response_config_json": 6, 40 | } 41 | 42 | func (GpbrpcType) GetRequestMsgid(msg proto.Message) uint32 { 43 | switch msg.(type) { 44 | case *RequestStats: 45 | return uint32(RequestMsgid_REQUEST_STATS) 46 | case *RequestVersion: 47 | return uint32(RequestMsgid_REQUEST_VERSION) 48 | case *RequestConfigJson: 49 | return uint32(RequestMsgid_REQUEST_CONFIG_JSON) 50 | default: 51 | panic("you gave me the wrong message") 52 | } 53 | } 54 | 55 | func (GpbrpcType) GetRequestNameToIdMap() map[string]uint32 { 56 | return RequestMsgid_gpbrpc_value 57 | } 58 | 59 | func (GpbrpcType) GetRequestIdToNameMap() map[uint32]string { 60 | return RequestMsgid_gpbrpc_name 61 | } 62 | 63 | func (GpbrpcType) GetResponseNameToIdMap() map[string]uint32 { 64 | return ResponseMsgid_gpbrpc_value 65 | } 66 | 67 | func (GpbrpcType) GetResponseIdToNameMap() map[uint32]string { 68 | return ResponseMsgid_gpbrpc_name 69 | } 70 | 71 | func (GpbrpcType) GetResponseMsgid(msg proto.Message) uint32 { 72 | switch msg.(type) { 73 | case *ResponseGeneric: 74 | return uint32(ResponseMsgid_RESPONSE_GENERIC) 75 | case *ResponseStats: 76 | return uint32(ResponseMsgid_RESPONSE_STATS) 77 | case *ResponseVersion: 78 | return uint32(ResponseMsgid_RESPONSE_VERSION) 79 | case *ResponseConfigJson: 80 | return uint32(ResponseMsgid_RESPONSE_CONFIG_JSON) 81 | default: 82 | panic("you gave me the wrong message") 83 | } 84 | } 85 | 86 | func (GpbrpcType) GetPackageName() string { 87 | return "badoo.service" 88 | } 89 | 90 | func (GpbrpcType) GetRequestMsg(request_msgid uint32) proto.Message { 91 | switch RequestMsgid(request_msgid) { 92 | case RequestMsgid_REQUEST_STATS: 93 | return &RequestStats{} 94 | case RequestMsgid_REQUEST_VERSION: 95 | return &RequestVersion{} 96 | case RequestMsgid_REQUEST_CONFIG_JSON: 97 | return &RequestConfigJson{} 98 | default: 99 | return nil 100 | } 101 | } 102 | 103 | func (GpbrpcType) GetResponseMsg(response_msgid uint32) proto.Message { 104 | switch ResponseMsgid(response_msgid) { 105 | case ResponseMsgid_RESPONSE_GENERIC: 106 | return &ResponseGeneric{} 107 | case ResponseMsgid_RESPONSE_STATS: 108 | return &ResponseStats{} 109 | case ResponseMsgid_RESPONSE_VERSION: 110 | return &ResponseVersion{} 111 | case ResponseMsgid_RESPONSE_CONFIG_JSON: 112 | return &ResponseConfigJson{} 113 | default: 114 | return nil 115 | } 116 | } 117 | 118 | type GpbrpcInterface interface { 119 | RequestStats(rctx gpbrpc.RequestT, request *RequestStats) gpbrpc.ResultT 120 | RequestVersion(rctx gpbrpc.RequestT, request *RequestVersion) gpbrpc.ResultT 121 | RequestConfigJson(rctx gpbrpc.RequestT, request *RequestConfigJson) gpbrpc.ResultT 122 | } 123 | 124 | func (GpbrpcType) Dispatch(rctx gpbrpc.RequestT, abstract_service interface{}) gpbrpc.ResultT { 125 | 126 | service := abstract_service.(GpbrpcInterface) 127 | 128 | switch RequestMsgid(rctx.MessageId) { 129 | case RequestMsgid_REQUEST_STATS: 130 | r := rctx.Message.(*RequestStats) 131 | return service.RequestStats(rctx, r) 132 | case RequestMsgid_REQUEST_VERSION: 133 | r := rctx.Message.(*RequestVersion) 134 | return service.RequestVersion(rctx, r) 135 | case RequestMsgid_REQUEST_CONFIG_JSON: 136 | r := rctx.Message.(*RequestConfigJson) 137 | return service.RequestConfigJson(rctx, r) 138 | default: 139 | panic("screw you") 140 | } 141 | } 142 | 143 | var okResult = gpbrpc.Result(&ResponseGeneric{ErrorCode: proto.Int32(0)}) 144 | 145 | func (GpbrpcType) OK(args ...interface{}) gpbrpc.ResultT { 146 | if len(args) == 0 { 147 | return okResult 148 | } 149 | return gpbrpc.Result(&ResponseGeneric{ErrorCode: proto.Int32(0), 150 | ErrorText: proto.String(fmt.Sprint(args...))}) 151 | } 152 | 153 | func (GpbrpcType) ErrorGeneric(args ...interface{}) gpbrpc.ResultT { 154 | return gpbrpc.Result(&ResponseGeneric{ErrorCode: proto.Int32(-int32(Errno_ERRNO_GENERIC)), 155 | ErrorText: proto.String(fmt.Sprint(args...))}) 156 | } 157 | 158 | /* 159 | func ($receiver$) RequestStats(rctx gpbrpc.RequestT, request *$proto$.RequestStats) gpbrpc.ResultT { 160 | return $proto$.Gpbrpc.ErrorGeneric("not implemented") 161 | } 162 | 163 | func ($receiver$) RequestVersion(rctx gpbrpc.RequestT, request *$proto$.RequestVersion) gpbrpc.ResultT { 164 | return $proto$.Gpbrpc.ErrorGeneric("not implemented") 165 | } 166 | 167 | func ($receiver$) RequestConfigJson(rctx gpbrpc.RequestT, request *$proto$.RequestConfigJson) gpbrpc.ResultT { 168 | return $proto$.Gpbrpc.ErrorGeneric("not implemented") 169 | } 170 | 171 | */ 172 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/infile.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import ( 12 | "fmt" 13 | "io" 14 | "os" 15 | "strings" 16 | "sync" 17 | ) 18 | 19 | var ( 20 | fileRegister map[string]bool 21 | fileRegisterLock sync.RWMutex 22 | readerRegister map[string]func() io.Reader 23 | readerRegisterLock sync.RWMutex 24 | ) 25 | 26 | // RegisterLocalFile adds the given file to the file whitelist, 27 | // so that it can be used by "LOAD DATA LOCAL INFILE ". 28 | // Alternatively you can allow the use of all local files with 29 | // the DSN parameter 'allowAllFiles=true' 30 | // 31 | // filePath := "/home/gopher/data.csv" 32 | // mysql.RegisterLocalFile(filePath) 33 | // err := db.Exec("LOAD DATA LOCAL INFILE '" + filePath + "' INTO TABLE foo") 34 | // if err != nil { 35 | // ... 36 | // 37 | func RegisterLocalFile(filePath string) { 38 | fileRegisterLock.Lock() 39 | // lazy map init 40 | if fileRegister == nil { 41 | fileRegister = make(map[string]bool) 42 | } 43 | 44 | fileRegister[strings.Trim(filePath, `"`)] = true 45 | fileRegisterLock.Unlock() 46 | } 47 | 48 | // DeregisterLocalFile removes the given filepath from the whitelist. 49 | func DeregisterLocalFile(filePath string) { 50 | fileRegisterLock.Lock() 51 | delete(fileRegister, strings.Trim(filePath, `"`)) 52 | fileRegisterLock.Unlock() 53 | } 54 | 55 | // RegisterReaderHandler registers a handler function which is used 56 | // to receive a io.Reader. 57 | // The Reader can be used by "LOAD DATA LOCAL INFILE Reader::". 58 | // If the handler returns a io.ReadCloser Close() is called when the 59 | // request is finished. 60 | // 61 | // mysql.RegisterReaderHandler("data", func() io.Reader { 62 | // var csvReader io.Reader // Some Reader that returns CSV data 63 | // ... // Open Reader here 64 | // return csvReader 65 | // }) 66 | // err := db.Exec("LOAD DATA LOCAL INFILE 'Reader::data' INTO TABLE foo") 67 | // if err != nil { 68 | // ... 69 | // 70 | func RegisterReaderHandler(name string, handler func() io.Reader) { 71 | readerRegisterLock.Lock() 72 | // lazy map init 73 | if readerRegister == nil { 74 | readerRegister = make(map[string]func() io.Reader) 75 | } 76 | 77 | readerRegister[name] = handler 78 | readerRegisterLock.Unlock() 79 | } 80 | 81 | // DeregisterReaderHandler removes the ReaderHandler function with 82 | // the given name from the registry. 83 | func DeregisterReaderHandler(name string) { 84 | readerRegisterLock.Lock() 85 | delete(readerRegister, name) 86 | readerRegisterLock.Unlock() 87 | } 88 | 89 | func deferredClose(err *error, closer io.Closer) { 90 | closeErr := closer.Close() 91 | if *err == nil { 92 | *err = closeErr 93 | } 94 | } 95 | 96 | func (mc *mysqlConn) handleInFileRequest(name string) (err error) { 97 | var rdr io.Reader 98 | var data []byte 99 | 100 | if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader 101 | // The server might return an an absolute path. See issue #355. 102 | name = name[idx+8:] 103 | 104 | readerRegisterLock.RLock() 105 | handler, inMap := readerRegister[name] 106 | readerRegisterLock.RUnlock() 107 | 108 | if inMap { 109 | rdr = handler() 110 | if rdr != nil { 111 | data = make([]byte, 4+mc.maxWriteSize) 112 | 113 | if cl, ok := rdr.(io.Closer); ok { 114 | defer deferredClose(&err, cl) 115 | } 116 | } else { 117 | err = fmt.Errorf("Reader '%s' is ", name) 118 | } 119 | } else { 120 | err = fmt.Errorf("Reader '%s' is not registered", name) 121 | } 122 | } else { // File 123 | name = strings.Trim(name, `"`) 124 | fileRegisterLock.RLock() 125 | fr := fileRegister[name] 126 | fileRegisterLock.RUnlock() 127 | if mc.cfg.allowAllFiles || fr { 128 | var file *os.File 129 | var fi os.FileInfo 130 | 131 | if file, err = os.Open(name); err == nil { 132 | defer deferredClose(&err, file) 133 | 134 | // get file size 135 | if fi, err = file.Stat(); err == nil { 136 | rdr = file 137 | if fileSize := int(fi.Size()); fileSize <= mc.maxWriteSize { 138 | data = make([]byte, 4+fileSize) 139 | } else if fileSize <= mc.maxPacketAllowed { 140 | data = make([]byte, 4+mc.maxWriteSize) 141 | } else { 142 | err = fmt.Errorf("Local File '%s' too large: Size: %d, Max: %d", name, fileSize, mc.maxPacketAllowed) 143 | } 144 | } 145 | } 146 | } else { 147 | err = fmt.Errorf("Local File '%s' is not registered. Use the DSN parameter 'allowAllFiles=true' to allow all files", name) 148 | } 149 | } 150 | 151 | // send content packets 152 | if err == nil { 153 | var n int 154 | for err == nil { 155 | n, err = rdr.Read(data[4:]) 156 | if n > 0 { 157 | if ioErr := mc.writePacket(data[:4+n]); ioErr != nil { 158 | return ioErr 159 | } 160 | } 161 | } 162 | if err == io.EOF { 163 | err = nil 164 | } 165 | } 166 | 167 | // send empty packet (termination) 168 | if data == nil { 169 | data = make([]byte, 4) 170 | } 171 | if ioErr := mc.writePacket(data[:4]); ioErr != nil { 172 | return ioErr 173 | } 174 | 175 | // read OK packet 176 | if err == nil { 177 | return mc.readResultOK() 178 | } else { 179 | mc.readPacket() 180 | } 181 | return err 182 | } 183 | -------------------------------------------------------------------------------- /vendor/github.com/gogo/protobuf/proto/decode_gogo.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 2 | // http://github.com/gogo/protobuf/gogoproto 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are 6 | // met: 7 | // 8 | // * Redistributions of source code must retain the above copyright 9 | // notice, this list of conditions and the following disclaimer. 10 | // * Redistributions in binary form must reproduce the above 11 | // copyright notice, this list of conditions and the following disclaimer 12 | // in the documentation and/or other materials provided with the 13 | // distribution. 14 | // 15 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 18 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | package proto 28 | 29 | import ( 30 | "reflect" 31 | ) 32 | 33 | // Decode a reference to a struct pointer. 34 | func (o *Buffer) dec_ref_struct_message(p *Properties, base structPointer) (err error) { 35 | raw, e := o.DecodeRawBytes(false) 36 | if e != nil { 37 | return e 38 | } 39 | 40 | // If the object can unmarshal itself, let it. 41 | if p.isUnmarshaler { 42 | panic("not supported, since this is a pointer receiver") 43 | } 44 | 45 | obuf := o.buf 46 | oi := o.index 47 | o.buf = raw 48 | o.index = 0 49 | 50 | bas := structPointer_FieldPointer(base, p.field) 51 | 52 | err = o.unmarshalType(p.stype, p.sprop, false, bas) 53 | o.buf = obuf 54 | o.index = oi 55 | 56 | return err 57 | } 58 | 59 | // Decode a slice of references to struct pointers ([]struct). 60 | func (o *Buffer) dec_slice_ref_struct(p *Properties, is_group bool, base structPointer) error { 61 | newBas := appendStructPointer(base, p.field, p.sstype) 62 | 63 | if is_group { 64 | panic("not supported, maybe in future, if requested.") 65 | } 66 | 67 | raw, err := o.DecodeRawBytes(false) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | // If the object can unmarshal itself, let it. 73 | if p.isUnmarshaler { 74 | panic("not supported, since this is not a pointer receiver.") 75 | } 76 | 77 | obuf := o.buf 78 | oi := o.index 79 | o.buf = raw 80 | o.index = 0 81 | 82 | err = o.unmarshalType(p.stype, p.sprop, is_group, newBas) 83 | 84 | o.buf = obuf 85 | o.index = oi 86 | 87 | return err 88 | } 89 | 90 | // Decode a slice of references to struct pointers. 91 | func (o *Buffer) dec_slice_ref_struct_message(p *Properties, base structPointer) error { 92 | return o.dec_slice_ref_struct(p, false, base) 93 | } 94 | 95 | func setPtrCustomType(base structPointer, f field, v interface{}) { 96 | if v == nil { 97 | return 98 | } 99 | structPointer_SetStructPointer(base, f, structPointer(reflect.ValueOf(v).Pointer())) 100 | } 101 | 102 | func setCustomType(base structPointer, f field, value interface{}) { 103 | if value == nil { 104 | return 105 | } 106 | v := reflect.ValueOf(value).Elem() 107 | t := reflect.TypeOf(value).Elem() 108 | kind := t.Kind() 109 | switch kind { 110 | case reflect.Slice: 111 | slice := reflect.MakeSlice(t, v.Len(), v.Cap()) 112 | reflect.Copy(slice, v) 113 | oldHeader := structPointer_GetSliceHeader(base, f) 114 | oldHeader.Data = slice.Pointer() 115 | oldHeader.Len = v.Len() 116 | oldHeader.Cap = v.Cap() 117 | default: 118 | l := 1 119 | size := reflect.TypeOf(value).Elem().Size() 120 | if kind == reflect.Array { 121 | l = reflect.TypeOf(value).Elem().Len() 122 | size = reflect.TypeOf(value).Size() 123 | } 124 | total := int(size) * l 125 | structPointer_Copy(toStructPointer(reflect.ValueOf(value)), structPointer_Add(base, f), total) 126 | } 127 | } 128 | 129 | func (o *Buffer) dec_custom_bytes(p *Properties, base structPointer) error { 130 | b, err := o.DecodeRawBytes(true) 131 | if err != nil { 132 | return err 133 | } 134 | i := reflect.New(p.ctype.Elem()).Interface() 135 | custom := (i).(Unmarshaler) 136 | if err := custom.Unmarshal(b); err != nil { 137 | return err 138 | } 139 | setPtrCustomType(base, p.field, custom) 140 | return nil 141 | } 142 | 143 | func (o *Buffer) dec_custom_ref_bytes(p *Properties, base structPointer) error { 144 | b, err := o.DecodeRawBytes(true) 145 | if err != nil { 146 | return err 147 | } 148 | i := reflect.New(p.ctype).Interface() 149 | custom := (i).(Unmarshaler) 150 | if err := custom.Unmarshal(b); err != nil { 151 | return err 152 | } 153 | if custom != nil { 154 | setCustomType(base, p.field, custom) 155 | } 156 | return nil 157 | } 158 | 159 | // Decode a slice of bytes ([]byte) into a slice of custom types. 160 | func (o *Buffer) dec_custom_slice_bytes(p *Properties, base structPointer) error { 161 | b, err := o.DecodeRawBytes(true) 162 | if err != nil { 163 | return err 164 | } 165 | i := reflect.New(p.ctype.Elem()).Interface() 166 | custom := (i).(Unmarshaler) 167 | if err := custom.Unmarshal(b); err != nil { 168 | return err 169 | } 170 | newBas := appendStructPointer(base, p.field, p.ctype) 171 | 172 | setCustomType(newBas, 0, custom) 173 | 174 | return nil 175 | } 176 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/log/exported.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | var ( 8 | // std is the name of the standard logger in stdlib `log` 9 | std = New() 10 | ) 11 | 12 | func StandardLogger() *Logger { 13 | return std 14 | } 15 | 16 | // SetOutput sets the standard logger output. 17 | func SetOutput(out io.Writer) { 18 | std.mu.Lock() 19 | defer std.mu.Unlock() 20 | std.Out = out 21 | } 22 | 23 | // SetFormatter sets the standard logger formatter. 24 | func SetFormatter(formatter Formatter) { 25 | std.mu.Lock() 26 | defer std.mu.Unlock() 27 | std.Formatter = formatter 28 | } 29 | 30 | // SetLevel sets the standard logger level. 31 | func SetLevel(level Level) { 32 | std.mu.Lock() 33 | defer std.mu.Unlock() 34 | std.Level = level 35 | } 36 | 37 | // GetLevel returns the standard logger level. 38 | func GetLevel() Level { 39 | std.mu.Lock() 40 | defer std.mu.Unlock() 41 | return std.Level 42 | } 43 | 44 | // AddHook adds a hook to the standard logger hooks. 45 | func AddHook(hook Hook) { 46 | std.mu.Lock() 47 | defer std.mu.Unlock() 48 | std.Hooks.Add(hook) 49 | } 50 | 51 | // WithoutHooks creates an entry from the standard logger and disables hooks for it 52 | func WithoutHooks() *Entry { 53 | return std.WithoutHooks() 54 | } 55 | 56 | // WithError creates an entry from the standard logger and adds an error to it, using the value defined in ErrorKey as key. 57 | func WithError(err error) *Entry { 58 | return std.WithField(ErrorKey, err) 59 | } 60 | 61 | // WithField creates an entry from the standard logger and adds a field to 62 | // it. If you want multiple fields, use `WithFields`. 63 | // 64 | // Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal 65 | // or Panic on the Entry it returns. 66 | func WithField(key string, value interface{}) *Entry { 67 | return std.WithField(key, value) 68 | } 69 | 70 | // WithFields creates an entry from the standard logger and adds multiple 71 | // fields to it. This is simply a helper for `WithField`, invoking it 72 | // once for each field. 73 | // 74 | // Note that it doesn't log until you call Debug, Print, Info, Warn, Fatal 75 | // or Panic on the Entry it returns. 76 | func WithFields(fields Fields) *Entry { 77 | return std.WithFields(fields) 78 | } 79 | 80 | // Debug logs a message at level Debug on the standard logger. 81 | func Debug(args ...interface{}) { 82 | std.Debug(args...) 83 | } 84 | 85 | // Print logs a message at level Info on the standard logger. 86 | func Print(args ...interface{}) { 87 | std.Print(args...) 88 | } 89 | 90 | // Info logs a message at level Info on the standard logger. 91 | func Info(args ...interface{}) { 92 | std.Info(args...) 93 | } 94 | 95 | // Warn logs a message at level Warn on the standard logger. 96 | func Warn(args ...interface{}) { 97 | std.Warn(args...) 98 | } 99 | 100 | // Warning logs a message at level Warn on the standard logger. 101 | func Warning(args ...interface{}) { 102 | std.Warning(args...) 103 | } 104 | 105 | // Error logs a message at level Error on the standard logger. 106 | func Error(args ...interface{}) { 107 | std.Error(args...) 108 | } 109 | 110 | // Panic logs a message at level Panic on the standard logger. 111 | func Panic(args ...interface{}) { 112 | std.Panic(args...) 113 | } 114 | 115 | // Fatal logs a message at level Fatal on the standard logger. 116 | func Fatal(args ...interface{}) { 117 | std.Fatal(args...) 118 | } 119 | 120 | // Debugf logs a message at level Debug on the standard logger. 121 | func Debugf(format string, args ...interface{}) { 122 | std.Debugf(format, args...) 123 | } 124 | 125 | // Printf logs a message at level Info on the standard logger. 126 | func Printf(format string, args ...interface{}) { 127 | std.Printf(format, args...) 128 | } 129 | 130 | // Infof logs a message at level Info on the standard logger. 131 | func Infof(format string, args ...interface{}) { 132 | std.Infof(format, args...) 133 | } 134 | 135 | // Warnf logs a message at level Warn on the standard logger. 136 | func Warnf(format string, args ...interface{}) { 137 | std.Warnf(format, args...) 138 | } 139 | 140 | // Warningf logs a message at level Warn on the standard logger. 141 | func Warningf(format string, args ...interface{}) { 142 | std.Warningf(format, args...) 143 | } 144 | 145 | // Errorf logs a message at level Error on the standard logger. 146 | func Errorf(format string, args ...interface{}) { 147 | std.Errorf(format, args...) 148 | } 149 | 150 | // Panicf logs a message at level Panic on the standard logger. 151 | func Panicf(format string, args ...interface{}) { 152 | std.Panicf(format, args...) 153 | } 154 | 155 | // Fatalf logs a message at level Fatal on the standard logger. 156 | func Fatalf(format string, args ...interface{}) { 157 | std.Fatalf(format, args...) 158 | } 159 | 160 | // Debugln logs a message at level Debug on the standard logger. 161 | func Debugln(args ...interface{}) { 162 | std.Debugln(args...) 163 | } 164 | 165 | // Println logs a message at level Info on the standard logger. 166 | func Println(args ...interface{}) { 167 | std.Println(args...) 168 | } 169 | 170 | // Infoln logs a message at level Info on the standard logger. 171 | func Infoln(args ...interface{}) { 172 | std.Infoln(args...) 173 | } 174 | 175 | // Warnln logs a message at level Warn on the standard logger. 176 | func Warnln(args ...interface{}) { 177 | std.Warnln(args...) 178 | } 179 | 180 | // Warningln logs a message at level Warn on the standard logger. 181 | func Warningln(args ...interface{}) { 182 | std.Warningln(args...) 183 | } 184 | 185 | // Errorln logs a message at level Error on the standard logger. 186 | func Errorln(args ...interface{}) { 187 | std.Errorln(args...) 188 | } 189 | 190 | // Panicln logs a message at level Panic on the standard logger. 191 | func Panicln(args ...interface{}) { 192 | std.Panicln(args...) 193 | } 194 | 195 | // Fatalln logs a message at level Fatal on the standard logger. 196 | func Fatalln(args ...interface{}) { 197 | std.Fatalln(args...) 198 | } 199 | -------------------------------------------------------------------------------- /vendor/badoo/_packages/gpbrpc/codec_gpbs.go: -------------------------------------------------------------------------------- 1 | package gpbrpc 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "net" 8 | "sync" 9 | 10 | "github.com/gogo/protobuf/proto" 11 | ) 12 | 13 | const ( 14 | GPBS_MaxMessageLength = 16 * 1024 * 1024 15 | ) 16 | 17 | var ( 18 | GpbsCodec = gpbsCodec{} 19 | headerPool = sync.Pool{ 20 | New: func() interface{} { 21 | return &buffer{} 22 | }, 23 | } 24 | ) 25 | 26 | type gpbsCodec struct { 27 | } 28 | 29 | type buffer struct { 30 | data [8]byte 31 | } 32 | 33 | // ServerCodec functions 34 | func (c *gpbsCodec) ReadRequest(p Protocol, conn net.Conn) (uint32, proto.Message, int, ConnStatus, error) { 35 | msgid, body, readln, status, err := ReadGpbsPacket(conn) 36 | if err != nil { 37 | return 0, nil, readln, status, err 38 | } 39 | 40 | msg := p.GetRequestMsg(msgid) 41 | if msg == nil { 42 | return 0, nil, readln, ConnOK, fmt.Errorf("parse error: unknown message_id: %d", msgid) 43 | } 44 | err = proto.Unmarshal(body, msg) 45 | if err != nil { 46 | return 0, nil, readln, ConnOK, fmt.Errorf("parse error: message %s, error: %s", MapRequestIdToName(p, msgid), err) 47 | } 48 | return msgid, msg, readln, status, nil 49 | } 50 | 51 | // FIXME: copy-paste of the above, just message getters are different and it does not return n_bytes_read 52 | // since exchange is packet based - we can factor this stuff out 53 | // OR completely remove all differences between requests and responses 54 | func (c *gpbsCodec) ReadResponse(p Protocol, conn net.Conn) (uint32, proto.Message, ConnStatus, error) { 55 | msgid, body, _, status, err := ReadGpbsPacket(conn) 56 | if err != nil { 57 | return 0, nil, status, err 58 | } 59 | 60 | msg := p.GetResponseMsg(msgid) 61 | if msg == nil { 62 | return 0, nil, ConnOK, fmt.Errorf("parse error: unknown message_id: %d", msgid) 63 | } 64 | err = proto.Unmarshal(body, msg) 65 | if err != nil { 66 | return 0, nil, ConnOK, fmt.Errorf("parse error: message %s, error: %s", MapRequestIdToName(p, msgid), err) 67 | } 68 | return msgid, msg, status, nil 69 | } 70 | 71 | func (c *gpbsCodec) WriteRequest(p Protocol, conn net.Conn, msg proto.Message) error { 72 | _, err := WriteGpbsPacket(conn, p.GetRequestMsgid(msg), msg) 73 | return err 74 | } 75 | 76 | func (c *gpbsCodec) WriteResponse(p Protocol, conn net.Conn, msg proto.Message) (int, error) { 77 | return WriteGpbsPacket(conn, p.GetResponseMsgid(msg), msg) 78 | } 79 | 80 | // --------------------------------------------------------------------------------------------------------------- 81 | // helper functions 82 | 83 | // FIXME: little mess: ReadGpbsPacket is about []byte, WriteGpbsPacket is about proto.Message 84 | 85 | // Reads a GPBS message from provided io.Reader. 86 | // Returns message id, message data, bytes read from the reader, connection status, error 87 | func ReadGpbsPacket(r io.Reader) (msgId uint32, data []byte, bytesRead int, connStatus ConnStatus, err error) { 88 | header := headerPool.Get().(*buffer) 89 | defer headerPool.Put(header) 90 | 91 | read_bytes, err := readAll(r, header.data[:]) 92 | if err != nil || read_bytes != len(header.data) { 93 | if err != nil { 94 | if read_bytes == 0 { 95 | return 0, nil, 0, ConnEOF, fmt.Errorf("read error: %s", err) 96 | } 97 | return 0, nil, read_bytes, ConnIOError, fmt.Errorf("read error: %s", err) 98 | } 99 | return 0, nil, read_bytes, ConnIOError, fmt.Errorf("read error: connection unexpectedly closed") 100 | } 101 | 102 | length := binary.BigEndian.Uint32(header.data[:4]) 103 | msgid := binary.BigEndian.Uint32(header.data[4:]) 104 | 105 | if length < 4 { 106 | return 0, nil, read_bytes, ConnParseError, fmt.Errorf("parse error: invalid packet length: %d", length) 107 | } 108 | 109 | if length > GPBS_MaxMessageLength { 110 | return 0, nil, read_bytes, ConnParseError, fmt.Errorf("parse error: packet is too long: %d (max: %d)", length, GPBS_MaxMessageLength) 111 | } 112 | 113 | // FIXME: don't need to allocate anything if body is empty (i.e. length == 4), rewrite the code around 114 | body := make([]byte, length-4) 115 | 116 | if length > 4 { 117 | 118 | read_bytes, err = readAll(r, body) 119 | if err != nil || read_bytes != len(body) { 120 | if err != nil { 121 | return 0, nil, read_bytes, ConnIOError, fmt.Errorf("read error: %s", err) 122 | } 123 | return 0, nil, read_bytes, ConnIOError, fmt.Errorf("read error: connection unexpectedly closed") 124 | } 125 | } 126 | 127 | return msgid, body, read_bytes, ConnOK, nil 128 | } 129 | 130 | // Writes a GPBS message to provided io.Writer. 131 | // Returns bytes written to the io.Writer and error if any. 132 | func WriteGpbsPacket(w io.Writer, msgid uint32, msg proto.Message) (int, error) { 133 | var body []byte 134 | var err error 135 | 136 | if msg != nil { 137 | body, err = proto.Marshal(msg) 138 | if err != nil { 139 | return 0, fmt.Errorf("encode error: %s", err) 140 | } 141 | } 142 | 143 | return WriteRawGpbsPacket(w, msgid, body) 144 | } 145 | 146 | func WriteRawGpbsPacket(w io.Writer, msgid uint32, body []byte) (int, error) { 147 | 148 | t := headerPool.Get().(*buffer) 149 | defer headerPool.Put(t) 150 | 151 | if len(t.data) != 8 { 152 | panic("could not happen!") 153 | } 154 | 155 | binary.BigEndian.PutUint32(t.data[:4], uint32(4+len(body))) 156 | binary.BigEndian.PutUint32(t.data[4:], uint32(msgid)) 157 | 158 | written_bytes1, err := writeAll(w, t.data[:]) 159 | if err != nil || written_bytes1 != len(t.data) { 160 | if err != nil { 161 | return written_bytes1, fmt.Errorf("write error: %s", err) 162 | } 163 | return written_bytes1, fmt.Errorf("write error: connection unexpectedly closed") 164 | } 165 | 166 | written_bytes2, err := writeAll(w, body) 167 | if err != nil || written_bytes2 != len(body) { 168 | if err != nil { 169 | return written_bytes1 + written_bytes2, fmt.Errorf("write error: %s", err) 170 | } 171 | return written_bytes1 + written_bytes2, fmt.Errorf("write error: connection unexpectedly closed") 172 | } 173 | 174 | return written_bytes1 + written_bytes2, nil 175 | } 176 | -------------------------------------------------------------------------------- /jobgen/api.go: -------------------------------------------------------------------------------- 1 | package jobgen 2 | 3 | import ( 4 | "github.com/badoo/thunder/db" 5 | "github.com/badoo/thunder/proto" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "time" 10 | ) 11 | 12 | const MAX_API_JOBS = 50000 13 | 14 | func APIAcceptTTJobs(jobs []*thunder.RequestAddJobsJobT) ([]uint64, error) { 15 | now := uint64(time.Now().Unix()) 16 | 17 | classLocType := make(map[string]string) // class_name => location_type 18 | ttRows := make([]*TimetableEntry, 0, len(jobs)) 19 | 20 | perClassLoc := make(map[string]map[string][]*TimetableEntry) 21 | 22 | for _, row := range jobs { 23 | settings, err := getScriptSettings(row.GetClassName()) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | classLocType[row.GetClassName()] = settings.location_type 29 | 30 | jrow := new(TimetableEntry) 31 | jrow.class_name = row.GetClassName() 32 | 33 | if row.Repeat == nil { 34 | jrow.repeat = settings.repeat_job 35 | } else { 36 | if row.Repeat.Value == nil { 37 | jrow.repeat.Valid = false 38 | } else { 39 | jrow.repeat.Valid = true 40 | jrow.repeat.Int64 = int64(row.Repeat.GetValue()) 41 | } 42 | } 43 | 44 | jrow.default_retry = settings.retry_job 45 | 46 | jrow.created = now 47 | 48 | if row.SettingsId == nil { 49 | jrow.settings_id = settings.id 50 | jrow.settings = settings 51 | } else { 52 | jrow.settings_id = row.GetSettingsId() 53 | allSettingsMutex.Lock() 54 | jrow.settings = allSettings[jrow.settings_id] 55 | allSettingsMutex.Unlock() 56 | 57 | if jrow.settings == nil { 58 | ids := make(map[uint64]bool) 59 | ids[jrow.settings_id] = true 60 | err := loadNewIds(ids) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | allSettingsMutex.Lock() 66 | jrow.settings = allSettings[jrow.settings_id] 67 | allSettingsMutex.Unlock() 68 | 69 | if jrow.settings == nil { 70 | return nil, errors.New(fmt.Sprintf("Incorrect value of settings_id: %v", jrow.settings_id)) 71 | } 72 | } 73 | 74 | if jrow.settings.location_type != settings.location_type { 75 | return nil, errors.New(fmt.Sprintf("You are not allowed to specify settings_id that has different location_type, row: %+v", row)) 76 | } 77 | } 78 | 79 | jrow.NextLaunchTs.Valid = true 80 | 81 | if row.NextLaunchTs == nil { 82 | jrow.NextLaunchTs.Int64 = int64(now) 83 | } else { 84 | jrow.NextLaunchTs.Int64 = row.GetNextLaunchTs() 85 | } 86 | 87 | if row.Location == nil { 88 | jrow.location = settings.location 89 | } else { 90 | jrow.location = row.GetLocation() 91 | 92 | if settings.location_type == LOCATION_TYPE_ANY && jrow.settings.location != settings.location { 93 | return nil, errors.New(fmt.Sprintf("For location_type=any scripts location field must be equal to current settings: %+v", row)) 94 | } 95 | } 96 | 97 | jrow.JobData = row.GetJobData() 98 | 99 | if row.Method == nil { 100 | jrow.method = METHOD_RUN 101 | } else { 102 | jrow.method = row.GetMethod() 103 | } 104 | 105 | if row.GenerationId == nil { 106 | jrow.generation_id.Valid = false 107 | } else { 108 | jrow.generation_id.Valid = true 109 | jrow.generation_id.Int64 = row.GetGenerationId() 110 | } 111 | 112 | ttRows = append(ttRows, jrow) 113 | 114 | el, ok := perClassLoc[jrow.class_name] 115 | if !ok { 116 | el = make(map[string][]*TimetableEntry) 117 | perClassLoc[jrow.class_name] = el 118 | } 119 | 120 | el[jrow.location] = append(el[jrow.location], jrow) 121 | } 122 | 123 | for className, locRows := range perClassLoc { 124 | for location, rows := range locRows { 125 | key := DEFAULT_LOCATION_IDX 126 | if classLocType[className] == LOCATION_TYPE_EACH { 127 | key = location 128 | } 129 | 130 | currentCnt := 0 131 | if ch := getDispatcherJobsCountCh(className, key); ch != nil { 132 | respCh := make(chan int, 1) 133 | ch <- &JobsCountRequest{RespCh: respCh} 134 | currentCnt = <-respCh 135 | } 136 | 137 | if currentCnt+len(rows) > MAX_API_JOBS { 138 | return nil, fmt.Errorf("Too many jobs: %d (current) + %d (adding) > %d (max)", currentCnt, len(rows), MAX_API_JOBS) 139 | } 140 | } 141 | } 142 | 143 | err := db.DoInLazyTransaction(func(tx *db.LazyTrx) error { 144 | return addToTimetable(tx, ttRows) 145 | }) 146 | 147 | if err != nil { 148 | return nil, err 149 | } 150 | 151 | for className, locRows := range perClassLoc { 152 | for location, rows := range locRows { 153 | key := DEFAULT_LOCATION_IDX 154 | if classLocType[className] == LOCATION_TYPE_EACH { 155 | key = location 156 | } 157 | 158 | notifyAboutNewTTRows(className, key, rows, false) 159 | } 160 | } 161 | 162 | ids := make([]uint64, 0, len(ttRows)) 163 | for _, row := range ttRows { 164 | ids = append(ids, row.id) 165 | } 166 | 167 | return ids, nil 168 | } 169 | 170 | func APIUpdateRunStatus(hostname string, runId uint64, prevStatus, status string) error { 171 | ch, err := getLauncherUpdateStatusCh(hostname) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | errCh := make(chan error, 1) 177 | 178 | ch <- &LauncherUpdateStatusRequest{ 179 | Hostname: hostname, 180 | RunId: runId, 181 | PrevStatus: prevStatus, 182 | Status: status, 183 | errCh: errCh, 184 | } 185 | 186 | return <-errCh 187 | } 188 | 189 | // string is json-encode'd run queue entry row 190 | func APILogFinish(hostname string, runId uint64, prevStatus string, success bool) (string, error) { 191 | ch, err := getLauncherLogFinishCh(hostname) 192 | if err != nil { 193 | return "", err 194 | } 195 | 196 | respCh := make(chan *LauncherLogFinishResponse, 1) 197 | 198 | ch <- &LauncherLogFinishRequest{ 199 | Hostname: hostname, 200 | RunId: runId, 201 | PrevStatus: prevStatus, 202 | Success: success, 203 | respCh: respCh, 204 | } 205 | 206 | res := <-respCh 207 | if res.err != nil { 208 | return "", res.err 209 | } 210 | 211 | data, err := json.Marshal(res.info) 212 | if err != nil { 213 | return "", err 214 | } 215 | 216 | return string(data), nil 217 | } 218 | -------------------------------------------------------------------------------- /vendor/github.com/go-sql-driver/mysql/benchmark_test.go: -------------------------------------------------------------------------------- 1 | // Go MySQL Driver - A MySQL-Driver for Go's database/sql package 2 | // 3 | // Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this file, 7 | // You can obtain one at http://mozilla.org/MPL/2.0/. 8 | 9 | package mysql 10 | 11 | import ( 12 | "bytes" 13 | "database/sql" 14 | "database/sql/driver" 15 | "math" 16 | "strings" 17 | "sync" 18 | "sync/atomic" 19 | "testing" 20 | "time" 21 | ) 22 | 23 | type TB testing.B 24 | 25 | func (tb *TB) check(err error) { 26 | if err != nil { 27 | tb.Fatal(err) 28 | } 29 | } 30 | 31 | func (tb *TB) checkDB(db *sql.DB, err error) *sql.DB { 32 | tb.check(err) 33 | return db 34 | } 35 | 36 | func (tb *TB) checkRows(rows *sql.Rows, err error) *sql.Rows { 37 | tb.check(err) 38 | return rows 39 | } 40 | 41 | func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { 42 | tb.check(err) 43 | return stmt 44 | } 45 | 46 | func initDB(b *testing.B, queries ...string) *sql.DB { 47 | tb := (*TB)(b) 48 | db := tb.checkDB(sql.Open("mysql", dsn)) 49 | for _, query := range queries { 50 | if _, err := db.Exec(query); err != nil { 51 | if w, ok := err.(MySQLWarnings); ok { 52 | b.Logf("Warning on %q: %v", query, w) 53 | } else { 54 | b.Fatalf("Error on %q: %v", query, err) 55 | } 56 | } 57 | } 58 | return db 59 | } 60 | 61 | const concurrencyLevel = 10 62 | 63 | func BenchmarkQuery(b *testing.B) { 64 | tb := (*TB)(b) 65 | b.StopTimer() 66 | b.ReportAllocs() 67 | db := initDB(b, 68 | "DROP TABLE IF EXISTS foo", 69 | "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", 70 | `INSERT INTO foo VALUES (1, "one")`, 71 | `INSERT INTO foo VALUES (2, "two")`, 72 | ) 73 | db.SetMaxIdleConns(concurrencyLevel) 74 | defer db.Close() 75 | 76 | stmt := tb.checkStmt(db.Prepare("SELECT val FROM foo WHERE id=?")) 77 | defer stmt.Close() 78 | 79 | remain := int64(b.N) 80 | var wg sync.WaitGroup 81 | wg.Add(concurrencyLevel) 82 | defer wg.Wait() 83 | b.StartTimer() 84 | 85 | for i := 0; i < concurrencyLevel; i++ { 86 | go func() { 87 | for { 88 | if atomic.AddInt64(&remain, -1) < 0 { 89 | wg.Done() 90 | return 91 | } 92 | 93 | var got string 94 | tb.check(stmt.QueryRow(1).Scan(&got)) 95 | if got != "one" { 96 | b.Errorf("query = %q; want one", got) 97 | wg.Done() 98 | return 99 | } 100 | } 101 | }() 102 | } 103 | } 104 | 105 | func BenchmarkExec(b *testing.B) { 106 | tb := (*TB)(b) 107 | b.StopTimer() 108 | b.ReportAllocs() 109 | db := tb.checkDB(sql.Open("mysql", dsn)) 110 | db.SetMaxIdleConns(concurrencyLevel) 111 | defer db.Close() 112 | 113 | stmt := tb.checkStmt(db.Prepare("DO 1")) 114 | defer stmt.Close() 115 | 116 | remain := int64(b.N) 117 | var wg sync.WaitGroup 118 | wg.Add(concurrencyLevel) 119 | defer wg.Wait() 120 | b.StartTimer() 121 | 122 | for i := 0; i < concurrencyLevel; i++ { 123 | go func() { 124 | for { 125 | if atomic.AddInt64(&remain, -1) < 0 { 126 | wg.Done() 127 | return 128 | } 129 | 130 | if _, err := stmt.Exec(); err != nil { 131 | b.Fatal(err.Error()) 132 | } 133 | } 134 | }() 135 | } 136 | } 137 | 138 | // data, but no db writes 139 | var roundtripSample []byte 140 | 141 | func initRoundtripBenchmarks() ([]byte, int, int) { 142 | if roundtripSample == nil { 143 | roundtripSample = []byte(strings.Repeat("0123456789abcdef", 1024*1024)) 144 | } 145 | return roundtripSample, 16, len(roundtripSample) 146 | } 147 | 148 | func BenchmarkRoundtripTxt(b *testing.B) { 149 | b.StopTimer() 150 | sample, min, max := initRoundtripBenchmarks() 151 | sampleString := string(sample) 152 | b.ReportAllocs() 153 | tb := (*TB)(b) 154 | db := tb.checkDB(sql.Open("mysql", dsn)) 155 | defer db.Close() 156 | b.StartTimer() 157 | var result string 158 | for i := 0; i < b.N; i++ { 159 | length := min + i 160 | if length > max { 161 | length = max 162 | } 163 | test := sampleString[0:length] 164 | rows := tb.checkRows(db.Query(`SELECT "` + test + `"`)) 165 | if !rows.Next() { 166 | rows.Close() 167 | b.Fatalf("crashed") 168 | } 169 | err := rows.Scan(&result) 170 | if err != nil { 171 | rows.Close() 172 | b.Fatalf("crashed") 173 | } 174 | if result != test { 175 | rows.Close() 176 | b.Errorf("mismatch") 177 | } 178 | rows.Close() 179 | } 180 | } 181 | 182 | func BenchmarkRoundtripBin(b *testing.B) { 183 | b.StopTimer() 184 | sample, min, max := initRoundtripBenchmarks() 185 | b.ReportAllocs() 186 | tb := (*TB)(b) 187 | db := tb.checkDB(sql.Open("mysql", dsn)) 188 | defer db.Close() 189 | stmt := tb.checkStmt(db.Prepare("SELECT ?")) 190 | defer stmt.Close() 191 | b.StartTimer() 192 | var result sql.RawBytes 193 | for i := 0; i < b.N; i++ { 194 | length := min + i 195 | if length > max { 196 | length = max 197 | } 198 | test := sample[0:length] 199 | rows := tb.checkRows(stmt.Query(test)) 200 | if !rows.Next() { 201 | rows.Close() 202 | b.Fatalf("crashed") 203 | } 204 | err := rows.Scan(&result) 205 | if err != nil { 206 | rows.Close() 207 | b.Fatalf("crashed") 208 | } 209 | if !bytes.Equal(result, test) { 210 | rows.Close() 211 | b.Errorf("mismatch") 212 | } 213 | rows.Close() 214 | } 215 | } 216 | 217 | func BenchmarkInterpolation(b *testing.B) { 218 | mc := &mysqlConn{ 219 | cfg: &config{ 220 | interpolateParams: true, 221 | loc: time.UTC, 222 | }, 223 | maxPacketAllowed: maxPacketSize, 224 | maxWriteSize: maxPacketSize - 1, 225 | buf: newBuffer(nil), 226 | } 227 | 228 | args := []driver.Value{ 229 | int64(42424242), 230 | float64(math.Pi), 231 | false, 232 | time.Unix(1423411542, 807015000), 233 | []byte("bytes containing special chars ' \" \a \x00"), 234 | "string containing special chars ' \" \a \x00", 235 | } 236 | q := "SELECT ?, ?, ?, ?, ?, ?" 237 | 238 | b.ReportAllocs() 239 | b.ResetTimer() 240 | for i := 0; i < b.N; i++ { 241 | _, err := mc.interpolateParams(q, args) 242 | if err != nil { 243 | b.Fatal(err) 244 | } 245 | } 246 | } 247 | --------------------------------------------------------------------------------