├── nimo-full-check ├── common │ └── common.go ├── go.mod ├── checker │ ├── sample.go │ └── checker.go ├── run │ └── run.go ├── configure │ └── conf.go └── main │ └── main.go ├── Makefile ├── nimo-shake ├── protocol │ ├── same_converter.go │ ├── protocal.go │ ├── mtype_converter.go │ ├── raw_converter.go │ └── type_converter.go ├── unit_test_common │ └── include.go ├── third_party │ └── log4go │ │ ├── examples │ │ ├── ConsoleLogWriter_Manual.go │ │ ├── XMLConfigurationExample.go │ │ ├── SocketLogWriter_Manual.go │ │ ├── SimpleNetLogServer.go │ │ ├── FileLogWriter_Manual.go │ │ └── example.xml │ │ ├── LICENSE │ │ ├── socklog.go │ │ ├── termlog.go │ │ ├── pattlog.go │ │ ├── filelog.go │ │ ├── config.go │ │ └── wrapper.go ├── common │ ├── http.go │ ├── callback.go │ ├── math.go │ ├── error.go │ ├── fcv.go │ ├── mix.go │ ├── operator.go │ ├── mongodb_community.go │ ├── unsafe.go │ ├── common.go │ ├── shard.go │ ├── dynamodb.go │ └── mongodb_mgo.go ├── go.mod ├── qps │ └── qps.go ├── configure │ ├── check.go │ └── conf.go ├── writer │ ├── writer.go │ └── dynamo_proxy.go ├── filter │ ├── filter.go │ └── filter_test.go ├── checkpoint │ ├── writer.go │ ├── struct.go │ ├── mongoWriter.go │ ├── fileWriter.go │ └── writer_test.go ├── full-sync │ ├── document-syncer_test.go │ ├── document-syncer.go │ ├── syncer.go │ └── table-syncer.go ├── run │ └── run.go ├── conf │ └── nimo-shake.conf ├── incr-sync │ └── fetcher.go ├── go.sum └── main │ └── main.go ├── scripts ├── stop.sh ├── start.sh ├── run_ut_test.py └── hypervisor.c ├── .gitignore ├── README.md └── ChangeLog /nimo-full-check/common/common.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | var ( 4 | Version = "$" 5 | ) -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | ./build.sh 3 | 4 | clean: 5 | rm -rf bin 6 | rm -rf *.pprof 7 | rm -rf *.output 8 | rm -rf logs 9 | rm -rf diagnostic/ 10 | rm -rf *.pid 11 | 12 | -------------------------------------------------------------------------------- /nimo-shake/protocol/same_converter.go: -------------------------------------------------------------------------------- 1 | package protocal 2 | 3 | import "github.com/aws/aws-sdk-go/service/dynamodb" 4 | 5 | type SameConverter struct { 6 | } 7 | 8 | func (sc *SameConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) { 9 | return input, nil 10 | } -------------------------------------------------------------------------------- /nimo-shake/unit_test_common/include.go: -------------------------------------------------------------------------------- 1 | package unit_test_common 2 | 3 | const ( 4 | TestUrl = "mongodb://100.81.164.186:31881,100.81.164.186:31882,100.81.164.186:31883" 5 | TestUrlServerless = "mongodb://100.81.164.181:32155" 6 | TestUrlServerlessTenant = "mongodb://100.81.164.181:36106" // sharding mongos with tenant 7 | TestUrlSharding = "mongodb://100.81.164.181:33010" 8 | ) 9 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/examples/ConsoleLogWriter_Manual.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | import l4g "github.com/alibaba/MongoShake/v2/third_party/log4go" 8 | 9 | func main() { 10 | log := l4g.NewLogger() 11 | defer log.Close() 12 | log.AddFilter("stdout", l4g.DEBUG, l4g.NewConsoleLogWriter()) 13 | log.Info("The time is now: %s", time.Now().Format("15:04:05 MST 2006/01/02")) 14 | } 15 | -------------------------------------------------------------------------------- /nimo-full-check/go.mod: -------------------------------------------------------------------------------- 1 | module nimo-full-check 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.44.61 7 | github.com/google/go-cmp v0.6.0 8 | github.com/jessevdk/go-flags v1.5.0 // indirect 9 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d 10 | go.mongodb.org/mongo-driver v1.16.1 11 | nimo-shake v0.0.0-00010101000000-000000000000 12 | ) 13 | 14 | replace nimo-shake => ../nimo-shake 15 | -------------------------------------------------------------------------------- /nimo-shake/common/http.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | nimo "github.com/gugemichael/nimo4go" 5 | ) 6 | 7 | var ( 8 | FullSyncHttpApi *nimo.HttpRestProvider 9 | IncrSyncHttpApi *nimo.HttpRestProvider 10 | ) 11 | 12 | func FullSyncInitHttpApi(port int) { 13 | FullSyncHttpApi = nimo.NewHttpRestProvider(port) 14 | } 15 | 16 | func IncrSyncInitHttpApi(port int) { 17 | IncrSyncHttpApi = nimo.NewHttpRestProvider(port) 18 | } 19 | -------------------------------------------------------------------------------- /scripts/stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | #kill -9 "$(cat "$1")" 3 | if [ $# != 1 ] ; then 4 | echo "USAGE: $0 [pid filename which by default is 'dynamo-shake.pid']" 5 | exit 0 6 | fi 7 | ppid=$(ps -ef | awk '{if ($2=='`cat $1`') print $3}') 8 | [ -z $ppid ] && echo "[Fail] No process number for $(cat "$1")." && exit 1 9 | if [ $ppid -eq 1 ];then 10 | kill -9 "$(cat "$1")" 11 | else 12 | kill -9 "$ppid" "$(cat "$1")" 13 | fi 14 | -------------------------------------------------------------------------------- /nimo-shake/go.mod: -------------------------------------------------------------------------------- 1 | module nimo-shake 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.44.61 7 | github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40 8 | github.com/jinzhu/copier v0.3.5 9 | github.com/nightlyone/lockfile v1.0.0 10 | github.com/stretchr/testify v1.6.1 11 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d 12 | github.com/vinllen/mgo v0.0.0-20220329061231-e5ecea62f194 13 | go.mongodb.org/mongo-driver v1.16.1 14 | ) 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gopath 2 | .idea 3 | *.iml 4 | logs 5 | *.pprof 6 | *.output 7 | *.data 8 | *.sw[ap] 9 | *.yml 10 | *.pid 11 | *.tar.gz 12 | *.log 13 | tags 14 | bin/* 15 | conf/* 16 | !conf/nimo-shake.conf 17 | 18 | runtime.trace 19 | 20 | .DS_Store 21 | 22 | data 23 | nimo-shake-v* 24 | 25 | .cache/ 26 | diagnostic/ 27 | *.pid 28 | src/vendor/* 29 | !src/vendor/vendor.json 30 | test_checkpoint_db 31 | 32 | pkg/ 33 | nimo-full-check/bin/ 34 | nimo-shake/bin/ 35 | scripts/hypervisor 36 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/examples/XMLConfigurationExample.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import l4g "github.com/alibaba/MongoShake/v2/third_party/log4go" 4 | 5 | func main() { 6 | // Load the configuration (isn't this easy?) 7 | l4g.LoadConfiguration("example.xml") 8 | 9 | // And now we're ready! 10 | l4g.Finest("This will only go to those of you really cool UDP kids! If you change enabled=true.") 11 | l4g.Debug("Oh no! %d + %d = %d!", 2, 2, 2+2) 12 | l4g.Info("About that time, eh chaps?") 13 | } 14 | -------------------------------------------------------------------------------- /scripts/start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | catalog=$(dirname "$0") 4 | 5 | cd "${catalog}" || exit 1 6 | 7 | if [ $# != 1 ] ; then 8 | echo "USAGE: $0 [conf]" 9 | exit 0 10 | fi 11 | 12 | name="nimo-shake.linux" 13 | 14 | if [ "Darwin" == "$(uname -s)" ];then 15 | printf "\\nWARNING !!! MacOs doesn't supply to use this script, please use \"./%s -conf=config_file_name\" manual command to run\\n" "$name" 16 | exit 1 17 | fi 18 | 19 | ./hypervisor --daemon --exec="./$name -conf=$1 1>>$name.output 2>&1" 1>>hypervisor.output 2>&1 20 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/examples/SocketLogWriter_Manual.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | import l4g "github.com/alibaba/MongoShake/v2/third_party/log4go" 8 | 9 | func main() { 10 | log := l4g.NewLogger() 11 | log.AddFilter("network", l4g.FINEST, l4g.NewSocketLogWriter("udp", "192.168.1.255:12124")) 12 | 13 | // Run `nc -u -l -p 12124` or similar before you run this to see the following message 14 | log.Info("The time is now: %s", time.Now().Format("15:04:05 MST 2006/01/02")) 15 | 16 | // This makes sure the output stream buffer is written 17 | log.Close() 18 | } 19 | -------------------------------------------------------------------------------- /nimo-shake/common/callback.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "time" 4 | 5 | /* 6 | * retry the callback function until successfully or overpass the threshold. 7 | * @input: 8 | * times: retry times 9 | * sleep: sleep time by ms interval 10 | * cb: callback 11 | * the callback should return true means need retry. 12 | */ 13 | func CallbackRetry(times int, sleep int, cb func() bool) bool { 14 | for i := 0; i < times; i++ { 15 | if cb() == false { // callback, true means retry 16 | return true 17 | } 18 | time.Sleep(time.Duration(sleep) * time.Millisecond) 19 | } 20 | return false 21 | } -------------------------------------------------------------------------------- /nimo-full-check/checker/sample.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import "math/rand" 4 | 5 | const ( 6 | SEED = 1 7 | ) 8 | 9 | type Sample struct { 10 | sampleCnt int64 11 | totalCnt int64 12 | source *rand.Rand 13 | } 14 | 15 | func NewSample(sampleCnt, totalCnt int64) *Sample { 16 | return &Sample{ 17 | sampleCnt: sampleCnt, 18 | totalCnt: totalCnt, 19 | source: rand.New(rand.NewSource(SEED)), 20 | } 21 | } 22 | 23 | func (s *Sample) Hit() bool { 24 | if s.sampleCnt >= s.totalCnt { 25 | return true 26 | } 27 | if s.sampleCnt == 0 { 28 | return false 29 | } 30 | 31 | return s.source.Int63n(s.totalCnt) < s.sampleCnt 32 | } 33 | -------------------------------------------------------------------------------- /nimo-shake/qps/qps.go: -------------------------------------------------------------------------------- 1 | package qps 2 | 3 | import "time" 4 | 5 | type Qos struct { 6 | Bucket chan struct{} 7 | 8 | limit int // qps 9 | close bool 10 | } 11 | 12 | func StartQoS(limit int) *Qos { 13 | q := new(Qos) 14 | q.limit = limit 15 | q.Bucket = make(chan struct{}, limit) 16 | 17 | go q.timer() 18 | return q 19 | } 20 | 21 | func (q *Qos) timer() { 22 | for range time.NewTicker(1 * time.Second).C { 23 | if q.close { 24 | return 25 | } 26 | for i := 0; i < q.limit; i++ { 27 | select { 28 | case q.Bucket <- struct{}{}: 29 | default: 30 | // break if bucket if full 31 | break 32 | } 33 | } 34 | } 35 | } 36 | 37 | func (q *Qos) Close() { 38 | q.close = true 39 | } 40 | -------------------------------------------------------------------------------- /nimo-shake/protocol/protocal.go: -------------------------------------------------------------------------------- 1 | package protocal 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/service/dynamodb" 5 | 6 | "nimo-shake/common" 7 | ) 8 | 9 | // convert DynamoDB attributeValue to bson 10 | type Converter interface { 11 | // run 12 | Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) 13 | } 14 | 15 | func NewConverter(tp string) Converter { 16 | switch tp { 17 | case utils.ConvertTypeRaw: 18 | return new(RawConverter) 19 | case utils.ConvertTypeChange: 20 | return new(TypeConverter) 21 | case utils.ConvertMTypeChange: 22 | return new(MTypeConverter) 23 | case utils.ConvertTypeSame: 24 | return new(SameConverter) 25 | default: 26 | return nil 27 | } 28 | } 29 | 30 | type RawData struct { 31 | Size int // fake size, only calculate real data 32 | Data interface{} // real data 33 | } 34 | -------------------------------------------------------------------------------- /nimo-shake/common/math.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/binary" 6 | ) 7 | 8 | /* 9 | // === FUNCTION ====================================================================== 10 | // Name: Md5 11 | // Description: 128位md5 12 | // ===================================================================================== 13 | */ 14 | func Md5(data []byte) [16]byte { 15 | return md5.Sum(data) 16 | } 17 | 18 | /* 19 | // === FUNCTION ====================================================================== 20 | // Name: Md5 21 | // Description: 64位md5 22 | // ===================================================================================== 23 | */ 24 | func Md5In64(data []byte) uint64 { 25 | var md5 = md5.Sum(data) 26 | var lowMd5 = md5[0:8] 27 | return binary.LittleEndian.Uint64(lowMd5) 28 | } 29 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/examples/SimpleNetLogServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net" 7 | "os" 8 | ) 9 | 10 | var ( 11 | port = flag.String("p", "12124", "Port number to listen on") 12 | ) 13 | 14 | func e(err error) { 15 | if err != nil { 16 | fmt.Printf("Erroring out: %s\n", err) 17 | os.Exit(1) 18 | } 19 | } 20 | 21 | func main() { 22 | flag.Parse() 23 | 24 | // Bind to the port 25 | bind, err := net.ResolveUDPAddr("udp", "0.0.0.0:"+*port) 26 | e(err) 27 | 28 | // Create listener 29 | listener, err := net.ListenUDP("udp", bind) 30 | e(err) 31 | 32 | fmt.Printf("Listening to port %s...\n", *port) 33 | for { 34 | // read into a new buffer 35 | buffer := make([]byte, 1024) 36 | _, _, err := listener.ReadFrom(buffer) 37 | e(err) 38 | 39 | // log to standard output 40 | fmt.Println(string(buffer)) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /nimo-shake/configure/check.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | import ( 4 | "bufio" 5 | "strings" 6 | "strconv" 7 | "fmt" 8 | "os" 9 | 10 | "nimo-shake/common" 11 | ) 12 | 13 | // read the given file and parse the fcv do comparison 14 | func CheckFcv(file string, fcv int) (int, error) { 15 | // read line by line and parse the version 16 | 17 | f, err := os.Open(file) 18 | if err != nil { 19 | return -1, err 20 | } 21 | 22 | scanner := bufio.NewScanner(f) 23 | versionName := "conf.version" 24 | version := 0 25 | for scanner.Scan() { 26 | field := strings.Split(scanner.Text(), "=") 27 | if len(field) >= 2 && strings.HasPrefix(field[0], versionName) { 28 | if value, err := strconv.Atoi(strings.Trim(field[1], " ")); err != nil { 29 | return 0, fmt.Errorf("illegal value[%v]", field[1]) 30 | } else { 31 | version = value 32 | break 33 | } 34 | } 35 | } 36 | 37 | if version < fcv { 38 | return version, fmt.Errorf("current required configuration version[%v] > input[%v], please upgrade NimoShake to version >= %v", 39 | fcv, version, utils.LowestConfigurationVersion[fcv]) 40 | } 41 | return version, nil 42 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | NimoShake: sync from dynamodb to mongodb, including full sync and increase sync.
2 | NimoFullCheck: check data consistency after syncing.
3 | These two tools are developed and maintained by NoSQL Team in Alibaba-Cloud Database department.
4 | 5 | * [中文文档介绍](https://yq.aliyun.com/articles/717439) 6 | * [binary download](https://github.com/alibaba/NimoShake/releases) 7 | 8 | #Usage 9 | --- 10 | NimoShake: 11 | `./nimo-shake -conf=nimo-shake.conf`
12 | NimoFullCheck: 13 | `./nimo-full-check --sourceAccessKeyID="xxx" --sourceSecretAccessKey="xxx" --sourceRegion="us-east-2" --filterTablesWhiteList="table list" -t="target mongodb address"` 14 | 15 | # Shake series tool 16 | --- 17 | We also provide some tools for synchronization in Shake series.
18 | 19 | * [MongoShake](https://github.com/aliyun/MongoShake): mongodb data synchronization tool. 20 | * [RedisShake](https://github.com/aliyun/RedisShake): redis data synchronization tool. 21 | * [RedisFullCheck](https://github.com/aliyun/RedisFullCheck): redis data synchronization verification tool. 22 | * [NimoShake](https://github.com/alibaba/NimoShake): sync dynamodb to mongodb. 23 | 24 | 25 | -------------------------------------------------------------------------------- /nimo-shake/writer/writer.go: -------------------------------------------------------------------------------- 1 | package writer 2 | 3 | import ( 4 | "nimo-shake/common" 5 | 6 | LOG "github.com/vinllen/log4go" 7 | "github.com/aws/aws-sdk-go/service/dynamodb" 8 | ) 9 | 10 | type Writer interface{ 11 | // create table 12 | CreateTable(tableDescribe *dynamodb.TableDescription) error 13 | // pass table description 14 | PassTableDesc(tableDescribe *dynamodb.TableDescription) 15 | // drop table 16 | DropTable() error 17 | // write bulk data, used in full sync 18 | WriteBulk(input []interface{}) error 19 | // insert 20 | Insert(input []interface{}, index []interface{}) error 21 | // delete 22 | Delete(input []interface{}) error 23 | // update 24 | Update(input []interface{}, index []interface{}) error 25 | // close 26 | Close() 27 | // get session 28 | GetSession() interface{} 29 | } 30 | 31 | func NewWriter(name, address string, ns utils.NS, logLevel string) Writer { 32 | switch name { 33 | case utils.TargetTypeMongo: 34 | // return NewMongoWriter(name, address, ns) 35 | return NewMongoCommunityWriter(name, address, ns) 36 | case utils.TargetTypeAliyunDynamoProxy: 37 | return NewDynamoProxyWriter(name, address, ns, logLevel) 38 | default: 39 | LOG.Crashf("unknown writer[%v]", name) 40 | } 41 | return nil 42 | } -------------------------------------------------------------------------------- /nimo-full-check/run/run.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import ( 4 | "nimo-full-check/checker" 5 | conf "nimo-full-check/configure" 6 | shakeUtils "nimo-shake/common" 7 | 8 | LOG "github.com/vinllen/log4go" 9 | ) 10 | 11 | func Start() { 12 | if err := shakeUtils.InitSession(conf.Opts.SourceAccessKeyID, conf.Opts.SourceSecretAccessKey, 13 | conf.Opts.SourceSessionToken, conf.Opts.SourceRegion, conf.Opts.SourceEndpointUrl, 14 | 3, 5000); err != nil { 15 | LOG.Crashf("init global session failed[%v]", err) 16 | } 17 | 18 | // create dynamo session 19 | dynamoSession, err := shakeUtils.CreateDynamoSession("info") 20 | if err != nil { 21 | LOG.Crashf("create dynamodb session failed[%v]", err) 22 | } 23 | 24 | // check mongodb connection 25 | mongoClient, err := shakeUtils.NewMongoCommunityConn(conf.Opts.TargetAddress, shakeUtils.ConnectModePrimary, true) 26 | if err != nil { 27 | LOG.Crashf("connect mongodb[%v] failed[%v]", conf.Opts.TargetAddress, err) 28 | } 29 | 30 | c := checker.NewChecker(dynamoSession, mongoClient) 31 | if c == nil { 32 | LOG.Crashf("create checker failed") 33 | } 34 | 35 | if err := c.Run(); err != nil { 36 | LOG.Crashf("checker runs failed[%v]", err) 37 | } 38 | 39 | LOG.Info("checker finishes!") 40 | } 41 | -------------------------------------------------------------------------------- /nimo-shake/protocol/mtype_converter.go: -------------------------------------------------------------------------------- 1 | package protocal 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/aws/aws-sdk-go/service/dynamodb" 8 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" 9 | 10 | conf "nimo-shake/configure" 11 | LOG "nimo-shake/third_party/log4go" 12 | ) 13 | 14 | type MTypeConverter struct { 15 | } 16 | 17 | func (tc *MTypeConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) { 18 | funcStartT := time.Now() 19 | 20 | outLen := 0 21 | for key, value := range input { 22 | outLen = outLen + len(key) + len(value.String()) 23 | } 24 | 25 | out := new(interface{}) 26 | if err := dynamodbattribute.UnmarshalMap(input, out); err == nil { 27 | 28 | for key, value := range (*out).(map[string]interface{}) { 29 | if key == "_id" { 30 | delete((*out).(map[string]interface{}), key) 31 | ((*out).(map[string]interface{}))[conf.ConvertIdFunc(key)] = value 32 | } 33 | } 34 | 35 | LOG.Debug("Run_func input[%v] out[%v] out_len[%v] duration[%v]", 36 | input, *out, outLen, time.Since(funcStartT)) 37 | 38 | return RawData{outLen, *out}, nil 39 | } else { 40 | LOG.Debug("Run_func input[%v] out[%v] err[%v]", input, *out, err) 41 | } 42 | 43 | return RawData{}, fmt.Errorf("parse failed, return nil") 44 | } 45 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Kyle Lemons . All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 5 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the 6 | documentation and/or other materials provided with the distribution. 7 | 8 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 9 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 10 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 11 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 12 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 13 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/socklog.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010, Kyle Lemons . All rights reserved. 2 | 3 | package log4go 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | "net" 9 | "os" 10 | ) 11 | 12 | // This log writer sends output to a socket 13 | type SocketLogWriter chan *LogRecord 14 | 15 | // This is the SocketLogWriter's output method 16 | func (w SocketLogWriter) LogWrite(rec *LogRecord) { 17 | w <- rec 18 | } 19 | 20 | func (w SocketLogWriter) Close() { 21 | close(w) 22 | } 23 | 24 | func NewSocketLogWriter(proto, hostport string) SocketLogWriter { 25 | sock, err := net.Dial(proto, hostport) 26 | if err != nil { 27 | fmt.Fprintf(os.Stderr, "NewSocketLogWriter(%q): %s\n", hostport, err) 28 | return nil 29 | } 30 | 31 | w := SocketLogWriter(make(chan *LogRecord, LogBufferLength)) 32 | 33 | go func() { 34 | defer func() { 35 | if sock != nil && proto == "tcp" { 36 | sock.Close() 37 | } 38 | }() 39 | 40 | for rec := range w { 41 | // Marshall into JSON 42 | js, err := json.Marshal(rec) 43 | if err != nil { 44 | _, _ = fmt.Fprintf(os.Stderr, "SocketLogWriter(%q): %s", hostport, err) 45 | return 46 | } 47 | 48 | _, err = sock.Write(js) 49 | if err != nil { 50 | _, _ = fmt.Fprintf(os.Stderr, "SocketLogWriter(%q): %s", hostport, err) 51 | return 52 | } 53 | } 54 | }() 55 | 56 | return w 57 | } 58 | -------------------------------------------------------------------------------- /nimo-shake/common/error.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/vinllen/mgo" 5 | ) 6 | 7 | // true means error can be ignored 8 | // https://github.com/mongodb/mongo/blob/master/src/mongo/base/error_codes.yml 9 | func MongodbIgnoreError(err error, op string, isFullSyncStage bool) bool { 10 | if err == nil { 11 | return true 12 | } 13 | 14 | errorCode := mgo.ErrorCodeList(err) 15 | if err != nil && len(errorCode) == 0 { 16 | return false 17 | } 18 | 19 | for _, err := range errorCode { 20 | switch op { 21 | case "i": 22 | case "u": 23 | if isFullSyncStage { 24 | if err == 28 { // PathNotViable 25 | continue 26 | } 27 | } 28 | case "d": 29 | if err == 26 { // NamespaceNotFound 30 | continue 31 | } 32 | case "c": 33 | if err == 26 { // NamespaceNotFound 34 | continue 35 | } 36 | default: 37 | return false 38 | } 39 | return false 40 | } 41 | 42 | return true 43 | } 44 | 45 | func DynamoIgnoreError(err error, op string, isFullSyncStage bool) bool { 46 | if err == nil { 47 | return true 48 | } 49 | 50 | switch op { 51 | case "i": 52 | case "u": 53 | if isFullSyncStage { 54 | if err.Error() == "xxxx" { // PathNotViable 55 | return true 56 | } 57 | } 58 | case "d": 59 | if err.Error() == "xxxx" { // NamespaceNotFound 60 | return true 61 | } 62 | case "c": 63 | if err.Error() == "xxxx" { // NamespaceNotFound 64 | return true 65 | } 66 | default: 67 | return false 68 | } 69 | return false 70 | } -------------------------------------------------------------------------------- /nimo-shake/filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | const ( 8 | Sep = ";" 9 | ) 10 | 11 | var ( 12 | f = &Filter{ 13 | collectionWhiteMap: make(map[string]bool), 14 | collectionBlackMap: make(map[string]bool), 15 | } 16 | ) 17 | 18 | type Filter struct { 19 | collectionWhiteMap map[string]bool 20 | collectionBlackMap map[string]bool 21 | } 22 | 23 | func Init(collectionWhite, collectBlack string) { 24 | var collectionWhiteList, collectionBlackList []string 25 | if collectionWhite != "" { 26 | collectionWhiteList = strings.Split(collectionWhite, Sep) 27 | } 28 | if collectBlack != "" { 29 | collectionBlackList = strings.Split(collectBlack, Sep) 30 | } 31 | 32 | for _, ele := range collectionWhiteList { 33 | f.collectionWhiteMap[ele] = true 34 | } 35 | for _, ele := range collectionBlackList { 36 | f.collectionBlackMap[ele] = true 37 | } 38 | } 39 | 40 | func IsFilter(collection string) bool { 41 | // fmt.Println(f.collectionWhiteMap, collection, f.collectionWhiteMap[collection]) 42 | if len(f.collectionWhiteMap) != 0 { 43 | return !f.collectionWhiteMap[collection] 44 | } else if len(f.collectionBlackMap) != 0 { 45 | return f.collectionBlackMap[collection] 46 | } 47 | return false 48 | } 49 | 50 | func FilterList(collectionList []string) []string { 51 | ret := make([]string, 0, len(collectionList)) 52 | for _, ele := range collectionList { 53 | if !IsFilter(ele) { 54 | ret = append(ret, ele) 55 | } 56 | } 57 | return ret 58 | } -------------------------------------------------------------------------------- /nimo-shake/checkpoint/writer.go: -------------------------------------------------------------------------------- 1 | package checkpoint 2 | 3 | import ( 4 | LOG "github.com/vinllen/log4go" 5 | ) 6 | 7 | type Writer interface { 8 | // find current status 9 | FindStatus() (string, error) 10 | 11 | // update status 12 | UpdateStatus(status string) error 13 | 14 | // extract all checkpoint 15 | ExtractCheckpoint() (map[string]map[string]*Checkpoint, error) 16 | 17 | // extract single checkpoint 18 | ExtractSingleCheckpoint(table string) (map[string]*Checkpoint, error) 19 | 20 | // insert checkpoint 21 | Insert(ckpt *Checkpoint, table string) error 22 | 23 | // update checkpoint 24 | Update(shardId string, ckpt *Checkpoint, table string) error 25 | 26 | // update with set 27 | UpdateWithSet(shardId string, input map[string]interface{}, table string) error 28 | 29 | // query 30 | Query(shardId string, table string) (*Checkpoint, error) 31 | 32 | // insert incrSyncCacheFile 33 | IncrCacheFileInsert(table string, shardId string, fileName string, lastSequenceNumber string, time string) error 34 | 35 | // drop 36 | DropAll() error 37 | } 38 | 39 | func NewWriter(name, address, db string) Writer { 40 | var w Writer 41 | switch name { 42 | case CheckpointWriterTypeMongo: 43 | w = NewMongoWriter(address, db) 44 | case CheckpointWriterTypeFile: 45 | w = NewFileWriter(db) 46 | default: 47 | LOG.Crashf("unknown checkpoint writer[%v]", name) 48 | } 49 | if w == nil { 50 | LOG.Crashf("create checkpoint writer[%v] failed", name) 51 | return nil 52 | } 53 | return w 54 | } 55 | -------------------------------------------------------------------------------- /nimo-shake/common/fcv.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | var ( 4 | FcvCheckpoint = Checkpoint{ 5 | CurrentVersion: 0, 6 | FeatureCompatibleVersion: 0, 7 | } 8 | FcvConfiguration = Configuration{ 9 | CurrentVersion: 4, 10 | FeatureCompatibleVersion: 1, 11 | } 12 | 13 | LowestCheckpointVersion = map[int]string{ 14 | 0: "1.0.0", 15 | } 16 | LowestConfigurationVersion = map[int]string{ 17 | 0: "1.0.0", 18 | 1: "1.0.6", // add full sync and incr sync http port 19 | 2: "1.0.8", // add full.read.concurrency 20 | 3: "1.0.9", // add full.document.write.batch 21 | 4: "1.0.11", // add source.endpoint_url 22 | } 23 | ) 24 | 25 | type Fcv interface { 26 | IsCompatible(int) bool 27 | } 28 | 29 | // for checkpoint 30 | type Checkpoint struct { 31 | /* 32 | * version: 0(or set not), MongoShake < 2.4, fcv == 0 33 | * version: 1, MongoShake == 2.4, 0 < fcv <= 1 34 | */ 35 | CurrentVersion int 36 | FeatureCompatibleVersion int 37 | } 38 | 39 | func (c Checkpoint) IsCompatible(v int) bool { 40 | return v >= c.FeatureCompatibleVersion && v <= c.CurrentVersion 41 | } 42 | 43 | // for configuration 44 | type Configuration struct { 45 | /* 46 | * version: 0(or set not), MongoShake < 2.4.0, fcv == 0 47 | * version: 1, MongoShake == 2.4.0, 0 <= fcv <= 1 48 | */ 49 | CurrentVersion int 50 | FeatureCompatibleVersion int 51 | } 52 | 53 | func (c Configuration) IsCompatible(v int) bool { 54 | return v >= c.FeatureCompatibleVersion && v <= c.CurrentVersion 55 | } 56 | -------------------------------------------------------------------------------- /nimo-shake/common/mix.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | "github.com/nightlyone/lockfile" 8 | LOG "github.com/vinllen/log4go" 9 | ) 10 | 11 | func WritePid(id string) (err error) { 12 | var lock lockfile.Lockfile 13 | lock, err = lockfile.New(id) 14 | if err != nil { 15 | return err 16 | } 17 | if err = lock.TryLock(); err != nil { 18 | return err 19 | } 20 | 21 | return nil 22 | } 23 | 24 | func WritePidById(id string, path string) error { 25 | var dir string 26 | var err error 27 | if path == "" { 28 | if dir, err = os.Getwd(); err != nil { 29 | return err 30 | } 31 | } else { 32 | dir = path 33 | if _, err := os.Stat(dir); os.IsNotExist(err) { 34 | os.Mkdir(dir, os.ModePerm) 35 | } 36 | } 37 | 38 | if dir, err = filepath.Abs(dir); err != nil { 39 | return err 40 | } 41 | 42 | pidfile := filepath.Join(dir, id) + ".pid" 43 | if err := WritePid(pidfile); err != nil { 44 | return err 45 | } 46 | return nil 47 | } 48 | 49 | func Welcome() { 50 | welcome := 51 | `______________________________ 52 | \ \ _ ______ | 53 | \ \ / \___-=O'/|O'/__| 54 | \ NimoShake, here we go !! \_______\ / | / ) 55 | / / '/-==__ _/__|/__=-| -GM 56 | / Alibaba Cloud / * \ | | 57 | / zhuzhao / (o) 58 | ------------------------------ 59 | ` 60 | startMsg := "if you have any problem, call aliyun" 61 | LOG.Warn("\n%s%s\n\n", welcome, startMsg) 62 | } -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/examples/FileLogWriter_Manual.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "time" 9 | ) 10 | 11 | import l4g "github.com/alibaba/MongoShake/v2/third_party/log4go" 12 | 13 | const ( 14 | filename = "flw.log" 15 | ) 16 | 17 | func main() { 18 | // Get a new logger instance 19 | log := l4g.NewLogger() 20 | 21 | // Create a default logger that is logging messages of FINE or higher 22 | log.AddFilter("file", l4g.FINE, l4g.NewFileLogWriter(filename, false)) 23 | log.Close() 24 | 25 | /* Can also specify manually via the following: (these are the defaults) */ 26 | flw := l4g.NewFileLogWriter(filename, false) 27 | flw.SetFormat("[%D %T] [%L] (%S) %M") 28 | flw.SetRotate(false) 29 | flw.SetRotateSize(0) 30 | flw.SetRotateLines(0) 31 | flw.SetRotateDaily(false) 32 | log.AddFilter("file", l4g.FINE, flw) 33 | 34 | // Log some experimental messages 35 | log.Finest("Everything is created now (notice that I will not be printing to the file)") 36 | log.Info("The time is now: %s", time.Now().Format("15:04:05 MST 2006/01/02")) 37 | log.Critical("Time to close out!") 38 | 39 | // Close the log 40 | log.Close() 41 | 42 | // Print what was logged to the file (yes, I know I'm skipping error checking) 43 | fd, _ := os.Open(filename) 44 | in := bufio.NewReader(fd) 45 | fmt.Print("Messages logged to file were: (line numbers not included)\n") 46 | for lineno := 1; ; lineno++ { 47 | line, err := in.ReadString('\n') 48 | if err == io.EOF { 49 | break 50 | } 51 | fmt.Printf("%3d:\t%s", lineno, line) 52 | } 53 | fd.Close() 54 | 55 | // Remove the file so it's not lying around 56 | os.Remove(filename) 57 | } 58 | -------------------------------------------------------------------------------- /nimo-shake/common/operator.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func AppendStringList(input []string, list []*string) ([]string) { 8 | for _, ele := range list { 9 | input = append(input, *ele) 10 | } 11 | return input 12 | } 13 | 14 | func StringListToMap(input []string) map[string]struct{} { 15 | mp := make(map[string]struct{}, len(input)) 16 | for _, ele := range input { 17 | mp[ele] = struct{}{} 18 | } 19 | return mp 20 | } 21 | 22 | // @vinllen. see BulkError.Error. -1 means not found 23 | func FindFirstErrorIndexAndMessage(error string) (int, string, bool) { 24 | subIndex := "index[" 25 | subMsg := "msg[" 26 | subDup := "dup[" 27 | index := strings.Index(error, subIndex) 28 | if index == -1 { 29 | return index, "", false 30 | } 31 | 32 | indexVal := 0 33 | for i := index + len(subIndex); i < len(error) && error[i] != ']'; i++ { 34 | // fmt.Printf("%c %d\n", rune(error[i]), int(error[i] - '0')) 35 | indexVal = indexVal * 10 + int(error[i] - '0') 36 | } 37 | 38 | index = strings.Index(error, subMsg) 39 | if index == -1 { 40 | return indexVal, "", false 41 | } 42 | 43 | i := index + len(subMsg) 44 | stack := 0 45 | for ; i < len(error); i++ { 46 | if error[i] == ']' { 47 | if stack == 0 { 48 | break 49 | } else { 50 | stack -= 1 51 | } 52 | } else if error[i] == '[' { 53 | stack += 1 54 | } 55 | } 56 | msg := error[index + len(subMsg): i] 57 | 58 | index = strings.Index(error, subDup) 59 | if index == -1 { 60 | return indexVal, msg, false 61 | } 62 | i = index + len(subMsg) 63 | for ; i < len(error) && error[i] != ']'; i++ {} 64 | dupVal := error[index + len(subMsg):i] 65 | 66 | return indexVal, msg, dupVal == "true" 67 | } -------------------------------------------------------------------------------- /scripts/run_ut_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # -*- coding:utf-8 -*- 4 | 5 | import os 6 | from os import listdir 7 | from os.path import isfile, join, isdir 8 | import subprocess 9 | 10 | go_path="." 11 | """ 12 | run unit test in recursive 13 | """ 14 | def run_ut(cur_path): 15 | print(os.path.abspath(os.path.curdir), cur_path) 16 | 17 | only_files = [f for f in listdir(".") if isfile(join(".", f))] 18 | only_dirs = [f for f in listdir(".") if isdir(join(".", f))] 19 | ut_files = [f for f in only_files if "_test.go" in f] 20 | print(only_files, only_dirs, ut_files) 21 | 22 | if len(ut_files) != 0: 23 | # with ut file, need run ut test 24 | print("----------- run ut test on dir[%s] -----------" % os.path.abspath(os.path.curdir)) 25 | ret = subprocess.call(["go", "test"]) 26 | # subprocess.check_output(["/bin/sh", "-c", "go", "test"]) 27 | 28 | print("********************************** %s *************************************" % ("OK" if ret == 0 else "FAIL")) 29 | if ret != 0: 30 | print("run failed") 31 | exit(ret) 32 | 33 | for dir in only_dirs: 34 | print("cd dir[%s]" % dir) 35 | 36 | # dfs 37 | os.chdir(dir) 38 | run_ut(dir) 39 | 40 | # backtracking 41 | os.chdir("..") 42 | 43 | if __name__ == "__main__": 44 | root_path = os.path.join("..", "src/nimo-shake") 45 | os.chdir(root_path) 46 | go_path=os.path.abspath("../..") 47 | print("GOPATH=%s" % go_path) 48 | #subprocess.call(["export GOPATH=%s" % go_path]) 49 | os.environ['GOPATH'] = go_path 50 | run_ut(".") 51 | 52 | print("-----------------------------------") 53 | print("all is well ^_^") 54 | -------------------------------------------------------------------------------- /nimo-shake/filter/filter_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | "fmt" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFilter(t *testing.T) { 11 | // test InsertCkpt, UpdateCkpt, QueryCkpt, DropCheckpoint 12 | 13 | var nr int 14 | { 15 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr) 16 | nr++ 17 | 18 | f = &Filter{ 19 | collectionWhiteMap: make(map[string]bool), 20 | collectionBlackMap: make(map[string]bool), 21 | } 22 | Init("abc;efg;a", "") 23 | 24 | assert.Equal(t, true, IsFilter("anc"), "should be equal") 25 | assert.Equal(t, true, IsFilter("ab"), "should be equal") 26 | assert.Equal(t, false, IsFilter("abc"), "should be equal") 27 | assert.Equal(t, false, IsFilter("efg"), "should be equal") 28 | } 29 | 30 | { 31 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr) 32 | nr++ 33 | 34 | f = &Filter{ 35 | collectionWhiteMap: make(map[string]bool), 36 | collectionBlackMap: make(map[string]bool), 37 | } 38 | Init("","abc;efg;a") 39 | 40 | assert.Equal(t, false, IsFilter("anc"), "should be equal") 41 | assert.Equal(t, false, IsFilter("ab"), "should be equal") 42 | assert.Equal(t, true, IsFilter("abc"), "should be equal") 43 | assert.Equal(t, true, IsFilter("efg"), "should be equal") 44 | } 45 | 46 | { 47 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr) 48 | nr++ 49 | 50 | f = &Filter{ 51 | collectionWhiteMap: make(map[string]bool), 52 | collectionBlackMap: make(map[string]bool), 53 | } 54 | Init("","") 55 | 56 | assert.Equal(t, false, IsFilter("anc"), "should be equal") 57 | assert.Equal(t, false, IsFilter("ab"), "should be equal") 58 | assert.Equal(t, false, IsFilter("abc"), "should be equal") 59 | assert.Equal(t, false, IsFilter("efg"), "should be equal") 60 | } 61 | } -------------------------------------------------------------------------------- /nimo-shake/common/mongodb_community.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | LOG "github.com/vinllen/log4go" 8 | "go.mongodb.org/mongo-driver/mongo" 9 | "go.mongodb.org/mongo-driver/mongo/options" 10 | "go.mongodb.org/mongo-driver/mongo/readconcern" 11 | "go.mongodb.org/mongo-driver/mongo/readpref" 12 | ) 13 | 14 | type MongoCommunityConn struct { 15 | Client *mongo.Client 16 | URL string 17 | ctx context.Context 18 | } 19 | 20 | func NewMongoCommunityConn(url string, connectMode string, timeout bool) (*MongoCommunityConn, error) { 21 | clientOps := options.Client().ApplyURI(url) 22 | 23 | clientOps.SetReadConcern(readconcern.New(readconcern.Level("local"))) 24 | 25 | // read pref 26 | if mode, err := readpref.ModeFromString(connectMode); err != nil { 27 | return nil, fmt.Errorf("create connectMode[%v] failed: %v", connectMode, err) 28 | } else if opts, err := readpref.New(mode); err != nil { 29 | return nil, fmt.Errorf("new mode with connectMode[%v] failed: %v", connectMode, err) 30 | } else { 31 | clientOps.SetReadPreference(opts) 32 | } 33 | 34 | // set timeout 35 | if !timeout { 36 | clientOps.SetConnectTimeout(0) 37 | } 38 | 39 | // create default context 40 | ctx := context.Background() 41 | 42 | // connect 43 | client, err := mongo.NewClient(clientOps) 44 | if err != nil { 45 | return nil, fmt.Errorf("new client failed: %v", err) 46 | } 47 | if err := client.Connect(ctx); err != nil { 48 | return nil, fmt.Errorf("connect to %s failed: %v", BlockMongoUrlPassword(url, "***"), err) 49 | } 50 | 51 | // ping 52 | if err = client.Ping(ctx, clientOps.ReadPreference); err != nil { 53 | return nil, fmt.Errorf("ping to %v failed: %v", BlockMongoUrlPassword(url, "***"), err) 54 | } 55 | 56 | LOG.Info("New session to %s successfully", BlockMongoUrlPassword(url, "***")) 57 | return &MongoCommunityConn{ 58 | Client: client, 59 | URL: url, 60 | }, nil 61 | } 62 | 63 | func (conn *MongoCommunityConn) Close() { 64 | LOG.Info("Close client with %s", BlockMongoUrlPassword(conn.URL, "***")) 65 | conn.Client.Disconnect(conn.ctx) 66 | } 67 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/termlog.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010, Kyle Lemons . All rights reserved. 2 | 3 | package log4go 4 | 5 | import ( 6 | "fmt" 7 | "io" 8 | "os" 9 | "time" 10 | ) 11 | 12 | var stdout io.Writer = os.Stdout 13 | 14 | // This is the standard writer that prints to standard output. 15 | type ConsoleLogWriter struct { 16 | format string 17 | w chan *LogRecord 18 | done chan struct{} 19 | } 20 | 21 | // This creates a new ConsoleLogWriter 22 | func NewConsoleLogWriter() *ConsoleLogWriter { 23 | consoleWriter := &ConsoleLogWriter{ 24 | format: "[%T %D] [%L] (%S) %M", 25 | w: make(chan *LogRecord, LogBufferLength), 26 | done: make(chan struct{}), 27 | } 28 | go consoleWriter.run(stdout) 29 | return consoleWriter 30 | } 31 | func (c *ConsoleLogWriter) SetFormat(format string) { 32 | c.format = format 33 | } 34 | func (c *ConsoleLogWriter) run(out io.Writer) { 35 | //for rec := range c.w { 36 | // fmt.Fprint(out, FormatLogRecord(c.format, rec)) 37 | //} 38 | for { 39 | select { 40 | case rec, ok := <-c.w: 41 | if !ok { 42 | return 43 | } 44 | fmt.Fprint(out, FormatLogRecord(c.format, rec)) 45 | case <-c.done: 46 | // drain remaining messages 47 | for { 48 | select { 49 | case rec, ok := <-c.w: 50 | if !ok { 51 | return 52 | } 53 | fmt.Fprint(out, FormatLogRecord(c.format, rec)) 54 | default: 55 | return 56 | } 57 | } 58 | } 59 | } 60 | } 61 | 62 | // This is the ConsoleLogWriter's output method. This will block if the output 63 | // buffer is full. 64 | func (c *ConsoleLogWriter) LogWrite(rec *LogRecord) { 65 | select { 66 | case c.w <- rec: 67 | // send succeed 68 | case <-c.done: 69 | // channel closed 70 | return 71 | } 72 | } 73 | 74 | // Close stops the logger from sending messages to standard output. Attempts to 75 | // send log messages to this logger after a Close have undefined behavior. 76 | func (c *ConsoleLogWriter) Close() { 77 | close(c.done) 78 | // close(c.w) 79 | time.Sleep(50 * time.Millisecond) // Try to give console I/O time to complete 80 | } 81 | -------------------------------------------------------------------------------- /nimo-full-check/configure/conf.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | var Opts struct { 4 | Id string `short:"i" long:"id" default:"nimo-shake" description:"target database collection name"` 5 | LogLevel string `short:"l" long:"logLevel" default:"info"` 6 | SourceAccessKeyID string `short:"s" long:"sourceAccessKeyID" description:"dynamodb source access key id"` 7 | SourceSecretAccessKey string `long:"sourceSecretAccessKey" description:"dynamodb source secret access key"` 8 | SourceSessionToken string `long:"sourceSessionToken" default:"" description:"dynamodb source session token"` 9 | SourceRegion string `long:"sourceRegion" default:"" description:"dynamodb source region"` 10 | SourceEndpointUrl string `long:"sourceEndpointUrl" default:"" description:"dynamodb source endpoint_url"` 11 | QpsFull int `long:"qpsFull" default:"10000" description:"qps of scan command, default is 10000"` 12 | QpsFullBatchNum int64 `long:"qpsFullBatchNum" default:"128" description:"batch number in each scan command, default is 128"` 13 | TargetAddress string `short:"t" long:"targetAddress" description:"mongodb target address"` 14 | DiffOutputFile string `short:"d" long:"diffOutputFile" default:"nimo-full-check-diff" description:"diff output file name"` 15 | Parallel int `short:"p" long:"parallel" default:"16" description:"how many threads used to compare, default is 16"` 16 | Sample int64 `short:"e" long:"sample" default:"1000" description:"comparison sample number for each table, 0 means disable"` 17 | //IndexPrimary bool `short:"m" long:"indexPrimary" description:"enable compare primary index"` 18 | //IndexUser bool `long:"indexUser" description:"enable compare user index"` 19 | FilterCollectionWhite string `long:"filterCollectionWhite" default:"" description:"only compare the given tables, split by ';'"` 20 | FilterCollectionBlack string `long:"filterCollectionBlack" default:"" description:"do not compare the given tables, split by ';'"` 21 | ConvertType string `short:"c" long:"convertType" default:"change" description:"convert type"` 22 | Version bool `short:"v" long:"version" description:"print version"` 23 | } 24 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/examples/example.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | stdout 4 | console 5 | 6 | DEBUG 7 | 8 | 9 | file 10 | file 11 | FINEST 12 | test.log 13 | 20 | [%D %T] [%L] (%S) %M false 21 | 0M 22 | 0K 23 | true 24 | 25 | 26 | xmllog 27 | xml 28 | TRACE 29 | trace.xml 30 | true 31 | 100M 32 | 6K 33 | false 34 | 35 | 36 | donotopen 37 | socket 38 | FINEST 39 | 192.168.1.255:12124 40 | udp 41 | 42 | 43 | -------------------------------------------------------------------------------- /nimo-shake/common/unsafe.go: -------------------------------------------------------------------------------- 1 | /* 2 | // ===================================================================================== 3 | // 4 | // Filename: BytesString.go 5 | // 6 | // Description: ref from fast http 7 | // 8 | // Version: 1.0 9 | // Created: 06/23/2018 02:34:41 PM 10 | // Revision: none 11 | // Compiler: go1.10.3 12 | // 13 | // Author: boyi.gw, boyi.gw@alibaba-inc.com 14 | // Company: Alibaba Group 15 | // 16 | // ===================================================================================== 17 | */ 18 | 19 | package utils 20 | 21 | import ( 22 | "reflect" 23 | "unsafe" 24 | ) 25 | 26 | /* 27 | // === FUNCTION ====================================================================== 28 | // Name: String2Bytes 29 | // Description: return GoString's buffer slice(enable modify string) 30 | // ===================================================================================== 31 | */ 32 | func String2Bytes(s string) []byte { 33 | sh := (*reflect.StringHeader)(unsafe.Pointer(&s)) 34 | bh := reflect.SliceHeader{ 35 | Data: sh.Data, 36 | Len: sh.Len, 37 | Cap: sh.Len, 38 | } 39 | return *(*[]byte)(unsafe.Pointer(&bh)) 40 | } 41 | 42 | /* 43 | // === FUNCTION ====================================================================== 44 | // Name: Bytes2String 45 | // Description: convert b to string without copy 46 | // ===================================================================================== 47 | */ 48 | func Bytes2String(b []byte) string { 49 | return *(*string)(unsafe.Pointer(&b)) 50 | } 51 | 52 | /* 53 | // === FUNCTION ====================================================================== 54 | // Name: StringPointer 55 | // Description: returns &s[0] 56 | // ===================================================================================== 57 | */ 58 | func StringPointer(s string) unsafe.Pointer { 59 | p := (*reflect.StringHeader)(unsafe.Pointer(&s)) 60 | return unsafe.Pointer(p.Data) 61 | } 62 | 63 | /* 64 | // === FUNCTION ====================================================================== 65 | // Name: BytesPointer 66 | // Description: returns &b[0] 67 | // ===================================================================================== 68 | */ 69 | func BytesPointer(b []byte) unsafe.Pointer { 70 | p := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 71 | return unsafe.Pointer(p.Data) 72 | } 73 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2025-07-04 2 | * VERSION: 1.0.14 3 | * IMPROVE: adapt shard split scene 4 | * REFINE: upgrade to go1.15.10 with mod to compile 5 | * REFINE: close writer when all writes are done 6 | * REFINE: delete useless bin files in project 7 | * IMPROVE: update shard status to finish when all incr data have written to dest DB 8 | * IMPROVE: wait for father shard in executor because of we are caching incremental data 9 | * REFINE: modify Node channel size, no need for large chan anymore when don't use memory cache 10 | * IMPROVE: incr cache file persist to db 11 | * IMPROVE: add parameter 'increase.persist.dir' 12 | * REFINE: deprecate mgo, replace it with mongo-driver 13 | * IMPROVE: add api qos 14 | * BUGFIX: fix index ele type parse 15 | * IMPROVE: support MongoDB 6.0+ 16 | * IMPROVE: support DynamoDB filter_expression 17 | 18 | 2021-12-07 19 | * VERSION: 1.0.13 20 | * IMPROVE: deprecated mongo.go.driver 21 | * IMPROVE: add convert._id, add a prefix to _id key name from DynamoDB 22 | * IMPROVE: add type mchange in convert.type, use function from aws-sdk-go to convert data, better performance 23 | * IMPROVE: adapt to LastEvaluatedStreamArn and LastEvaluatedShardId not nil 24 | * IMPROVE: add incr_sync_parallel to do parallel full&incr 25 | 26 | 2021-05-18 27 | * VERSION: 1.0.12 28 | * BUGFIX: parse int value empty. 29 | 30 | 2021-04-07 31 | * VERSION: 1.0.11 32 | * IMPROVE: data source support endpoint_url. 33 | 34 | 2021-03-02 35 | * VERSION: 1.0.10 36 | * IMPROVE: replace mgo driver with mongo.go.driver 37 | 38 | 2021-03-01 39 | * VERSION: 1.0.9 40 | * IMPROVE: add full.document.write.batch. 41 | * IMPROVE: enable MongoDB unordered bulk write. 42 | * IMPROVE: create range index instead of hash index when target mongodb 43 | type is replcaSet. 44 | * BUGFIX: create union index for gsi and lsi. 45 | 46 | 2021-02-25 47 | * VERSION: 1.0.8 48 | * IMPROVE: add pprof. 49 | * IMPROVE: support parallel scan. 50 | 51 | 2021-01-28 52 | * VERSION: 1.0.7 53 | * BUGFIX: some corner cases. 54 | 55 | 2021-01-25 56 | * VERSION: 1.0.6 57 | * IMPROVE: add metric for both full sync and incr sync. 58 | 59 | 2020-12-22 60 | * VERSION: 1.0.5 61 | * BUGFIX: shard iterator expired based on full sync timeout. 62 | 63 | 2020-12-18 64 | * VERSION: 1.0.4 65 | * BUGFIX: duplicate value fetched in document-syncer. 66 | * BUGFIX: primary key is not passed to writer. 67 | * IMPROVE: store N as decimal into mongodb. 68 | 69 | 2020-10-20 70 | * VERSION: 1.0.3 71 | * IMPROVE: support migrating dynamodb meta info. 72 | * IMPROVE: support migrating gsi and lsi. 73 | 74 | 2020-07-15 75 | * VERSION: 1.0.2 76 | * IMPROVE: support aliyun-dynamo-proxy 77 | * IMPROVE: add checkpoint address with type file 78 | 79 | 2019-10-15 Alibaba Cloud. 80 | * VERSION: 1.0.0 81 | * FEATURE: first release 82 | 83 | -------------------------------------------------------------------------------- /nimo-full-check/checker/checker.go: -------------------------------------------------------------------------------- 1 | package checker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | conf "nimo-full-check/configure" 9 | shakeUtils "nimo-shake/common" 10 | shakeFilter "nimo-shake/filter" 11 | 12 | "github.com/aws/aws-sdk-go/service/dynamodb" 13 | LOG "github.com/vinllen/log4go" 14 | "go.mongodb.org/mongo-driver/bson" 15 | ) 16 | 17 | type Checker struct { 18 | dynamoSession *dynamodb.DynamoDB 19 | mongoClient *shakeUtils.MongoCommunityConn 20 | } 21 | 22 | func NewChecker(dynamoSession *dynamodb.DynamoDB, mongoClient *shakeUtils.MongoCommunityConn) *Checker { 23 | return &Checker{ 24 | dynamoSession: dynamoSession, 25 | mongoClient: mongoClient, 26 | } 27 | } 28 | 29 | func (c *Checker) Run() error { 30 | // fetch all tables 31 | LOG.Info("start fetching table list") 32 | rawTableList, err := shakeUtils.FetchTableList(c.dynamoSession) 33 | if err != nil { 34 | return fmt.Errorf("fetch table list failed[%v]", err) 35 | } 36 | LOG.Info("finish fetching table list: %v", rawTableList) 37 | 38 | tableList := shakeFilter.FilterList(rawTableList) 39 | LOG.Info("filter table list: %v", tableList) 40 | 41 | // check table exist 42 | if err := c.checkTableExist(tableList); err != nil { 43 | return fmt.Errorf("check table exist failed[%v]", err) 44 | } 45 | 46 | // reset parallel if needed 47 | parallel := conf.Opts.Parallel 48 | if parallel > len(tableList) { 49 | parallel = len(tableList) 50 | } 51 | 52 | execChan := make(chan string, len(tableList)) 53 | for _, table := range tableList { 54 | execChan <- table 55 | } 56 | 57 | var wg sync.WaitGroup 58 | wg.Add(len(tableList)) 59 | for i := 0; i < parallel; i++ { 60 | go func(id int) { 61 | for { 62 | table, ok := <-execChan 63 | if !ok { 64 | break 65 | } 66 | 67 | LOG.Info("documentChecker[%v] starts checking table[%v]", id, table) 68 | dc := NewDocumentChecker(id, table, c.dynamoSession) 69 | dc.Run() 70 | 71 | LOG.Info("documentChecker[%v] finishes checking table[%v]", id, table) 72 | wg.Done() 73 | } 74 | }(i) 75 | } 76 | wg.Wait() 77 | 78 | LOG.Info("all documentCheckers finish") 79 | return nil 80 | } 81 | 82 | func (c *Checker) checkTableExist(tableList []string) error { 83 | collections, err := c.mongoClient.Client.Database(conf.Opts.Id).ListCollectionNames(context.TODO(), bson.M{}) 84 | if err != nil { 85 | return fmt.Errorf("get target collection names error[%v]", err) 86 | } 87 | 88 | LOG.Info("all table: %v", collections) 89 | 90 | collectionsMp := shakeUtils.StringListToMap(collections) 91 | notExist := make([]string, 0) 92 | for _, table := range tableList { 93 | if _, ok := collectionsMp[table]; !ok { 94 | notExist = append(notExist, table) 95 | } 96 | } 97 | 98 | if len(notExist) != 0 { 99 | return fmt.Errorf("table not exist on the target side: %v", notExist) 100 | } 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /nimo-shake/full-sync/document-syncer_test.go: -------------------------------------------------------------------------------- 1 | package full_sync 2 | 3 | import ( 4 | "testing" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "time" 8 | "nimo-shake/configure" 9 | ) 10 | 11 | func TestDocumentSyncer(t *testing.T) { 12 | // test documentSyncer main function 13 | 14 | conf.Options.FullDocumentWriteBatch = 25 15 | batchNumber := int(conf.Options.FullDocumentWriteBatch) 16 | 17 | var nr int 18 | { 19 | fmt.Printf("TestDocumentSyncer case %d.\n", nr) 20 | nr++ 21 | 22 | UT_TestDocumentSyncer = true 23 | UT_TestDocumentSyncer_Chan = make(chan []interface{}, 1000) 24 | 25 | docSyncer := &documentSyncer{ 26 | inputChan: make(chan interface{}, 10), 27 | } 28 | go docSyncer.Run() 29 | 30 | for i := 0; i < batchNumber - 5; i++ { 31 | docSyncer.inputChan<-i 32 | } 33 | 34 | out := <-UT_TestDocumentSyncer_Chan 35 | assert.Equal(t, batchNumber - 5, len(out), "should be equal") 36 | for i := 0; i < len(out); i++ { 37 | assert.Equal(t, i, out[i].(int), "should be equal") 38 | } 39 | } 40 | 41 | // output length > batchNumber 42 | { 43 | fmt.Printf("TestDocumentSyncer case %d.\n", nr) 44 | nr++ 45 | 46 | UT_TestDocumentSyncer = true 47 | UT_TestDocumentSyncer_Chan = make(chan []interface{}, 1000) 48 | 49 | docSyncer := &documentSyncer{ 50 | inputChan: make(chan interface{}, 10), 51 | } 52 | go docSyncer.Run() 53 | 54 | for i := 0; i < batchNumber + 5; i++ { 55 | docSyncer.inputChan<-i 56 | } 57 | 58 | out := make([]interface{}, 0) 59 | out1 := <-UT_TestDocumentSyncer_Chan 60 | assert.Equal(t, batchNumber, len(out1), "should be equal") 61 | out = append(out, out1...) 62 | 63 | out2 := <-UT_TestDocumentSyncer_Chan 64 | assert.Equal(t, 5, len(out2), "should be equal") 65 | out = append(out, out2...) 66 | 67 | for i := 0; i < len(out); i++ { 68 | assert.Equal(t, i, out[i].(int), "should be equal") 69 | } 70 | } 71 | 72 | // output timeout 73 | { 74 | fmt.Printf("TestDocumentSyncer case %d.\n", nr) 75 | nr++ 76 | 77 | UT_TestDocumentSyncer = true 78 | UT_TestDocumentSyncer_Chan = make(chan []interface{}, 1000) 79 | 80 | docSyncer := &documentSyncer{ 81 | inputChan: make(chan interface{}, 10), 82 | } 83 | go docSyncer.Run() 84 | 85 | for i := 0; i < batchNumber - 10; i++ { 86 | docSyncer.inputChan<-i 87 | } 88 | time.Sleep((batchTimeout + 2) * time.Second) 89 | for i := batchNumber - 10; i < batchNumber + 5; i++ { 90 | docSyncer.inputChan<-i 91 | } 92 | 93 | out := make([]interface{}, 0) 94 | out1 := <-UT_TestDocumentSyncer_Chan 95 | assert.Equal(t, batchNumber - 10, len(out1), "should be equal") 96 | out = append(out, out1...) 97 | 98 | out2 := <-UT_TestDocumentSyncer_Chan 99 | assert.Equal(t, 15, len(out2), "should be equal") 100 | out = append(out, out2...) 101 | fmt.Println(out) 102 | 103 | for i := 0; i < len(out); i++ { 104 | assert.Equal(t, i, out[i].(int), "should be equal") 105 | } 106 | } 107 | } -------------------------------------------------------------------------------- /nimo-shake/common/common.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | LOG "github.com/vinllen/log4go" 9 | ) 10 | 11 | const ( 12 | GolangSecurityTime = "2006-01-02T15:04:05Z" 13 | 14 | ConvertTypeRaw = "raw" 15 | ConvertTypeChange = "change" 16 | ConvertMTypeChange = "mchange" 17 | ConvertTypeSame = "same" // used in dynamodb -> dynamo-proxy 18 | 19 | SyncModeAll = "all" 20 | SyncModeFull = "full" 21 | SyncModeIncr = "incr" 22 | 23 | TargetTypeMongo = "mongodb" 24 | TargetTypeAliyunDynamoProxy = "aliyun_dynamo_proxy" 25 | 26 | TargetMongoDBTypeReplica = "replica" 27 | TargetMongoDBTypeSharding = "sharding" 28 | 29 | TargetDBExistRename = "rename" 30 | TargetDBExistDrop = "drop" 31 | 32 | SIGNALPROFILE = 31 33 | SIGNALSTACK = 30 34 | ) 35 | 36 | var ( 37 | Version = "$" 38 | StartTime string 39 | ) 40 | 41 | func InitialLogger(logFile string, level string, logBuffer bool) bool { 42 | logLevel := parseLogLevel(level) 43 | if len(logFile) != 0 { 44 | // create logs folder for log4go. because of its mistake that doesn't create ! 45 | if err := os.MkdirAll("logs", os.ModeDir|os.ModePerm); err != nil { 46 | return false 47 | } 48 | if logBuffer { 49 | LOG.LogBufferLength = 32 50 | } else { 51 | LOG.LogBufferLength = 0 52 | } 53 | fileLogger := LOG.NewFileLogWriter(fmt.Sprintf("logs/%s", logFile), true) 54 | //fileLogger.SetRotateDaily(true) 55 | fileLogger.SetRotateSize(500 * 1024 * 1024) 56 | // fileLogger.SetFormat("[%D %T] [%L] [%s] %M") 57 | fileLogger.SetFormat("[%D %T] [%L] %M") 58 | fileLogger.SetRotateMaxBackup(100) 59 | LOG.AddFilter("file", logLevel, fileLogger) 60 | } else { 61 | LOG.AddFilter("console", logLevel, LOG.NewConsoleLogWriter()) 62 | } 63 | return true 64 | } 65 | 66 | func parseLogLevel(level string) LOG.Level { 67 | switch strings.ToLower(level) { 68 | case "debug": 69 | return LOG.DEBUG 70 | case "info": 71 | return LOG.INFO 72 | case "warning": 73 | return LOG.WARNING 74 | case "error": 75 | return LOG.ERROR 76 | default: 77 | return LOG.DEBUG 78 | } 79 | } 80 | 81 | /** 82 | * block password in mongo_urls: 83 | * two kind mongo_urls: 84 | * 1. mongodb://username:password@address 85 | * 2. username:password@address 86 | */ 87 | func BlockMongoUrlPassword(url, replace string) string { 88 | colon := strings.Index(url, ":") 89 | if colon == -1 || colon == len(url)-1 { 90 | return url 91 | } else if url[colon+1] == '/' { 92 | // find the second '/' 93 | for colon++; colon < len(url); colon++ { 94 | if url[colon] == ':' { 95 | break 96 | } 97 | } 98 | 99 | if colon == len(url) { 100 | return url 101 | } 102 | } 103 | 104 | at := strings.Index(url, "@") 105 | if at == -1 || at == len(url)-1 || at <= colon { 106 | return url 107 | } 108 | 109 | newUrl := make([]byte, 0, len(url)) 110 | for i := 0; i < len(url); i++ { 111 | if i <= colon || i > at { 112 | newUrl = append(newUrl, byte(url[i])) 113 | } else if i == at { 114 | newUrl = append(newUrl, []byte(replace)...) 115 | newUrl = append(newUrl, byte(url[i])) 116 | } 117 | } 118 | return string(newUrl) 119 | } 120 | -------------------------------------------------------------------------------- /nimo-shake/common/shard.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/aws/aws-sdk-go/service/dynamodbstreams" 9 | ) 10 | 11 | var ( 12 | StopTraverseSonErr = fmt.Errorf("stop traverse") 13 | ) 14 | 15 | type ShardNode struct { 16 | Shard *dynamodbstreams.Shard 17 | ShardArn string 18 | Sons map[string]*ShardNode 19 | Table string 20 | } 21 | 22 | // build shard tree base on the input shard list, return root node 23 | func BuildShardTree(shards []*dynamodbstreams.Shard, table string, ShardArn string) *ShardNode { 24 | pathMap := make(map[string]*ShardNode, len(shards)) // store the inheritance relationship 25 | inDegree := make(map[string]bool, len(shards)) 26 | // initial 27 | for _, shard := range shards { 28 | pathMap[*shard.ShardId] = &ShardNode{ 29 | Shard: shard, 30 | ShardArn: ShardArn, 31 | Sons: make(map[string]*ShardNode), 32 | Table: table, 33 | } 34 | 35 | inDegree[*shard.ShardId] = false 36 | } 37 | 38 | // build path 39 | for _, shard := range shards { 40 | node := pathMap[*shard.ShardId] 41 | father := shard.ParentShardId 42 | if father != nil { 43 | if _, ok := pathMap[*father]; !ok { 44 | // father node isn't exist on the pathMap, which means deleted 45 | continue 46 | } 47 | inDegree[*shard.ShardId] = true 48 | pathMap[*father].Sons[*shard.ShardId] = node 49 | } 50 | } 51 | 52 | // root is fake node 53 | rootNode := &ShardNode{ 54 | Shard: nil, 55 | ShardArn: "", 56 | Sons: make(map[string]*ShardNode), 57 | Table: table, 58 | } 59 | for key, val := range inDegree { 60 | if val == false { 61 | rootNode.Sons[key] = pathMap[key] 62 | // fmt.Printf("root->%s\n", key) 63 | } 64 | } 65 | return rootNode 66 | } 67 | 68 | // dfs 69 | func TraverseShard(node *ShardNode, callback func(node *ShardNode) error) error { 70 | if node == nil { 71 | return nil 72 | } 73 | 74 | if node.Shard != nil { 75 | // skip root node 76 | if err := callback(node); err != nil { 77 | if err != StopTraverseSonErr { 78 | return err 79 | } else { 80 | // return if error == StopTraverseSonErr 81 | return nil 82 | } 83 | } 84 | // fmt.Printf("%s->%s\n", *node.Shard.ParentShardId, *node.Shard.ShardId) 85 | } 86 | 87 | for _, son := range node.Sons { 88 | if err := TraverseShard(son, callback); err != nil { 89 | return err 90 | } 91 | } 92 | 93 | return nil 94 | } 95 | 96 | // calculate md5. TODO: add UT 97 | func CalMd5(root *ShardNode) uint64 { 98 | if root == nil { 99 | return 0 100 | } 101 | 102 | list := make([]string, 0, len(root.Sons)) 103 | for _, son := range root.Sons { 104 | ret := CalMd5(son) 105 | list = append(list, fmt.Sprintf("%s->%d", *son.Shard.ShardId, ret)) 106 | } 107 | 108 | sort.Strings(list) 109 | concat := strings.Join(list, ";") 110 | // fmt.Println("concat: ", concat) 111 | return Md5In64(String2Bytes(concat)) 112 | } 113 | 114 | func PrintShardTree(node *ShardNode) (string, error) { 115 | newSet := make([]string, 0) 116 | err := TraverseShard(node, func(node *ShardNode) error { 117 | var father string 118 | if node.Shard.ParentShardId != nil { 119 | father = *node.Shard.ParentShardId 120 | } else { 121 | father = "nil" 122 | } 123 | inheritance := strings.Join([]string{father, *node.Shard.ShardId}, "->") 124 | newSet = append(newSet, inheritance) 125 | 126 | return nil 127 | }) 128 | if err != nil { 129 | return "", err 130 | } 131 | 132 | return strings.Join(newSet, "\n"), nil 133 | } -------------------------------------------------------------------------------- /nimo-full-check/main/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | utils "nimo-full-check/common" 9 | conf "nimo-full-check/configure" 10 | "nimo-full-check/run" 11 | shakeUtils "nimo-shake/common" 12 | shakeFilter "nimo-shake/filter" 13 | 14 | "github.com/jessevdk/go-flags" 15 | LOG "github.com/vinllen/log4go" 16 | ) 17 | 18 | type Exit struct{ Code int } 19 | 20 | func main() { 21 | defer LOG.Close() 22 | 23 | // parse conf.Opts 24 | args, err := flags.Parse(&conf.Opts) 25 | 26 | if conf.Opts.Version { 27 | fmt.Println(utils.Version) 28 | os.Exit(0) 29 | } 30 | 31 | // 若err != nil, 会自动打印错误到 stderr 32 | if err != nil { 33 | if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { 34 | os.Exit(0) 35 | } else { 36 | fmt.Fprintf(os.Stderr, "flag err %s\n", flagsErr) 37 | os.Exit(1) 38 | } 39 | } 40 | 41 | if len(args) != 0 { 42 | fmt.Fprintf(os.Stderr, "unexpected args %+v", args) 43 | os.Exit(1) 44 | } 45 | 46 | shakeUtils.InitialLogger("", conf.Opts.LogLevel, false) 47 | 48 | if err := sanitizeOptions(); err != nil { 49 | crash(fmt.Sprintf("Conf.Options check failed: %s", err.Error()), -4) 50 | } 51 | 52 | LOG.Info("full-check starts") 53 | run.Start() 54 | LOG.Info("full-check completes!") 55 | } 56 | 57 | func sanitizeOptions() error { 58 | if len(conf.Opts.SourceAccessKeyID) == 0 || len(conf.Opts.SourceSecretAccessKey) == 0 { 59 | return fmt.Errorf("sourceAccessKeyID and sourceSecretAccessKey can't be empty") 60 | } 61 | 62 | if len(conf.Opts.TargetAddress) == 0 { 63 | return fmt.Errorf("targetAddress can't be empty") 64 | } 65 | 66 | if conf.Opts.Parallel <= 0 { 67 | return fmt.Errorf("parallel should >= 1, default is 16") 68 | } 69 | 70 | if conf.Opts.QpsFull == 0 { 71 | conf.Opts.QpsFull = 10000 72 | } else if conf.Opts.QpsFull < 0 { 73 | return fmt.Errorf("qps.full should > 0, default is 10000") 74 | } 75 | 76 | if conf.Opts.QpsFullBatchNum <= 0 { 77 | conf.Opts.QpsFullBatchNum = 128 78 | } 79 | 80 | shakeFilter.Init(conf.Opts.FilterCollectionWhite, conf.Opts.FilterCollectionBlack) 81 | 82 | if len(conf.Opts.DiffOutputFile) == 0 { 83 | return fmt.Errorf("diff output file shouldn't be empty") 84 | } else { 85 | _, err := os.Stat(conf.Opts.DiffOutputFile) 86 | if os.IsNotExist(err) { 87 | if err = os.Mkdir(conf.Opts.DiffOutputFile, os.ModePerm); err != nil { 88 | return fmt.Errorf("mkdir diffOutputFile[%v] failed[%v]", conf.Opts.DiffOutputFile, err) 89 | } 90 | } else { 91 | newName := fmt.Sprintf("%v-%v", conf.Opts.DiffOutputFile, time.Now().Format(shakeUtils.GolangSecurityTime)) 92 | if err := os.Rename(conf.Opts.DiffOutputFile, newName); err != nil { 93 | return fmt.Errorf("diffOutputFile dir[%v] rename to newFile[%v] failed[%v], need to be remvoed manullay", 94 | conf.Opts.DiffOutputFile, newName, err) 95 | } 96 | if err = os.Mkdir(conf.Opts.DiffOutputFile, os.ModePerm); err != nil { 97 | return fmt.Errorf("mkdir diffOutputFile[%v] again failed[%v]", conf.Opts.DiffOutputFile, err) 98 | } 99 | } 100 | } 101 | 102 | if conf.Opts.ConvertType == "" { 103 | conf.Opts.ConvertType = shakeUtils.ConvertMTypeChange 104 | } else if conf.Opts.ConvertType != shakeUtils.ConvertMTypeChange { 105 | return fmt.Errorf("convertType[%v] illegal, only support %v", 106 | conf.Opts.ConvertType, shakeUtils.ConvertMTypeChange) 107 | } 108 | 109 | return nil 110 | } 111 | 112 | func crash(msg string, errCode int) { 113 | fmt.Println(msg) 114 | panic(Exit{errCode}) 115 | } 116 | -------------------------------------------------------------------------------- /nimo-shake/configure/conf.go: -------------------------------------------------------------------------------- 1 | package conf 2 | 3 | type Configuration struct { 4 | ConfVersion uint `config:"conf.version"` // do not modify the tag name 5 | Id string `config:"id"` 6 | LogFile string `config:"log.file"` 7 | LogLevel string `config:"log.level"` 8 | LogBuffer bool `config:"log.buffer"` 9 | PidPath string `config:"pid_path"` 10 | SystemProfile int `config:"system_profile"` 11 | FullSyncHTTPListenPort int `config:"full_sync.http_port"` 12 | IncrSyncHTTPListenPort int `config:"incr_sync.http_port"` 13 | SyncMode string `config:"sync_mode"` 14 | SourceAccessKeyID string `config:"source.access_key_id"` 15 | SourceSecretAccessKey string `config:"source.secret_access_key"` 16 | SourceSessionToken string `config:"source.session_token"` 17 | SourceRegion string `config:"source.region"` 18 | SourceEndpointUrl string `config:"source.endpoint_url"` 19 | SourceSessionMaxRetries uint `config:"source.session.max_retries"` 20 | SourceSessionTimeout uint `config:"source.session.timeout"` 21 | QpsFull uint `config:"qps.full"` 22 | QpsFullBatchNum int64 `config:"qps.full.batch_num"` 23 | QpsIncr uint `config:"qps.incr"` 24 | QpsIncrBatchNum int64 `config:"qps.incr.batch_num"` 25 | FilterCollectionWhite string `config:"filter.collection.white"` 26 | FilterCollectionBlack string `config:"filter.collection.black"` 27 | TargetType string `config:"target.type"` 28 | TargetAddress string `config:"target.address"` 29 | TargetMongoDBType string `config:"target.mongodb.type"` 30 | TargetDBExist string `config:"target.db.exist"` 31 | SyncSchemaOnly bool `config:"sync_schema_only"` 32 | FullConcurrency uint `config:"full.concurrency"` 33 | FullReadConcurrency uint `config:"full.read.concurrency"` 34 | FullDocumentConcurrency uint `config:"full.document.concurrency"` 35 | FullDocumentWriteBatch uint `config:"full.document.write.batch"` 36 | FullDocumentParser uint `config:"full.document.parser"` 37 | FullEnableIndexPrimary bool `config:"full.enable_index.primary"` 38 | FullEnableIndexUser bool `config:"full.enable_index.user"` 39 | FullExecutorInsertOnDupUpdate bool `config:"full.executor.insert_on_dup_update"` 40 | FullFilterExpression string `config:"full.read.filter_expression"` 41 | FullFilterAttributeValues string `config:"full.read.filter_attributevalues"` 42 | ConvertType string `config:"convert.type"` 43 | ConvertId string `config:"convert._id"` 44 | IncreaseConcurrency uint `config:"increase.concurrency"` 45 | IncreaseExecutorInsertOnDupUpdate bool `config:"increase.executor.insert_on_dup_update"` 46 | IncreaseExecutorUpsert bool `config:"increase.executor.upsert"` 47 | IncreasePersistDir string `config:"increase.persist.dir"` 48 | CheckpointType string `config:"checkpoint.type"` 49 | CheckpointAddress string `config:"checkpoint.address"` 50 | CheckpointDb string `config:"checkpoint.db"` 51 | IncrSyncParallel bool `config:"incr_sync_parallel"` 52 | 53 | /*---------------------------------------------------------*/ 54 | // generated variables 55 | Version string // version 56 | } 57 | 58 | var Options Configuration 59 | 60 | func ConvertIdFunc(name string) string { 61 | if Options.ConvertId != "" && name == "_id" { 62 | return Options.ConvertId + name 63 | } 64 | 65 | return name 66 | } 67 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/pattlog.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010, Kyle Lemons . All rights reserved. 2 | 3 | package log4go 4 | 5 | import ( 6 | "bytes" 7 | "fmt" 8 | "io" 9 | "strings" 10 | "sync" 11 | ) 12 | 13 | const ( 14 | FORMAT_DEFAULT = "[%D %T] [%L] (%S) %M" 15 | FORMAT_SHORT = "[%t %d] [%L] %M" 16 | FORMAT_ABBREV = "[%L] %M" 17 | ) 18 | 19 | type formatCacheType struct { 20 | LastUpdateSeconds int64 21 | shortTime, shortDate string 22 | longTime, longDate string 23 | } 24 | 25 | var formatCache = &formatCacheType{} 26 | var muFormatCache = sync.Mutex{} 27 | 28 | func setFormatCache(f *formatCacheType) { 29 | muFormatCache.Lock() 30 | defer muFormatCache.Unlock() 31 | formatCache = f 32 | } 33 | func getFormatCache() *formatCacheType { 34 | muFormatCache.Lock() 35 | defer muFormatCache.Unlock() 36 | return formatCache 37 | } 38 | // Known format codes: 39 | // %T - Time (15:04:05 MST) 40 | // %t - Time (15:04) 41 | // %D - Date (2006/01/02) 42 | // %d - Date (01/02/06) 43 | // %L - Level (FNST, FINE, DEBG, TRAC, WARN, EROR, CRIT) 44 | // %S - Source 45 | // %M - Message 46 | // Ignores unknown formats 47 | // Recommended: "[%D %T] [%L] (%S) %M" 48 | func FormatLogRecord(format string, rec *LogRecord) string { 49 | if rec == nil { 50 | return "" 51 | } 52 | if len(format) == 0 { 53 | return "" 54 | } 55 | 56 | out := bytes.NewBuffer(make([]byte, 0, 64)) 57 | secs := rec.Created.UnixNano() / 1e9 58 | 59 | cache := getFormatCache() 60 | if cache.LastUpdateSeconds != secs { 61 | month, day, year := rec.Created.Month(), rec.Created.Day(), rec.Created.Year() 62 | hour, minute, second := rec.Created.Hour(), rec.Created.Minute(), rec.Created.Second() 63 | zone, _ := rec.Created.Zone() 64 | updated := &formatCacheType{ 65 | LastUpdateSeconds: secs, 66 | shortTime: fmt.Sprintf("%02d:%02d", hour, minute), 67 | shortDate: fmt.Sprintf("%02d/%02d/%02d", day, month, year%100), 68 | longTime: fmt.Sprintf("%02d:%02d:%02d %s", hour, minute, second, zone), 69 | longDate: fmt.Sprintf("%04d/%02d/%02d", year, month, day), 70 | } 71 | cache = updated 72 | setFormatCache(updated) 73 | } 74 | 75 | // Split the string into pieces by % signs 76 | pieces := bytes.Split([]byte(format), []byte{'%'}) 77 | 78 | // Iterate over the pieces, replacing known formats 79 | for i, piece := range pieces { 80 | if i > 0 && len(piece) > 0 { 81 | switch piece[0] { 82 | case 'T': 83 | out.WriteString(cache.longTime) 84 | case 't': 85 | out.WriteString(cache.shortTime) 86 | case 'D': 87 | out.WriteString(cache.longDate) 88 | case 'd': 89 | out.WriteString(cache.shortDate) 90 | case 'L': 91 | out.WriteString(levelStrings[rec.Level]) 92 | case 'S': 93 | out.WriteString(rec.Source) 94 | case 's': 95 | slice := strings.Split(rec.Source, "/") 96 | out.WriteString(slice[len(slice)-1]) 97 | case 'M': 98 | out.WriteString(rec.Message) 99 | } 100 | if len(piece) > 1 { 101 | out.Write(piece[1:]) 102 | } 103 | } else if len(piece) > 0 { 104 | out.Write(piece) 105 | } 106 | } 107 | out.WriteByte('\n') 108 | 109 | return out.String() 110 | } 111 | 112 | // This is the standard writer that prints to standard output. 113 | type FormatLogWriter chan *LogRecord 114 | 115 | // This creates a new FormatLogWriter 116 | func NewFormatLogWriter(out io.Writer, format string) FormatLogWriter { 117 | records := make(FormatLogWriter, LogBufferLength) 118 | go records.run(out, format) 119 | return records 120 | } 121 | 122 | func (w FormatLogWriter) run(out io.Writer, format string) { 123 | for rec := range w { 124 | fmt.Fprint(out, FormatLogRecord(format, rec)) 125 | } 126 | } 127 | 128 | // This is the FormatLogWriter's output method. This will block if the output 129 | // buffer is full. 130 | func (w FormatLogWriter) LogWrite(rec *LogRecord) { 131 | w <- rec 132 | } 133 | 134 | // Close stops the logger from sending messages to standard output. Attempts to 135 | // send log messages to this logger after a Close have undefined behavior. 136 | func (w FormatLogWriter) Close() { 137 | close(w) 138 | } 139 | -------------------------------------------------------------------------------- /nimo-shake/checkpoint/struct.go: -------------------------------------------------------------------------------- 1 | package checkpoint 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "nimo-shake/filter" 7 | "sync" 8 | ) 9 | 10 | const ( 11 | CheckpointWriterTypeMongo = "mongodb" 12 | CheckpointWriterTypeFile = "file" 13 | CheckpointStatusTable = "status_table" 14 | CheckpointStatusKey = "status_key" 15 | CheckpointStatusValueEmpty = "" 16 | CheckpointStatusValueFullSync = "full_sync" 17 | CheckpointStatusValueIncrSync = "incr_sync" 18 | 19 | // 0: not process; 1: no need to process; 2: prepare stage 3: in processing; 4: wait father finish, 5: done 20 | StatusNotProcess = "not process" 21 | StatusNoNeedProcess = "no need to process" 22 | StatusPrepareProcess = "prepare stage" 23 | StatusInProcessing = "in processing" 24 | StatusWaitFather = "wait father finish" 25 | StatusDone = "done" 26 | 27 | IteratorTypeLatest = "LATEST" 28 | IteratorTypeAtSequence = "AT_SEQUENCE_NUMBER" 29 | IteratorTypeAfterSequence = "AFTER_SEQUENCE_NUMBER" 30 | IteratorTypeTrimHorizon = "TRIM_HORIZON" 31 | 32 | InitShardIt = "see sequence number" 33 | 34 | StreamViewType = "NEW_AND_OLD_IMAGES" 35 | 36 | FieldShardId = "ShardId" 37 | FieldShardIt = "ShardIt" 38 | FieldStatus = "Status" 39 | FieldSeqNum = "SequenceNumber" 40 | FieldIteratorType = "IteratorType" 41 | FieldTimestamp = "UpdateDate" 42 | FieldApproximateTime = "ApproximateTime" 43 | ) 44 | 45 | type Checkpoint struct { 46 | ShardId string `bson:"ShardId" json:"ShardId"` // shard id 47 | FatherId string `bson:"FatherId" json:"FatherId"` // father id 48 | SequenceNumber string `bson:"SequenceNumber" json:"SequenceNumber"` // checkpoint 49 | Status string `bson:"Status" json:"Status"` // status 50 | WorkerId string `bson:"WorkerId" json:"WorkerId"` // thread number 51 | IteratorType string `bson:"IteratorType" json:"IteratorType"` // "LATEST" or "AT_SEQUENCE_NUMBER" 52 | ShardIt string `bson:"ShardIt" json:"ShardIt"` // only used when IteratorType == "LATEST" 53 | UpdateDate string `bson:"UpdateDate" json:"UpdateDate"` // update checkpoint time 54 | ApproximateTime string `bson:"ApproximateTime" json:"ApproximateTime"` // approximate time of records 55 | } 56 | 57 | func (c *Checkpoint) String() string { 58 | out, _ := json.Marshal(c) 59 | return fmt.Sprintf("%s", out) 60 | } 61 | 62 | type Status struct { 63 | Key string `bson:"Key" json:"Key"` // key -> CheckpointStatusKey 64 | Value string `bson:"StatusValue" json:"StatusValue"` // CheckpointStatusValueFullSync or CheckpointStatusValueIncrSync 65 | } 66 | 67 | /*---------------------------------------*/ 68 | 69 | var ( 70 | GlobalShardIteratorMap = ShardIteratorMap{ 71 | mp: make(map[string]string), 72 | } 73 | ) 74 | 75 | type ShardIteratorMap struct { 76 | mp map[string]string 77 | lock sync.Mutex 78 | } 79 | 80 | func (sim *ShardIteratorMap) Set(key, iterator string) bool { 81 | sim.lock.Lock() 82 | defer sim.lock.Unlock() 83 | 84 | if _, ok := sim.mp[key]; ok { 85 | return false 86 | } 87 | 88 | sim.mp[key] = iterator 89 | return false 90 | } 91 | 92 | func (sim *ShardIteratorMap) Get(key string) (string, bool) { 93 | sim.lock.Lock() 94 | defer sim.lock.Unlock() 95 | 96 | it, ok := sim.mp[key] 97 | return it, ok 98 | } 99 | 100 | func (sim *ShardIteratorMap) Delete(key string) bool { 101 | sim.lock.Lock() 102 | defer sim.lock.Unlock() 103 | 104 | if _, ok := sim.mp[key]; ok { 105 | delete(sim.mp, key) 106 | return true 107 | } 108 | return false 109 | } 110 | 111 | /*---------------------------------------*/ 112 | 113 | func FilterCkptCollection(collection string) bool { 114 | return collection == CheckpointStatusTable || filter.IsFilter(collection) 115 | } 116 | 117 | func IsStatusProcessing(status string) bool { 118 | return status == StatusPrepareProcess || status == StatusInProcessing || status == StatusWaitFather 119 | } 120 | 121 | func IsStatusNoNeedProcess(status string) bool { 122 | return status == StatusDone || status == StatusNoNeedProcess 123 | } 124 | -------------------------------------------------------------------------------- /nimo-shake/common/dynamodb.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strings" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/credentials" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/dynamodb" 13 | "github.com/aws/aws-sdk-go/service/dynamodbstreams" 14 | ) 15 | 16 | var ( 17 | globalSession *session.Session 18 | ) 19 | 20 | /* 21 | * all client share the same session. 22 | * Sessions can be shared across all service clients that share the same base configuration 23 | * refer: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/ 24 | */ 25 | func InitSession(accessKeyID, secretAccessKey, sessionToken, region, endpoint string, maxRetries, timeout uint) error { 26 | config := &aws.Config{ 27 | Region: aws.String(region), 28 | Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken), 29 | MaxRetries: aws.Int(int(maxRetries)), 30 | HTTPClient: &http.Client{ 31 | Timeout: time.Duration(timeout) * time.Millisecond, 32 | }, 33 | } 34 | 35 | if endpoint != "" { 36 | config.Endpoint = aws.String(endpoint) 37 | } 38 | 39 | var err error 40 | globalSession, err = session.NewSession(config) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | return nil 46 | } 47 | 48 | func CreateDynamoSession(logLevel string) (*dynamodb.DynamoDB, error) { 49 | if logLevel == "debug" { 50 | svc := dynamodb.New(globalSession, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody)) 51 | return svc, nil 52 | } 53 | svc := dynamodb.New(globalSession) 54 | return svc, nil 55 | } 56 | 57 | func CreateDynamoStreamSession(logLevel string) (*dynamodbstreams.DynamoDBStreams, error) { 58 | if logLevel == "debug" { 59 | svc := dynamodbstreams.New(globalSession, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody)) 60 | return svc, nil 61 | } 62 | svc := dynamodbstreams.New(globalSession) 63 | return svc, nil 64 | } 65 | 66 | func ParseIndexType(input []*dynamodb.AttributeDefinition) map[string]string { 67 | mp := make(map[string]string, len(input)) 68 | 69 | for _, ele := range input { 70 | mp[*ele.AttributeName] = *ele.AttributeType 71 | } 72 | 73 | return mp 74 | } 75 | 76 | // fetch dynamodb table list 77 | func FetchTableList(dynamoSession *dynamodb.DynamoDB) ([]string, error) { 78 | ans := make([]string, 0) 79 | var lastEvaluatedTableName *string 80 | 81 | for { 82 | out, err := dynamoSession.ListTables(&dynamodb.ListTablesInput{ 83 | ExclusiveStartTableName: lastEvaluatedTableName, 84 | }) 85 | 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | ans = AppendStringList(ans, out.TableNames) 91 | if out.LastEvaluatedTableName == nil { 92 | // finish 93 | break 94 | } 95 | lastEvaluatedTableName = out.LastEvaluatedTableName 96 | } 97 | 98 | return ans, nil 99 | } 100 | 101 | func ParsePrimaryAndSortKey(primaryIndexes []*dynamodb.KeySchemaElement, parseMap map[string]string) (string, string, error) { 102 | var primaryKey string 103 | var sortKey string 104 | for _, index := range primaryIndexes { 105 | if *(index.KeyType) == "HASH" { 106 | if primaryKey != "" { 107 | return "", "", fmt.Errorf("duplicate primary key type[%v]", *(index.AttributeName)) 108 | } 109 | primaryKey = *(index.AttributeName) 110 | } else if *(index.KeyType) == "RANGE" { 111 | if sortKey != "" { 112 | return "", "", fmt.Errorf("duplicate sort key type[%v]", *(index.AttributeName)) 113 | } 114 | sortKey = *(index.AttributeName) 115 | } else { 116 | return "", "", fmt.Errorf("unknonw key type[%v]", *(index.KeyType)) 117 | } 118 | } 119 | return primaryKey, sortKey, nil 120 | } 121 | 122 | /* 123 | input: 124 | 125 | "begin```N```1646724207280~~~end```S```1646724207283" 126 | 127 | output: 128 | 129 | map[string]*dynamodb.AttributeValue{ 130 | ":begin": &dynamodb.AttributeValue{N: aws.String("1646724207280")}, 131 | ":end": &dynamodb.AttributeValue{S: aws.String("1646724207283")}, 132 | } 133 | */ 134 | func ParseAttributes(input string) map[string]*dynamodb.AttributeValue { 135 | result := make(map[string]*dynamodb.AttributeValue) 136 | pairs := strings.Split(input, "~~~") 137 | 138 | for _, pair := range pairs { 139 | parts := strings.Split(pair, "```") 140 | if len(parts) != 3 { 141 | continue 142 | } 143 | 144 | key := ":" + parts[0] 145 | dataType := parts[1] 146 | value := parts[2] 147 | 148 | switch dataType { 149 | case "N": 150 | result[key] = &dynamodb.AttributeValue{N: aws.String(value)} 151 | case "S": 152 | result[key] = &dynamodb.AttributeValue{S: aws.String(value)} 153 | } 154 | } 155 | 156 | return result 157 | } 158 | -------------------------------------------------------------------------------- /nimo-shake/common/mongodb_mgo.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "time" 5 | "fmt" 6 | "reflect" 7 | 8 | "github.com/vinllen/mgo" 9 | "github.com/vinllen/mgo/bson" 10 | LOG "github.com/vinllen/log4go" 11 | "github.com/jinzhu/copier" 12 | ) 13 | 14 | const ( 15 | ConnectModePrimary = "primary" 16 | ConnectModeSecondaryPreferred = "secondaryPreferred" 17 | ConnectModeStandalone = "standalone" 18 | 19 | OplogNS = "oplog.rs" 20 | ) 21 | 22 | var ( 23 | NotFountErr = "not found" 24 | NsNotFountErr = "ns not found" 25 | ) 26 | 27 | type NS struct { 28 | Database string 29 | Collection string 30 | } 31 | 32 | func (ns NS) Str() string { 33 | return fmt.Sprintf("%s.%s", ns.Database, ns.Collection) 34 | } 35 | 36 | type MongoConn struct { 37 | Session *mgo.Session 38 | URL string 39 | } 40 | 41 | func NewMongoConn(url string, connectMode string, timeout bool) (*MongoConn, error) { 42 | if connectMode == ConnectModeStandalone { 43 | url += "?connect=direct" 44 | } 45 | 46 | session, err := mgo.Dial(url) 47 | if err != nil { 48 | LOG.Critical("Connect to [%s] failed. %v", url, err) 49 | return nil, err 50 | } 51 | // maximum pooled connections. the overall established sockets 52 | // should be lower than this value(will block otherwise) 53 | session.SetPoolLimit(256) 54 | if timeout { 55 | session.SetSocketTimeout(10 * time.Minute) 56 | } else { 57 | session.SetSocketTimeout(0) 58 | } 59 | 60 | // already ping in the session 61 | /*if err := session.Ping(); err != nil { 62 | LOG.Critical("Verify ping command to %s failed. %v", url, err) 63 | return nil, err 64 | }*/ 65 | 66 | // Switch the session to a eventually behavior. In that case session 67 | // may read for any secondary node. default mode is mgo.Strong 68 | switch connectMode { 69 | case ConnectModePrimary: 70 | session.SetMode(mgo.Primary, true) 71 | case ConnectModeSecondaryPreferred: 72 | session.SetMode(mgo.SecondaryPreferred, true) 73 | case ConnectModeStandalone: 74 | session.SetMode(mgo.Monotonic, true) 75 | default: 76 | err = fmt.Errorf("unknown connect mode[%v]", connectMode) 77 | return nil, err 78 | } 79 | 80 | LOG.Info("New session to %s successfully", url) 81 | return &MongoConn{Session: session, URL: url}, nil 82 | } 83 | 84 | func (conn *MongoConn) Close() { 85 | LOG.Info("Close session with %s", conn.URL) 86 | conn.Session.Close() 87 | } 88 | 89 | func (conn *MongoConn) IsGood() bool { 90 | if err := conn.Session.Ping(); err != nil { 91 | return false 92 | } 93 | 94 | return true 95 | } 96 | 97 | func (conn *MongoConn) AcquireReplicaSetName() string { 98 | var replicaset struct { 99 | Id string `bson:"set"` 100 | } 101 | if err := conn.Session.DB("admin").Run(bson.M{"replSetGetStatus": 1}, &replicaset); err != nil { 102 | LOG.Warn("Replica set name not found in system.replset, %v", err) 103 | } 104 | return replicaset.Id 105 | } 106 | 107 | func (conn *MongoConn) HasOplogNs() bool { 108 | if ns, err := conn.Session.DB("local").CollectionNames(); err == nil { 109 | for _, table := range ns { 110 | if table == OplogNS { 111 | return true 112 | } 113 | } 114 | } 115 | return false 116 | } 117 | 118 | func (conn *MongoConn) HasUniqueIndex() bool { 119 | checkNs := make([]NS, 0, 128) 120 | var databases []string 121 | var err error 122 | if databases, err = conn.Session.DatabaseNames(); err != nil { 123 | LOG.Critical("Couldn't get databases from remote server %v", err) 124 | return false 125 | } 126 | 127 | for _, db := range databases { 128 | if db != "admin" && db != "local" { 129 | coll, _ := conn.Session.DB(db).CollectionNames() 130 | for _, c := range coll { 131 | if c != "system.profile" { 132 | // push all collections 133 | checkNs = append(checkNs, NS{Database: db, Collection: c}) 134 | } 135 | } 136 | } 137 | } 138 | 139 | for _, ns := range checkNs { 140 | indexes, _ := conn.Session.DB(ns.Database).C(ns.Collection).Indexes() 141 | for _, idx := range indexes { 142 | // has unique index 143 | if idx.Unique { 144 | LOG.Info("Found unique index %s on %s.%s in auto shard mode", idx.Name, ns.Database, ns.Collection) 145 | return true 146 | } 147 | } 148 | } 149 | 150 | return false 151 | } 152 | 153 | // first is from dynamo, second is from mongo 154 | func CompareBson(first, second bson.M) (bool, error) { 155 | v2 := make(bson.M, 0) 156 | if err := copier.Copy(&v2, &second); err != nil { 157 | return false, fmt.Errorf("copy[%v] failed[%v]", second, err) 158 | } 159 | 160 | if _, ok := v2["_id"]; ok { 161 | delete(v2, "_id") 162 | } 163 | 164 | return reflect.DeepEqual(first, v2), nil 165 | // return DeepEqual(first, v2), nil 166 | } 167 | -------------------------------------------------------------------------------- /nimo-shake/protocol/raw_converter.go: -------------------------------------------------------------------------------- 1 | package protocal 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | conf "nimo-shake/configure" 8 | 9 | bson2 "go.mongodb.org/mongo-driver/bson" 10 | 11 | "github.com/aws/aws-sdk-go/service/dynamodb" 12 | 13 | LOG "nimo-shake/third_party/log4go" 14 | ) 15 | 16 | type RawConverter struct { 17 | } 18 | 19 | // use dfs to convert to bson.M 20 | func (rc *RawConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) { 21 | v := reflect.ValueOf(input) 22 | if output := rc.dfs(v); output == nil { 23 | return RawData{}, fmt.Errorf("parse failed, return nil") 24 | } else if out, ok := output.(RawData); !ok { 25 | return RawData{}, fmt.Errorf("parse failed, return type isn't RawData") 26 | } else if _, ok := out.Data.(bson2.M); !ok { 27 | return RawData{}, fmt.Errorf("parse failed, return data isn't bson.M") 28 | } else { 29 | return out, nil 30 | } 31 | } 32 | 33 | func (rc *RawConverter) dfs(v reflect.Value) interface{} { 34 | switch v.Kind() { 35 | case reflect.Invalid: 36 | return nil 37 | case reflect.Slice, reflect.Array: 38 | if v.Len() == 0 { 39 | return nil 40 | } 41 | 42 | size := 0 43 | ret := make([]interface{}, 0, v.Len()) 44 | for i := 0; i < v.Len(); i++ { 45 | out := rc.dfs(v.Index(i)) 46 | md := out.(RawData) 47 | size += md.Size 48 | ret = append(ret, md.Data) 49 | } 50 | return RawData{size, ret} 51 | case reflect.Struct: 52 | if v.NumField() == 0 { 53 | return nil 54 | } 55 | 56 | size := 0 57 | ret := make(bson2.M) 58 | for i := 0; i < v.NumField(); i++ { 59 | name := v.Type().Field(i).Name 60 | if out := rc.dfs(v.Field(i)); out != nil { 61 | md := out.(RawData) 62 | size += md.Size 63 | size += len(name) 64 | if _, ok := md.Data.([]interface{}); ok { 65 | // is type array 66 | md.Data = rc.convertListToDetailList(name, md.Data) 67 | } 68 | ret[name] = md.Data 69 | } 70 | } 71 | return RawData{size, ret} 72 | case reflect.Map: 73 | if len(v.MapKeys()) == 0 { 74 | return nil 75 | } 76 | 77 | size := 0 78 | ret := make(bson2.M) 79 | for _, key := range v.MapKeys() { 80 | name := key.String() 81 | name = conf.ConvertIdFunc(name) 82 | if out := rc.dfs(v.MapIndex(key)); out != nil { 83 | md := out.(RawData) 84 | size += md.Size 85 | size += len(name) 86 | if _, ok := md.Data.([]interface{}); ok { 87 | // is type array 88 | out = rc.convertListToDetailList(name, md.Data) 89 | } 90 | ret[name] = md.Data 91 | } 92 | } 93 | return RawData{size, ret} 94 | case reflect.Ptr: 95 | if v.IsNil() { 96 | return nil 97 | } else { 98 | return rc.dfs(v.Elem()) 99 | } 100 | case reflect.Interface: 101 | if v.IsNil() { 102 | return nil 103 | } else { 104 | return rc.dfs(v.Elem()) 105 | } 106 | case reflect.String: 107 | out := v.String() 108 | return RawData{len(out), out} 109 | case reflect.Int: 110 | fallthrough 111 | case reflect.Int64: 112 | return RawData{8, v.Int()} 113 | case reflect.Int8: 114 | return RawData{1, int8(v.Int())} 115 | case reflect.Int16: 116 | return RawData{2, int16(v.Int())} 117 | case reflect.Int32: 118 | return RawData{4, int32(v.Int())} 119 | case reflect.Uint: 120 | fallthrough 121 | case reflect.Uint64: 122 | return RawData{8, v.Uint()} 123 | case reflect.Uint8: 124 | return RawData{1, uint8(v.Uint())} 125 | case reflect.Uint16: 126 | return RawData{2, uint16(v.Uint())} 127 | case reflect.Uint32: 128 | return RawData{4, uint32(v.Uint())} 129 | case reflect.Bool: 130 | // fake size 131 | return RawData{1, v.Bool()} 132 | default: 133 | // not support 134 | LOG.Error("unknown type[%v]", v.Kind()) 135 | return nil 136 | } 137 | } 138 | 139 | func (rc *RawConverter) convertListToDetailList(name string, input interface{}) interface{} { 140 | list := input.([]interface{}) 141 | switch name { 142 | case "B": 143 | output := make([]byte, 0, len(list)) 144 | for _, ele := range list { 145 | output = append(output, ele.(byte)) 146 | } 147 | return output 148 | case "BS": 149 | output := make([][]byte, 0, len(list)) 150 | for _, ele := range list { 151 | inner := rc.convertListToDetailList("B", ele) 152 | output = append(output, inner.([]byte)) 153 | } 154 | return output 155 | case "NS": 156 | fallthrough 157 | case "SS": 158 | output := make([]interface{}, 0, len(list)) 159 | for _, ele := range list { 160 | output = append(output, ele.(string)) 161 | } 162 | return output 163 | case "L": 164 | output := make([]interface{}, 0, len(list)) 165 | for _, ele := range list { 166 | output = append(output, ele.(bson2.M)) 167 | } 168 | return output 169 | } 170 | return list 171 | } 172 | -------------------------------------------------------------------------------- /nimo-shake/checkpoint/mongoWriter.go: -------------------------------------------------------------------------------- 1 | package checkpoint 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | utils "nimo-shake/common" 8 | 9 | LOG "github.com/vinllen/log4go" 10 | "go.mongodb.org/mongo-driver/bson" 11 | "go.mongodb.org/mongo-driver/mongo" 12 | "go.mongodb.org/mongo-driver/mongo/options" 13 | ) 14 | 15 | type MongoWriter struct { 16 | address string 17 | //conn *utils.MongoConn 18 | nconn *utils.MongoCommunityConn 19 | db string 20 | } 21 | 22 | func NewMongoWriter(address, db string) *MongoWriter { 23 | targetConn, err := utils.NewMongoCommunityConn(address, utils.ConnectModePrimary, true) 24 | if err != nil { 25 | LOG.Error("create mongodb with address[%v] db[%v] connection error[%v]", address, db, err) 26 | return nil 27 | } 28 | 29 | return &MongoWriter{ 30 | address: address, 31 | nconn: targetConn, 32 | db: db, 33 | } 34 | } 35 | 36 | func (mw *MongoWriter) FindStatus() (string, error) { 37 | var query Status 38 | if err := mw.nconn.Client.Database(mw.db).Collection(CheckpointStatusTable).FindOne(context.TODO(), 39 | bson.M{"Key": CheckpointStatusKey}).Decode(&query); err != nil { 40 | if err == mongo.ErrNoDocuments { 41 | return CheckpointStatusValueEmpty, nil 42 | } 43 | return "", err 44 | } else { 45 | return query.Value, nil 46 | } 47 | } 48 | 49 | func (mw *MongoWriter) UpdateStatus(status string) error { 50 | update := Status{ 51 | Key: CheckpointStatusKey, 52 | Value: status, 53 | } 54 | 55 | opts := options.Update().SetUpsert(true) 56 | filter := bson.M{"Key": CheckpointStatusKey} 57 | updateStr := bson.M{"$set": update} 58 | _, err := mw.nconn.Client.Database(mw.db).Collection(CheckpointStatusTable).UpdateOne(context.TODO(), filter, updateStr, opts) 59 | return err 60 | } 61 | 62 | func (mw *MongoWriter) ExtractCheckpoint() (map[string]map[string]*Checkpoint, error) { 63 | // extract checkpoint from mongodb, every collection checkpoint have independent collection(table) 64 | ckptMap := make(map[string]map[string]*Checkpoint) 65 | 66 | collectionList, err := mw.nconn.Client.Database(mw.db).ListCollectionNames(context.TODO(), bson.M{}) 67 | if err != nil { 68 | return nil, fmt.Errorf("fetch checkpoint collection list failed[%v]", err) 69 | } 70 | for _, table := range collectionList { 71 | if FilterCkptCollection(table) { 72 | continue 73 | } 74 | 75 | innerMap, err := mw.ExtractSingleCheckpoint(table) 76 | if err != nil { 77 | return nil, err 78 | } 79 | ckptMap[table] = innerMap 80 | } 81 | 82 | return ckptMap, nil 83 | } 84 | 85 | func (mw *MongoWriter) ExtractSingleCheckpoint(table string) (map[string]*Checkpoint, error) { 86 | innerMap := make(map[string]*Checkpoint) 87 | data := make([]*Checkpoint, 0) 88 | 89 | cursor, err := mw.nconn.Client.Database(mw.db).Collection(table).Find(context.TODO(), bson.M{}) 90 | if err != nil { 91 | return nil, err 92 | } 93 | defer cursor.Close(context.TODO()) 94 | 95 | for cursor.Next(context.TODO()) { 96 | var elem Checkpoint 97 | if err := cursor.Decode(&elem); err != nil { 98 | return nil, err 99 | } 100 | data = append(data, &elem) 101 | } 102 | 103 | for _, ele := range data { 104 | innerMap[ele.ShardId] = ele 105 | } 106 | 107 | return innerMap, nil 108 | } 109 | 110 | func (mw *MongoWriter) Insert(ckpt *Checkpoint, table string) error { 111 | _, err := mw.nconn.Client.Database(mw.db).Collection(table).InsertOne(context.TODO(), ckpt) 112 | 113 | return err 114 | } 115 | 116 | func (mw *MongoWriter) Update(shardId string, ckpt *Checkpoint, table string) error { 117 | 118 | filter := bson.M{"ShardId": shardId} 119 | updateStr := bson.M{"$set": ckpt} 120 | _, err := mw.nconn.Client.Database(mw.db).Collection(table).UpdateOne(context.TODO(), filter, updateStr) 121 | return err 122 | } 123 | 124 | func (mw *MongoWriter) UpdateWithSet(shardId string, input map[string]interface{}, table string) error { 125 | 126 | filter := bson.M{"ShardId": shardId} 127 | updateStr := bson.M{"$set": input} 128 | _, err := mw.nconn.Client.Database(mw.db).Collection(table).UpdateOne(context.TODO(), filter, updateStr) 129 | return err 130 | } 131 | 132 | func (mw *MongoWriter) Query(shardId string, table string) (*Checkpoint, error) { 133 | var res Checkpoint 134 | err := mw.nconn.Client.Database(mw.db).Collection(table).FindOne(context.TODO(), bson.M{"ShardId": shardId}).Decode(&res) 135 | 136 | return &res, err 137 | } 138 | 139 | func (mw *MongoWriter) DropAll() error { 140 | return mw.nconn.Client.Database(mw.db).Drop(context.TODO()) 141 | } 142 | 143 | func (fw *MongoWriter) IncrCacheFileInsert(table string, shardId string, fileName string, 144 | lastSequenceNumber string, time string) error { 145 | 146 | // write cachefile struct to db 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /nimo-shake/full-sync/document-syncer.go: -------------------------------------------------------------------------------- 1 | package full_sync 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "nimo-shake/common" 8 | "nimo-shake/configure" 9 | "nimo-shake/writer" 10 | 11 | "github.com/aws/aws-sdk-go/service/dynamodb" 12 | LOG "github.com/vinllen/log4go" 13 | "nimo-shake/protocal" 14 | "sync/atomic" 15 | ) 16 | 17 | const ( 18 | batchSize = 2 * utils.MB // mongodb limit: 16MB 19 | batchTimeout = 1 // seconds 20 | ) 21 | 22 | var ( 23 | UT_TestDocumentSyncer = false 24 | UT_TestDocumentSyncer_Chan chan []interface{} 25 | ) 26 | 27 | /*------------------------------------------------------*/ 28 | // one document link corresponding to one documentSyncer 29 | type documentSyncer struct { 30 | tableSyncerId int 31 | id int // documentSyncer id 32 | ns utils.NS 33 | inputChan chan interface{} // parserChan in table-syncer 34 | writer writer.Writer 35 | collectionMetric *utils.CollectionMetric 36 | } 37 | 38 | func NewDocumentSyncer(tableSyncerId int, table string, id int, inputChan chan interface{}, 39 | tableDescribe *dynamodb.TableDescription, collectionMetric *utils.CollectionMetric) *documentSyncer { 40 | ns := utils.NS{ 41 | Database: conf.Options.Id, 42 | Collection: table, 43 | } 44 | 45 | w := writer.NewWriter(conf.Options.TargetType, conf.Options.TargetAddress, ns, conf.Options.LogLevel) 46 | if w == nil { 47 | LOG.Crashf("tableSyncer[%v] documentSyncer[%v] create writer failed", tableSyncerId, table) 48 | } 49 | 50 | w.PassTableDesc(tableDescribe) 51 | 52 | return &documentSyncer{ 53 | tableSyncerId: tableSyncerId, 54 | id: id, 55 | inputChan: inputChan, 56 | writer: w, 57 | ns: ns, 58 | collectionMetric: collectionMetric, 59 | } 60 | } 61 | 62 | func (ds *documentSyncer) String() string { 63 | return fmt.Sprintf("tableSyncer[%v] documentSyncer[%v] ns[%v]", ds.tableSyncerId, ds.id, ds.ns) 64 | } 65 | 66 | func (ds *documentSyncer) Close() { 67 | ds.writer.Close() 68 | } 69 | 70 | func (ds *documentSyncer) Run() { 71 | batchNumber := int(conf.Options.FullDocumentWriteBatch) 72 | LOG.Info("%s start with batchSize[%v]", ds.String(), batchNumber) 73 | 74 | var data interface{} 75 | var ok bool 76 | batchGroup := make([]interface{}, 0, batchNumber) 77 | timeout := false 78 | batchGroupSize := 0 79 | exit := false 80 | for { 81 | StartT := time.Now() 82 | select { 83 | case data, ok = <-ds.inputChan: 84 | if !ok { 85 | exit = true 86 | LOG.Info("%s channel already closed, flushing cache and exiting...", ds.String()) 87 | } 88 | case <-time.After(time.Second * batchTimeout): 89 | // timeout 90 | timeout = true 91 | data = nil 92 | } 93 | readParserChanDuration := time.Since(StartT) 94 | 95 | LOG.Debug("exit[%v], timeout[%v], len(batchGroup)[%v], batchGroupSize[%v], data[%v]", exit, timeout, 96 | len(batchGroup), batchGroupSize, data) 97 | 98 | if data != nil { 99 | if UT_TestDocumentSyncer { 100 | batchGroup = append(batchGroup, data) 101 | } else { 102 | switch v := data.(type) { 103 | case protocal.RawData: 104 | if v.Size > 0 { 105 | batchGroup = append(batchGroup, v.Data) 106 | batchGroupSize += v.Size 107 | } 108 | case map[string]*dynamodb.AttributeValue: 109 | batchGroup = append(batchGroup, v) 110 | // meaningless batchGroupSize 111 | } 112 | } 113 | } 114 | 115 | if exit || timeout || len(batchGroup) >= batchNumber || batchGroupSize >= batchSize { 116 | StartT = time.Now() 117 | batchGroupLen := len(batchGroup) 118 | if len(batchGroup) != 0 { 119 | if err := ds.write(batchGroup); err != nil { 120 | LOG.Crashf("%s write data failed[%v]", ds.String(), err) 121 | } 122 | 123 | batchGroup = make([]interface{}, 0, batchNumber) 124 | batchGroupSize = 0 125 | } 126 | writeDestDBDuration := time.Since(StartT) 127 | LOG.Info("%s write db batch[%v] parserChan.len[%v] readParserChanTime[%v] writeDestDbTime[%v]", 128 | ds.String(), batchGroupLen, len(ds.inputChan), readParserChanDuration, writeDestDBDuration) 129 | 130 | if exit { 131 | break 132 | } 133 | timeout = false 134 | } 135 | } 136 | 137 | go func() { 138 | <-time.NewTimer(time.Minute * 5).C 139 | ds.writer.Close() 140 | LOG.Info("%s full-sync writer close", ds.String()) 141 | }() 142 | LOG.Info("%s finish writing", ds.String()) 143 | } 144 | 145 | func (ds *documentSyncer) write(input []interface{}) error { 146 | LOG.Debug("%s writing data with length[%v]", ds.String(), len(input)) 147 | if len(input) == 0 { 148 | return nil 149 | } 150 | 151 | if UT_TestDocumentSyncer { 152 | UT_TestDocumentSyncer_Chan <- input 153 | return nil 154 | } 155 | 156 | defer atomic.AddUint64(&ds.collectionMetric.FinishCount, uint64(len(input))) 157 | return ds.writer.WriteBulk(input) 158 | } 159 | -------------------------------------------------------------------------------- /nimo-shake/run/run.go: -------------------------------------------------------------------------------- 1 | package run 2 | 3 | import ( 4 | "nimo-shake/checkpoint" 5 | "nimo-shake/common" 6 | "nimo-shake/configure" 7 | "nimo-shake/filter" 8 | "nimo-shake/full-sync" 9 | "nimo-shake/incr-sync" 10 | "nimo-shake/writer" 11 | 12 | "github.com/aws/aws-sdk-go/service/dynamodbstreams" 13 | "github.com/gugemichael/nimo4go" 14 | LOG "github.com/vinllen/log4go" 15 | ) 16 | 17 | func incrStart(streamMap map[string]*dynamodbstreams.Stream, ckptWriter checkpoint.Writer) { 18 | LOG.Info("start increase sync") 19 | 20 | // register restful api 21 | incr_sync.RestAPI() 22 | 23 | // start http server. 24 | nimo.GoRoutine(func() { 25 | // before starting, we must register all interface 26 | if err := utils.IncrSyncHttpApi.Listen(); err != nil { 27 | LOG.Critical("start incr sync server with port[%v] failed: %v", conf.Options.IncrSyncHTTPListenPort, 28 | err) 29 | } 30 | }) 31 | 32 | LOG.Info("------------------------start incr sync------------------------") 33 | incr_sync.Start(streamMap, ckptWriter) 34 | LOG.Info("------------------------end incr sync------------------------") 35 | } 36 | 37 | func Start() { 38 | LOG.Info("check connections") 39 | 40 | utils.FullSyncInitHttpApi(conf.Options.FullSyncHTTPListenPort) 41 | utils.IncrSyncInitHttpApi(conf.Options.IncrSyncHTTPListenPort) 42 | 43 | // init filter 44 | filter.Init(conf.Options.FilterCollectionWhite, conf.Options.FilterCollectionBlack) 45 | 46 | if err := utils.InitSession(conf.Options.SourceAccessKeyID, conf.Options.SourceSecretAccessKey, 47 | conf.Options.SourceSessionToken, conf.Options.SourceRegion, conf.Options.SourceEndpointUrl, 48 | conf.Options.SourceSessionMaxRetries, conf.Options.SourceSessionTimeout); err != nil { 49 | LOG.Crashf("init global session failed[%v]", err) 50 | } 51 | 52 | // check writer connection 53 | w := writer.NewWriter(conf.Options.TargetType, conf.Options.TargetAddress, 54 | utils.NS{"nimo-shake", "shake_writer_test"}, conf.Options.LogLevel) 55 | if w == nil { 56 | LOG.Crashf("connect type[%v] address[%v] failed[%v]", 57 | conf.Options.TargetType, conf.Options.TargetAddress) 58 | } 59 | 60 | // create dynamo session 61 | dynamoSession, err := utils.CreateDynamoSession(conf.Options.LogLevel) 62 | if err != nil { 63 | LOG.Crashf("create dynamodb session failed[%v]", err) 64 | } 65 | 66 | // create dynamo stream client 67 | dynamoStreamSession, err := utils.CreateDynamoStreamSession(conf.Options.LogLevel) 68 | if err != nil { 69 | LOG.Crashf("create dynamodb stream session failed[%v]", err) 70 | } 71 | 72 | LOG.Info("create checkpoint writer: type=%v", conf.Options.CheckpointType) 73 | ckptWriter := checkpoint.NewWriter(conf.Options.CheckpointType, conf.Options.CheckpointAddress, 74 | conf.Options.CheckpointDb) 75 | 76 | var skipFull bool 77 | var streamMap map[string]*dynamodbstreams.Stream 78 | if conf.Options.SyncMode == utils.SyncModeAll { 79 | LOG.Info("------------------------check checkpoint------------------------") 80 | skipFull, streamMap, err = checkpoint.CheckCkpt(ckptWriter, dynamoStreamSession) 81 | if err != nil { 82 | LOG.Crashf("check checkpoint failed[%v]", err) 83 | } 84 | LOG.Info("------------------------end check checkpoint------------------------") 85 | } 86 | 87 | // full sync 88 | skipIncrSync := false 89 | if skipFull == false { 90 | // register restful api 91 | full_sync.RestAPI() 92 | 93 | // start http server. 94 | nimo.GoRoutine(func() { 95 | // before starting, we must register all interface 96 | if err := utils.FullSyncHttpApi.Listen(); err != nil { 97 | LOG.Critical("start full sync server with port[%v] failed: %v", conf.Options.FullSyncHTTPListenPort, 98 | err) 99 | } 100 | }) 101 | 102 | if conf.Options.SyncMode == utils.SyncModeAll { 103 | LOG.Info("------------------------drop old checkpoint------------------------") 104 | if err := ckptWriter.DropAll(); err != nil && err.Error() != utils.NotFountErr { 105 | LOG.Crashf("drop checkpoint failed[%v]", err) 106 | } 107 | 108 | LOG.Info("------------------------prepare checkpoint start------------------------") 109 | streamMap, err = checkpoint.PrepareFullSyncCkpt(ckptWriter, dynamoSession, dynamoStreamSession) 110 | if err != nil { 111 | LOG.Crashf("prepare checkpoint failed[%v]", err) 112 | } 113 | LOG.Info("------------------------prepare checkpoint done------------------------") 114 | 115 | // select{} 116 | } else { 117 | LOG.Info("sync.mode is 'full', no need to check checkpoint") 118 | } 119 | 120 | if conf.Options.IncrSyncParallel == true { 121 | skipIncrSync = true 122 | go incrStart(streamMap, ckptWriter) 123 | } 124 | 125 | // update checkpoint 126 | if err := ckptWriter.UpdateStatus(checkpoint.CheckpointStatusValueFullSync); err != nil { 127 | LOG.Crashf("set checkpoint to [%v] failed[%v]", checkpoint.CheckpointStatusValueFullSync, err) 128 | } 129 | 130 | LOG.Info("------------------------start full sync------------------------") 131 | full_sync.Start(dynamoSession, w) 132 | LOG.Info("------------------------full sync done!------------------------") 133 | } 134 | 135 | if conf.Options.SyncMode == utils.SyncModeFull { 136 | LOG.Info("sync.mode is 'full', finish") 137 | return 138 | } 139 | 140 | if conf.Options.SyncSchemaOnly { 141 | LOG.Info("sync_schema_only enabled, finish") 142 | return 143 | } 144 | 145 | // update checkpoint 146 | if err := ckptWriter.UpdateStatus(checkpoint.CheckpointStatusValueIncrSync); err != nil { 147 | LOG.Crashf("set checkpoint to [%v] failed[%v]", checkpoint.CheckpointStatusValueIncrSync, err) 148 | } 149 | 150 | if skipIncrSync == false { 151 | go incrStart(streamMap, ckptWriter) 152 | } 153 | 154 | select {} 155 | } 156 | -------------------------------------------------------------------------------- /nimo-shake/conf/nimo-shake.conf: -------------------------------------------------------------------------------- 1 | # current configuration version, do not modify. 2 | # 当前配置文件的版本号，请不要修改该值。 3 | conf.version = 3 4 | 5 | # id 6 | id = nimo-shake 7 | 8 | # log file name，all log will be printed in stdout if log.file is empty 9 | # 日志文件，不配置将打印到stdout (e.g. dynamo-shake.log ) 10 | log.file = 11 | # log level: "none", "error", "warn", "info", "debug". default is "info". 12 | log.level = info 13 | # log buffer，enabling it to make every log print but performance maybe decrease. 14 | # log buffer选项，不启用将会降低性能但保证退出时每条log都被打印，否则，退出时可能有最后几条log丢失 15 | log.buffer = true 16 | 17 | # pprof port. 18 | system_profile = 9330 19 | # restful port, not used currently. 查看metric 20 | # 全量和增量的restful监控端口，可以用curl查看内部监控metric统计情况。详见wiki。 21 | full_sync.http_port = 9341 22 | incr_sync.http_port = 9340 23 | 24 | # sync mode. currently, only support "full". 25 | # all: full sync and increase sync. 26 | # full: full sync only. 27 | # incr: increase sync only.(not supported for now) 28 | # 同步的类型。 29 | # all: 全量+增量同步 30 | # full: 全量同步 31 | # incr: 增量同步(暂不支持) 32 | sync_mode = all 33 | 34 | # incr sync parallel 35 | # 是否并行做增量同步: true为开启，false为关闭。开启后会消耗更多内存 36 | incr_sync_parallel = false 37 | 38 | # dynamodb configuration. leave empty if not set. 39 | # 源端dynamodb的账号信息配置，source.session_token和source.region没有可以留空 40 | source.access_key_id = 41 | source.secret_access_key = 42 | source.session_token = 43 | source.region = 44 | # 源端如果是endpoint类型，可以配置该参数，启用该参数则上述source参数失效 45 | # 例如：http://100.123.124.125:1010 46 | source.endpoint_url = 47 | # max_retries in session once failed 48 | source.session.max_retries = 3 49 | # session timeout, 0 means disable. unit: ms. 50 | source.session.timeout = 3000 51 | 52 | # filter collection split by semicolon(;). at most one of these two parameters can be given. 53 | # if the filter.collection.black is not empty, the given collection will be filtered while others collection passed. 54 | # if the filter.collection.white is not empty, the given collection will be passed while others collection filtered. 55 | # all the namespace will be passed if no condition given. 56 | # E.g., "filter.collection.white = c1;c2" means only c1 and c2 passed while the others filtered. 57 | # 表粒度黑白名单过滤，白名单表示通过，黑名单表示过滤，这两个参数不能同时指定，都指定表示全部通过。分号分隔不同表。 58 | # 举例："filter.collection.white = c1;c2"表示c1和c2表通过，剩下的都被过滤掉。 59 | filter.collection.white = 60 | filter.collection.black = 61 | 62 | # qps limit for each table. 63 | # 对表级别限速 64 | # the scan call(Scan) per second. 65 | # 全量阶段，我们调用的是scan命令，这个参数表示一秒钟最多调用的scan个数 66 | qps.full = 1000 67 | # the limit batch number in one query. default is 128. 68 | # 1次query内部的条数大小 69 | qps.full.batch_num = 128 70 | # the query call(GetRecords) per second. 71 | # 增量阶段，我们调用的是GetRecords命令，这个参数表示一秒钟最多调用的GetRecords个数 72 | qps.incr = 1000 73 | # the limit batch number in one query. default is 128. 74 | # 1次query内部的条数大小 75 | qps.incr.batch_num = 128 76 | 77 | # target mongodb configuration, currently, only supports sync to mongodb. 78 | # 目的端配置, 目前支持mongodb和aliyun_dynamo_proxy 79 | target.type = mongodb 80 | # target mongodb address, e.g., mongodb://username:password@10.1.1.1:3791,10.1.1.2:3792 81 | # 如果是mongodb，此处需要配置目的mongodb的连接串，否则请配置阿里云dynamodb的连接串 82 | # 例如：http://dds-xxxx:3717 83 | target.address = 84 | # target mongodb type, replica or sharding. 85 | # 目的mongodb类型, 副本集选择replica，分片集群请选择sharding 86 | target.mongodb.type = sharding 87 | # how to solve if target mongodb has the same name table. 88 | # "drop" means drop the table before syncing. 89 | # "rename" means rename current table which timestamp suffix, e.g., c1 -> c1.2019-07-01Z12:10:11 90 | # rename仅支持target.type=mongodb的情况 91 | # 如果目的端已经有重名的表，rename将会对原来的表进行重命名，添加 92 | # 时间戳后缀，比如c1变为c1.2019-07-01Z12:10:11；drop表示删除目的表；留空表示不处理。 93 | target.db.exist = drop 94 | 95 | # only sync schema without any data. 96 | # 仅同步数据结构，不同步任何数据。 97 | sync_schema_only = false 98 | 99 | # full sync configuration. 100 | # 全量同步参数 101 | # how many tables will be synced at the same time. 102 | # 表级别并发度，1次最多同步表的数目 103 | full.concurrency = 4 104 | # how many reading threads working in one table. 105 | # 表内文档的并发度，1个表最多有几个线程同时并发读取源端，对应Scan接口的TotalSegments 106 | full.read.concurrency = 1 107 | # how many writing threads working in one table. 108 | # 表内文档的并发度，1个表最多有几个线程同时并发写入目的端 109 | full.document.concurrency = 4 110 | # how many doc batched in one writing request 111 | # 一次聚合写入多少条数据，如果目的端是DynamoDB协议最大配置25 112 | full.document.write.batch = 25 113 | # the number of parsers which do parse dynamodb to mongodb. 114 | # 表内解析线程个数，用户转换dynamo协议到目的端对应协议 115 | full.document.parser = 2 116 | # enable sync user created indexes? 117 | # 主键索引primaryKey会默认创建索引，除此以外是否同步用户自建的索引， 118 | full.enable_index.user = true 119 | # change insert to update when duplicate key error found 120 | # 全量同步，目的端碰到相同的key，是否将insert改为update 121 | full.executor.insert_on_dup_update = true 122 | 123 | # increase sync configuration. 124 | # 增量同步参数 125 | 126 | # 增量同步并发参数，1个表最多有几个线程来读取这个表的shard数据，1个线程同时抓取一个shard 127 | increase.concurrency = 16 128 | # 增量同步，insert语句目的端碰到相同的key，是否将insert改为update 129 | increase.executor.insert_on_dup_update = true 130 | # 增量同步，update语句目的端不存在key，是否将update改为upsert 131 | increase.executor.upsert = true 132 | 133 | # checkpoint存储的类型，可以是mongodb（target.type必须是mongodb），也可以是 134 | # file，本地落盘 135 | checkpoint.type = mongodb 136 | # checkpoint存储的地址，如果是目的端是mongodb则存储目的端的mongodb的地址，不配置默认目的mongodb 137 | # 如果是file则填写相对路径（例如checkpoint），不配置默认名为checkpoint 138 | checkpoint.address = 139 | # checkpoint存储在目的mongodb中的db名，默认以"$id-checkpoint"存储 140 | checkpoint.db = 141 | 142 | # 给DynamoDB中的_id字段增加前缀，不和MongoDB的_id冲突 143 | convert._id = pre 144 | 145 | # :begin 和 :end，这两个冒号开头的是变量，实际的值在 filter_attributevalues 中 146 | # begin:N:1646724207280，begin 是变量名，N 为类型，1646724207280 为值内容 147 | # N 为 Number, S 为 String 148 | full.read.filter_expression = create_time > :begin AND create_time < :end 149 | full.read.filter_attributevalues = begin```N```1646724207280~~~end```N```1646724207283 -------------------------------------------------------------------------------- /nimo-shake/protocol/type_converter.go: -------------------------------------------------------------------------------- 1 | package protocal 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strconv" 7 | "time" 8 | 9 | conf "nimo-shake/configure" 10 | 11 | "github.com/aws/aws-sdk-go/service/dynamodb" 12 | bson2 "go.mongodb.org/mongo-driver/bson" 13 | "go.mongodb.org/mongo-driver/bson/primitive" 14 | 15 | LOG "nimo-shake/third_party/log4go" 16 | ) 17 | 18 | type TypeConverter struct { 19 | } 20 | 21 | // use dfs to convert to bson.M 22 | func (tc *TypeConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) { 23 | v := reflect.ValueOf(input) 24 | if output := tc.dfs(v); output == nil { 25 | return RawData{}, fmt.Errorf("parse failed, return nil") 26 | } else if out, ok := output.(RawData); !ok { 27 | return RawData{}, fmt.Errorf("parse failed, return type isn't RawData") 28 | } else if _, ok := out.Data.(bson2.M); !ok { 29 | return RawData{}, fmt.Errorf("parse failed, return data isn't bson.M") 30 | } else { 31 | return out, nil 32 | } 33 | } 34 | 35 | func (tc *TypeConverter) dfs(v reflect.Value) interface{} { 36 | 37 | funcStartT := time.Now() 38 | defer LOG.Debug("dfs_func kind[%v] value[%v] duration[%v]", 39 | v.Kind().String(), v, time.Since(funcStartT)) 40 | 41 | switch v.Kind() { 42 | case reflect.Invalid: 43 | return nil 44 | case reflect.Slice, reflect.Array: 45 | if v.Len() == 0 { 46 | return nil 47 | } 48 | 49 | size := 0 50 | ret := make([]interface{}, 0, v.Len()) 51 | for i := 0; i < v.Len(); i++ { 52 | out := tc.dfs(v.Index(i)) 53 | md := out.(RawData) 54 | size += md.Size 55 | ret = append(ret, md.Data) 56 | } 57 | return RawData{size, ret} 58 | case reflect.Struct: 59 | if v.NumField() == 0 { 60 | return nil 61 | } 62 | if v.Type().Name() == "bson.Decimal128" { 63 | return RawData{16, v} 64 | } 65 | 66 | size := 0 67 | var ret interface{} 68 | cnt := 0 69 | // at most one field in AttributeValue 70 | for i := 0; i < v.NumField(); i++ { 71 | name := v.Type().Field(i).Name 72 | if out := tc.dfs(v.Field(i)); out != nil { 73 | cnt++ 74 | if cnt > 2 { 75 | LOG.Crashf("illegal struct field number") 76 | } 77 | 78 | md := out.(RawData) 79 | size += md.Size 80 | md.Data = tc.convertToDetail(name, md.Data) 81 | ret = md.Data 82 | } 83 | } 84 | return RawData{size, ret} 85 | case reflect.Map: 86 | if len(v.MapKeys()) == 0 { 87 | return nil 88 | } 89 | 90 | size := 0 91 | ret := make(bson2.M) 92 | for _, key := range v.MapKeys() { 93 | name := key.String() 94 | name = conf.ConvertIdFunc(name) 95 | if out := tc.dfs(v.MapIndex(key)); out != nil { 96 | md := out.(RawData) 97 | size += md.Size 98 | size += len(name) 99 | // out = tc.convertToDetail(name, md.Data, false) 100 | ret[name] = md.Data 101 | } 102 | } 103 | return RawData{size, ret} 104 | case reflect.Ptr: 105 | if v.IsNil() { 106 | return nil 107 | } else { 108 | return tc.dfs(v.Elem()) 109 | } 110 | case reflect.Interface: 111 | if v.IsNil() { 112 | return nil 113 | } else { 114 | return tc.dfs(v.Elem()) 115 | } 116 | case reflect.String: 117 | out := v.String() 118 | return RawData{len(out), out} 119 | case reflect.Int: 120 | fallthrough 121 | case reflect.Int64: 122 | return RawData{8, v.Int()} 123 | case reflect.Int8: 124 | return RawData{1, int8(v.Int())} 125 | case reflect.Int16: 126 | return RawData{2, int16(v.Int())} 127 | case reflect.Int32: 128 | return RawData{4, int32(v.Int())} 129 | case reflect.Uint: 130 | fallthrough 131 | case reflect.Uint64: 132 | return RawData{8, v.Uint()} 133 | case reflect.Uint8: 134 | return RawData{1, uint8(v.Uint())} 135 | case reflect.Uint16: 136 | return RawData{2, uint16(v.Uint())} 137 | case reflect.Uint32: 138 | return RawData{4, uint32(v.Uint())} 139 | case reflect.Bool: 140 | // fake size 141 | return RawData{1, v.Bool()} 142 | default: 143 | // not support 144 | LOG.Error("unknown type[%v]", v.Kind()) 145 | return nil 146 | } 147 | } 148 | 149 | func (tc *TypeConverter) convertToDetail(name string, input interface{}) interface{} { 150 | 151 | funcStartT := time.Now() 152 | defer LOG.Debug("convertToDetail_func name[%v] input[%v] duration[%v]", name, input, time.Since(funcStartT)) 153 | 154 | switch name { 155 | case "B": 156 | list := input.([]interface{}) 157 | output := make([]byte, 0, len(list)) 158 | for _, ele := range list { 159 | output = append(output, ele.(byte)) 160 | } 161 | return output 162 | case "BS": 163 | list := input.([]interface{}) 164 | output := make([][]byte, 0, len(list)) 165 | for _, ele := range list { 166 | inner := tc.convertToDetail("B", ele) 167 | output = append(output, inner.([]byte)) 168 | } 169 | return output 170 | case "NS": 171 | list := input.([]interface{}) 172 | 173 | var nType reflect.Type 174 | for _, ele := range list { 175 | inner := tc.convertToDetail("N", ele) 176 | nType = reflect.TypeOf(inner) 177 | break 178 | } 179 | 180 | if nType.Name() == "int" { 181 | 182 | output := make([]int, 0, len(list)) 183 | for _, ele := range list { 184 | inner := tc.convertToDetail("N", ele) 185 | output = append(output, inner.(int)) 186 | } 187 | 188 | return output 189 | } else { 190 | output := make([]primitive.Decimal128, 0, len(list)) 191 | for _, ele := range list { 192 | inner := tc.convertToDetail("N", ele) 193 | output = append(output, inner.(primitive.Decimal128)) 194 | } 195 | 196 | return output 197 | } 198 | 199 | case "SS": 200 | list := input.([]interface{}) 201 | output := make([]string, 0, len(list)) 202 | for _, ele := range list { 203 | inner := tc.convertToDetail("S", ele) 204 | output = append(output, inner.(string)) 205 | } 206 | return output 207 | case "L": 208 | list := input.([]interface{}) 209 | output := make([]interface{}, 0, len(list)) 210 | for _, ele := range list { 211 | output = append(output, ele) 212 | } 213 | return output 214 | case "BOOL": 215 | fallthrough 216 | case "NULL": 217 | return input.(bool) 218 | case "N": 219 | v := input.(string) 220 | 221 | val_int, err := strconv.Atoi(v) 222 | if err == nil { 223 | return val_int 224 | } 225 | 226 | val, err := primitive.ParseDecimal128(v) 227 | if err != nil { 228 | LOG.Error("convert N to decimal128 failed[%v]", err) 229 | val2, err := strconv.ParseFloat(v, 64) 230 | if err != nil { 231 | LOG.Crashf("convert N to decimal128 and float64 both failed[%v]", err) 232 | } 233 | 234 | val, _ = primitive.ParseDecimal128(fmt.Sprintf("%v", val2)) 235 | return val 236 | } 237 | return val 238 | case "S": 239 | return input.(string) 240 | } 241 | 242 | // "M" 243 | return input 244 | } 245 | -------------------------------------------------------------------------------- /scripts/hypervisor.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Compilation: 3 | * gcc -Wall -O3 hypervisor.c -o hypervisor 4 | * 5 | * */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #ifdef __linux__ 13 | #include 14 | #else 15 | #include 16 | #endif 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | 26 | #define MAXOPT 256 27 | #define INTERVAL 5 28 | #define MAXINTERVAL 180 29 | 30 | #define USAGE() do{ \ 31 | fprintf(stderr, "Usage : %s [--daemon] --exec=\"command arg1 arg2 arg3 ...\"\n", argv[0]); \ 32 | }while(0) 33 | 34 | static char *cmd, *cmdopt[MAXOPT + 1]; 35 | static int daemonize; 36 | 37 | static int parseopt(int argc, char *argv[]); 38 | static int set_nonblock(int fd); 39 | static int getstatus(char *buf, int size, int status); 40 | 41 | static int parseopt(int argc, char *argv[]) 42 | { 43 | int ch, i; 44 | char *token, *tmpptr, *cmdstr; 45 | 46 | cmdstr = cmd = NULL; 47 | daemonize = 0; 48 | for(i = 0; i < MAXOPT + 1; i++){ 49 | cmdopt[i] = NULL; 50 | } 51 | 52 | struct option long_options[] = { 53 | {"daemon",optional_argument,NULL,'d'}, 54 | {"exec",required_argument,NULL,'e'}, 55 | {0,0,0,0}, 56 | }; 57 | 58 | while((ch=getopt_long(argc, argv, "dec:", long_options, NULL)) != -1) { 59 | switch(ch) 60 | { 61 | case 'e': 62 | if((cmdstr = strdup(optarg)) == NULL ) 63 | return -1; 64 | break; 65 | case 'd': 66 | daemonize = 1; 67 | break; 68 | default: 69 | USAGE(); 70 | return -1; 71 | } 72 | } 73 | 74 | if(cmdstr == NULL){ 75 | USAGE(); 76 | return -1; 77 | } 78 | 79 | for(i = 0;i < MAXOPT + 1;cmdstr = NULL, i++){ 80 | token = strtok_r(cmdstr, " \t\n", &tmpptr); 81 | if(token == NULL){ 82 | break; 83 | } else { 84 | cmdopt[i] = strdup(token); 85 | 86 | if(i == 0){ 87 | cmd = strdup(token); 88 | } 89 | } 90 | } 91 | 92 | if( (cmd == NULL) || (strlen(cmd) == 0) ){ 93 | fprintf(stderr, "Error, cmd should not be empty.\n"); 94 | return -1; 95 | } 96 | 97 | if(i == MAXOPT + 1){ 98 | fprintf(stderr, "Argument too long\n"); 99 | return -1; 100 | } 101 | 102 | cmdopt[i] = NULL; 103 | 104 | return 0; 105 | } 106 | 107 | static int set_nonblock(int fd) 108 | { 109 | int flags = fcntl(fd, F_GETFL, 0); 110 | if (flags == -1) { 111 | return -1; 112 | } 113 | return fcntl(fd, F_SETFL, flags | O_NONBLOCK); 114 | } 115 | 116 | static int getstatus(char *buf, int size, int status) 117 | { 118 | int n, len; 119 | 120 | len = size; 121 | 122 | if(WIFEXITED(status)){ 123 | n = snprintf(buf, len, "- normal termination, exit status = %d\n", WEXITSTATUS(status)); 124 | } else if(WIFSIGNALED(status)) { 125 | n = snprintf(buf, len, "- abnormal termination, signal number = %d%s", 126 | WTERMSIG(status), 127 | #ifdef WCOREDUMP 128 | WCOREDUMP(status) ? "-> core file generated" : ""); 129 | #else 130 | ""); 131 | #endif 132 | } else if(WIFSTOPPED(status)) { 133 | n = snprintf(buf, len, "child stopped, signal number = %d\n", WSTOPSIG(status)); 134 | } 135 | 136 | return n; 137 | } 138 | 139 | 140 | 141 | void go_daemon() { 142 | int fd; 143 | 144 | if (fork() != 0) exit(0); /* parent exits */ 145 | setsid(); /* create a new session */ 146 | 147 | /* Every output goes to /dev/null. If Redis is daemonized but 148 | * * the 'logfile' is set to 'stdout' in the configuration file 149 | * * it will not log at all. */ 150 | if ((fd = open("/tmp/mongo4bls.output", O_RDWR, 0)) != -1) { 151 | dup2(fd, STDIN_FILENO); 152 | dup2(fd, STDOUT_FILENO); 153 | dup2(fd, STDERR_FILENO); 154 | if (fd > STDERR_FILENO) close(fd); 155 | } 156 | } 157 | 158 | 159 | int main(int argc, char *argv[]) 160 | { 161 | int ssec = INTERVAL, ret, status; 162 | int first_start = 1; 163 | int pipefd[2], waited, alive, isdaemon; 164 | char buf[1024], info[4096]; 165 | pid_t pid; 166 | time_t now, last = time(NULL); 167 | 168 | if((ret = parseopt(argc, argv)) < 0 ) 169 | exit(ret); 170 | 171 | daemonize ? go_daemon() : 0; 172 | 173 | while(1){ 174 | if(pipe(pipefd) < 0){ 175 | fprintf(stderr, "- make pipe error : %s\n", strerror(errno)); 176 | exit(-1); 177 | } 178 | 179 | if( (ret = set_nonblock(pipefd[0])) < 0 ){ 180 | fprintf(stderr, "- set read nonblock error : %s\n", strerror(errno)); 181 | exit(-1); 182 | } 183 | 184 | if((pid = fork()) < 0){ 185 | fprintf(stderr, "- call fork() error : %s\n", strerror(errno)); 186 | exit(-1); 187 | } else if (pid > 0){ 188 | close(pipefd[1]); 189 | alive = waited = 1; 190 | isdaemon = 0; 191 | while(alive){ 192 | if(waited){ 193 | if(pid != waitpid(pid, &status, 0)){ 194 | sleep(INTERVAL); 195 | continue; 196 | } else { 197 | fprintf(stderr, "- child process[%d] terminated .\n",pid); 198 | if (first_start && (time(NULL)-last)<=5) { 199 | fprintf(stderr,"- child process killed in %ld seconds , may wrong ! exit !\n",(time(NULL)-last)); 200 | exit(-1); 201 | } else 202 | first_start = 0; 203 | waited = 0; 204 | } 205 | } 206 | 207 | ret = read(pipefd[0], buf, sizeof(buf)); 208 | if(ret < 0){ 209 | if(errno == EAGAIN){ 210 | if(isdaemon == 0){ 211 | fprintf(stderr, "- this daemon process has no output !.\n"); 212 | isdaemon = 1; 213 | } 214 | sleep(INTERVAL); 215 | continue; 216 | } else { 217 | fprintf(stderr, "- read pipe error : %s\n", strerror(errno)); 218 | exit(-1); 219 | } 220 | } else if(ret == 0) { 221 | alive = 0; 222 | close(pipefd[0]); 223 | fprintf(stderr, "- read zero from pipe of children.\n"); 224 | if(isdaemon == 0){ 225 | getstatus(info, sizeof(info), status); 226 | fprintf(stderr, "- extra info: %s\n", info); 227 | } else { 228 | strcpy(info, ""); 229 | } 230 | continue; 231 | } else { 232 | fprintf(stderr, " - read pipe return: %d bytes\n", ret); 233 | exit(-1); 234 | } 235 | } 236 | 237 | fprintf(stderr, "- process: \"%s\" exit, restart it\n", cmd); 238 | 239 | sleep(ssec); 240 | 241 | now = time(NULL); 242 | if(now - last > 3600){ 243 | ssec = INTERVAL; 244 | last = now; 245 | } else { 246 | ssec = (ssec << 1) < MAXINTERVAL ? (ssec << 1) : MAXINTERVAL; 247 | } 248 | } else { 249 | close(pipefd[0]); 250 | fprintf(stderr, "- execute: \"%s\"\n", cmd); 251 | if(execvp(cmd, cmdopt) < 0){ 252 | fprintf(stderr, "- execute: \"%s\" error, %s\n", cmd, strerror(errno)); 253 | exit(-1); 254 | } 255 | 256 | } 257 | } 258 | 259 | return 0; 260 | } 261 | -------------------------------------------------------------------------------- /nimo-shake/incr-sync/fetcher.go: -------------------------------------------------------------------------------- 1 | package incr_sync 2 | 3 | import ( 4 | "time" 5 | 6 | "nimo-shake/checkpoint" 7 | utils "nimo-shake/common" 8 | conf "nimo-shake/configure" 9 | "nimo-shake/qps" 10 | 11 | "fmt" 12 | 13 | "github.com/aws/aws-sdk-go/service/dynamodbstreams" 14 | LOG "github.com/vinllen/log4go" 15 | ) 16 | 17 | const ( 18 | FetcherInterval = 60 // seconds 19 | ) 20 | 21 | type Fetcher struct { 22 | dynamoClient *dynamodbstreams.DynamoDBStreams 23 | table string 24 | stream *dynamodbstreams.Stream 25 | shardChan chan *utils.ShardNode 26 | ckptWriter checkpoint.Writer 27 | metric *utils.ReplicationMetric 28 | } 29 | 30 | func NewFetcher(table string, stream *dynamodbstreams.Stream, shardChan chan *utils.ShardNode, ckptWriter checkpoint.Writer, metric *utils.ReplicationMetric) *Fetcher { 31 | // create dynamo stream client 32 | dynamoStreamSession, err := utils.CreateDynamoStreamSession(conf.Options.LogLevel) 33 | if err != nil { 34 | LOG.Error("create dynamodb stream session failed[%v]", err) 35 | return nil 36 | } 37 | 38 | return &Fetcher{ 39 | dynamoClient: dynamoStreamSession, 40 | table: table, 41 | stream: stream, 42 | shardChan: shardChan, 43 | ckptWriter: ckptWriter, 44 | metric: metric, 45 | } 46 | } 47 | 48 | func (f *Fetcher) Run() { 49 | md5Map := make(map[string]uint64) 50 | tableEpoch := make(map[string]int) // GlobalFetcherMoreFlag, restore previous epoch 51 | 52 | qos := qps.StartQoS(10) 53 | defer qos.Close() 54 | 55 | for range time.NewTicker(FetcherInterval * time.Second).C { 56 | shardList := make([]*utils.ShardNode, 0) 57 | // LOG.Debug("fetch table[%v] stream", table) 58 | 59 | preEpoch, ok := tableEpoch[f.table] 60 | if !ok { 61 | tableEpoch[f.table] = 0 62 | } 63 | 64 | var allShards []*dynamodbstreams.Shard 65 | var lastShardIdString *string = nil 66 | for { 67 | var describeStreamInput *dynamodbstreams.DescribeStreamInput 68 | if lastShardIdString != nil { 69 | describeStreamInput = &dynamodbstreams.DescribeStreamInput{ 70 | StreamArn: f.stream.StreamArn, 71 | ExclusiveStartShardId: lastShardIdString, 72 | } 73 | } else { 74 | describeStreamInput = &dynamodbstreams.DescribeStreamInput{ 75 | StreamArn: f.stream.StreamArn, 76 | } 77 | } 78 | 79 | // limit qos of api DescribeStreamInput 80 | <-qos.Bucket 81 | 82 | desStream, err := f.dynamoClient.DescribeStream(describeStreamInput) 83 | if err != nil { 84 | LOG.Crashf("describe table[%v] with stream[%v] failed[%v]", f.table, *f.stream.StreamArn, err) 85 | } 86 | if *desStream.StreamDescription.StreamStatus == "DISABLED" { 87 | LOG.Crashf("table[%v] with stream[%v] has already been disabled", f.table, *f.stream.StreamArn) 88 | } 89 | 90 | allShards = append(allShards, desStream.StreamDescription.Shards...) 91 | 92 | if desStream.StreamDescription.LastEvaluatedShardId == nil { 93 | break 94 | } else { 95 | lastShardIdString = desStream.StreamDescription.LastEvaluatedShardId 96 | LOG.Info("table[%v] have next shardId,LastEvaluatedShardId[%v]", 97 | f.table, *desStream.StreamDescription.LastEvaluatedShardId) 98 | } 99 | } 100 | LOG.Info("fetch.Run table[%v] allShards(len:%d)[%v]", f.table, len(allShards), allShards) 101 | 102 | rootNode := utils.BuildShardTree(allShards, f.table, *f.stream.StreamArn) 103 | md5 := utils.CalMd5(rootNode) 104 | 105 | GlobalFetcherLock.Lock() 106 | curEpoch := GlobalFetcherMoreFlag[f.table] 107 | GlobalFetcherLock.Unlock() 108 | 109 | if val, ok := md5Map[f.table]; !ok || val != md5 { 110 | // shards is changed 111 | LOG.Info("table[%v] md5 changed from old[%v] to new[%v], need fetch shard", f.table, val, md5) 112 | md5Map[f.table] = md5 113 | } else if preEpoch != curEpoch { 114 | // old shard has already been finished 115 | LOG.Info("table[%v] curEpoch[%v] != preEpoch[%v]", f.table, curEpoch, preEpoch) 116 | tableEpoch[f.table] = curEpoch 117 | } else { 118 | LOG.Info("table[%v] md5-old[%v] md5-new[%v]", f.table, val, md5) 119 | 120 | continue 121 | } 122 | 123 | // extract checkpoint from mongodb 124 | ckptSingleMap, err := f.ckptWriter.ExtractSingleCheckpoint(f.table) 125 | if err != nil { 126 | LOG.Crashf("extract checkpoint failed[%v]", err) 127 | } else { 128 | LOG.Info("table:[%v] ckptSingleMap:[%v]", f.table, ckptSingleMap) 129 | } 130 | 131 | if tree, err := utils.PrintShardTree(rootNode); err != nil { 132 | LOG.Info("table[%v] traverse to print tree failed[%v]", f.table, err) 133 | } else { 134 | LOG.Info("traverse stream tree for table[%v](father->child): \n-----\n%v\n-----", f.table, tree) 135 | } 136 | 137 | // traverse shards 138 | err = utils.TraverseShard(rootNode, func(node *utils.ShardNode) error { 139 | LOG.Info("traverse shard[%v]", *node.Shard.ShardId) 140 | id := *node.Shard.ShardId 141 | var father string 142 | if node.Shard.ParentShardId != nil { 143 | father = *node.Shard.ParentShardId 144 | } 145 | 146 | ckpt, ok := ckptSingleMap[id] 147 | if !ok { 148 | // insert checkpoint 149 | newCkpt := &checkpoint.Checkpoint{ 150 | ShardId: id, 151 | SequenceNumber: *node.Shard.SequenceNumberRange.StartingSequenceNumber, 152 | Status: checkpoint.StatusPrepareProcess, 153 | WorkerId: "unknown", 154 | FatherId: father, 155 | IteratorType: checkpoint.IteratorTypeTrimHorizon, 156 | UpdateDate: "", // empty at first 157 | } 158 | f.ckptWriter.Insert(newCkpt, f.table) 159 | shardList = append(shardList, node) 160 | LOG.Info("insert new checkpoint: %v ckptSingleMap[id]:%v", *newCkpt, ckptSingleMap[id]) 161 | return utils.StopTraverseSonErr 162 | } 163 | switch ckpt.Status { 164 | case checkpoint.StatusNoNeedProcess: 165 | LOG.Info("no need to process: %v", *ckpt) 166 | return nil 167 | case checkpoint.StatusPrepareProcess: 168 | LOG.Info("status already in prepare: %v", *ckpt) 169 | shardList = append(shardList, node) 170 | return utils.StopTraverseSonErr 171 | case checkpoint.StatusInProcessing: 172 | LOG.Info("status already in processing: %v", *ckpt) 173 | shardList = append(shardList, node) 174 | return utils.StopTraverseSonErr 175 | case checkpoint.StatusNotProcess: 176 | fallthrough 177 | case checkpoint.StatusWaitFather: 178 | LOG.Info("status need to process: %v", *ckpt) 179 | ckpt.SequenceNumber = *node.Shard.SequenceNumberRange.StartingSequenceNumber 180 | ckpt.Status = checkpoint.StatusPrepareProcess 181 | ckpt.IteratorType = checkpoint.IteratorTypeTrimHorizon 182 | f.ckptWriter.Update(ckpt.ShardId, ckpt, f.table) 183 | shardList = append(shardList, node) 184 | return utils.StopTraverseSonErr 185 | case checkpoint.StatusDone: 186 | LOG.Info("already done: %v", *ckpt) 187 | return nil 188 | default: 189 | return fmt.Errorf("unknown checkpoint status[%v]", ckpt.Status) 190 | } 191 | 192 | return nil 193 | }) 194 | if err != nil { 195 | LOG.Crashf("traverse shard tree failed[%v]", err) 196 | } 197 | 198 | // dispatch shard list 199 | for _, shard := range shardList { 200 | LOG.Info("need to dispatch shard[%v]", *shard.Shard.ShardId) 201 | f.shardChan <- shard 202 | } 203 | } 204 | LOG.Crashf("can't see me!") 205 | } 206 | -------------------------------------------------------------------------------- /nimo-shake/full-sync/syncer.go: -------------------------------------------------------------------------------- 1 | package full_sync 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "sync" 7 | 8 | utils "nimo-shake/common" 9 | conf "nimo-shake/configure" 10 | "nimo-shake/filter" 11 | "nimo-shake/writer" 12 | 13 | "time" 14 | 15 | "github.com/aws/aws-sdk-go/aws" 16 | "github.com/aws/aws-sdk-go/service/dynamodb" 17 | nimo "github.com/gugemichael/nimo4go" 18 | LOG "github.com/vinllen/log4go" 19 | bson2 "go.mongodb.org/mongo-driver/bson" 20 | "go.mongodb.org/mongo-driver/mongo" 21 | ) 22 | 23 | var ( 24 | metricNsMapLock sync.Mutex 25 | metricNsMap = make(map[string]*utils.CollectionMetric) // namespace map: db.collection -> collection metric 26 | ) 27 | 28 | func Start(dynamoSession *dynamodb.DynamoDB, w writer.Writer) { 29 | // fetch all tables 30 | LOG.Info("start fetching table list") 31 | tableList, err := utils.FetchTableList(dynamoSession) 32 | if err != nil { 33 | LOG.Crashf("fetch table list failed[%v]", err) 34 | } 35 | LOG.Info("finish fetching table list: %v", tableList) 36 | 37 | tableList = filter.FilterList(tableList) 38 | 39 | if err := checkTableExists(tableList, w); err != nil { 40 | if !strings.Contains(err.Error(), "ResourceNotFoundException") { 41 | LOG.Crashf("check table exists failed[%v]", err) 42 | return 43 | } 44 | } 45 | 46 | LOG.Info("start syncing: %v", tableList) 47 | 48 | metricNsMapLock.Lock() 49 | for _, table := range tableList { 50 | metricNsMap[table] = utils.NewCollectionMetric() 51 | } 52 | metricNsMapLock.Unlock() 53 | 54 | fullChan := make(chan string, len(tableList)) 55 | for _, table := range tableList { 56 | fullChan <- table 57 | } 58 | 59 | var wg sync.WaitGroup 60 | wg.Add(len(tableList)) 61 | for i := 0; i < int(conf.Options.FullConcurrency); i++ { 62 | go func(id int) { 63 | for { 64 | table, ok := <-fullChan 65 | if !ok { 66 | // chan closed 67 | break 68 | } 69 | 70 | // no need to lock map because the map size won't change 71 | ts := NewTableSyncer(id, table, metricNsMap[table]) 72 | if ts == nil { 73 | LOG.Crashf("tableSyncer[%v] create failed", id) 74 | } 75 | 76 | LOG.Info("tableSyncer[%v] starts sync table[%v]", id, table) 77 | ts.Sync() 78 | LOG.Info("tableSyncer[%v] finish sync table[%v]", id, table) 79 | ts.Close() 80 | 81 | wg.Done() 82 | } 83 | }(i) 84 | } 85 | 86 | wg.Wait() 87 | close(fullChan) 88 | 89 | LOG.Info("finish syncing all tables and indexes!") 90 | } 91 | 92 | func checkTableExists(tableList []string, w writer.Writer) error { 93 | LOG.Info("target.db.exist is set[%v]", conf.Options.TargetDBExist) 94 | switch conf.Options.TargetType { 95 | case utils.TargetTypeMongo: 96 | 97 | sess := w.GetSession().(*mongo.Client) 98 | 99 | now := time.Now().Format(utils.GolangSecurityTime) 100 | collections, err := sess.Database(conf.Options.Id).ListCollectionNames(nil, bson2.M{}) 101 | if err != nil { 102 | return fmt.Errorf("get target collection names error[%v]", err) 103 | } 104 | 105 | collectionsMp := utils.StringListToMap(collections) 106 | for _, table := range tableList { 107 | // check exist on the target mongodb 108 | if _, ok := collectionsMp[table]; ok { 109 | // exist 110 | LOG.Info("table[%v] exists", table) 111 | if conf.Options.TargetDBExist == utils.TargetDBExistDrop { 112 | if err := sess.Database(conf.Options.Id).Collection(table).Drop(nil); err != nil { 113 | return fmt.Errorf("drop target collection[%v] failed[%v]", table, err) 114 | } 115 | } else if conf.Options.TargetDBExist == utils.TargetDBExistRename { 116 | fromCollection := fmt.Sprintf("%s.%s", conf.Options.Id, table) 117 | toCollection := fmt.Sprintf("%s.%s_%v", conf.Options.Id, table, now) 118 | res := sess.Database("admin").RunCommand(nil, bson2.D{ 119 | {"renameCollection", fromCollection}, 120 | {"to", toCollection}, 121 | {"dropTarget", false}, 122 | }) 123 | if err := res.Err(); err != nil { 124 | return fmt.Errorf("rename target collection[%v] failed[%v]", table, err) 125 | } 126 | } else { 127 | //return fmt.Errorf("collection[%v] exists on the target", table) 128 | } 129 | } 130 | } 131 | case utils.TargetTypeAliyunDynamoProxy: 132 | sess := w.GetSession().(*dynamodb.DynamoDB) 133 | 134 | // query table list 135 | collections := make([]string, 0, 16) 136 | 137 | // dynamo-proxy is not support Limit and ExclusiveStartTableName 138 | /*lastTableName := aws.String("") 139 | var count int64 = 100 140 | for i := 0; ; i++ { 141 | LOG.Debug("list table round[%v]", i) 142 | var input *dynamodb.ListTablesInput 143 | if i == 0 { 144 | input = &dynamodb.ListTablesInput{ 145 | Limit: aws.Int64(count), 146 | } 147 | } else { 148 | input = &dynamodb.ListTablesInput{ 149 | ExclusiveStartTableName: lastTableName, 150 | Limit: aws.Int64(count), 151 | } 152 | } 153 | out, err := sess.ListTables(input) 154 | if err != nil { 155 | return fmt.Errorf("list table failed: %v", err) 156 | } 157 | 158 | for _, collection := range out.TableNames { 159 | collections = append(collections, *collection) 160 | } 161 | 162 | lastTableName = out.LastEvaluatedTableName 163 | if len(out.TableNames) < int(count) { 164 | break 165 | } 166 | }*/ 167 | out, err := sess.ListTables(&dynamodb.ListTablesInput{}) 168 | if err != nil { 169 | return fmt.Errorf("list table failed: %v", err) 170 | } 171 | for _, collection := range out.TableNames { 172 | collections = append(collections, *collection) 173 | } 174 | 175 | collectionsMp := utils.StringListToMap(collections) 176 | LOG.Info("target exit db list: %v", collections) 177 | for _, table := range tableList { 178 | // check exist on the target 179 | if _, ok := collectionsMp[table]; ok { 180 | // exist 181 | LOG.Info("table[%v] exists, try [%v]", table, conf.Options.TargetDBExist) 182 | if conf.Options.TargetDBExist == utils.TargetDBExistDrop { 183 | if _, err := sess.DeleteTable(&dynamodb.DeleteTableInput{ 184 | TableName: aws.String(table), 185 | }); err != nil { 186 | return fmt.Errorf("drop target collection[%v] failed[%v]", table, err) 187 | } 188 | } else { 189 | return fmt.Errorf("collection[%v] exists on the target", table) 190 | } 191 | } 192 | } 193 | } 194 | 195 | LOG.Info("finish handling table exists") 196 | 197 | return nil 198 | } 199 | 200 | func RestAPI() { 201 | type FullSyncInfo struct { 202 | Progress string `json:"progress"` // synced_collection_number / total_collection_number 203 | TotalCollection int `json:"total_collection_number"` // total collection 204 | FinishedCollection int `json:"finished_collection_number"` // finished 205 | ProcessingCollection int `json:"processing_collection_number"` // in processing 206 | WaitCollection int `json:"wait_collection_number"` // wait start 207 | CollectionMetric map[string]string `json:"collection_metric"` // collection_name -> process 208 | } 209 | 210 | utils.FullSyncHttpApi.RegisterAPI("/progress", nimo.HttpGet, func([]byte) interface{} { 211 | ret := FullSyncInfo{ 212 | CollectionMetric: make(map[string]string), 213 | } 214 | 215 | metricNsMapLock.Lock() 216 | defer metricNsMapLock.Unlock() 217 | 218 | ret.TotalCollection = len(metricNsMap) 219 | for ns, collectionMetric := range metricNsMap { 220 | ret.CollectionMetric[ns] = collectionMetric.String() 221 | switch collectionMetric.CollectionStatus { 222 | case utils.StatusWaitStart: 223 | ret.WaitCollection += 1 224 | case utils.StatusProcessing: 225 | ret.ProcessingCollection += 1 226 | case utils.StatusFinish: 227 | ret.FinishedCollection += 1 228 | } 229 | } 230 | 231 | if ret.TotalCollection == 0 { 232 | ret.Progress = "-%" 233 | } else { 234 | ret.Progress = fmt.Sprintf("%.2f%%", float64(ret.FinishedCollection)/float64(ret.TotalCollection)*100) 235 | } 236 | 237 | return ret 238 | }) 239 | 240 | } 241 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/filelog.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010, Kyle Lemons . All rights reserved. 2 | 3 | package log4go 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "time" 9 | ) 10 | 11 | // This log writer sends output to a file 12 | type FileLogWriter struct { 13 | rec chan *LogRecord 14 | rot chan bool 15 | 16 | // The opened file 17 | filename string 18 | file *os.File 19 | 20 | // The logging format 21 | format string 22 | 23 | // File header/trailer 24 | header, trailer string 25 | 26 | // Rotate at linecount 27 | maxlines int 28 | maxlines_curlines int 29 | 30 | // Rotate at size 31 | maxsize int 32 | maxsize_cursize int 33 | 34 | // Rotate daily 35 | daily bool 36 | daily_opendate int 37 | 38 | // Keep old logfiles (.001, .002, etc) 39 | rotate bool 40 | maxbackup int 41 | } 42 | 43 | // This is the FileLogWriter's output method 44 | func (w *FileLogWriter) LogWrite(rec *LogRecord) { 45 | w.rec <- rec 46 | } 47 | 48 | func (w *FileLogWriter) Close() { 49 | close(w.rec) 50 | w.file.Sync() 51 | } 52 | 53 | // NewFileLogWriter creates a new LogWriter which writes to the given file and 54 | // has rotation enabled if rotate is true. 55 | // 56 | // If rotate is true, any time a new log file is opened, the old one is renamed 57 | // with a .### extension to preserve it. The various Set* methods can be used 58 | // to configure log rotation based on lines, size, and daily. 59 | // 60 | // The standard log-line format is: 61 | // [%D %T] [%L] (%S) %M 62 | func NewFileLogWriter(fname string, rotate bool) *FileLogWriter { 63 | w := &FileLogWriter{ 64 | rec: make(chan *LogRecord, LogBufferLength), 65 | rot: make(chan bool), 66 | filename: fname, 67 | format: "[%D %T] [%L] (%S) %M", 68 | rotate: rotate, 69 | maxbackup: 999, 70 | } 71 | 72 | // open the file for the first time 73 | if err := w.intRotate(); err != nil { 74 | fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.filename, err) 75 | return nil 76 | } 77 | 78 | go func() { 79 | defer func() { 80 | if w.file != nil { 81 | fmt.Fprint(w.file, FormatLogRecord(w.trailer, &LogRecord{Created: time.Now()})) 82 | w.file.Close() 83 | } 84 | }() 85 | 86 | for { 87 | select { 88 | case <-w.rot: 89 | if err := w.intRotate(); err != nil { 90 | fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.filename, err) 91 | return 92 | } 93 | case rec, ok := <-w.rec: 94 | if !ok { 95 | return 96 | } 97 | now := time.Now() 98 | if (w.maxlines > 0 && w.maxlines_curlines >= w.maxlines) || 99 | (w.maxsize > 0 && w.maxsize_cursize >= w.maxsize) || 100 | (w.daily && now.Day() != w.daily_opendate) { 101 | if err := w.intRotate(); err != nil { 102 | fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.filename, err) 103 | return 104 | } 105 | } 106 | 107 | // Perform the write 108 | n, err := fmt.Fprint(w.file, FormatLogRecord(w.format, rec)) 109 | if err != nil { 110 | fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.filename, err) 111 | return 112 | } 113 | 114 | // Update the counts 115 | w.maxlines_curlines++ 116 | w.maxsize_cursize += n 117 | } 118 | } 119 | }() 120 | 121 | return w 122 | } 123 | 124 | // Request that the logs rotate 125 | func (w *FileLogWriter) Rotate() { 126 | w.rot <- true 127 | } 128 | 129 | // If this is called in a threaded context, it MUST be synchronized 130 | func (w *FileLogWriter) intRotate() error { 131 | // Close any log file that may be open 132 | if w.file != nil { 133 | fmt.Fprint(w.file, FormatLogRecord(w.trailer, &LogRecord{Created: time.Now()})) 134 | w.file.Close() 135 | } 136 | 137 | // If we are keeping log files, move it to the next available number 138 | if w.rotate { 139 | _, err := os.Lstat(w.filename) 140 | if err == nil { // file exists 141 | // Find the next available number 142 | num := 1 143 | fname := "" 144 | if w.daily && time.Now().Day() != w.daily_opendate { 145 | yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") 146 | 147 | for ; err == nil && num <= w.maxbackup; num++ { 148 | fname = w.filename + fmt.Sprintf(".%s.%03d", yesterday, num) 149 | _, err = os.Lstat(fname) 150 | } 151 | // return error if the last file checked still existed 152 | if err == nil { 153 | return fmt.Errorf("Rotate: Cannot find free log number to rename %s\n", w.filename) 154 | } 155 | } else { 156 | num = w.maxbackup - 1 157 | for ; num >= 1; num-- { 158 | fname = w.filename + fmt.Sprintf(".%d", num) 159 | nfname := w.filename + fmt.Sprintf(".%d", num+1) 160 | _, err = os.Lstat(fname) 161 | if err == nil { 162 | os.Rename(fname, nfname) 163 | } 164 | } 165 | } 166 | 167 | w.file.Close() 168 | // Rename the file to its newfound home 169 | err = os.Rename(w.filename, fname) 170 | if err != nil { 171 | return fmt.Errorf("Rotate: %s\n", err) 172 | } 173 | } 174 | } 175 | 176 | // Open the log file 177 | fd, err := os.OpenFile(w.filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0660) 178 | if err != nil { 179 | return err 180 | } 181 | w.file = fd 182 | 183 | now := time.Now() 184 | fmt.Fprint(w.file, FormatLogRecord(w.header, &LogRecord{Created: now})) 185 | 186 | // Set the daily open date to the current date 187 | w.daily_opendate = now.Day() 188 | 189 | // initialize rotation values 190 | w.maxlines_curlines = 0 191 | w.maxsize_cursize = 0 192 | 193 | return nil 194 | } 195 | 196 | // Set the logging format (chainable). Must be called before the first log 197 | // message is written. 198 | func (w *FileLogWriter) SetFormat(format string) *FileLogWriter { 199 | w.format = format 200 | return w 201 | } 202 | 203 | // Set the logfile header and footer (chainable). Must be called before the first log 204 | // message is written. These are formatted similar to the FormatLogRecord (e.g. 205 | // you can use %D and %T in your header/footer for date and time). 206 | func (w *FileLogWriter) SetHeadFoot(head, foot string) *FileLogWriter { 207 | w.header, w.trailer = head, foot 208 | if w.maxlines_curlines == 0 { 209 | fmt.Fprint(w.file, FormatLogRecord(w.header, &LogRecord{Created: time.Now()})) 210 | } 211 | return w 212 | } 213 | 214 | // Set rotate at linecount (chainable). Must be called before the first log 215 | // message is written. 216 | func (w *FileLogWriter) SetRotateLines(maxlines int) *FileLogWriter { 217 | //fmt.Fprintf(os.Stderr, "FileLogWriter.SetRotateLines: %v\n", maxlines) 218 | w.maxlines = maxlines 219 | return w 220 | } 221 | 222 | // Set rotate at size (chainable). Must be called before the first log message 223 | // is written. 224 | func (w *FileLogWriter) SetRotateSize(maxsize int) *FileLogWriter { 225 | //fmt.Fprintf(os.Stderr, "FileLogWriter.SetRotateSize: %v\n", maxsize) 226 | w.maxsize = maxsize 227 | return w 228 | } 229 | 230 | // Set rotate daily (chainable). Must be called before the first log message is 231 | // written. 232 | func (w *FileLogWriter) SetRotateDaily(daily bool) *FileLogWriter { 233 | //fmt.Fprintf(os.Stderr, "FileLogWriter.SetRotateDaily: %v\n", daily) 234 | w.daily = daily 235 | return w 236 | } 237 | 238 | // Set max backup files. Must be called before the first log message 239 | // is written. 240 | func (w *FileLogWriter) SetRotateMaxBackup(maxbackup int) *FileLogWriter { 241 | w.maxbackup = maxbackup 242 | return w 243 | } 244 | 245 | // SetRotate changes whether or not the old logs are kept. (chainable) Must be 246 | // called before the first log message is written. If rotate is false, the 247 | // files are overwritten; otherwise, they are rotated to another file before the 248 | // new log is opened. 249 | func (w *FileLogWriter) SetRotate(rotate bool) *FileLogWriter { 250 | //fmt.Fprintf(os.Stderr, "FileLogWriter.SetRotate: %v\n", rotate) 251 | w.rotate = rotate 252 | return w 253 | } 254 | 255 | // NewXMLLogWriter is a utility method for creating a FileLogWriter set up to 256 | // output XML record log messages instead of line-based ones. 257 | func NewXMLLogWriter(fname string, rotate bool) *FileLogWriter { 258 | return NewFileLogWriter(fname, rotate).SetFormat( 259 | ` 260 | %D %T 261 | %S 262 | %M 263 | `).SetHeadFoot("", "") 264 | } 265 | -------------------------------------------------------------------------------- /nimo-shake/writer/dynamo_proxy.go: -------------------------------------------------------------------------------- 1 | package writer 2 | 3 | import ( 4 | "fmt" 5 | "bytes" 6 | "net/http" 7 | "time" 8 | 9 | "nimo-shake/common" 10 | "nimo-shake/configure" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | LOG "github.com/vinllen/log4go" 15 | "github.com/aws/aws-sdk-go/service/dynamodb" 16 | ) 17 | 18 | type DynamoProxyWriter struct { 19 | Name string 20 | svc *dynamodb.DynamoDB 21 | ns utils.NS 22 | } 23 | 24 | func NewDynamoProxyWriter(name, address string, ns utils.NS, logLevel string) *DynamoProxyWriter { 25 | config := &aws.Config{ 26 | Region: aws.String("us-east-2"), // meaningless 27 | Endpoint: aws.String(address), 28 | MaxRetries: aws.Int(3), 29 | DisableSSL: aws.Bool(true), 30 | HTTPClient: &http.Client{ 31 | Timeout: time.Duration(5000) * time.Millisecond, 32 | }, 33 | } 34 | 35 | var err error 36 | sess, err := session.NewSession(config) 37 | if err != nil { 38 | LOG.Crashf("create dynamo connection error[%v]", err) 39 | return nil 40 | } 41 | 42 | var svc *dynamodb.DynamoDB 43 | if logLevel == "debug" { 44 | svc = dynamodb.New(sess, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody)) 45 | } else { 46 | svc = dynamodb.New(sess) 47 | } 48 | 49 | return &DynamoProxyWriter{ 50 | Name: name, 51 | svc: svc, 52 | ns: ns, 53 | } 54 | } 55 | 56 | func (dpw *DynamoProxyWriter) String() string { 57 | return dpw.Name 58 | } 59 | 60 | func (dpw *DynamoProxyWriter) GetSession() interface{} { 61 | return dpw.svc 62 | } 63 | 64 | func (dpw *DynamoProxyWriter) PassTableDesc(tableDescribe *dynamodb.TableDescription) { 65 | } 66 | 67 | func (dpw *DynamoProxyWriter) CreateTable(tableDescribe *dynamodb.TableDescription) error { 68 | createTableInput := &dynamodb.CreateTableInput{ 69 | AttributeDefinitions: tableDescribe.AttributeDefinitions, 70 | KeySchema: tableDescribe.KeySchema, 71 | TableName: tableDescribe.TableName, 72 | } 73 | 74 | LOG.Info("try create table: %v", *tableDescribe) 75 | 76 | if conf.Options.FullEnableIndexUser { 77 | // convert []*GlobalSecondaryIndexDescription => []*GlobalSecondaryIndex 78 | gsiList := make([]*dynamodb.GlobalSecondaryIndex, 0, len(tableDescribe.GlobalSecondaryIndexes)) 79 | for _, gsiDesc := range tableDescribe.GlobalSecondaryIndexes { 80 | gsiIndex := &dynamodb.GlobalSecondaryIndex{ 81 | IndexName: gsiDesc.IndexName, 82 | KeySchema: gsiDesc.KeySchema, 83 | Projection: gsiDesc.Projection, 84 | // ProvisionedThroughput: gsiDesc.ProvisionedThroughput, 85 | } 86 | 87 | // meaningless, support aliyun_dynamodb 88 | if gsiDesc.Projection == nil { 89 | gsiIndex.Projection = &dynamodb.Projection{} 90 | } 91 | gsiList = append(gsiList, gsiIndex) 92 | } 93 | createTableInput.SetGlobalSecondaryIndexes(gsiList) 94 | 95 | // convert []*LocalSecondaryIndexDescription => []*LocalSecondaryIndex 96 | lsiList := make([]*dynamodb.LocalSecondaryIndex, 0, len(tableDescribe.LocalSecondaryIndexes)) 97 | for _, lsiDesc := range tableDescribe.LocalSecondaryIndexes { 98 | lsiIndex := &dynamodb.LocalSecondaryIndex{ 99 | IndexName: lsiDesc.IndexName, 100 | KeySchema: lsiDesc.KeySchema, 101 | Projection: lsiDesc.Projection, 102 | } 103 | 104 | // meaningless, support aliyun_dynamodb 105 | if lsiDesc.Projection == nil { 106 | lsiIndex.Projection = &dynamodb.Projection{} 107 | } 108 | lsiList = append(lsiList, lsiIndex) 109 | } 110 | createTableInput.SetLocalSecondaryIndexes(lsiList) 111 | } 112 | 113 | _, err := dpw.svc.CreateTable(createTableInput) 114 | if err != nil { 115 | LOG.Error("create table[%v] fail: %v", *tableDescribe.TableName, err) 116 | return err 117 | } 118 | 119 | checkReady := func() bool { 120 | // check table is ready 121 | out, err := dpw.svc.DescribeTable(&dynamodb.DescribeTableInput{ 122 | TableName: tableDescribe.TableName, 123 | }) 124 | if err != nil { 125 | LOG.Warn("create table[%v] ok but describe failed: %v", *tableDescribe.TableName, err) 126 | return true 127 | } 128 | 129 | if *out.Table.TableStatus != "ACTIVE" { 130 | LOG.Warn("create table[%v] ok but describe not ready: %v", *tableDescribe.TableName, *out.Table.TableStatus) 131 | return true 132 | } 133 | 134 | return false 135 | } 136 | 137 | // check with retry 5 times and 1s gap 138 | ok := utils.CallbackRetry(5, 1000, checkReady) 139 | if !ok { 140 | return fmt.Errorf("create table[%v] fail: check ready fail", dpw.ns.Collection) 141 | } 142 | 143 | return nil 144 | } 145 | 146 | func (dpw *DynamoProxyWriter) DropTable() error { 147 | _, err := dpw.svc.DeleteTable(&dynamodb.DeleteTableInput{ 148 | TableName: aws.String(dpw.ns.Collection), 149 | }) 150 | return err 151 | } 152 | 153 | func (dpw *DynamoProxyWriter) WriteBulk(input []interface{}) error { 154 | if len(input) == 0 { 155 | return nil 156 | } 157 | 158 | // convert to WriteRequest 159 | request := make([]*dynamodb.WriteRequest, len(input)) 160 | for i, ele := range input { 161 | request[i] = &dynamodb.WriteRequest{ 162 | PutRequest: &dynamodb.PutRequest{ 163 | Item: ele.(map[string]*dynamodb.AttributeValue), 164 | }, 165 | } 166 | } 167 | 168 | _, err := dpw.svc.BatchWriteItem(&dynamodb.BatchWriteItemInput{ 169 | RequestItems: map[string][]*dynamodb.WriteRequest{ 170 | dpw.ns.Collection: request, 171 | }, 172 | }) 173 | return err 174 | } 175 | 176 | func (dpw *DynamoProxyWriter) Close() { 177 | 178 | } 179 | 180 | // input type is map[string]*dynamodb.AttributeValue 181 | func (dpw *DynamoProxyWriter) Insert(input []interface{}, index []interface{}) error { 182 | if len(input) == 0 { 183 | return nil 184 | } 185 | 186 | request := make([]*dynamodb.WriteRequest, len(index)) 187 | for i, ele := range input { 188 | request[i] = &dynamodb.WriteRequest{ 189 | PutRequest: &dynamodb.PutRequest{ 190 | Item: ele.(map[string]*dynamodb.AttributeValue), 191 | }, 192 | } 193 | } 194 | 195 | _, err := dpw.svc.BatchWriteItem(&dynamodb.BatchWriteItemInput{ 196 | RequestItems: map[string][]*dynamodb.WriteRequest{ 197 | dpw.ns.Collection: request, 198 | }, 199 | }) 200 | 201 | if err != nil && utils.DynamoIgnoreError(err, "i", true) { 202 | LOG.Warn("%s ignore error[%v] when insert", dpw, err) 203 | return nil 204 | } 205 | 206 | return err 207 | } 208 | 209 | func (dpw *DynamoProxyWriter) Delete(index []interface{}) error { 210 | if len(index) == 0 { 211 | return nil 212 | } 213 | 214 | request := make([]*dynamodb.WriteRequest, len(index)) 215 | for i, ele := range index { 216 | request[i] = &dynamodb.WriteRequest{ 217 | DeleteRequest: &dynamodb.DeleteRequest{ 218 | Key: ele.(map[string]*dynamodb.AttributeValue), 219 | }, 220 | } 221 | } 222 | 223 | _, err := dpw.svc.BatchWriteItem(&dynamodb.BatchWriteItemInput{ 224 | RequestItems: map[string][]*dynamodb.WriteRequest{ 225 | dpw.ns.Collection: request, 226 | }, 227 | }) 228 | 229 | if utils.DynamoIgnoreError(err, "d", true) { 230 | LOG.Warn("%s ignore error[%v] when delete", dpw, err) 231 | return nil 232 | } 233 | 234 | return err 235 | } 236 | 237 | // upsert 238 | func (dpw *DynamoProxyWriter) Update(input []interface{}, index []interface{}) error { 239 | if len(input) == 0 { 240 | return nil 241 | } 242 | 243 | // fmt.Println(input, index) 244 | 245 | for i := range input { 246 | val := input[i].(map[string]*dynamodb.AttributeValue) 247 | key := index[i].(map[string]*dynamodb.AttributeValue) 248 | 249 | // why no update interface like BatchWriteItem !!!! 250 | // generate new map(expression-attribute-values) and expression(update-expression) 251 | newMap := make(map[string]*dynamodb.AttributeValue, len(val)) 252 | expressionBuffer := new(bytes.Buffer) 253 | expressionBuffer.WriteString("SET") 254 | cnt := 1 255 | for k, v := range val { 256 | newKey := fmt.Sprintf(":v%d", cnt) 257 | newMap[newKey] = v 258 | 259 | if cnt == 1 { 260 | expressionBuffer.WriteString(fmt.Sprintf(" %s=%s", k, newKey)) 261 | } else { 262 | expressionBuffer.WriteString(fmt.Sprintf(",%s=%s", k, newKey)) 263 | } 264 | 265 | cnt++ 266 | } 267 | 268 | // fmt.Println(newMap) 269 | _, err := dpw.svc.UpdateItem(&dynamodb.UpdateItemInput{ 270 | TableName: aws.String(dpw.ns.Collection), 271 | Key: key, 272 | UpdateExpression: aws.String(expressionBuffer.String()), 273 | ExpressionAttributeValues: newMap, 274 | }) 275 | if err != nil && utils.DynamoIgnoreError(err, "u", true) { 276 | LOG.Warn("%s ignore error[%v] when insert", dpw, err) 277 | return nil 278 | } 279 | } 280 | 281 | return nil 282 | } 283 | -------------------------------------------------------------------------------- /nimo-shake/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRco= 2 | github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= 3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 7 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 8 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 9 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 10 | github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40 h1:6TWAiHVyKs75ZHEn7XtVv7SO7M4rHwvY/5Tf7xdJBkc= 11 | github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40/go.mod h1:ibO7uKpO8fOH/bKD4trmwm5tHhHKiAjC0u288Rd+GnI= 12 | github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= 13 | github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= 14 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 15 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 16 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 17 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 18 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= 19 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 20 | github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= 21 | github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= 22 | github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= 23 | github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= 24 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 28 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 29 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 30 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d h1:V+5NxRH9I8rVJyazHI25yXaszF4JA64Isr4ilU+nnck= 31 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d/go.mod h1:F8d+yTmuUlynKo7Wn26s+Q8bCZBfcymR5bYSydymIlc= 32 | github.com/vinllen/mgo v0.0.0-20220329061231-e5ecea62f194 h1:UlCPbcl8pHR5vAFP3qPx7HItWNa7+wjxRVySHstMsec= 33 | github.com/vinllen/mgo v0.0.0-20220329061231-e5ecea62f194/go.mod h1:nwebO37lLx+dkv3Dzp24vuzrapu+JudOSYpue46unDk= 34 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 35 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 36 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 37 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 38 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 39 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 40 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= 41 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 42 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 43 | go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= 44 | go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= 45 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 46 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 47 | golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= 48 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= 49 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= 50 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 51 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 52 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 53 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 54 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 55 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 56 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 57 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 58 | golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= 59 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= 60 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 61 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 63 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 64 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 65 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 66 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 67 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 68 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 69 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 70 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 71 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 72 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 74 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 75 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 76 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 77 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 78 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 79 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= 80 | golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= 81 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 82 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 83 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 84 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 85 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 86 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 87 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 88 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 89 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 90 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 91 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 92 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 93 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 94 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 95 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 96 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 97 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 98 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 99 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 100 | -------------------------------------------------------------------------------- /nimo-shake/main/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | _ "net/http/pprof" 8 | "os" 9 | "runtime" 10 | "syscall" 11 | "time" 12 | 13 | "nimo-shake/checkpoint" 14 | "nimo-shake/common" 15 | "nimo-shake/configure" 16 | "nimo-shake/run" 17 | 18 | "github.com/gugemichael/nimo4go" 19 | LOG "github.com/vinllen/log4go" 20 | ) 21 | 22 | type Exit struct{ Code int } 23 | 24 | func main() { 25 | defer LOG.Close() 26 | 27 | //http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler()) 28 | //go func() { 29 | // http.ListenAndServe(":6060", nil) 30 | //}() 31 | 32 | runtime.GOMAXPROCS(256) 33 | fmt.Println("max process:", runtime.GOMAXPROCS(0)) 34 | 35 | var err error 36 | // argument options 37 | configuration := flag.String("conf", "", "configuration path") 38 | version := flag.Bool("version", false, "show version") 39 | flag.Parse() 40 | 41 | if *version { 42 | fmt.Println(utils.Version) 43 | return 44 | } 45 | 46 | if *configuration == "" { 47 | fmt.Println(utils.Version) 48 | flag.PrintDefaults() 49 | return 50 | } 51 | 52 | conf.Options.Version = utils.Version 53 | 54 | var file *os.File 55 | if file, err = os.Open(*configuration); err != nil { 56 | crash(fmt.Sprintf("Configure file open failed. %v", err), -1) 57 | } 58 | 59 | configure := nimo.NewConfigLoader(file) 60 | configure.SetDateFormat(utils.GolangSecurityTime) 61 | if err := configure.Load(&conf.Options); err != nil { 62 | crash(fmt.Sprintf("Configure file %s parse failed. %v", *configuration, err), -2) 63 | } 64 | 65 | utils.InitialLogger(conf.Options.LogFile, conf.Options.LogLevel, conf.Options.LogBuffer) 66 | 67 | // sanitize options 68 | if err := sanitizeOptions(); err != nil { 69 | crash(fmt.Sprintf("Conf.Options check failed: %s", err.Error()), -4) 70 | } 71 | 72 | // read fcv and do comparison 73 | if _, err := conf.CheckFcv(*configuration, utils.FcvConfiguration.FeatureCompatibleVersion); err != nil { 74 | crash(err.Error(), -5) 75 | } 76 | 77 | utils.Welcome() 78 | utils.StartTime = fmt.Sprintf("%v", time.Now().Format(utils.GolangSecurityTime)) 79 | 80 | // write pid 81 | if err = utils.WritePidById(conf.Options.Id, "."); err != nil { 82 | crash(fmt.Sprintf("write pid failed. %v", err), -5) 83 | } 84 | 85 | // print configuration 86 | if opts, err := json.Marshal(conf.Options); err != nil { 87 | crash(fmt.Sprintf("marshal configuration failed[%v]", err), -6) 88 | } else { 89 | LOG.Info("%v configuration: %s", conf.Options.Id, string(opts)) 90 | } 91 | 92 | nimo.Profiling(int(conf.Options.SystemProfile)) 93 | nimo.RegisterSignalForProfiling(syscall.Signal(utils.SIGNALPROFILE)) // syscall.SIGUSR2 94 | nimo.RegisterSignalForPrintStack(syscall.Signal(utils.SIGNALSTACK), func(bytes []byte) { // syscall.SIGUSR1 95 | LOG.Info(string(bytes)) 96 | }) 97 | 98 | run.Start() 99 | 100 | LOG.Info("sync complete!") 101 | } 102 | 103 | func sanitizeOptions() error { 104 | if len(conf.Options.Id) == 0 { 105 | return fmt.Errorf("id[%v] shouldn't be empty", conf.Options.Id) 106 | } 107 | 108 | if conf.Options.SyncMode != utils.SyncModeAll && conf.Options.SyncMode != utils.SyncModeFull { 109 | return fmt.Errorf("sync_mode[%v] illegal, should in {all, full}", conf.Options.SyncMode) 110 | } 111 | 112 | if conf.Options.IncrSyncParallel != true { 113 | conf.Options.IncrSyncParallel = false 114 | } else { 115 | if conf.Options.SyncMode != utils.SyncModeAll { 116 | return fmt.Errorf("sync_mode must be all when incr_sync_parallel is true") 117 | } 118 | } 119 | if conf.Options.IncreasePersistDir == "" { 120 | conf.Options.IncreasePersistDir = "/tmp/" 121 | } 122 | 123 | if conf.Options.SourceAccessKeyID == "" { 124 | return fmt.Errorf("source.access_key_id shouldn't be empty") 125 | } 126 | 127 | if conf.Options.SourceSecretAccessKey == "" { 128 | return fmt.Errorf("source.secret_access_key shouldn't be empty") 129 | } 130 | 131 | if conf.Options.FilterCollectionBlack != "" && conf.Options.FilterCollectionWhite != "" { 132 | return fmt.Errorf("filter.collection.white and filter.collection.black can't both be given") 133 | } 134 | 135 | if conf.Options.QpsFull <= 0 || conf.Options.QpsIncr <= 0 { 136 | return fmt.Errorf("qps should > 0") 137 | } 138 | 139 | if conf.Options.QpsFullBatchNum <= 0 { 140 | conf.Options.QpsFullBatchNum = 128 141 | } 142 | if conf.Options.QpsIncrBatchNum <= 0 { 143 | conf.Options.QpsIncrBatchNum = 128 144 | } 145 | 146 | if conf.Options.TargetType != utils.TargetTypeMongo && conf.Options.TargetType != utils.TargetTypeAliyunDynamoProxy { 147 | return fmt.Errorf("conf.Options.TargetType[%v] supports {mongodb, aliyun_dynamo_proxy} currently", conf.Options.TargetType) 148 | } 149 | 150 | if len(conf.Options.TargetAddress) == 0 { 151 | return fmt.Errorf("target.address[%v] illegal", conf.Options.TargetAddress) 152 | } 153 | 154 | if conf.Options.FullConcurrency > 4096 || conf.Options.FullConcurrency == 0 { 155 | return fmt.Errorf("full.concurrency[%v] should in (0, 4096]", conf.Options.FullConcurrency) 156 | } 157 | 158 | if conf.Options.FullDocumentConcurrency > 4096 || conf.Options.FullDocumentConcurrency == 0 { 159 | return fmt.Errorf("full.document.concurrency[%v] should in (0, 4096]", conf.Options.FullDocumentConcurrency) 160 | } 161 | 162 | if conf.Options.FullDocumentWriteBatch <= 0 { 163 | if conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy { 164 | conf.Options.FullDocumentWriteBatch = 25 165 | } else { 166 | conf.Options.FullDocumentWriteBatch = 128 167 | } 168 | } else if conf.Options.FullDocumentWriteBatch > 25 && conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy { 169 | conf.Options.FullDocumentWriteBatch = 25 170 | } 171 | 172 | if conf.Options.FullReadConcurrency <= 0 { 173 | conf.Options.FullReadConcurrency = 1 174 | } else if conf.Options.FullReadConcurrency > 8192 { 175 | return fmt.Errorf("full.read.concurrency[%v] should in (0, 8192]", conf.Options.FullReadConcurrency) 176 | } 177 | 178 | if conf.Options.FullDocumentParser > 4096 || conf.Options.FullDocumentParser == 0 { 179 | return fmt.Errorf("full.document.parser[%v] should in (0, 4096]", conf.Options.FullDocumentParser) 180 | } 181 | 182 | // always enable 183 | conf.Options.FullEnableIndexPrimary = true 184 | 185 | if conf.Options.ConvertType == "" { 186 | conf.Options.ConvertType = utils.ConvertMTypeChange 187 | } 188 | if conf.Options.ConvertType != utils.ConvertTypeRaw && 189 | conf.Options.ConvertType != utils.ConvertTypeChange && 190 | conf.Options.ConvertType != utils.ConvertMTypeChange { 191 | return fmt.Errorf("convert.type[%v] illegal", conf.Options.ConvertType) 192 | } 193 | 194 | if conf.Options.IncreaseConcurrency == 0 { 195 | return fmt.Errorf("increase.concurrency should > 0") 196 | } 197 | 198 | if conf.Options.TargetMongoDBType != "" && conf.Options.TargetMongoDBType != utils.TargetMongoDBTypeReplica && 199 | conf.Options.TargetMongoDBType != utils.TargetMongoDBTypeSharding { 200 | return fmt.Errorf("illegal target.mongodb.type[%v]", conf.Options.TargetMongoDBType) 201 | } 202 | 203 | if conf.Options.TargetType == utils.TargetTypeMongo && conf.Options.TargetDBExist != "" && 204 | conf.Options.TargetDBExist != utils.TargetDBExistRename && 205 | conf.Options.TargetDBExist != utils.TargetDBExistDrop || 206 | conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy && conf.Options.TargetDBExist != "" && 207 | conf.Options.TargetDBExist != utils.TargetDBExistDrop { 208 | return fmt.Errorf("target.mongodb.exist[%v] should be 'drop' when target.type=%v", 209 | conf.Options.TargetDBExist, conf.Options.TargetType) 210 | } 211 | // set ConvertType 212 | if conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy { 213 | conf.Options.ConvertType = utils.ConvertTypeSame 214 | } 215 | 216 | // checkpoint 217 | if conf.Options.CheckpointType == "" { 218 | conf.Options.CheckpointType = checkpoint.CheckpointWriterTypeFile 219 | } 220 | if conf.Options.CheckpointType == checkpoint.CheckpointWriterTypeMongo && 221 | conf.Options.CheckpointAddress == "" && 222 | conf.Options.TargetType != utils.TargetTypeMongo { 223 | return fmt.Errorf("checkpoint.type should == file when checkpoint.address is empty and target.type != mongodb") 224 | } 225 | 226 | if conf.Options.CheckpointAddress == "" { 227 | if conf.Options.TargetType == utils.TargetTypeMongo { 228 | conf.Options.CheckpointAddress = conf.Options.TargetAddress 229 | } else { 230 | conf.Options.CheckpointAddress = "checkpoint" 231 | } 232 | } 233 | if conf.Options.CheckpointDb == "" { 234 | conf.Options.CheckpointDb = fmt.Sprintf("%s-%s", conf.Options.Id, "checkpoint") 235 | } 236 | 237 | if conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy && 238 | (!conf.Options.IncreaseExecutorUpsert || !conf.Options.IncreaseExecutorInsertOnDupUpdate) { 239 | return fmt.Errorf("increase.executor.upsert and increase.executor.insert_on_dup_update should be "+ 240 | "enable when target type is %v", utils.TargetTypeAliyunDynamoProxy) 241 | } 242 | 243 | if conf.Options.SourceEndpointUrl != "" && conf.Options.SyncMode != "full" { 244 | return fmt.Errorf("only support sync.mode=full when source.endpoint_url is set") 245 | } 246 | 247 | return nil 248 | } 249 | 250 | func crash(msg string, errCode int) { 251 | fmt.Println(msg) 252 | panic(Exit{errCode}) 253 | } 254 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010, Kyle Lemons . All rights reserved. 2 | 3 | package log4go 4 | 5 | import ( 6 | "encoding/xml" 7 | "fmt" 8 | "io/ioutil" 9 | "os" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | type xmlProperty struct { 15 | Name string `xml:"name,attr"` 16 | Value string `xml:",chardata"` 17 | } 18 | 19 | type xmlFilter struct { 20 | Enabled string `xml:"enabled,attr"` 21 | Tag string `xml:"tag"` 22 | Level string `xml:"level"` 23 | Type string `xml:"type"` 24 | Property []xmlProperty `xml:"property"` 25 | } 26 | 27 | type xmlLoggerConfig struct { 28 | Filter []xmlFilter `xml:"filter"` 29 | } 30 | 31 | // Load XML configuration; see examples/example.xml for documentation 32 | func (log Logger) LoadConfiguration(filename string) { 33 | log.Close() 34 | 35 | // Open the configuration file 36 | fd, err := os.Open(filename) 37 | if err != nil { 38 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Could not open %q for reading: %s\n", filename, err) 39 | os.Exit(1) 40 | } 41 | 42 | contents, err := ioutil.ReadAll(fd) 43 | if err != nil { 44 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Could not read %q: %s\n", filename, err) 45 | os.Exit(1) 46 | } 47 | 48 | xc := new(xmlLoggerConfig) 49 | if err := xml.Unmarshal(contents, xc); err != nil { 50 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Could not parse XML configuration in %q: %s\n", filename, err) 51 | os.Exit(1) 52 | } 53 | 54 | for _, xmlfilt := range xc.Filter { 55 | var filt LogWriter 56 | var lvl Level 57 | bad, good, enabled := false, true, false 58 | 59 | // Check required children 60 | if len(xmlfilt.Enabled) == 0 { 61 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required attribute %s for filter missing in %s\n", "enabled", filename) 62 | bad = true 63 | } else { 64 | enabled = xmlfilt.Enabled != "false" 65 | } 66 | if len(xmlfilt.Tag) == 0 { 67 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required child <%s> for filter missing in %s\n", "tag", filename) 68 | bad = true 69 | } 70 | if len(xmlfilt.Type) == 0 { 71 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required child <%s> for filter missing in %s\n", "type", filename) 72 | bad = true 73 | } 74 | if len(xmlfilt.Level) == 0 { 75 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required child <%s> for filter missing in %s\n", "level", filename) 76 | bad = true 77 | } 78 | 79 | switch xmlfilt.Level { 80 | case "FINEST": 81 | lvl = FINEST 82 | case "FINE": 83 | lvl = FINE 84 | case "DEBUG": 85 | lvl = DEBUG 86 | case "TRACE": 87 | lvl = TRACE 88 | case "INFO": 89 | lvl = INFO 90 | case "WARNING": 91 | lvl = WARNING 92 | case "ERROR": 93 | lvl = ERROR 94 | case "CRITICAL": 95 | lvl = CRITICAL 96 | default: 97 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required child <%s> for filter has unknown value in %s: %s\n", "level", filename, xmlfilt.Level) 98 | bad = true 99 | } 100 | 101 | // Just so all of the required attributes are errored at the same time if missing 102 | if bad { 103 | os.Exit(1) 104 | } 105 | 106 | switch xmlfilt.Type { 107 | case "console": 108 | filt, good = xmlToConsoleLogWriter(filename, xmlfilt.Property, enabled) 109 | case "file": 110 | filt, good = xmlToFileLogWriter(filename, xmlfilt.Property, enabled) 111 | case "xml": 112 | filt, good = xmlToXMLLogWriter(filename, xmlfilt.Property, enabled) 113 | case "socket": 114 | filt, good = xmlToSocketLogWriter(filename, xmlfilt.Property, enabled) 115 | default: 116 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Could not load XML configuration in %s: unknown filter type \"%s\"\n", filename, xmlfilt.Type) 117 | os.Exit(1) 118 | } 119 | 120 | // Just so all of the required params are errored at the same time if wrong 121 | if !good { 122 | os.Exit(1) 123 | } 124 | 125 | // If we're disabled (syntax and correctness checks only), don't add to logger 126 | if !enabled { 127 | continue 128 | } 129 | 130 | log[xmlfilt.Tag] = &Filter{lvl, filt} 131 | } 132 | } 133 | 134 | func xmlToConsoleLogWriter(filename string, props []xmlProperty, enabled bool) (*ConsoleLogWriter, bool) { 135 | 136 | format := "[%D %T] [%L] (%S) %M" 137 | 138 | // Parse properties 139 | for _, prop := range props { 140 | switch prop.Name { 141 | case "format": 142 | format = strings.Trim(prop.Value, " \r\n") 143 | default: 144 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Warning: Unknown property \"%s\" for console filter in %s\n", prop.Name, filename) 145 | } 146 | } 147 | 148 | // If it's disabled, we're just checking syntax 149 | if !enabled { 150 | return nil, true 151 | } 152 | 153 | clw := NewConsoleLogWriter() 154 | clw.SetFormat(format) 155 | 156 | return clw, true 157 | } 158 | 159 | // Parse a number with K/M/G suffixes based on thousands (1000) or 2^10 (1024) 160 | func strToNumSuffix(str string, mult int) int { 161 | num := 1 162 | if len(str) > 1 { 163 | switch str[len(str)-1] { 164 | case 'G', 'g': 165 | num *= mult 166 | fallthrough 167 | case 'M', 'm': 168 | num *= mult 169 | fallthrough 170 | case 'K', 'k': 171 | num *= mult 172 | str = str[0 : len(str)-1] 173 | } 174 | } 175 | parsed, _ := strconv.Atoi(str) 176 | return parsed * num 177 | } 178 | func xmlToFileLogWriter(filename string, props []xmlProperty, enabled bool) (*FileLogWriter, bool) { 179 | file := "" 180 | format := "[%D %T] [%L] (%S) %M" 181 | maxlines := 0 182 | maxsize := 0 183 | daily := false 184 | rotate := false 185 | 186 | // Parse properties 187 | for _, prop := range props { 188 | switch prop.Name { 189 | case "filename": 190 | file = strings.Trim(prop.Value, " \r\n") 191 | case "format": 192 | format = strings.Trim(prop.Value, " \r\n") 193 | case "maxlines": 194 | maxlines = strToNumSuffix(strings.Trim(prop.Value, " \r\n"), 1000) 195 | case "maxsize": 196 | maxsize = strToNumSuffix(strings.Trim(prop.Value, " \r\n"), 1024) 197 | case "daily": 198 | daily = strings.Trim(prop.Value, " \r\n") != "false" 199 | case "rotate": 200 | rotate = strings.Trim(prop.Value, " \r\n") != "false" 201 | default: 202 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Warning: Unknown property \"%s\" for file filter in %s\n", prop.Name, filename) 203 | } 204 | } 205 | 206 | // Check properties 207 | if len(file) == 0 { 208 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required property \"%s\" for file filter missing in %s\n", "filename", filename) 209 | return nil, false 210 | } 211 | 212 | // If it's disabled, we're just checking syntax 213 | if !enabled { 214 | return nil, true 215 | } 216 | 217 | flw := NewFileLogWriter(file, rotate) 218 | flw.SetFormat(format) 219 | flw.SetRotateLines(maxlines) 220 | flw.SetRotateSize(maxsize) 221 | flw.SetRotateDaily(daily) 222 | return flw, true 223 | } 224 | 225 | func xmlToXMLLogWriter(filename string, props []xmlProperty, enabled bool) (*FileLogWriter, bool) { 226 | file := "" 227 | maxrecords := 0 228 | maxsize := 0 229 | daily := false 230 | rotate := false 231 | 232 | // Parse properties 233 | for _, prop := range props { 234 | switch prop.Name { 235 | case "filename": 236 | file = strings.Trim(prop.Value, " \r\n") 237 | case "maxrecords": 238 | maxrecords = strToNumSuffix(strings.Trim(prop.Value, " \r\n"), 1000) 239 | case "maxsize": 240 | maxsize = strToNumSuffix(strings.Trim(prop.Value, " \r\n"), 1024) 241 | case "daily": 242 | daily = strings.Trim(prop.Value, " \r\n") != "false" 243 | case "rotate": 244 | rotate = strings.Trim(prop.Value, " \r\n") != "false" 245 | default: 246 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Warning: Unknown property \"%s\" for xml filter in %s\n", prop.Name, filename) 247 | } 248 | } 249 | 250 | // Check properties 251 | if len(file) == 0 { 252 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required property \"%s\" for xml filter missing in %s\n", "filename", filename) 253 | return nil, false 254 | } 255 | 256 | // If it's disabled, we're just checking syntax 257 | if !enabled { 258 | return nil, true 259 | } 260 | 261 | xlw := NewXMLLogWriter(file, rotate) 262 | xlw.SetRotateLines(maxrecords) 263 | xlw.SetRotateSize(maxsize) 264 | xlw.SetRotateDaily(daily) 265 | return xlw, true 266 | } 267 | 268 | func xmlToSocketLogWriter(filename string, props []xmlProperty, enabled bool) (SocketLogWriter, bool) { 269 | endpoint := "" 270 | protocol := "udp" 271 | 272 | // Parse properties 273 | for _, prop := range props { 274 | switch prop.Name { 275 | case "endpoint": 276 | endpoint = strings.Trim(prop.Value, " \r\n") 277 | case "protocol": 278 | protocol = strings.Trim(prop.Value, " \r\n") 279 | default: 280 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Warning: Unknown property \"%s\" for file filter in %s\n", prop.Name, filename) 281 | } 282 | } 283 | 284 | // Check properties 285 | if len(endpoint) == 0 { 286 | fmt.Fprintf(os.Stderr, "LoadConfiguration: Error: Required property \"%s\" for file filter missing in %s\n", "endpoint", filename) 287 | return nil, false 288 | } 289 | 290 | // If it's disabled, we're just checking syntax 291 | if !enabled { 292 | return nil, true 293 | } 294 | 295 | return NewSocketLogWriter(protocol, endpoint), true 296 | } 297 | -------------------------------------------------------------------------------- /nimo-shake/full-sync/table-syncer.go: -------------------------------------------------------------------------------- 1 | package full_sync 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws/awserr" 9 | "github.com/aws/aws-sdk-go/aws/request" 10 | 11 | utils "nimo-shake/common" 12 | conf "nimo-shake/configure" 13 | "nimo-shake/protocal" 14 | "nimo-shake/qps" 15 | "nimo-shake/writer" 16 | 17 | "github.com/aws/aws-sdk-go/aws" 18 | "github.com/aws/aws-sdk-go/service/dynamodb" 19 | LOG "github.com/vinllen/log4go" 20 | ) 21 | 22 | const ( 23 | fetcherChanSize = 1024 24 | parserChanSize = 81920 25 | ) 26 | 27 | type tableSyncer struct { 28 | id int 29 | ns utils.NS 30 | sourceConn *dynamodb.DynamoDB 31 | sourceTableDescribe *dynamodb.TableDescription 32 | fetcherChan chan *dynamodb.ScanOutput // chan between fetcher and parser 33 | parserChan chan interface{} // chan between parser and writer 34 | converter protocal.Converter // converter 35 | collectionMetric *utils.CollectionMetric 36 | } 37 | 38 | func NewTableSyncer(id int, table string, collectionMetric *utils.CollectionMetric) *tableSyncer { 39 | sourceConn, err := utils.CreateDynamoSession(conf.Options.LogLevel) 40 | if err != nil { 41 | LOG.Error("tableSyncer[%v] with table[%v] create dynamodb session error[%v]", id, table, err) 42 | return nil 43 | } 44 | 45 | // describe source table 46 | tableDescription, err := sourceConn.DescribeTable(&dynamodb.DescribeTableInput{ 47 | TableName: aws.String(table), 48 | }) 49 | if err != nil { 50 | LOG.Error("tableSyncer[%v] with table[%v] describe failed[%v]", id, table, err) 51 | return nil 52 | } 53 | 54 | converter := protocal.NewConverter(conf.Options.ConvertType) 55 | if converter == nil { 56 | LOG.Error("tableSyncer[%v] with table[%v] create converter failed", id, table) 57 | return nil 58 | } 59 | 60 | return &tableSyncer{ 61 | id: id, 62 | sourceConn: sourceConn, 63 | sourceTableDescribe: tableDescription.Table, 64 | converter: converter, 65 | ns: utils.NS{ 66 | Database: conf.Options.Id, 67 | Collection: table, 68 | }, 69 | collectionMetric: collectionMetric, 70 | } 71 | } 72 | 73 | func (ts *tableSyncer) String() string { 74 | return fmt.Sprintf("tableSyncer[%v] with table[%v]", ts.id, ts.ns.Collection) 75 | } 76 | 77 | func (ts *tableSyncer) Sync() { 78 | ts.fetcherChan = make(chan *dynamodb.ScanOutput, fetcherChanSize) 79 | ts.parserChan = make(chan interface{}, parserChanSize) 80 | 81 | targetWriter := writer.NewWriter(conf.Options.TargetType, conf.Options.TargetAddress, ts.ns, conf.Options.LogLevel) 82 | if targetWriter == nil { 83 | LOG.Crashf("%s create writer failed", ts) 84 | return 85 | } 86 | // create table and index with description 87 | if err := targetWriter.CreateTable(ts.sourceTableDescribe); err != nil { 88 | LOG.Crashf("%s create table failed: %v", ts, err) 89 | return 90 | } 91 | 92 | // wait dynamo proxy to sync cache 93 | time.Sleep(10 * time.Second) 94 | 95 | if conf.Options.SyncSchemaOnly { 96 | LOG.Info("sync_schema_only enabled, %s exits", ts) 97 | return 98 | } 99 | 100 | // total table item count 101 | totalCount := ts.count() 102 | 103 | ts.collectionMetric.CollectionStatus = utils.StatusProcessing 104 | ts.collectionMetric.TotalCount = totalCount 105 | 106 | // start fetcher to fetch all data from DynamoDB 107 | go ts.fetcher() 108 | 109 | // start parser to get data from fetcher and write into writer. 110 | // we can also start several parsers to accelerate 111 | var wgParser sync.WaitGroup 112 | wgParser.Add(int(conf.Options.FullDocumentParser)) 113 | for i := 0; i < int(conf.Options.FullDocumentParser); i++ { 114 | go func(id int) { 115 | ts.parser(id) 116 | wgParser.Done() 117 | }(i) 118 | } 119 | 120 | // start writer 121 | var wgWriter sync.WaitGroup 122 | wgWriter.Add(int(conf.Options.FullDocumentConcurrency)) 123 | for i := 0; i < int(conf.Options.FullDocumentConcurrency); i++ { 124 | go func(id int) { 125 | LOG.Info("%s create document syncer with id[%v]", ts, id) 126 | ds := NewDocumentSyncer(ts.id, ts.ns.Collection, id, ts.parserChan, ts.sourceTableDescribe, 127 | ts.collectionMetric) 128 | ds.Run() 129 | LOG.Info("%s document syncer with id[%v] exit", ts, id) 130 | wgWriter.Done() 131 | }(i) 132 | } 133 | 134 | LOG.Info("%s wait all parsers exiting", ts.String()) 135 | wgParser.Wait() // wait all parser exit 136 | close(ts.parserChan) // close parser channel 137 | 138 | LOG.Info("%s all parsers exited, wait all writers exiting", ts.String()) 139 | wgWriter.Wait() // wait all writer exit 140 | 141 | ts.collectionMetric.CollectionStatus = utils.StatusFinish 142 | LOG.Info("%s finish syncing table", ts.String()) 143 | } 144 | 145 | func (ts *tableSyncer) Close() { 146 | // TODO, dynamo-session doesn't have close function? 147 | } 148 | 149 | func (ts *tableSyncer) fetcher() { 150 | LOG.Info("%s start fetcher with %v reader", ts.String(), conf.Options.FullReadConcurrency) 151 | 152 | qos := qps.StartQoS(int(conf.Options.QpsFull)) 153 | defer qos.Close() 154 | 155 | var wg sync.WaitGroup 156 | wg.Add(int(conf.Options.FullReadConcurrency)) 157 | for i := 0; i < int(conf.Options.FullReadConcurrency); i++ { 158 | go func(segmentId int64) { 159 | LOG.Info("%s start reader[%v]", ts.String(), segmentId) 160 | defer LOG.Info("%s stop reader[%v]", ts.String(), segmentId) 161 | 162 | // init nil 163 | var previousKey map[string]*dynamodb.AttributeValue 164 | for { 165 | <-qos.Bucket 166 | 167 | startT := time.Now() 168 | scanInput := &dynamodb.ScanInput{ 169 | TableName: aws.String(ts.ns.Collection), 170 | TotalSegments: aws.Int64(int64(conf.Options.FullReadConcurrency)), 171 | Segment: aws.Int64(segmentId), 172 | ExclusiveStartKey: previousKey, 173 | Limit: aws.Int64(conf.Options.QpsFullBatchNum), 174 | } 175 | if len(conf.Options.FullFilterExpression) > 0 { 176 | scanInput.FilterExpression = aws.String(conf.Options.FullFilterExpression) 177 | scanInput.ExpressionAttributeValues = utils.ParseAttributes(conf.Options.FullFilterAttributeValues) 178 | } 179 | out, err := ts.sourceConn.Scan(scanInput) 180 | if err != nil { 181 | // TODO check network error and retry 182 | if aerr, ok := err.(awserr.Error); ok { 183 | 184 | switch aerr.Code() { 185 | case dynamodb.ErrCodeProvisionedThroughputExceededException: 186 | LOG.Warn("%s fetcher reader[%v] recv ProvisionedThroughputExceededException continue", 187 | ts.String(), segmentId) 188 | time.Sleep(5 * time.Second) 189 | continue 190 | 191 | case request.ErrCodeSerialization: 192 | LOG.Warn("%s fetcher reader[%v] recv SerializationError[%v] continue", 193 | ts.String(), segmentId, err) 194 | time.Sleep(5 * time.Second) 195 | continue 196 | 197 | case request.ErrCodeRequestError, request.CanceledErrorCode, 198 | request.ErrCodeResponseTimeout, request.HandlerResponseTimeout, 199 | request.WaiterResourceNotReadyErrorCode, request.ErrCodeRead: 200 | LOG.Warn("%s fetcher reader[%v] recv Error[%v] continue", 201 | ts.String(), segmentId, err) 202 | time.Sleep(5 * time.Second) 203 | continue 204 | 205 | default: 206 | LOG.Crashf("%s fetcher scan failed[%v] errcode[%v]", ts.String(), err, aerr.Code()) 207 | } 208 | } else { 209 | LOG.Crashf("%s fetcher scan failed[%v]", ts.String(), err) 210 | } 211 | } 212 | scanDuration := time.Since(startT) 213 | 214 | // LOG.Info(*out.Count) 215 | 216 | // pass result to parser 217 | startT = time.Now() 218 | ts.fetcherChan <- out 219 | writeFetcherChan := time.Since(startT) 220 | 221 | LOG.Info("%s fetcher reader[%v] ts.fetcherChan.len[%v] "+ 222 | "scanTime[%v] scanCount[%v] writeFetcherChanTime[%v]", 223 | ts.String(), segmentId, len(ts.fetcherChan), scanDuration, *out.Count, writeFetcherChan) 224 | 225 | previousKey = out.LastEvaluatedKey 226 | if previousKey == nil { 227 | // complete 228 | break 229 | } 230 | } 231 | wg.Done() 232 | }(int64(i)) 233 | } 234 | wg.Wait() 235 | 236 | LOG.Info("%s close fetcher", ts.String()) 237 | close(ts.fetcherChan) 238 | } 239 | 240 | func (ts *tableSyncer) parser(id int) { 241 | LOG.Info("%s start parser[%v]", ts.String(), id) 242 | 243 | for { 244 | startT := time.Now() 245 | data, ok := <-ts.fetcherChan 246 | if !ok { 247 | break 248 | } 249 | readFetcherChanDuration := time.Since(startT) 250 | 251 | LOG.Debug("%s parser[%v] read data[%v]", ts.String(), id, data) 252 | 253 | var parserDuration, writeParseChanDuration time.Duration = 0, 0 254 | 255 | list := data.Items 256 | for _, ele := range list { 257 | startT = time.Now() 258 | out, err := ts.converter.Run(ele) 259 | parserDuration = parserDuration + time.Since(startT) 260 | if err != nil { 261 | LOG.Crashf("%s parser[%v] parse ele[%v] failed[%v]", ts.String(), id, ele, err) 262 | } 263 | 264 | startT = time.Now() 265 | ts.parserChan <- out 266 | writeParseChanDuration = writeParseChanDuration + time.Since(startT) 267 | } 268 | 269 | LOG.Info("%s parser parser[%v] readFetcherChanTime[%v] parserTime[%v]"+ 270 | " writeParseChantime[%v] parserChan.len[%v]", 271 | ts.String(), id, readFetcherChanDuration, parserDuration, writeParseChanDuration, len(ts.parserChan)) 272 | 273 | } 274 | LOG.Info("%s close parser", ts.String()) 275 | } 276 | 277 | func (ts *tableSyncer) count() uint64 { 278 | return uint64(*ts.sourceTableDescribe.ItemCount) 279 | } 280 | -------------------------------------------------------------------------------- /nimo-shake/third_party/log4go/wrapper.go: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2010, Kyle Lemons . All rights reserved. 2 | 3 | package log4go 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "os" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | Global Logger 14 | ) 15 | 16 | func init() { 17 | Global = NewDefaultLogger(CRITICAL) 18 | } 19 | 20 | // Wrapper for (*Logger).LoadConfiguration 21 | func LoadConfiguration(filename string) { 22 | Global.LoadConfiguration(filename) 23 | } 24 | 25 | // Wrapper for (*Logger).AddFilter 26 | func AddFilter(name string, lvl Level, writer LogWriter) { 27 | Global.AddFilter(name, lvl, writer) 28 | } 29 | 30 | // Wrapper for (*Logger).Close (closes and removes all logwriters) 31 | func Close() { 32 | Global.Close() 33 | } 34 | 35 | func Crash(args ...interface{}) { 36 | if len(args) > 0 { 37 | Global.intLogf(CRITICAL, strings.Repeat(" %v", len(args))[1:], args...) 38 | } 39 | panic(args) 40 | } 41 | 42 | // Logs the given message and crashes the program 43 | func Crashf(format string, args ...interface{}) { 44 | Global.intLogf(CRITICAL, format, args...) 45 | Global.Close() // so that hopefully the messages get logged 46 | panic(fmt.Sprintf(format, args...)) 47 | } 48 | 49 | // Compatibility with `log` 50 | func Exit(args ...interface{}) { 51 | if len(args) > 0 { 52 | Global.intLogf(ERROR, strings.Repeat(" %v", len(args))[1:], args...) 53 | } 54 | Global.Close() // so that hopefully the messages get logged 55 | os.Exit(0) 56 | } 57 | 58 | // Compatibility with `log` 59 | func Exitf(format string, args ...interface{}) { 60 | Global.intLogf(ERROR, format, args...) 61 | Global.Close() // so that hopefully the messages get logged 62 | os.Exit(0) 63 | } 64 | 65 | // Compatibility with `log` 66 | func Stderr(args ...interface{}) { 67 | if len(args) > 0 { 68 | Global.intLogf(ERROR, strings.Repeat(" %v", len(args))[1:], args...) 69 | } 70 | } 71 | 72 | // Compatibility with `log` 73 | func Stderrf(format string, args ...interface{}) { 74 | Global.intLogf(ERROR, format, args...) 75 | } 76 | 77 | // Compatibility with `log` 78 | func Stdout(args ...interface{}) { 79 | if len(args) > 0 { 80 | Global.intLogf(INFO, strings.Repeat(" %v", len(args))[1:], args...) 81 | } 82 | } 83 | 84 | // Compatibility with `log` 85 | func Stdoutf(format string, args ...interface{}) { 86 | Global.intLogf(INFO, format, args...) 87 | } 88 | 89 | // Send a log message manually 90 | // Wrapper for (*Logger).Log 91 | func Log(lvl Level, source, message string) { 92 | Global.Log(lvl, source, message) 93 | } 94 | 95 | // Send a formatted log message easily 96 | // Wrapper for (*Logger).Logf 97 | func Logf(lvl Level, format string, args ...interface{}) { 98 | Global.intLogf(lvl, format, args...) 99 | } 100 | 101 | // Send a closure log message 102 | // Wrapper for (*Logger).Logc 103 | func Logc(lvl Level, closure func() string) { 104 | Global.intLogc(lvl, closure) 105 | } 106 | 107 | // Utility for finest log messages (see Debug() for parameter explanation) 108 | // Wrapper for (*Logger).Finest 109 | func Finest(arg0 interface{}, args ...interface{}) { 110 | const ( 111 | lvl = FINEST 112 | ) 113 | switch first := arg0.(type) { 114 | case string: 115 | // Use the string as a format string 116 | Global.intLogf(lvl, first, args...) 117 | case func() string: 118 | // Log the closure (no other arguments used) 119 | Global.intLogc(lvl, first) 120 | default: 121 | // Build a format string so that it will be similar to Sprint 122 | Global.intLogf(lvl, fmt.Sprint(arg0)+strings.Repeat(" %v", len(args)), args...) 123 | } 124 | } 125 | 126 | // Utility for fine log messages (see Debug() for parameter explanation) 127 | // Wrapper for (*Logger).Fine 128 | func Fine(arg0 interface{}, args ...interface{}) { 129 | const ( 130 | lvl = FINE 131 | ) 132 | switch first := arg0.(type) { 133 | case string: 134 | // Use the string as a format string 135 | Global.intLogf(lvl, first, args...) 136 | case func() string: 137 | // Log the closure (no other arguments used) 138 | Global.intLogc(lvl, first) 139 | default: 140 | // Build a format string so that it will be similar to Sprint 141 | Global.intLogf(lvl, fmt.Sprint(arg0)+strings.Repeat(" %v", len(args)), args...) 142 | } 143 | } 144 | 145 | // Utility for debug log messages 146 | // When given a string as the first argument, this behaves like Logf but with the DEBUG log level (e.g. the first argument is interpreted as a format for the latter arguments) 147 | // When given a closure of type func()string, this logs the string returned by the closure iff it will be logged. The closure runs at most one time. 148 | // When given anything else, the log message will be each of the arguments formatted with %v and separated by spaces (ala Sprint). 149 | // Wrapper for (*Logger).Debug 150 | func Debug(arg0 interface{}, args ...interface{}) { 151 | const ( 152 | lvl = DEBUG 153 | ) 154 | switch first := arg0.(type) { 155 | case string: 156 | // Use the string as a format string 157 | Global.intLogf(lvl, first, args...) 158 | case func() string: 159 | // Log the closure (no other arguments used) 160 | Global.intLogc(lvl, first) 161 | default: 162 | // Build a format string so that it will be similar to Sprint 163 | Global.intLogf(lvl, fmt.Sprint(arg0)+strings.Repeat(" %v", len(args)), args...) 164 | } 165 | } 166 | 167 | // Utility for trace log messages (see Debug() for parameter explanation) 168 | // Wrapper for (*Logger).Trace 169 | func Trace(arg0 interface{}, args ...interface{}) { 170 | const ( 171 | lvl = TRACE 172 | ) 173 | switch first := arg0.(type) { 174 | case string: 175 | // Use the string as a format string 176 | Global.intLogf(lvl, first, args...) 177 | case func() string: 178 | // Log the closure (no other arguments used) 179 | Global.intLogc(lvl, first) 180 | default: 181 | // Build a format string so that it will be similar to Sprint 182 | Global.intLogf(lvl, fmt.Sprint(arg0)+strings.Repeat(" %v", len(args)), args...) 183 | } 184 | } 185 | 186 | // Utility for info log messages (see Debug() for parameter explanation) 187 | // Wrapper for (*Logger).Info 188 | func Info(arg0 interface{}, args ...interface{}) { 189 | const ( 190 | lvl = INFO 191 | ) 192 | switch first := arg0.(type) { 193 | case string: 194 | // Use the string as a format string 195 | Global.intLogf(lvl, first, args...) 196 | case func() string: 197 | // Log the closure (no other arguments used) 198 | Global.intLogc(lvl, first) 199 | default: 200 | // Build a format string so that it will be similar to Sprint 201 | Global.intLogf(lvl, fmt.Sprint(arg0)+strings.Repeat(" %v", len(args)), args...) 202 | } 203 | } 204 | 205 | // Utility for warn log messages (returns an error for easy function returns) (see Debug() for parameter explanation) 206 | // These functions will execute a closure exactly once, to build the error message for the return 207 | // Wrapper for (*Logger).Warn 208 | func Warn(arg0 interface{}, args ...interface{}) error { 209 | const ( 210 | lvl = WARNING 211 | ) 212 | switch first := arg0.(type) { 213 | case string: 214 | // Use the string as a format string 215 | Global.intLogf(lvl, first, args...) 216 | return errors.New(fmt.Sprintf(first, args...)) 217 | case func() string: 218 | // Log the closure (no other arguments used) 219 | str := first() 220 | Global.intLogf(lvl, "%s", str) 221 | return errors.New(str) 222 | default: 223 | // Build a format string so that it will be similar to Sprint 224 | Global.intLogf(lvl, fmt.Sprint(first)+strings.Repeat(" %v", len(args)), args...) 225 | return errors.New(fmt.Sprint(first) + fmt.Sprintf(strings.Repeat(" %v", len(args)), args...)) 226 | } 227 | return nil 228 | } 229 | 230 | // Utility for error log messages (returns an error for easy function returns) (see Debug() for parameter explanation) 231 | // These functions will execute a closure exactly once, to build the error message for the return 232 | // Wrapper for (*Logger).Error 233 | func Error(arg0 interface{}, args ...interface{}) error { 234 | const ( 235 | lvl = ERROR 236 | ) 237 | switch first := arg0.(type) { 238 | case string: 239 | // Use the string as a format string 240 | Global.intLogf(lvl, first, args...) 241 | return errors.New(fmt.Sprintf(first, args...)) 242 | case func() string: 243 | // Log the closure (no other arguments used) 244 | str := first() 245 | Global.intLogf(lvl, "%s", str) 246 | return errors.New(str) 247 | default: 248 | // Build a format string so that it will be similar to Sprint 249 | Global.intLogf(lvl, fmt.Sprint(first)+strings.Repeat(" %v", len(args)), args...) 250 | return errors.New(fmt.Sprint(first) + fmt.Sprintf(strings.Repeat(" %v", len(args)), args...)) 251 | } 252 | return nil 253 | } 254 | 255 | // Utility for critical log messages (returns an error for easy function returns) (see Debug() for parameter explanation) 256 | // These functions will execute a closure exactly once, to build the error message for the return 257 | // Wrapper for (*Logger).Critical 258 | func Critical(arg0 interface{}, args ...interface{}) error { 259 | const ( 260 | lvl = CRITICAL 261 | ) 262 | switch first := arg0.(type) { 263 | case string: 264 | // Use the string as a format string 265 | Global.intLogf(lvl, first, args...) 266 | return errors.New(fmt.Sprintf(first, args...)) 267 | case func() string: 268 | // Log the closure (no other arguments used) 269 | str := first() 270 | Global.intLogf(lvl, "%s", str) 271 | return errors.New(str) 272 | default: 273 | // Build a format string so that it will be similar to Sprint 274 | Global.intLogf(lvl, fmt.Sprint(first)+strings.Repeat(" %v", len(args)), args...) 275 | return errors.New(fmt.Sprint(first) + fmt.Sprintf(strings.Repeat(" %v", len(args)), args...)) 276 | } 277 | return nil 278 | } 279 | -------------------------------------------------------------------------------- /nimo-shake/checkpoint/fileWriter.go: -------------------------------------------------------------------------------- 1 | package checkpoint 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | 7 | "bytes" 8 | "encoding/json" 9 | "fmt" 10 | LOG "github.com/vinllen/log4go" 11 | "io/ioutil" 12 | "path/filepath" 13 | "reflect" 14 | "strings" 15 | ) 16 | 17 | // marshal in json 18 | type FileWriter struct { 19 | dir string 20 | fileHandler *sync.Map // file name -> fd 21 | fileLock *sync.Map // file name -> lock 22 | } 23 | 24 | func NewFileWriter(dir string) *FileWriter { 25 | // create dir if not exist 26 | if _, err := os.Stat(dir); err != nil { 27 | if os.IsNotExist(err) { 28 | // create dir 29 | if err = os.Mkdir(dir, 0755); err != nil { 30 | LOG.Crashf("create dir[%v] failed[%v]", dir, err) 31 | return nil 32 | } 33 | } else { 34 | LOG.Crashf("stat dir[%v] failed[%v]", dir, err) 35 | return nil 36 | } 37 | } 38 | 39 | return &FileWriter{ 40 | dir: dir, 41 | fileHandler: new(sync.Map), 42 | fileLock: new(sync.Map), 43 | } 44 | } 45 | 46 | // find current status 47 | func (fw *FileWriter) FindStatus() (string, error) { 48 | // lock file 49 | fw.lockFile(CheckpointStatusTable) 50 | defer fw.unlockFile(CheckpointStatusTable) 51 | 52 | file := fmt.Sprintf("%s/%s", fw.dir, CheckpointStatusTable) 53 | if _, err := os.Stat(file); err != nil { 54 | if os.IsNotExist(err) { 55 | return CheckpointStatusValueEmpty, nil 56 | } 57 | } 58 | 59 | jsonFile, err := os.Open(file) 60 | if err != nil { 61 | return "", err 62 | } 63 | defer jsonFile.Close() 64 | 65 | byteValue, err := ioutil.ReadAll(jsonFile) 66 | if err != nil { 67 | return "", err 68 | } 69 | 70 | var ret Status 71 | if err := json.Unmarshal(byteValue, &ret); err != nil { 72 | return "", err 73 | } 74 | 75 | return ret.Value, nil 76 | } 77 | 78 | // update status 79 | func (fw *FileWriter) UpdateStatus(status string) error { 80 | // lock file 81 | fw.lockFile(CheckpointStatusTable) 82 | defer fw.unlockFile(CheckpointStatusTable) 83 | 84 | file := fmt.Sprintf("%s/%s", fw.dir, CheckpointStatusTable) 85 | input := &Status{ 86 | Key: CheckpointStatusKey, 87 | Value: status, 88 | } 89 | 90 | val, err := json.Marshal(input) 91 | if err != nil { 92 | return nil 93 | } 94 | 95 | f, err := os.Create(file) 96 | if err != nil { 97 | return err 98 | } 99 | defer f.Close() 100 | 101 | _, err = f.Write(val) 102 | return err 103 | } 104 | 105 | // extract all checkpoint 106 | func (fw *FileWriter) ExtractCheckpoint() (map[string]map[string]*Checkpoint, error) { 107 | ckptMap := make(map[string]map[string]*Checkpoint) 108 | // fileList isn't include directory 109 | var fileList []string 110 | err := filepath.Walk(fw.dir, func(path string, info os.FileInfo, err error) error { 111 | if path != fw.dir { 112 | pathList := strings.Split(path, "/") 113 | fileList = append(fileList, pathList[len(pathList)-1]) 114 | } 115 | return nil 116 | }) 117 | 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | for _, file := range fileList { 123 | if FilterCkptCollection(file) { 124 | continue 125 | } 126 | 127 | innerMap, err := fw.ExtractSingleCheckpoint(file) 128 | if err != nil { 129 | return nil, err 130 | } 131 | ckptMap[file] = innerMap 132 | } 133 | 134 | return ckptMap, nil 135 | } 136 | 137 | // extract single checkpoint 138 | func (fw *FileWriter) ExtractSingleCheckpoint(table string) (map[string]*Checkpoint, error) { 139 | fw.lockFile(table) 140 | defer fw.unlockFile(table) 141 | 142 | file := fmt.Sprintf("%s/%s", fw.dir, table) 143 | jsonFile, err := os.Open(file) 144 | if err != nil { 145 | return nil, err 146 | } 147 | defer jsonFile.Close() 148 | 149 | data, err := fw.readJsonList(jsonFile) 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | innerMap := make(map[string]*Checkpoint) 155 | for _, ele := range data { 156 | innerMap[ele.ShardId] = ele 157 | } 158 | 159 | return innerMap, nil 160 | } 161 | 162 | // insert checkpoint 163 | func (fw *FileWriter) Insert(ckpt *Checkpoint, table string) error { 164 | fw.lockFile(table) 165 | defer fw.unlockFile(table) 166 | 167 | file := fmt.Sprintf("%s/%s", fw.dir, table) 168 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 169 | if err != nil { 170 | return err 171 | } 172 | defer jsonFile.Close() 173 | 174 | LOG.Debug("file[%s] insert data: %v", file, *ckpt) 175 | 176 | return fw.writeJsonList(jsonFile, []*Checkpoint{ckpt}) 177 | } 178 | 179 | // update checkpoint 180 | func (fw *FileWriter) Update(shardId string, ckpt *Checkpoint, table string) error { 181 | fw.lockFile(table) 182 | defer fw.unlockFile(table) 183 | 184 | file := fmt.Sprintf("%s/%s", fw.dir, table) 185 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666) 186 | if err != nil { 187 | return err 188 | } 189 | defer jsonFile.Close() 190 | 191 | data, err := fw.readJsonList(jsonFile) 192 | if err != nil { 193 | return err 194 | } 195 | 196 | if len(data) == 0 { 197 | return fmt.Errorf("empty data") 198 | } 199 | 200 | match := false 201 | for i := range data { 202 | if data[i].ShardId == shardId { 203 | match = true 204 | data[i] = ckpt 205 | break 206 | } 207 | } 208 | if !match { 209 | return fmt.Errorf("shardId[%v] not exists", shardId) 210 | } 211 | 212 | // truncate file 213 | jsonFile.Truncate(0) 214 | jsonFile.Seek(0, 0) 215 | 216 | // write 217 | return fw.writeJsonList(jsonFile, data) 218 | } 219 | 220 | // update with set 221 | func (fw *FileWriter) UpdateWithSet(shardId string, input map[string]interface{}, table string) error { 222 | fw.lockFile(table) 223 | defer fw.unlockFile(table) 224 | 225 | file := fmt.Sprintf("%s/%s", fw.dir, table) 226 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666) 227 | if err != nil { 228 | return err 229 | } 230 | defer jsonFile.Close() 231 | 232 | data, err := fw.readJsonList(jsonFile) 233 | if err != nil { 234 | return err 235 | } 236 | 237 | if len(data) == 0 { 238 | return fmt.Errorf("empty data") 239 | } 240 | 241 | match := false 242 | for i := range data { 243 | if data[i].ShardId == shardId { 244 | match = true 245 | // set partial 246 | for key, val := range input { 247 | field := reflect.ValueOf(data[i]).Elem().FieldByName(key) 248 | switch field.Kind() { 249 | case reflect.String: 250 | v, _ := val.(string) 251 | field.SetString(v) 252 | case reflect.Invalid: 253 | printData, _ := json.Marshal(data[i]) 254 | return fmt.Errorf("invalid field[%v], current checkpoint[%s], input checkpoint[%v]", 255 | key, printData, input) 256 | default: 257 | printData, _ := json.Marshal(data[i]) 258 | return fmt.Errorf("unknown type[%v] of field[%v], current checkpoint[%s], input checkpoint[%v]", 259 | field.Kind(), key, printData, input) 260 | } 261 | } 262 | 263 | break 264 | } 265 | } 266 | if !match { 267 | return fmt.Errorf("shardId[%v] not exists", shardId) 268 | } 269 | 270 | // truncate file 271 | jsonFile.Truncate(0) 272 | jsonFile.Seek(0, 0) 273 | 274 | // write 275 | return fw.writeJsonList(jsonFile, data) 276 | } 277 | 278 | // query 279 | func (fw *FileWriter) Query(shardId string, table string) (*Checkpoint, error) { 280 | fw.lockFile(table) 281 | defer fw.unlockFile(table) 282 | 283 | file := fmt.Sprintf("%s/%s", fw.dir, table) 284 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666) 285 | if err != nil { 286 | return nil, err 287 | } 288 | defer jsonFile.Close() 289 | 290 | data, err := fw.readJsonList(jsonFile) 291 | if err != nil { 292 | return nil, err 293 | } 294 | 295 | for _, ele := range data { 296 | if ele.ShardId == shardId { 297 | return ele, nil 298 | } 299 | } 300 | 301 | return nil, fmt.Errorf("not found") 302 | } 303 | 304 | // drop 305 | func (fw *FileWriter) DropAll() error { 306 | var fileList []string 307 | err := filepath.Walk(fw.dir, func(path string, info os.FileInfo, err error) error { 308 | if path != fw.dir { 309 | fileList = append(fileList, path) 310 | } 311 | return nil 312 | }) 313 | 314 | if err != nil { 315 | return err 316 | } 317 | 318 | LOG.Info("drop file list: %v", fileList) 319 | 320 | for _, file := range fileList { 321 | fw.lockFile(file) 322 | if err := os.Remove(file); err != nil { 323 | fw.unlockFile(file) 324 | return err 325 | } 326 | fw.unlockFile(file) 327 | } 328 | 329 | return nil 330 | } 331 | 332 | func (fw *FileWriter) lockFile(table string) { 333 | val, ok := fw.fileLock.Load(CheckpointStatusTable) 334 | if !ok { 335 | val = new(sync.Mutex) 336 | fw.fileLock.Store(CheckpointStatusTable, val) 337 | } 338 | 339 | lock := val.(*sync.Mutex) 340 | lock.Lock() 341 | } 342 | 343 | func (fw *FileWriter) unlockFile(table string) { 344 | val, ok := fw.fileLock.Load(CheckpointStatusTable) 345 | if !ok { 346 | val = new(sync.Mutex) 347 | fw.fileLock.Store(CheckpointStatusTable, val) 348 | } 349 | 350 | lock := val.(*sync.Mutex) 351 | lock.Unlock() 352 | } 353 | 354 | func (fw *FileWriter) readJsonList(f *os.File) ([]*Checkpoint, error) { 355 | byteValue, err := ioutil.ReadAll(f) 356 | if err != nil { 357 | return nil, err 358 | } 359 | 360 | byteList := bytes.Split(byteValue, []byte{10}) 361 | ret := make([]*Checkpoint, 0, len(byteList)) 362 | for i := 0; i < len(byteList)-1; i++ { 363 | var ele Checkpoint 364 | if err := json.Unmarshal(byteList[i], &ele); err != nil { 365 | return nil, err 366 | } 367 | ret = append(ret, &ele) 368 | } 369 | 370 | return ret, nil 371 | } 372 | 373 | func (fw *FileWriter) writeJsonList(f *os.File, input []*Checkpoint) error { 374 | for _, single := range input { 375 | val, err := json.Marshal(single) 376 | if err != nil { 377 | return nil 378 | } 379 | 380 | val = append(val, byte(10)) // suffix 381 | if _, err := f.Write(val); err != nil { 382 | return err 383 | } 384 | } 385 | return nil 386 | } 387 | 388 | func (fw *FileWriter) IncrCacheFileInsert(table string, shardId string, fileName string, 389 | lastSequenceNumber string, time string) error { 390 | return nil 391 | } 392 | -------------------------------------------------------------------------------- /nimo-shake/checkpoint/writer_test.go: -------------------------------------------------------------------------------- 1 | package checkpoint 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | utils "nimo-shake/common" 10 | ) 11 | 12 | const ( 13 | TestMongoAddress = "mongodb://100.81.164.181:18901" 14 | TestCheckpointDb = "test_checkpoint_db" 15 | TestCheckpointTable = "test_checkpoint_table" 16 | ) 17 | 18 | func TestStatus(t *testing.T) { 19 | 20 | utils.InitialLogger("", "debug", false) 21 | 22 | var err error 23 | mongoWriter := NewWriter(CheckpointWriterTypeMongo, TestMongoAddress, TestCheckpointDb) 24 | assert.Equal(t, true, mongoWriter != nil, "should be equal") 25 | 26 | fileWriter := NewWriter(CheckpointWriterTypeFile, TestMongoAddress, TestCheckpointDb) 27 | assert.Equal(t, true, fileWriter != nil, "should be equal") 28 | 29 | var nr int 30 | // test status: mongo 31 | { 32 | fmt.Printf("TestStatus case %d.\n", nr) 33 | nr++ 34 | 35 | err = mongoWriter.DropAll() 36 | assert.Equal(t, nil, err, "should be equal") 37 | 38 | status, err := mongoWriter.FindStatus() 39 | assert.Equal(t, nil, err, "should be equal") 40 | assert.Equal(t, CheckpointStatusValueEmpty, status, "should be equal") 41 | 42 | err = mongoWriter.UpdateStatus(CheckpointStatusValueIncrSync) 43 | assert.Equal(t, nil, err, "should be equal") 44 | 45 | status, err = mongoWriter.FindStatus() 46 | assert.Equal(t, nil, err, "should be equal") 47 | assert.Equal(t, CheckpointStatusValueIncrSync, status, "should be equal") 48 | } 49 | 50 | // test status: file 51 | { 52 | fmt.Printf("TestStatus case %d.\n", nr) 53 | nr++ 54 | 55 | err = fileWriter.DropAll() 56 | assert.Equal(t, nil, err, "should be equal") 57 | 58 | status, err := fileWriter.FindStatus() 59 | assert.Equal(t, nil, err, "should be equal") 60 | assert.Equal(t, CheckpointStatusValueEmpty, status, "should be equal") 61 | 62 | err = fileWriter.UpdateStatus(CheckpointStatusValueIncrSync) 63 | assert.Equal(t, nil, err, "should be equal") 64 | 65 | status, err = fileWriter.FindStatus() 66 | assert.Equal(t, nil, err, "should be equal") 67 | assert.Equal(t, CheckpointStatusValueIncrSync, status, "should be equal") 68 | } 69 | } 70 | 71 | func TestCheckpointCRUD(t *testing.T) { 72 | var err error 73 | mongoWriter := NewWriter(CheckpointWriterTypeMongo, TestMongoAddress, TestCheckpointDb) 74 | assert.Equal(t, true, mongoWriter != nil, "should be equal") 75 | 76 | fileWriter := NewWriter(CheckpointWriterTypeFile, TestMongoAddress, TestCheckpointDb) 77 | assert.Equal(t, true, fileWriter != nil, "should be equal") 78 | 79 | // utils.InitialLogger("", "info", false) 80 | 81 | var nr int 82 | // test CRUD: mongo 83 | { 84 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr) 85 | nr++ 86 | 87 | err = mongoWriter.DropAll() 88 | assert.Equal(t, nil, err, "should be equal") 89 | 90 | cpkt := &Checkpoint{ 91 | ShardId: "test_id", 92 | FatherId: "test_father", 93 | Status: StatusNotProcess, 94 | } 95 | 96 | err = mongoWriter.Update("test_id", cpkt, TestCheckpointTable) 97 | assert.Equal(t, nil, err, "should be equal") 98 | 99 | err = mongoWriter.UpdateWithSet("test_id", map[string]interface{}{ 100 | "Status": StatusNotProcess, 101 | }, TestCheckpointTable) 102 | assert.Equal(t, nil, err, "should be equal") 103 | 104 | err = mongoWriter.Insert(cpkt, TestCheckpointTable) 105 | assert.Equal(t, nil, err, "should be equal") 106 | 107 | ckptRet, err := mongoWriter.Query("test_id", TestCheckpointTable) 108 | assert.Equal(t, nil, err, "should be equal") 109 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal") 110 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal") 111 | assert.Equal(t, cpkt.Status, ckptRet.Status, "should be equal") 112 | 113 | err = mongoWriter.UpdateWithSet("test_id", map[string]interface{}{ 114 | "Status": StatusInProcessing, 115 | }, TestCheckpointTable) 116 | assert.Equal(t, nil, err, "should be equal") 117 | 118 | ckptRet, err = mongoWriter.Query("test_id", TestCheckpointTable) 119 | assert.Equal(t, nil, err, "should be equal") 120 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal") 121 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal") 122 | assert.Equal(t, StatusInProcessing, ckptRet.Status, "should be equal") 123 | } 124 | 125 | // test CRUD: file 126 | { 127 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr) 128 | nr++ 129 | 130 | err = fileWriter.DropAll() 131 | assert.Equal(t, nil, err, "should be equal") 132 | 133 | cpkt := &Checkpoint{ 134 | ShardId: "test_id", 135 | FatherId: "test_father", 136 | Status: StatusNotProcess, 137 | SequenceNumber: "seq-123", 138 | } 139 | 140 | err = fileWriter.Update("test_id", cpkt, TestCheckpointTable) 141 | assert.Equal(t, true, err != nil, "should be equal") 142 | fmt.Println(err) 143 | 144 | err = fileWriter.UpdateWithSet("test_id", map[string]interface{}{ 145 | "Status": StatusNotProcess, 146 | }, TestCheckpointTable) 147 | assert.Equal(t, true, err != nil, "should be equal") 148 | fmt.Println(err) 149 | 150 | err = fileWriter.Insert(cpkt, TestCheckpointTable) 151 | assert.Equal(t, nil, err, "should be equal") 152 | 153 | ckptRet, err := fileWriter.Query("test_id", TestCheckpointTable) 154 | assert.Equal(t, nil, err, "should be equal") 155 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal") 156 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal") 157 | assert.Equal(t, cpkt.Status, ckptRet.Status, "should be equal") 158 | assert.Equal(t, cpkt.SequenceNumber, "seq-123", "should be equal") 159 | 160 | err = fileWriter.UpdateWithSet("test_id", map[string]interface{}{ 161 | "Status": StatusInProcessing, 162 | "SequenceNumber": "seq-456", 163 | }, TestCheckpointTable) 164 | assert.Equal(t, nil, err, "should be equal") 165 | 166 | ckptRet, err = fileWriter.Query("test_id", TestCheckpointTable) 167 | assert.Equal(t, nil, err, "should be equal") 168 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal") 169 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal") 170 | assert.Equal(t, StatusInProcessing, ckptRet.Status, "should be equal") 171 | assert.Equal(t, "seq-456", ckptRet.SequenceNumber, "should be equal") 172 | } 173 | } 174 | 175 | func TestExtractCheckpoint(t *testing.T) { 176 | var err error 177 | mongoWriter := NewWriter(CheckpointWriterTypeMongo, TestMongoAddress, TestCheckpointDb) 178 | assert.Equal(t, true, mongoWriter != nil, "should be equal") 179 | 180 | fileWriter := NewWriter(CheckpointWriterTypeFile, TestMongoAddress, TestCheckpointDb) 181 | assert.Equal(t, true, fileWriter != nil, "should be equal") 182 | 183 | // utils.InitialLogger("", "info", false) 184 | 185 | var nr int 186 | // test CRUD: mongo 187 | { 188 | fmt.Printf("TestExtractCheckpoint case %d.\n", nr) 189 | nr++ 190 | 191 | err = mongoWriter.DropAll() 192 | assert.Equal(t, nil, err, "should be equal") 193 | 194 | err = mongoWriter.Insert(&Checkpoint{ 195 | ShardId: "id1", 196 | Status: StatusNotProcess, 197 | }, "table1") 198 | assert.Equal(t, nil, err, "should be equal") 199 | 200 | err = mongoWriter.Insert(&Checkpoint{ 201 | ShardId: "id2", 202 | Status: StatusInProcessing, 203 | }, "table1") 204 | assert.Equal(t, nil, err, "should be equal") 205 | 206 | err = mongoWriter.Insert(&Checkpoint{ 207 | ShardId: "id3", 208 | Status: StatusPrepareProcess, 209 | }, "table1") 210 | assert.Equal(t, nil, err, "should be equal") 211 | 212 | err = mongoWriter.Insert(&Checkpoint{ 213 | ShardId: "id1", 214 | Status: StatusDone, 215 | }, "table2") 216 | assert.Equal(t, nil, err, "should be equal") 217 | 218 | err = mongoWriter.Insert(&Checkpoint{ 219 | ShardId: "id10", 220 | Status: StatusWaitFather, 221 | }, "table2") 222 | assert.Equal(t, nil, err, "should be equal") 223 | 224 | mp, err := mongoWriter.ExtractCheckpoint() 225 | assert.Equal(t, nil, err, "should be equal") 226 | assert.Equal(t, 2, len(mp), "should be equal") 227 | assert.Equal(t, 3, len(mp["table1"]), "should be equal") 228 | assert.Equal(t, 2, len(mp["table2"]), "should be equal") 229 | assert.Equal(t, StatusNotProcess, mp["table1"]["id1"].Status, "should be equal") 230 | assert.Equal(t, StatusInProcessing, mp["table1"]["id2"].Status, "should be equal") 231 | assert.Equal(t, StatusPrepareProcess, mp["table1"]["id3"].Status, "should be equal") 232 | assert.Equal(t, StatusDone, mp["table2"]["id1"].Status, "should be equal") 233 | assert.Equal(t, StatusWaitFather, mp["table2"]["id10"].Status, "should be equal") 234 | } 235 | 236 | // test CRUD: file 237 | { 238 | fmt.Printf("TestExtractCheckpoint case %d.\n", nr) 239 | nr++ 240 | 241 | err = fileWriter.DropAll() 242 | assert.Equal(t, nil, err, "should be equal") 243 | 244 | err = fileWriter.Insert(&Checkpoint{ 245 | ShardId: "id1", 246 | Status: StatusNotProcess, 247 | }, "table1") 248 | assert.Equal(t, nil, err, "should be equal") 249 | 250 | err = fileWriter.Insert(&Checkpoint{ 251 | ShardId: "id2", 252 | Status: StatusInProcessing, 253 | }, "table1") 254 | assert.Equal(t, nil, err, "should be equal") 255 | 256 | err = fileWriter.Insert(&Checkpoint{ 257 | ShardId: "id3", 258 | Status: StatusPrepareProcess, 259 | }, "table1") 260 | assert.Equal(t, nil, err, "should be equal") 261 | 262 | err = fileWriter.Insert(&Checkpoint{ 263 | ShardId: "id1", 264 | Status: StatusDone, 265 | }, "table2") 266 | assert.Equal(t, nil, err, "should be equal") 267 | 268 | err = fileWriter.Insert(&Checkpoint{ 269 | ShardId: "id10", 270 | Status: StatusWaitFather, 271 | }, "table2") 272 | assert.Equal(t, nil, err, "should be equal") 273 | 274 | mp, err := fileWriter.ExtractCheckpoint() 275 | assert.Equal(t, nil, err, "should be equal") 276 | assert.Equal(t, 2, len(mp), "should be equal") 277 | assert.Equal(t, 3, len(mp["table1"]), "should be equal") 278 | assert.Equal(t, 2, len(mp["table2"]), "should be equal") 279 | assert.Equal(t, StatusNotProcess, mp["table1"]["id1"].Status, "should be equal") 280 | assert.Equal(t, StatusInProcessing, mp["table1"]["id2"].Status, "should be equal") 281 | assert.Equal(t, StatusPrepareProcess, mp["table1"]["id3"].Status, "should be equal") 282 | assert.Equal(t, StatusDone, mp["table2"]["id1"].Status, "should be equal") 283 | assert.Equal(t, StatusWaitFather, mp["table2"]["id10"].Status, "should be equal") 284 | } 285 | } 286 | --------------------------------------------------------------------------------