├── .gitignore
├── ChangeLog
├── LICENSE
├── Makefile
├── README.md
├── build.sh
├── nimo-full-check
├── checker
│ ├── checker.go
│ ├── document-checker.go
│ └── sample.go
├── common
│ └── common.go
├── configure
│ └── conf.go
├── go.mod
├── go.sum
├── main
│ └── main.go
└── run
│ └── run.go
├── nimo-shake
├── checkpoint
│ ├── fileWriter.go
│ ├── manager.go
│ ├── mongoWriter.go
│ ├── struct.go
│ ├── writer.go
│ └── writer_test.go
├── common
│ ├── callback.go
│ ├── common.go
│ ├── dynamodb.go
│ ├── error.go
│ ├── fcv.go
│ ├── http.go
│ ├── math.go
│ ├── metric.go
│ ├── mix.go
│ ├── mongodb_community.go
│ ├── mongodb_mgo.go
│ ├── operator.go
│ ├── shard.go
│ ├── unsafe.go
│ └── utils_test.go
├── conf
│ └── nimo-shake.conf
├── configure
│ ├── check.go
│ └── conf.go
├── filter
│ ├── filter.go
│ └── filter_test.go
├── full-sync
│ ├── document-syncer.go
│ ├── document-syncer_test.go
│ ├── syncer.go
│ └── table-syncer.go
├── go.mod
├── go.sum
├── incr-sync
│ ├── fetcher.go
│ ├── syncer.go
│ └── syncer_test.go
├── main
│ └── main.go
├── protocal
│ ├── converter_test.go
│ ├── mtype_converter.go
│ ├── protocal.go
│ ├── raw_converter.go
│ ├── same_converter.go
│ └── type_converter.go
├── qps
│ └── qps.go
├── run
│ └── run.go
├── unit_test_common
│ └── include.go
└── writer
│ ├── dynamo_proxy.go
│ ├── mongodb_community_driver.go
│ ├── mongodb_mgo_driver.go
│ ├── writer.go
│ └── writer_test.go
└── scripts
├── hypervisor.c
├── run_ut_test.py
├── start.sh
└── stop.sh
/.gitignore:
--------------------------------------------------------------------------------
1 | .gopath
2 | .idea
3 | *.iml
4 | logs
5 | *.pprof
6 | *.output
7 | *.data
8 | *.sw[ap]
9 | *.yml
10 | *.pid
11 | *.tar.gz
12 | *.log
13 | tags
14 | bin/*
15 | conf/*
16 | !conf/nimo-shake.conf
17 |
18 | runtime.trace
19 |
20 | .DS_Store
21 |
22 | data
23 | nimo-shake-v*
24 |
25 | .cache/
26 | diagnostic/
27 | *.pid
28 | src/vendor/*
29 | !src/vendor/vendor.json
30 | test_checkpoint_db
31 |
--------------------------------------------------------------------------------
/ChangeLog:
--------------------------------------------------------------------------------
1 | 2021-12-07
2 | * VERSION: 1.0.13
3 | * IMPROVE: deprecated mongo.go.driver
4 | * IMPROVE: add convert._id, add a prefix to _id key name from DynamoDB
5 | * IMPROVE: add type mchange in convert.type, use function from aws-sdk-go to convert data, better performance
6 | * IMPROVE: adapt to LastEvaluatedStreamArn and LastEvaluatedShardId not nil
7 | * IMPROVE: add incr_sync_parallel to do parallel full&incr
8 |
9 | 2021-05-18
10 | * VERSION: 1.0.12
11 | * BUGFIX: parse int value empty.
12 |
13 | 2021-04-07
14 | * VERSION: 1.0.11
15 | * IMPROVE: data source support endpoint_url.
16 |
17 | 2021-03-02
18 | * VERSION: 1.0.10
19 | * IMPROVE: replace mgo driver with mongo.go.driver
20 |
21 | 2021-03-01
22 | * VERSION: 1.0.9
23 | * IMPROVE: add full.document.write.batch.
24 | * IMPROVE: enable MongoDB unordered bulk write.
25 | * IMPROVE: create range index instead of hash index when target mongodb
26 | type is replcaSet.
27 | * BUGFIX: create union index for gsi and lsi.
28 |
29 | 2021-02-25
30 | * VERSION: 1.0.8
31 | * IMPROVE: add pprof.
32 | * IMPROVE: support parallel scan.
33 |
34 | 2021-01-28
35 | * VERSION: 1.0.7
36 | * BUGFIX: some corner cases.
37 |
38 | 2021-01-25
39 | * VERSION: 1.0.6
40 | * IMPROVE: add metric for both full sync and incr sync.
41 |
42 | 2020-12-22
43 | * VERSION: 1.0.5
44 | * BUGFIX: shard iterator expired based on full sync timeout.
45 |
46 | 2020-12-18
47 | * VERSION: 1.0.4
48 | * BUGFIX: duplicate value fetched in document-syncer.
49 | * BUGFIX: primary key is not passed to writer.
50 | * IMPROVE: store N as decimal into mongodb.
51 |
52 | 2020-10-20
53 | * VERSION: 1.0.3
54 | * IMPROVE: support migrating dynamodb meta info.
55 | * IMPROVE: support migrating gsi and lsi.
56 |
57 | 2020-07-15
58 | * VERSION: 1.0.2
59 | * IMPROVE: support aliyun-dynamo-proxy
60 | * IMPROVE: add checkpoint address with type file
61 |
62 | 2019-10-15 Alibaba Cloud.
63 | * VERSION: 1.0.0
64 | * FEATURE: first release
65 |
66 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | all:
2 | ./build.sh
3 |
4 | clean:
5 | rm -rf bin
6 | rm -rf *.pprof
7 | rm -rf *.output
8 | rm -rf logs
9 | rm -rf diagnostic/
10 | rm -rf *.pid
11 |
12 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | NimoShake: sync from dynamodb to mongodb, including full sync and increase sync.
2 | NimoFullCheck: check data consistency after syncing.
3 | These two tools are developed and maintained by NoSQL Team in Alibaba-Cloud Database department.
4 |
5 | * [中文文档介绍](https://yq.aliyun.com/articles/717439)
6 | * [binary download](https://github.com/alibaba/NimoShake/releases)
7 |
8 | #Usage
9 | ---
10 | NimoShake:
11 | `./nimo-shake -conf=nimo-shake.conf`
12 | NimoFullCheck:
13 | `./nimo-full-check --sourceAccessKeyID="xxx" --sourceSecretAccessKey="xxx" --sourceRegion="us-east-2" --filterTablesWhiteList="table list" -t="target mongodb address"`
14 |
15 | # Shake series tool
16 | ---
17 | We also provide some tools for synchronization in Shake series.
18 |
19 | * [MongoShake](https://github.com/aliyun/MongoShake): mongodb data synchronization tool.
20 | * [RedisShake](https://github.com/aliyun/RedisShake): redis data synchronization tool.
21 | * [RedisFullCheck](https://github.com/aliyun/RedisFullCheck): redis data synchronization verification tool.
22 | * [NimoShake](https://github.com/alibaba/NimoShake): sync dynamodb to mongodb.
23 |
24 |
25 |
--------------------------------------------------------------------------------
/build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -o errexit
4 |
5 | # older version Git don't support --short !
6 | if [ -d ".git" ];then
7 | #branch=`git symbolic-ref --short -q HEAD`
8 | branch=$(git symbolic-ref -q HEAD | awk -F'/' '{print $3;}')
9 | cid=$(git rev-parse HEAD)
10 | else
11 | branch="unknown"
12 | cid="0.0"
13 | fi
14 | branch=$branch","$cid
15 |
16 | output=./bin/
17 | rm -rf ${output}
18 | mkdir ${output}
19 |
20 | # make sure we're in the directory where the script lives
21 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
22 | cd "$SCRIPT_DIR"
23 |
24 | GOPATH=$(pwd)
25 | export GOPATH
26 |
27 | # golang version
28 | goversion=$(go version | awk -F' ' '{print $3;}')
29 | bigVersion=$(echo $goversion | awk -F'[o.]' '{print $2}')
30 | midVersion=$(echo $goversion | awk -F'[o.]' '{print $3}')
31 | if [ $bigVersion -lt "1" -o $bigVersion -eq "1" -a $midVersion -lt "6" ]; then
32 | echo "go version[$goversion] must >= 1.6"
33 | exit 1
34 | fi
35 |
36 | t=$(date "+%Y-%m-%d_%H:%M:%S")
37 |
38 | echo "[ BUILD RELEASE ]"
39 | run_builder='go build -v'
40 |
41 | modules=(nimo-shake nimo-full-check)
42 | #modules=(nimo-shake)
43 |
44 | goos=(linux darwin)
45 | for g in "${goos[@]}"; do
46 | export GOOS=$g
47 | export GOARCH=amd64
48 | echo "try build goos=$g"
49 | for i in "${modules[@]}" ; do
50 | echo "build "$i
51 | cd $i
52 | info="$i/common.Version=$branch"
53 | info=$info","$goversion
54 | info=$info","$t
55 | $run_builder -ldflags "-X $info" -o "${output}/$i.$g" "./main/main.go"
56 | echo "build $i successfully!"
57 | cd ..
58 | cp "$i/${output}/$i.$g" ${output}/
59 | done
60 | echo "build goos=$g: all modules successfully!"
61 | done
62 |
63 | # copy scripts
64 | cp scripts/start.sh ${output}/
65 | cp scripts/stop.sh ${output}/
66 |
--------------------------------------------------------------------------------
/nimo-full-check/checker/checker.go:
--------------------------------------------------------------------------------
1 | package checker
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "sync"
7 |
8 | conf "nimo-full-check/configure"
9 | shakeUtils "nimo-shake/common"
10 | shakeFilter "nimo-shake/filter"
11 |
12 | "github.com/aws/aws-sdk-go/service/dynamodb"
13 | LOG "github.com/vinllen/log4go"
14 | "go.mongodb.org/mongo-driver/bson"
15 | )
16 |
17 | type Checker struct {
18 | dynamoSession *dynamodb.DynamoDB
19 | mongoClient *shakeUtils.MongoCommunityConn
20 | }
21 |
22 | func NewChecker(dynamoSession *dynamodb.DynamoDB, mongoClient *shakeUtils.MongoCommunityConn) *Checker {
23 | return &Checker{
24 | dynamoSession: dynamoSession,
25 | mongoClient: mongoClient,
26 | }
27 | }
28 |
29 | func (c *Checker) Run() error {
30 | // fetch all tables
31 | LOG.Info("start fetching table list")
32 | rawTableList, err := shakeUtils.FetchTableList(c.dynamoSession)
33 | if err != nil {
34 | return fmt.Errorf("fetch table list failed[%v]", err)
35 | }
36 | LOG.Info("finish fetching table list: %v", rawTableList)
37 |
38 | tableList := shakeFilter.FilterList(rawTableList)
39 | LOG.Info("filter table list: %v", tableList)
40 |
41 | // check table exist
42 | if err := c.checkTableExist(tableList); err != nil {
43 | return fmt.Errorf("check table exist failed[%v]", err)
44 | }
45 |
46 | // reset parallel if needed
47 | parallel := conf.Opts.Parallel
48 | if parallel > len(tableList) {
49 | parallel = len(tableList)
50 | }
51 |
52 | execChan := make(chan string, len(tableList))
53 | for _, table := range tableList {
54 | execChan <- table
55 | }
56 |
57 | var wg sync.WaitGroup
58 | wg.Add(len(tableList))
59 | for i := 0; i < parallel; i++ {
60 | go func(id int) {
61 | for {
62 | table, ok := <-execChan
63 | if !ok {
64 | break
65 | }
66 |
67 | LOG.Info("documentChecker[%v] starts checking table[%v]", id, table)
68 | dc := NewDocumentChecker(id, table, c.dynamoSession)
69 | dc.Run()
70 |
71 | LOG.Info("documentChecker[%v] finishes checking table[%v]", id, table)
72 | wg.Done()
73 | }
74 | }(i)
75 | }
76 | wg.Wait()
77 |
78 | LOG.Info("all documentCheckers finish")
79 | return nil
80 | }
81 |
82 | func (c *Checker) checkTableExist(tableList []string) error {
83 | collections, err := c.mongoClient.Client.Database(conf.Opts.Id).ListCollectionNames(context.TODO(), bson.M{})
84 | if err != nil {
85 | return fmt.Errorf("get target collection names error[%v]", err)
86 | }
87 |
88 | LOG.Info("all table: %v", collections)
89 |
90 | collectionsMp := shakeUtils.StringListToMap(collections)
91 | notExist := make([]string, 0)
92 | for _, table := range tableList {
93 | if _, ok := collectionsMp[table]; !ok {
94 | notExist = append(notExist, table)
95 | }
96 | }
97 |
98 | if len(notExist) != 0 {
99 | return fmt.Errorf("table not exist on the target side: %v", notExist)
100 | }
101 | return nil
102 | }
103 |
--------------------------------------------------------------------------------
/nimo-full-check/checker/document-checker.go:
--------------------------------------------------------------------------------
1 | package checker
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | conf "nimo-full-check/configure"
8 | "os"
9 | "reflect"
10 |
11 | shakeUtils "nimo-shake/common"
12 | "nimo-shake/protocal"
13 | shakeQps "nimo-shake/qps"
14 |
15 | "github.com/aws/aws-sdk-go/aws"
16 | "github.com/aws/aws-sdk-go/service/dynamodb"
17 | "github.com/google/go-cmp/cmp"
18 | LOG "github.com/vinllen/log4go"
19 | "go.mongodb.org/mongo-driver/bson"
20 | "go.mongodb.org/mongo-driver/bson/primitive"
21 | )
22 |
23 | func convertToMap(data interface{}) interface{} {
24 | switch v := data.(type) {
25 | case primitive.D:
26 | result := make(map[string]interface{})
27 | for _, elem := range v {
28 | result[elem.Key] = convertToMap(elem.Value)
29 | }
30 | return result
31 |
32 | case primitive.ObjectID:
33 | return v.Hex()
34 |
35 | case []interface{}:
36 | var newSlice []interface{}
37 | for _, item := range v {
38 | newSlice = append(newSlice, convertToMap(item))
39 | }
40 | return newSlice
41 |
42 | default:
43 | return v
44 | }
45 | }
46 |
47 | func interIsEqual(dynamoData, convertedMongo interface{}) bool {
48 |
49 | //convertedMongo := convertToMap(mongoData)
50 | if m, ok := convertedMongo.(map[string]interface{}); ok {
51 | delete(m, "_id")
52 | } else {
53 | LOG.Warn("don't have _id in mongodb document")
54 | return false
55 | }
56 |
57 | opts := cmp.Options{
58 | cmp.FilterPath(func(p cmp.Path) bool {
59 | if len(p) > 0 {
60 | switch p.Last().(type) {
61 | case cmp.MapIndex, cmp.SliceIndex:
62 | return true
63 | }
64 | }
65 | return false
66 | }, cmp.Transformer("NormalizeNumbers", func(in interface{}) interface{} {
67 | switch v := in.(type) {
68 | case int, int32, int64, float32, float64:
69 | return reflect.ValueOf(v).Convert(reflect.TypeOf(float64(0))).Float()
70 | default:
71 | return in
72 | }
73 | })),
74 | }
75 | // LOG.Warn("tmp2 %v", cmp.Diff(dynamoData, convertedMongo, opts))
76 |
77 | return cmp.Equal(dynamoData, convertedMongo, opts)
78 | }
79 |
80 | const (
81 | fetcherChanSize = 512
82 | parserChanSize = 4096
83 | )
84 |
85 | type KeyUnion struct {
86 | name string
87 | tp string
88 | union string
89 | }
90 |
91 | type DocumentChecker struct {
92 | id int
93 | ns shakeUtils.NS
94 | sourceConn *dynamodb.DynamoDB
95 | mongoClient *shakeUtils.MongoCommunityConn
96 | fetcherChan chan *dynamodb.ScanOutput // chan between fetcher and parser
97 | parserChan chan protocal.RawData // chan between parser and writer
98 | converter protocal.Converter // converter
99 | sampler *Sample // use to sample
100 | primaryKeyWithType KeyUnion
101 | sortKeyWithType KeyUnion
102 | }
103 |
104 | func NewDocumentChecker(id int, table string, dynamoSession *dynamodb.DynamoDB) *DocumentChecker {
105 | // check mongodb connection
106 | mongoClient, err := shakeUtils.NewMongoCommunityConn(conf.Opts.TargetAddress, shakeUtils.ConnectModePrimary, true)
107 | if err != nil {
108 | LOG.Crashf("documentChecker[%v] with table[%v] connect mongodb[%v] failed[%v]", id, table,
109 | conf.Opts.TargetAddress, err)
110 | }
111 |
112 | converter := protocal.NewConverter(conf.Opts.ConvertType)
113 | if converter == nil {
114 | LOG.Error("documentChecker[%v] with table[%v] create converter failed", id, table)
115 | return nil
116 | }
117 |
118 | return &DocumentChecker{
119 | id: id,
120 | sourceConn: dynamoSession,
121 | mongoClient: mongoClient,
122 | converter: converter,
123 | ns: shakeUtils.NS{
124 | Collection: table,
125 | Database: conf.Opts.Id,
126 | },
127 | }
128 | }
129 |
130 | func (dc *DocumentChecker) String() string {
131 | return fmt.Sprintf("documentChecker[%v] with table[%s]", dc.id, dc.ns)
132 | }
133 |
134 | func (dc *DocumentChecker) Run() {
135 | // check outline
136 | if err := dc.checkOutline(); err != nil {
137 | LOG.Crashf("%s check outline failed[%v]", dc.String(), err)
138 | }
139 |
140 | LOG.Info("%s check outline finish, starts checking details", dc.String())
141 |
142 | dc.fetcherChan = make(chan *dynamodb.ScanOutput, fetcherChanSize)
143 | dc.parserChan = make(chan protocal.RawData, parserChanSize)
144 |
145 | // start fetcher to fetch all data from DynamoDB
146 | go dc.fetcher()
147 |
148 | // start parser to get data from fetcher and write into exector.
149 | go dc.parser()
150 |
151 | // start executor to check
152 | dc.executor()
153 | }
154 |
155 | func (dc *DocumentChecker) fetcher() {
156 | LOG.Info("%s start fetcher", dc.String())
157 |
158 | qos := shakeQps.StartQoS(int(conf.Opts.QpsFull))
159 | defer qos.Close()
160 |
161 | // init nil
162 | var previousKey map[string]*dynamodb.AttributeValue
163 | for {
164 | <-qos.Bucket
165 |
166 | out, err := dc.sourceConn.Scan(&dynamodb.ScanInput{
167 | TableName: aws.String(dc.ns.Collection),
168 | ExclusiveStartKey: previousKey,
169 | Limit: aws.Int64(conf.Opts.QpsFullBatchNum),
170 | })
171 | if err != nil {
172 | // TODO check network error and retry
173 | LOG.Crashf("%s fetcher scan failed[%v]", dc.String(), err)
174 | }
175 |
176 | // pass result to parser
177 | dc.fetcherChan <- out
178 |
179 | previousKey = out.LastEvaluatedKey
180 | if previousKey == nil {
181 | // complete
182 | break
183 | }
184 | }
185 |
186 | LOG.Info("%s close fetcher", dc.String())
187 | close(dc.fetcherChan)
188 | }
189 |
190 | func (dc *DocumentChecker) parser() {
191 | LOG.Info("%s start parser", dc.String())
192 |
193 | for {
194 | data, ok := <-dc.fetcherChan
195 | if !ok {
196 | break
197 | }
198 |
199 | LOG.Debug("%s reads data[%v]", dc.String(), data)
200 |
201 | list := data.Items
202 | for _, ele := range list {
203 | out, err := dc.converter.Run(ele)
204 | if err != nil {
205 | LOG.Crashf("%s parses ele[%v] failed[%v]", dc.String(), ele, err)
206 | }
207 |
208 | // sample
209 | if dc.sampler.Hit() == false {
210 | continue
211 | }
212 |
213 | dc.parserChan <- out.(protocal.RawData)
214 | }
215 | }
216 |
217 | LOG.Info("%s close parser", dc.String())
218 | close(dc.parserChan)
219 | }
220 |
221 | func (dc *DocumentChecker) executor() {
222 | LOG.Info("%s start executor", dc.String())
223 |
224 | diffFile := fmt.Sprintf("%s/%s", conf.Opts.DiffOutputFile, dc.ns.Collection)
225 | f, err := os.Create(diffFile)
226 | if err != nil {
227 | LOG.Crashf("%s create diff output file[%v] failed", dc.String(), diffFile)
228 | return
229 | }
230 |
231 | for {
232 | data, ok := <-dc.parserChan
233 | if !ok {
234 | break
235 | }
236 |
237 | //var query map[string]interface{}
238 | query := make(map[string]interface{})
239 | if dc.primaryKeyWithType.name != "" {
240 | // find by union key
241 | if conf.Opts.ConvertType == shakeUtils.ConvertMTypeChange {
242 | query[dc.primaryKeyWithType.name] = data.Data.(map[string]interface{})[dc.primaryKeyWithType.name]
243 | } else {
244 | LOG.Crashf("unknown convert type[%v]", conf.Opts.ConvertType)
245 | }
246 | }
247 | if dc.sortKeyWithType.name != "" {
248 | if conf.Opts.ConvertType == shakeUtils.ConvertMTypeChange {
249 | query[dc.sortKeyWithType.name] = data.Data.(map[string]interface{})[dc.sortKeyWithType.name]
250 | } else {
251 | LOG.Crashf("unknown convert type[%v]", conf.Opts.ConvertType)
252 | }
253 | }
254 |
255 | LOG.Info("query: %v", query)
256 |
257 | // query
258 | var output, outputMap interface{}
259 | isSame := true
260 | err := dc.mongoClient.Client.Database(dc.ns.Database).Collection(dc.ns.Collection).
261 | FindOne(context.TODO(), query).Decode(&output)
262 | if err != nil {
263 | err = fmt.Errorf("target query failed[%v][%v][%v]", err, output, query)
264 | LOG.Error("%s %v", dc.String(), err)
265 | } else {
266 | outputMap = convertToMap(output)
267 | isSame = interIsEqual(data.Data, outputMap)
268 | }
269 |
270 | inputJson, _ := json.Marshal(data.Data)
271 | outputJson, _ := json.Marshal(outputMap)
272 | if err != nil {
273 | f.WriteString(fmt.Sprintf("compare src[%s] to dst[%s] failed: %v\n", inputJson, outputJson, err))
274 | } else if isSame == false {
275 | LOG.Warn("compare src[%s] and dst[%s] failed", inputJson, outputJson)
276 | f.WriteString(fmt.Sprintf("src[%s] != dst[%s]\n", inputJson, outputJson))
277 | }
278 | }
279 |
280 | LOG.Info("%s close executor", dc.String())
281 | f.Close()
282 |
283 | // remove file if size == 0
284 | if fi, err := os.Stat(diffFile); err != nil {
285 | LOG.Warn("stat diffFile[%v] failed[%v]", diffFile, err)
286 | return
287 | } else if fi.Size() == 0 {
288 | if err := os.Remove(diffFile); err != nil {
289 | LOG.Warn("remove diffFile[%v] failed[%v]", diffFile, err)
290 | }
291 | }
292 | }
293 |
294 | func (dc *DocumentChecker) checkOutline() error {
295 | // describe dynamodb table
296 | out, err := dc.sourceConn.DescribeTable(&dynamodb.DescribeTableInput{
297 | TableName: aws.String(dc.ns.Collection),
298 | })
299 | if err != nil {
300 | return fmt.Errorf("describe table failed[%v]", err)
301 | }
302 |
303 | LOG.Info("describe table[%v] result: %v", dc.ns.Collection, out)
304 |
305 | // 1. check total number
306 | // dynamo count
307 | dynamoCount := out.Table.ItemCount
308 |
309 | // mongo count
310 | cnt, err := dc.mongoClient.Client.Database(dc.ns.Database).Collection(dc.ns.Collection).CountDocuments(context.Background(), bson.M{})
311 | if err != nil {
312 | return fmt.Errorf("get mongo count failed[%v]", err)
313 | }
314 |
315 | if *dynamoCount != cnt {
316 | // return fmt.Errorf("dynamo count[%v] != mongo count[%v]", *dynamoCount, cnt)
317 | LOG.Warn("dynamo count[%v] != mongo count[%v]", *dynamoCount, cnt)
318 | }
319 |
320 | // set sampler
321 | dc.sampler = NewSample(conf.Opts.Sample, cnt)
322 |
323 | // 2. check index
324 | // TODO
325 |
326 | // parse index
327 | // parse primary key with sort key
328 | allIndexes := out.Table.AttributeDefinitions
329 | primaryIndexes := out.Table.KeySchema
330 |
331 | // parse index type
332 | parseMap := shakeUtils.ParseIndexType(allIndexes)
333 | primaryKey, sortKey, err := shakeUtils.ParsePrimaryAndSortKey(primaryIndexes, parseMap)
334 | if err != nil {
335 | return fmt.Errorf("parse primary and sort key failed[%v]", err)
336 | }
337 | dc.primaryKeyWithType = KeyUnion{
338 | name: primaryKey,
339 | tp: parseMap[primaryKey],
340 | union: fmt.Sprintf("%s.%s", primaryKey, parseMap[primaryKey]),
341 | }
342 | dc.sortKeyWithType = KeyUnion{
343 | name: sortKey,
344 | tp: parseMap[sortKey],
345 | union: fmt.Sprintf("%s.%s", sortKey, parseMap[sortKey]),
346 | }
347 |
348 | return nil
349 | }
350 |
--------------------------------------------------------------------------------
/nimo-full-check/checker/sample.go:
--------------------------------------------------------------------------------
1 | package checker
2 |
3 | import "math/rand"
4 |
5 | const (
6 | SEED = 1
7 | )
8 |
9 | type Sample struct {
10 | sampleCnt int64
11 | totalCnt int64
12 | source *rand.Rand
13 | }
14 |
15 | func NewSample(sampleCnt, totalCnt int64) *Sample {
16 | return &Sample{
17 | sampleCnt: sampleCnt,
18 | totalCnt: totalCnt,
19 | source: rand.New(rand.NewSource(SEED)),
20 | }
21 | }
22 |
23 | func (s *Sample) Hit() bool {
24 | if s.sampleCnt >= s.totalCnt {
25 | return true
26 | }
27 | if s.sampleCnt == 0 {
28 | return false
29 | }
30 |
31 | return s.source.Int63n(s.totalCnt) < s.sampleCnt
32 | }
33 |
--------------------------------------------------------------------------------
/nimo-full-check/common/common.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | var (
4 | Version = "$"
5 | )
--------------------------------------------------------------------------------
/nimo-full-check/configure/conf.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | var Opts struct {
4 | Id string `short:"i" long:"id" default:"nimo-shake" description:"target database collection name"`
5 | LogLevel string `short:"l" long:"logLevel" default:"info"`
6 | SourceAccessKeyID string `short:"s" long:"sourceAccessKeyID" description:"dynamodb source access key id"`
7 | SourceSecretAccessKey string `long:"sourceSecretAccessKey" description:"dynamodb source secret access key"`
8 | SourceSessionToken string `long:"sourceSessionToken" default:"" description:"dynamodb source session token"`
9 | SourceRegion string `long:"sourceRegion" default:"" description:"dynamodb source region"`
10 | SourceEndpointUrl string `long:"sourceEndpointUrl" default:"" description:"dynamodb source endpoint_url"`
11 | QpsFull int `long:"qpsFull" default:"10000" description:"qps of scan command, default is 10000"`
12 | QpsFullBatchNum int64 `long:"qpsFullBatchNum" default:"128" description:"batch number in each scan command, default is 128"`
13 | TargetAddress string `short:"t" long:"targetAddress" description:"mongodb target address"`
14 | DiffOutputFile string `short:"d" long:"diffOutputFile" default:"nimo-full-check-diff" description:"diff output file name"`
15 | Parallel int `short:"p" long:"parallel" default:"16" description:"how many threads used to compare, default is 16"`
16 | Sample int64 `short:"e" long:"sample" default:"1000" description:"comparison sample number for each table, 0 means disable"`
17 | //IndexPrimary bool `short:"m" long:"indexPrimary" description:"enable compare primary index"`
18 | //IndexUser bool `long:"indexUser" description:"enable compare user index"`
19 | FilterCollectionWhite string `long:"filterCollectionWhite" default:"" description:"only compare the given tables, split by ';'"`
20 | FilterCollectionBlack string `long:"filterCollectionBlack" default:"" description:"do not compare the given tables, split by ';'"`
21 | ConvertType string `short:"c" long:"convertType" default:"change" description:"convert type"`
22 | Version bool `short:"v" long:"version" description:"print version"`
23 | }
24 |
--------------------------------------------------------------------------------
/nimo-full-check/go.mod:
--------------------------------------------------------------------------------
1 | module nimo-full-check
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/aws/aws-sdk-go v1.44.61
7 | github.com/google/go-cmp v0.6.0
8 | github.com/jessevdk/go-flags v1.5.0 // indirect
9 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d
10 | go.mongodb.org/mongo-driver v1.16.1
11 | nimo-shake v0.0.0-00010101000000-000000000000
12 | )
13 |
14 | replace nimo-shake => ../nimo-shake
15 |
--------------------------------------------------------------------------------
/nimo-full-check/main/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "time"
7 |
8 | utils "nimo-full-check/common"
9 | conf "nimo-full-check/configure"
10 | "nimo-full-check/run"
11 | shakeUtils "nimo-shake/common"
12 | shakeFilter "nimo-shake/filter"
13 |
14 | "github.com/jessevdk/go-flags"
15 | LOG "github.com/vinllen/log4go"
16 | )
17 |
18 | type Exit struct{ Code int }
19 |
20 | func main() {
21 | defer LOG.Close()
22 |
23 | // parse conf.Opts
24 | args, err := flags.Parse(&conf.Opts)
25 |
26 | if conf.Opts.Version {
27 | fmt.Println(utils.Version)
28 | os.Exit(0)
29 | }
30 |
31 | // 若err != nil, 会自动打印错误到 stderr
32 | if err != nil {
33 | if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
34 | os.Exit(0)
35 | } else {
36 | fmt.Fprintf(os.Stderr, "flag err %s\n", flagsErr)
37 | os.Exit(1)
38 | }
39 | }
40 |
41 | if len(args) != 0 {
42 | fmt.Fprintf(os.Stderr, "unexpected args %+v", args)
43 | os.Exit(1)
44 | }
45 |
46 | shakeUtils.InitialLogger("", conf.Opts.LogLevel, false)
47 |
48 | if err := sanitizeOptions(); err != nil {
49 | crash(fmt.Sprintf("Conf.Options check failed: %s", err.Error()), -4)
50 | }
51 |
52 | LOG.Info("full-check starts")
53 | run.Start()
54 | LOG.Info("full-check completes!")
55 | }
56 |
57 | func sanitizeOptions() error {
58 | if len(conf.Opts.SourceAccessKeyID) == 0 || len(conf.Opts.SourceSecretAccessKey) == 0 {
59 | return fmt.Errorf("sourceAccessKeyID and sourceSecretAccessKey can't be empty")
60 | }
61 |
62 | if len(conf.Opts.TargetAddress) == 0 {
63 | return fmt.Errorf("targetAddress can't be empty")
64 | }
65 |
66 | if conf.Opts.Parallel <= 0 {
67 | return fmt.Errorf("parallel should >= 1, default is 16")
68 | }
69 |
70 | if conf.Opts.QpsFull == 0 {
71 | conf.Opts.QpsFull = 10000
72 | } else if conf.Opts.QpsFull < 0 {
73 | return fmt.Errorf("qps.full should > 0, default is 10000")
74 | }
75 |
76 | if conf.Opts.QpsFullBatchNum <= 0 {
77 | conf.Opts.QpsFullBatchNum = 128
78 | }
79 |
80 | shakeFilter.Init(conf.Opts.FilterCollectionWhite, conf.Opts.FilterCollectionBlack)
81 |
82 | if len(conf.Opts.DiffOutputFile) == 0 {
83 | return fmt.Errorf("diff output file shouldn't be empty")
84 | } else {
85 | _, err := os.Stat(conf.Opts.DiffOutputFile)
86 | if os.IsNotExist(err) {
87 | if err = os.Mkdir(conf.Opts.DiffOutputFile, os.ModePerm); err != nil {
88 | return fmt.Errorf("mkdir diffOutputFile[%v] failed[%v]", conf.Opts.DiffOutputFile, err)
89 | }
90 | } else {
91 | newName := fmt.Sprintf("%v-%v", conf.Opts.DiffOutputFile, time.Now().Format(shakeUtils.GolangSecurityTime))
92 | if err := os.Rename(conf.Opts.DiffOutputFile, newName); err != nil {
93 | return fmt.Errorf("diffOutputFile dir[%v] rename to newFile[%v] failed[%v], need to be remvoed manullay",
94 | conf.Opts.DiffOutputFile, newName, err)
95 | }
96 | if err = os.Mkdir(conf.Opts.DiffOutputFile, os.ModePerm); err != nil {
97 | return fmt.Errorf("mkdir diffOutputFile[%v] again failed[%v]", conf.Opts.DiffOutputFile, err)
98 | }
99 | }
100 | }
101 |
102 | if conf.Opts.ConvertType == "" {
103 | conf.Opts.ConvertType = shakeUtils.ConvertMTypeChange
104 | } else if conf.Opts.ConvertType != shakeUtils.ConvertMTypeChange {
105 | return fmt.Errorf("convertType[%v] illegal, only support %v",
106 | conf.Opts.ConvertType, shakeUtils.ConvertMTypeChange)
107 | }
108 |
109 | return nil
110 | }
111 |
112 | func crash(msg string, errCode int) {
113 | fmt.Println(msg)
114 | panic(Exit{errCode})
115 | }
116 |
--------------------------------------------------------------------------------
/nimo-full-check/run/run.go:
--------------------------------------------------------------------------------
1 | package run
2 |
3 | import (
4 | "nimo-full-check/checker"
5 | conf "nimo-full-check/configure"
6 | shakeUtils "nimo-shake/common"
7 |
8 | LOG "github.com/vinllen/log4go"
9 | )
10 |
11 | func Start() {
12 | if err := shakeUtils.InitSession(conf.Opts.SourceAccessKeyID, conf.Opts.SourceSecretAccessKey,
13 | conf.Opts.SourceSessionToken, conf.Opts.SourceRegion, conf.Opts.SourceEndpointUrl,
14 | 3, 5000); err != nil {
15 | LOG.Crashf("init global session failed[%v]", err)
16 | }
17 |
18 | // create dynamo session
19 | dynamoSession, err := shakeUtils.CreateDynamoSession("info")
20 | if err != nil {
21 | LOG.Crashf("create dynamodb session failed[%v]", err)
22 | }
23 |
24 | // check mongodb connection
25 | mongoClient, err := shakeUtils.NewMongoCommunityConn(conf.Opts.TargetAddress, shakeUtils.ConnectModePrimary, true)
26 | if err != nil {
27 | LOG.Crashf("connect mongodb[%v] failed[%v]", conf.Opts.TargetAddress, err)
28 | }
29 |
30 | c := checker.NewChecker(dynamoSession, mongoClient)
31 | if c == nil {
32 | LOG.Crashf("create checker failed")
33 | }
34 |
35 | if err := c.Run(); err != nil {
36 | LOG.Crashf("checker runs failed[%v]", err)
37 | }
38 |
39 | LOG.Info("checker finishes!")
40 | }
41 |
--------------------------------------------------------------------------------
/nimo-shake/checkpoint/fileWriter.go:
--------------------------------------------------------------------------------
1 | package checkpoint
2 |
3 | import (
4 | "os"
5 | "sync"
6 |
7 | "bytes"
8 | "encoding/json"
9 | "fmt"
10 | LOG "github.com/vinllen/log4go"
11 | "io/ioutil"
12 | "path/filepath"
13 | "reflect"
14 | "strings"
15 | )
16 |
17 | // marshal in json
18 | type FileWriter struct {
19 | dir string
20 | fileHandler *sync.Map // file name -> fd
21 | fileLock *sync.Map // file name -> lock
22 | }
23 |
24 | func NewFileWriter(dir string) *FileWriter {
25 | // create dir if not exist
26 | if _, err := os.Stat(dir); err != nil {
27 | if os.IsNotExist(err) {
28 | // create dir
29 | if err = os.Mkdir(dir, 0755); err != nil {
30 | LOG.Crashf("create dir[%v] failed[%v]", dir, err)
31 | return nil
32 | }
33 | } else {
34 | LOG.Crashf("stat dir[%v] failed[%v]", dir, err)
35 | return nil
36 | }
37 | }
38 |
39 | return &FileWriter{
40 | dir: dir,
41 | fileHandler: new(sync.Map),
42 | fileLock: new(sync.Map),
43 | }
44 | }
45 |
46 | // find current status
47 | func (fw *FileWriter) FindStatus() (string, error) {
48 | // lock file
49 | fw.lockFile(CheckpointStatusTable)
50 | defer fw.unlockFile(CheckpointStatusTable)
51 |
52 | file := fmt.Sprintf("%s/%s", fw.dir, CheckpointStatusTable)
53 | if _, err := os.Stat(file); err != nil {
54 | if os.IsNotExist(err) {
55 | return CheckpointStatusValueEmpty, nil
56 | }
57 | }
58 |
59 | jsonFile, err := os.Open(file)
60 | if err != nil {
61 | return "", err
62 | }
63 | defer jsonFile.Close()
64 |
65 | byteValue, err := ioutil.ReadAll(jsonFile)
66 | if err != nil {
67 | return "", err
68 | }
69 |
70 | var ret Status
71 | if err := json.Unmarshal(byteValue, &ret); err != nil {
72 | return "", err
73 | }
74 |
75 | return ret.Value, nil
76 | }
77 |
78 | // update status
79 | func (fw *FileWriter) UpdateStatus(status string) error {
80 | // lock file
81 | fw.lockFile(CheckpointStatusTable)
82 | defer fw.unlockFile(CheckpointStatusTable)
83 |
84 | file := fmt.Sprintf("%s/%s", fw.dir, CheckpointStatusTable)
85 | input := &Status{
86 | Key: CheckpointStatusKey,
87 | Value: status,
88 | }
89 |
90 | val, err := json.Marshal(input)
91 | if err != nil {
92 | return nil
93 | }
94 |
95 | f, err := os.Create(file)
96 | if err != nil {
97 | return err
98 | }
99 | defer f.Close()
100 |
101 | _, err = f.Write(val)
102 | return err
103 | }
104 |
105 | // extract all checkpoint
106 | func (fw *FileWriter) ExtractCheckpoint() (map[string]map[string]*Checkpoint, error) {
107 | ckptMap := make(map[string]map[string]*Checkpoint)
108 | // fileList isn't include directory
109 | var fileList []string
110 | err := filepath.Walk(fw.dir, func(path string, info os.FileInfo, err error) error {
111 | if path != fw.dir {
112 | pathList := strings.Split(path, "/")
113 | fileList = append(fileList, pathList[len(pathList)-1])
114 | }
115 | return nil
116 | })
117 |
118 | if err != nil {
119 | return nil, err
120 | }
121 |
122 | for _, file := range fileList {
123 | if FilterCkptCollection(file) {
124 | continue
125 | }
126 |
127 | innerMap, err := fw.ExtractSingleCheckpoint(file)
128 | if err != nil {
129 | return nil, err
130 | }
131 | ckptMap[file] = innerMap
132 | }
133 |
134 | return ckptMap, nil
135 | }
136 |
137 | // extract single checkpoint
138 | func (fw *FileWriter) ExtractSingleCheckpoint(table string) (map[string]*Checkpoint, error) {
139 | fw.lockFile(table)
140 | defer fw.unlockFile(table)
141 |
142 | file := fmt.Sprintf("%s/%s", fw.dir, table)
143 | jsonFile, err := os.Open(file)
144 | if err != nil {
145 | return nil, err
146 | }
147 | defer jsonFile.Close()
148 |
149 | data, err := fw.readJsonList(jsonFile)
150 | if err != nil {
151 | return nil, err
152 | }
153 |
154 | innerMap := make(map[string]*Checkpoint)
155 | for _, ele := range data {
156 | innerMap[ele.ShardId] = ele
157 | }
158 |
159 | return innerMap, nil
160 | }
161 |
162 | // insert checkpoint
163 | func (fw *FileWriter) Insert(ckpt *Checkpoint, table string) error {
164 | fw.lockFile(table)
165 | defer fw.unlockFile(table)
166 |
167 | file := fmt.Sprintf("%s/%s", fw.dir, table)
168 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
169 | if err != nil {
170 | return err
171 | }
172 | defer jsonFile.Close()
173 |
174 | LOG.Debug("file[%s] insert data: %v", file, *ckpt)
175 |
176 | return fw.writeJsonList(jsonFile, []*Checkpoint{ckpt})
177 | }
178 |
179 | // update checkpoint
180 | func (fw *FileWriter) Update(shardId string, ckpt *Checkpoint, table string) error {
181 | fw.lockFile(table)
182 | defer fw.unlockFile(table)
183 |
184 | file := fmt.Sprintf("%s/%s", fw.dir, table)
185 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666)
186 | if err != nil {
187 | return err
188 | }
189 | defer jsonFile.Close()
190 |
191 | data, err := fw.readJsonList(jsonFile)
192 | if err != nil {
193 | return err
194 | }
195 |
196 | if len(data) == 0 {
197 | return fmt.Errorf("empty data")
198 | }
199 |
200 | match := false
201 | for i := range data {
202 | if data[i].ShardId == shardId {
203 | match = true
204 | data[i] = ckpt
205 | break
206 | }
207 | }
208 | if !match {
209 | return fmt.Errorf("shardId[%v] not exists", shardId)
210 | }
211 |
212 | // truncate file
213 | jsonFile.Truncate(0)
214 | jsonFile.Seek(0, 0)
215 |
216 | // write
217 | return fw.writeJsonList(jsonFile, data)
218 | }
219 |
220 | // update with set
221 | func (fw *FileWriter) UpdateWithSet(shardId string, input map[string]interface{}, table string) error {
222 | fw.lockFile(table)
223 | defer fw.unlockFile(table)
224 |
225 | file := fmt.Sprintf("%s/%s", fw.dir, table)
226 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666)
227 | if err != nil {
228 | return err
229 | }
230 | defer jsonFile.Close()
231 |
232 | data, err := fw.readJsonList(jsonFile)
233 | if err != nil {
234 | return err
235 | }
236 |
237 | if len(data) == 0 {
238 | return fmt.Errorf("empty data")
239 | }
240 |
241 | match := false
242 | for i := range data {
243 | if data[i].ShardId == shardId {
244 | match = true
245 | // set partial
246 | for key, val := range input {
247 | field := reflect.ValueOf(data[i]).Elem().FieldByName(key)
248 | switch field.Kind() {
249 | case reflect.String:
250 | v, _ := val.(string)
251 | field.SetString(v)
252 | case reflect.Invalid:
253 | printData, _ := json.Marshal(data[i])
254 | return fmt.Errorf("invalid field[%v], current checkpoint[%s], input checkpoint[%v]",
255 | key, printData, input)
256 | default:
257 | printData, _ := json.Marshal(data[i])
258 | return fmt.Errorf("unknown type[%v] of field[%v], current checkpoint[%s], input checkpoint[%v]",
259 | field.Kind(), key, printData, input)
260 | }
261 | }
262 |
263 | break
264 | }
265 | }
266 | if !match {
267 | return fmt.Errorf("shardId[%v] not exists", shardId)
268 | }
269 |
270 | // truncate file
271 | jsonFile.Truncate(0)
272 | jsonFile.Seek(0, 0)
273 |
274 | // write
275 | return fw.writeJsonList(jsonFile, data)
276 | }
277 |
278 | // query
279 | func (fw *FileWriter) Query(shardId string, table string) (*Checkpoint, error) {
280 | fw.lockFile(table)
281 | defer fw.unlockFile(table)
282 |
283 | file := fmt.Sprintf("%s/%s", fw.dir, table)
284 | jsonFile, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666)
285 | if err != nil {
286 | return nil, err
287 | }
288 | defer jsonFile.Close()
289 |
290 | data, err := fw.readJsonList(jsonFile)
291 | if err != nil {
292 | return nil, err
293 | }
294 |
295 | for _, ele := range data {
296 | if ele.ShardId == shardId {
297 | return ele, nil
298 | }
299 | }
300 |
301 | return nil, fmt.Errorf("not found")
302 | }
303 |
304 | // drop
305 | func (fw *FileWriter) DropAll() error {
306 | var fileList []string
307 | err := filepath.Walk(fw.dir, func(path string, info os.FileInfo, err error) error {
308 | if path != fw.dir {
309 | fileList = append(fileList, path)
310 | }
311 | return nil
312 | })
313 |
314 | if err != nil {
315 | return err
316 | }
317 |
318 | LOG.Info("drop file list: %v", fileList)
319 |
320 | for _, file := range fileList {
321 | fw.lockFile(file)
322 | if err := os.Remove(file); err != nil {
323 | fw.unlockFile(file)
324 | return err
325 | }
326 | fw.unlockFile(file)
327 | }
328 |
329 | return nil
330 | }
331 |
332 | func (fw *FileWriter) lockFile(table string) {
333 | val, ok := fw.fileLock.Load(CheckpointStatusTable)
334 | if !ok {
335 | val = new(sync.Mutex)
336 | fw.fileLock.Store(CheckpointStatusTable, val)
337 | }
338 |
339 | lock := val.(*sync.Mutex)
340 | lock.Lock()
341 | }
342 |
343 | func (fw *FileWriter) unlockFile(table string) {
344 | val, ok := fw.fileLock.Load(CheckpointStatusTable)
345 | if !ok {
346 | val = new(sync.Mutex)
347 | fw.fileLock.Store(CheckpointStatusTable, val)
348 | }
349 |
350 | lock := val.(*sync.Mutex)
351 | lock.Unlock()
352 | }
353 |
354 | func (fw *FileWriter) readJsonList(f *os.File) ([]*Checkpoint, error) {
355 | byteValue, err := ioutil.ReadAll(f)
356 | if err != nil {
357 | return nil, err
358 | }
359 |
360 | byteList := bytes.Split(byteValue, []byte{10})
361 | ret := make([]*Checkpoint, 0, len(byteList))
362 | for i := 0; i < len(byteList)-1; i++ {
363 | var ele Checkpoint
364 | if err := json.Unmarshal(byteList[i], &ele); err != nil {
365 | return nil, err
366 | }
367 | ret = append(ret, &ele)
368 | }
369 |
370 | return ret, nil
371 | }
372 |
373 | func (fw *FileWriter) writeJsonList(f *os.File, input []*Checkpoint) error {
374 | for _, single := range input {
375 | val, err := json.Marshal(single)
376 | if err != nil {
377 | return nil
378 | }
379 |
380 | val = append(val, byte(10)) // suffix
381 | if _, err := f.Write(val); err != nil {
382 | return err
383 | }
384 | }
385 | return nil
386 | }
387 |
388 | func (fw *FileWriter) IncrCacheFileInsert(table string, shardId string, fileName string,
389 | lastSequenceNumber string, time string) error {
390 | return nil
391 | }
392 |
--------------------------------------------------------------------------------
/nimo-shake/checkpoint/mongoWriter.go:
--------------------------------------------------------------------------------
1 | package checkpoint
2 |
3 | import (
4 | "context"
5 | "fmt"
6 |
7 | utils "nimo-shake/common"
8 |
9 | LOG "github.com/vinllen/log4go"
10 | "go.mongodb.org/mongo-driver/bson"
11 | "go.mongodb.org/mongo-driver/mongo"
12 | "go.mongodb.org/mongo-driver/mongo/options"
13 | )
14 |
15 | type MongoWriter struct {
16 | address string
17 | //conn *utils.MongoConn
18 | nconn *utils.MongoCommunityConn
19 | db string
20 | }
21 |
22 | func NewMongoWriter(address, db string) *MongoWriter {
23 | targetConn, err := utils.NewMongoCommunityConn(address, utils.ConnectModePrimary, true)
24 | if err != nil {
25 | LOG.Error("create mongodb with address[%v] db[%v] connection error[%v]", address, db, err)
26 | return nil
27 | }
28 |
29 | return &MongoWriter{
30 | address: address,
31 | nconn: targetConn,
32 | db: db,
33 | }
34 | }
35 |
36 | func (mw *MongoWriter) FindStatus() (string, error) {
37 | var query Status
38 | if err := mw.nconn.Client.Database(mw.db).Collection(CheckpointStatusTable).FindOne(context.TODO(),
39 | bson.M{"Key": CheckpointStatusKey}).Decode(&query); err != nil {
40 | if err == mongo.ErrNoDocuments {
41 | return CheckpointStatusValueEmpty, nil
42 | }
43 | return "", err
44 | } else {
45 | return query.Value, nil
46 | }
47 | }
48 |
49 | func (mw *MongoWriter) UpdateStatus(status string) error {
50 | update := Status{
51 | Key: CheckpointStatusKey,
52 | Value: status,
53 | }
54 |
55 | opts := options.Update().SetUpsert(true)
56 | filter := bson.M{"Key": CheckpointStatusKey}
57 | updateStr := bson.M{"$set": update}
58 | _, err := mw.nconn.Client.Database(mw.db).Collection(CheckpointStatusTable).UpdateOne(context.TODO(), filter, updateStr, opts)
59 | return err
60 | }
61 |
62 | func (mw *MongoWriter) ExtractCheckpoint() (map[string]map[string]*Checkpoint, error) {
63 | // extract checkpoint from mongodb, every collection checkpoint have independent collection(table)
64 | ckptMap := make(map[string]map[string]*Checkpoint)
65 |
66 | collectionList, err := mw.nconn.Client.Database(mw.db).ListCollectionNames(context.TODO(), bson.M{})
67 | if err != nil {
68 | return nil, fmt.Errorf("fetch checkpoint collection list failed[%v]", err)
69 | }
70 | for _, table := range collectionList {
71 | if FilterCkptCollection(table) {
72 | continue
73 | }
74 |
75 | innerMap, err := mw.ExtractSingleCheckpoint(table)
76 | if err != nil {
77 | return nil, err
78 | }
79 | ckptMap[table] = innerMap
80 | }
81 |
82 | return ckptMap, nil
83 | }
84 |
85 | func (mw *MongoWriter) ExtractSingleCheckpoint(table string) (map[string]*Checkpoint, error) {
86 | innerMap := make(map[string]*Checkpoint)
87 | data := make([]*Checkpoint, 0)
88 |
89 | cursor, err := mw.nconn.Client.Database(mw.db).Collection(table).Find(context.TODO(), bson.M{})
90 | if err != nil {
91 | return nil, err
92 | }
93 | defer cursor.Close(context.TODO())
94 |
95 | for cursor.Next(context.TODO()) {
96 | var elem Checkpoint
97 | if err := cursor.Decode(&elem); err != nil {
98 | return nil, err
99 | }
100 | data = append(data, &elem)
101 | }
102 |
103 | for _, ele := range data {
104 | innerMap[ele.ShardId] = ele
105 | }
106 |
107 | return innerMap, nil
108 | }
109 |
110 | func (mw *MongoWriter) Insert(ckpt *Checkpoint, table string) error {
111 | _, err := mw.nconn.Client.Database(mw.db).Collection(table).InsertOne(context.TODO(), ckpt)
112 |
113 | return err
114 | }
115 |
116 | func (mw *MongoWriter) Update(shardId string, ckpt *Checkpoint, table string) error {
117 |
118 | filter := bson.M{"ShardId": shardId}
119 | updateStr := bson.M{"$set": ckpt}
120 | _, err := mw.nconn.Client.Database(mw.db).Collection(table).UpdateOne(context.TODO(), filter, updateStr)
121 | return err
122 | }
123 |
124 | func (mw *MongoWriter) UpdateWithSet(shardId string, input map[string]interface{}, table string) error {
125 |
126 | filter := bson.M{"ShardId": shardId}
127 | updateStr := bson.M{"$set": input}
128 | _, err := mw.nconn.Client.Database(mw.db).Collection(table).UpdateOne(context.TODO(), filter, updateStr)
129 | return err
130 | }
131 |
132 | func (mw *MongoWriter) Query(shardId string, table string) (*Checkpoint, error) {
133 | var res Checkpoint
134 | err := mw.nconn.Client.Database(mw.db).Collection(table).FindOne(context.TODO(), bson.M{"ShardId": shardId}).Decode(&res)
135 |
136 | return &res, err
137 | }
138 |
139 | func (mw *MongoWriter) DropAll() error {
140 | return mw.nconn.Client.Database(mw.db).Drop(context.TODO())
141 | }
142 |
143 | func (fw *MongoWriter) IncrCacheFileInsert(table string, shardId string, fileName string,
144 | lastSequenceNumber string, time string) error {
145 |
146 | // write cachefile struct to db
147 | return nil
148 | }
149 |
--------------------------------------------------------------------------------
/nimo-shake/checkpoint/struct.go:
--------------------------------------------------------------------------------
1 | package checkpoint
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "nimo-shake/filter"
7 | "sync"
8 | )
9 |
10 | const (
11 | CheckpointWriterTypeMongo = "mongodb"
12 | CheckpointWriterTypeFile = "file"
13 | CheckpointStatusTable = "status_table"
14 | CheckpointStatusKey = "status_key"
15 | CheckpointStatusValueEmpty = ""
16 | CheckpointStatusValueFullSync = "full_sync"
17 | CheckpointStatusValueIncrSync = "incr_sync"
18 |
19 | // 0: not process; 1: no need to process; 2: prepare stage 3: in processing; 4: wait father finish, 5: done
20 | StatusNotProcess = "not process"
21 | StatusNoNeedProcess = "no need to process"
22 | StatusPrepareProcess = "prepare stage"
23 | StatusInProcessing = "in processing"
24 | StatusWaitFather = "wait father finish"
25 | StatusDone = "done"
26 |
27 | IteratorTypeLatest = "LATEST"
28 | IteratorTypeAtSequence = "AT_SEQUENCE_NUMBER"
29 | IteratorTypeAfterSequence = "AFTER_SEQUENCE_NUMBER"
30 | IteratorTypeTrimHorizon = "TRIM_HORIZON"
31 |
32 | InitShardIt = "see sequence number"
33 |
34 | StreamViewType = "NEW_AND_OLD_IMAGES"
35 |
36 | FieldShardId = "ShardId"
37 | FieldShardIt = "ShardIt"
38 | FieldStatus = "Status"
39 | FieldSeqNum = "SequenceNumber"
40 | FieldIteratorType = "IteratorType"
41 | FieldTimestamp = "UpdateDate"
42 | FieldApproximateTime = "ApproximateTime"
43 | )
44 |
45 | type Checkpoint struct {
46 | ShardId string `bson:"ShardId" json:"ShardId"` // shard id
47 | FatherId string `bson:"FatherId" json:"FatherId"` // father id
48 | SequenceNumber string `bson:"SequenceNumber" json:"SequenceNumber"` // checkpoint
49 | Status string `bson:"Status" json:"Status"` // status
50 | WorkerId string `bson:"WorkerId" json:"WorkerId"` // thread number
51 | IteratorType string `bson:"IteratorType" json:"IteratorType"` // "LATEST" or "AT_SEQUENCE_NUMBER"
52 | ShardIt string `bson:"ShardIt" json:"ShardIt"` // only used when IteratorType == "LATEST"
53 | UpdateDate string `bson:"UpdateDate" json:"UpdateDate"` // update checkpoint time
54 | ApproximateTime string `bson:"ApproximateTime" json:"ApproximateTime"` // approximate time of records
55 | }
56 |
57 | func (c *Checkpoint) String() string {
58 | out, _ := json.Marshal(c)
59 | return fmt.Sprintf("%s", out)
60 | }
61 |
62 | type Status struct {
63 | Key string `bson:"Key" json:"Key"` // key -> CheckpointStatusKey
64 | Value string `bson:"StatusValue" json:"StatusValue"` // CheckpointStatusValueFullSync or CheckpointStatusValueIncrSync
65 | }
66 |
67 | /*---------------------------------------*/
68 |
69 | var (
70 | GlobalShardIteratorMap = ShardIteratorMap{
71 | mp: make(map[string]string),
72 | }
73 | )
74 |
75 | type ShardIteratorMap struct {
76 | mp map[string]string
77 | lock sync.Mutex
78 | }
79 |
80 | func (sim *ShardIteratorMap) Set(key, iterator string) bool {
81 | sim.lock.Lock()
82 | defer sim.lock.Unlock()
83 |
84 | if _, ok := sim.mp[key]; ok {
85 | return false
86 | }
87 |
88 | sim.mp[key] = iterator
89 | return false
90 | }
91 |
92 | func (sim *ShardIteratorMap) Get(key string) (string, bool) {
93 | sim.lock.Lock()
94 | defer sim.lock.Unlock()
95 |
96 | it, ok := sim.mp[key]
97 | return it, ok
98 | }
99 |
100 | func (sim *ShardIteratorMap) Delete(key string) bool {
101 | sim.lock.Lock()
102 | defer sim.lock.Unlock()
103 |
104 | if _, ok := sim.mp[key]; ok {
105 | delete(sim.mp, key)
106 | return true
107 | }
108 | return false
109 | }
110 |
111 | /*---------------------------------------*/
112 |
113 | func FilterCkptCollection(collection string) bool {
114 | return collection == CheckpointStatusTable || filter.IsFilter(collection)
115 | }
116 |
117 | func IsStatusProcessing(status string) bool {
118 | return status == StatusPrepareProcess || status == StatusInProcessing || status == StatusWaitFather
119 | }
120 |
121 | func IsStatusNoNeedProcess(status string) bool {
122 | return status == StatusDone || status == StatusNoNeedProcess
123 | }
124 |
--------------------------------------------------------------------------------
/nimo-shake/checkpoint/writer.go:
--------------------------------------------------------------------------------
1 | package checkpoint
2 |
3 | import (
4 | LOG "github.com/vinllen/log4go"
5 | )
6 |
7 | type Writer interface {
8 | // find current status
9 | FindStatus() (string, error)
10 |
11 | // update status
12 | UpdateStatus(status string) error
13 |
14 | // extract all checkpoint
15 | ExtractCheckpoint() (map[string]map[string]*Checkpoint, error)
16 |
17 | // extract single checkpoint
18 | ExtractSingleCheckpoint(table string) (map[string]*Checkpoint, error)
19 |
20 | // insert checkpoint
21 | Insert(ckpt *Checkpoint, table string) error
22 |
23 | // update checkpoint
24 | Update(shardId string, ckpt *Checkpoint, table string) error
25 |
26 | // update with set
27 | UpdateWithSet(shardId string, input map[string]interface{}, table string) error
28 |
29 | // query
30 | Query(shardId string, table string) (*Checkpoint, error)
31 |
32 | // insert incrSyncCacheFile
33 | IncrCacheFileInsert(table string, shardId string, fileName string, lastSequenceNumber string, time string) error
34 |
35 | // drop
36 | DropAll() error
37 | }
38 |
39 | func NewWriter(name, address, db string) Writer {
40 | var w Writer
41 | switch name {
42 | case CheckpointWriterTypeMongo:
43 | w = NewMongoWriter(address, db)
44 | case CheckpointWriterTypeFile:
45 | w = NewFileWriter(db)
46 | default:
47 | LOG.Crashf("unknown checkpoint writer[%v]", name)
48 | }
49 | if w == nil {
50 | LOG.Crashf("create checkpoint writer[%v] failed", name)
51 | return nil
52 | }
53 | return w
54 | }
55 |
--------------------------------------------------------------------------------
/nimo-shake/checkpoint/writer_test.go:
--------------------------------------------------------------------------------
1 | package checkpoint
2 |
3 | import (
4 | "fmt"
5 | "testing"
6 |
7 | "github.com/stretchr/testify/assert"
8 |
9 | utils "nimo-shake/common"
10 | )
11 |
12 | const (
13 | TestMongoAddress = "mongodb://100.81.164.181:18901"
14 | TestCheckpointDb = "test_checkpoint_db"
15 | TestCheckpointTable = "test_checkpoint_table"
16 | )
17 |
18 | func TestStatus(t *testing.T) {
19 |
20 | utils.InitialLogger("", "debug", false)
21 |
22 | var err error
23 | mongoWriter := NewWriter(CheckpointWriterTypeMongo, TestMongoAddress, TestCheckpointDb)
24 | assert.Equal(t, true, mongoWriter != nil, "should be equal")
25 |
26 | fileWriter := NewWriter(CheckpointWriterTypeFile, TestMongoAddress, TestCheckpointDb)
27 | assert.Equal(t, true, fileWriter != nil, "should be equal")
28 |
29 | var nr int
30 | // test status: mongo
31 | {
32 | fmt.Printf("TestStatus case %d.\n", nr)
33 | nr++
34 |
35 | err = mongoWriter.DropAll()
36 | assert.Equal(t, nil, err, "should be equal")
37 |
38 | status, err := mongoWriter.FindStatus()
39 | assert.Equal(t, nil, err, "should be equal")
40 | assert.Equal(t, CheckpointStatusValueEmpty, status, "should be equal")
41 |
42 | err = mongoWriter.UpdateStatus(CheckpointStatusValueIncrSync)
43 | assert.Equal(t, nil, err, "should be equal")
44 |
45 | status, err = mongoWriter.FindStatus()
46 | assert.Equal(t, nil, err, "should be equal")
47 | assert.Equal(t, CheckpointStatusValueIncrSync, status, "should be equal")
48 | }
49 |
50 | // test status: file
51 | {
52 | fmt.Printf("TestStatus case %d.\n", nr)
53 | nr++
54 |
55 | err = fileWriter.DropAll()
56 | assert.Equal(t, nil, err, "should be equal")
57 |
58 | status, err := fileWriter.FindStatus()
59 | assert.Equal(t, nil, err, "should be equal")
60 | assert.Equal(t, CheckpointStatusValueEmpty, status, "should be equal")
61 |
62 | err = fileWriter.UpdateStatus(CheckpointStatusValueIncrSync)
63 | assert.Equal(t, nil, err, "should be equal")
64 |
65 | status, err = fileWriter.FindStatus()
66 | assert.Equal(t, nil, err, "should be equal")
67 | assert.Equal(t, CheckpointStatusValueIncrSync, status, "should be equal")
68 | }
69 | }
70 |
71 | func TestCheckpointCRUD(t *testing.T) {
72 | var err error
73 | mongoWriter := NewWriter(CheckpointWriterTypeMongo, TestMongoAddress, TestCheckpointDb)
74 | assert.Equal(t, true, mongoWriter != nil, "should be equal")
75 |
76 | fileWriter := NewWriter(CheckpointWriterTypeFile, TestMongoAddress, TestCheckpointDb)
77 | assert.Equal(t, true, fileWriter != nil, "should be equal")
78 |
79 | // utils.InitialLogger("", "info", false)
80 |
81 | var nr int
82 | // test CRUD: mongo
83 | {
84 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr)
85 | nr++
86 |
87 | err = mongoWriter.DropAll()
88 | assert.Equal(t, nil, err, "should be equal")
89 |
90 | cpkt := &Checkpoint{
91 | ShardId: "test_id",
92 | FatherId: "test_father",
93 | Status: StatusNotProcess,
94 | }
95 |
96 | err = mongoWriter.Update("test_id", cpkt, TestCheckpointTable)
97 | assert.Equal(t, nil, err, "should be equal")
98 |
99 | err = mongoWriter.UpdateWithSet("test_id", map[string]interface{}{
100 | "Status": StatusNotProcess,
101 | }, TestCheckpointTable)
102 | assert.Equal(t, nil, err, "should be equal")
103 |
104 | err = mongoWriter.Insert(cpkt, TestCheckpointTable)
105 | assert.Equal(t, nil, err, "should be equal")
106 |
107 | ckptRet, err := mongoWriter.Query("test_id", TestCheckpointTable)
108 | assert.Equal(t, nil, err, "should be equal")
109 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal")
110 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal")
111 | assert.Equal(t, cpkt.Status, ckptRet.Status, "should be equal")
112 |
113 | err = mongoWriter.UpdateWithSet("test_id", map[string]interface{}{
114 | "Status": StatusInProcessing,
115 | }, TestCheckpointTable)
116 | assert.Equal(t, nil, err, "should be equal")
117 |
118 | ckptRet, err = mongoWriter.Query("test_id", TestCheckpointTable)
119 | assert.Equal(t, nil, err, "should be equal")
120 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal")
121 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal")
122 | assert.Equal(t, StatusInProcessing, ckptRet.Status, "should be equal")
123 | }
124 |
125 | // test CRUD: file
126 | {
127 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr)
128 | nr++
129 |
130 | err = fileWriter.DropAll()
131 | assert.Equal(t, nil, err, "should be equal")
132 |
133 | cpkt := &Checkpoint{
134 | ShardId: "test_id",
135 | FatherId: "test_father",
136 | Status: StatusNotProcess,
137 | SequenceNumber: "seq-123",
138 | }
139 |
140 | err = fileWriter.Update("test_id", cpkt, TestCheckpointTable)
141 | assert.Equal(t, true, err != nil, "should be equal")
142 | fmt.Println(err)
143 |
144 | err = fileWriter.UpdateWithSet("test_id", map[string]interface{}{
145 | "Status": StatusNotProcess,
146 | }, TestCheckpointTable)
147 | assert.Equal(t, true, err != nil, "should be equal")
148 | fmt.Println(err)
149 |
150 | err = fileWriter.Insert(cpkt, TestCheckpointTable)
151 | assert.Equal(t, nil, err, "should be equal")
152 |
153 | ckptRet, err := fileWriter.Query("test_id", TestCheckpointTable)
154 | assert.Equal(t, nil, err, "should be equal")
155 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal")
156 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal")
157 | assert.Equal(t, cpkt.Status, ckptRet.Status, "should be equal")
158 | assert.Equal(t, cpkt.SequenceNumber, "seq-123", "should be equal")
159 |
160 | err = fileWriter.UpdateWithSet("test_id", map[string]interface{}{
161 | "Status": StatusInProcessing,
162 | "SequenceNumber": "seq-456",
163 | }, TestCheckpointTable)
164 | assert.Equal(t, nil, err, "should be equal")
165 |
166 | ckptRet, err = fileWriter.Query("test_id", TestCheckpointTable)
167 | assert.Equal(t, nil, err, "should be equal")
168 | assert.Equal(t, cpkt.ShardId, ckptRet.ShardId, "should be equal")
169 | assert.Equal(t, cpkt.FatherId, ckptRet.FatherId, "should be equal")
170 | assert.Equal(t, StatusInProcessing, ckptRet.Status, "should be equal")
171 | assert.Equal(t, "seq-456", ckptRet.SequenceNumber, "should be equal")
172 | }
173 | }
174 |
175 | func TestExtractCheckpoint(t *testing.T) {
176 | var err error
177 | mongoWriter := NewWriter(CheckpointWriterTypeMongo, TestMongoAddress, TestCheckpointDb)
178 | assert.Equal(t, true, mongoWriter != nil, "should be equal")
179 |
180 | fileWriter := NewWriter(CheckpointWriterTypeFile, TestMongoAddress, TestCheckpointDb)
181 | assert.Equal(t, true, fileWriter != nil, "should be equal")
182 |
183 | // utils.InitialLogger("", "info", false)
184 |
185 | var nr int
186 | // test CRUD: mongo
187 | {
188 | fmt.Printf("TestExtractCheckpoint case %d.\n", nr)
189 | nr++
190 |
191 | err = mongoWriter.DropAll()
192 | assert.Equal(t, nil, err, "should be equal")
193 |
194 | err = mongoWriter.Insert(&Checkpoint{
195 | ShardId: "id1",
196 | Status: StatusNotProcess,
197 | }, "table1")
198 | assert.Equal(t, nil, err, "should be equal")
199 |
200 | err = mongoWriter.Insert(&Checkpoint{
201 | ShardId: "id2",
202 | Status: StatusInProcessing,
203 | }, "table1")
204 | assert.Equal(t, nil, err, "should be equal")
205 |
206 | err = mongoWriter.Insert(&Checkpoint{
207 | ShardId: "id3",
208 | Status: StatusPrepareProcess,
209 | }, "table1")
210 | assert.Equal(t, nil, err, "should be equal")
211 |
212 | err = mongoWriter.Insert(&Checkpoint{
213 | ShardId: "id1",
214 | Status: StatusDone,
215 | }, "table2")
216 | assert.Equal(t, nil, err, "should be equal")
217 |
218 | err = mongoWriter.Insert(&Checkpoint{
219 | ShardId: "id10",
220 | Status: StatusWaitFather,
221 | }, "table2")
222 | assert.Equal(t, nil, err, "should be equal")
223 |
224 | mp, err := mongoWriter.ExtractCheckpoint()
225 | assert.Equal(t, nil, err, "should be equal")
226 | assert.Equal(t, 2, len(mp), "should be equal")
227 | assert.Equal(t, 3, len(mp["table1"]), "should be equal")
228 | assert.Equal(t, 2, len(mp["table2"]), "should be equal")
229 | assert.Equal(t, StatusNotProcess, mp["table1"]["id1"].Status, "should be equal")
230 | assert.Equal(t, StatusInProcessing, mp["table1"]["id2"].Status, "should be equal")
231 | assert.Equal(t, StatusPrepareProcess, mp["table1"]["id3"].Status, "should be equal")
232 | assert.Equal(t, StatusDone, mp["table2"]["id1"].Status, "should be equal")
233 | assert.Equal(t, StatusWaitFather, mp["table2"]["id10"].Status, "should be equal")
234 | }
235 |
236 | // test CRUD: file
237 | {
238 | fmt.Printf("TestExtractCheckpoint case %d.\n", nr)
239 | nr++
240 |
241 | err = fileWriter.DropAll()
242 | assert.Equal(t, nil, err, "should be equal")
243 |
244 | err = fileWriter.Insert(&Checkpoint{
245 | ShardId: "id1",
246 | Status: StatusNotProcess,
247 | }, "table1")
248 | assert.Equal(t, nil, err, "should be equal")
249 |
250 | err = fileWriter.Insert(&Checkpoint{
251 | ShardId: "id2",
252 | Status: StatusInProcessing,
253 | }, "table1")
254 | assert.Equal(t, nil, err, "should be equal")
255 |
256 | err = fileWriter.Insert(&Checkpoint{
257 | ShardId: "id3",
258 | Status: StatusPrepareProcess,
259 | }, "table1")
260 | assert.Equal(t, nil, err, "should be equal")
261 |
262 | err = fileWriter.Insert(&Checkpoint{
263 | ShardId: "id1",
264 | Status: StatusDone,
265 | }, "table2")
266 | assert.Equal(t, nil, err, "should be equal")
267 |
268 | err = fileWriter.Insert(&Checkpoint{
269 | ShardId: "id10",
270 | Status: StatusWaitFather,
271 | }, "table2")
272 | assert.Equal(t, nil, err, "should be equal")
273 |
274 | mp, err := fileWriter.ExtractCheckpoint()
275 | assert.Equal(t, nil, err, "should be equal")
276 | assert.Equal(t, 2, len(mp), "should be equal")
277 | assert.Equal(t, 3, len(mp["table1"]), "should be equal")
278 | assert.Equal(t, 2, len(mp["table2"]), "should be equal")
279 | assert.Equal(t, StatusNotProcess, mp["table1"]["id1"].Status, "should be equal")
280 | assert.Equal(t, StatusInProcessing, mp["table1"]["id2"].Status, "should be equal")
281 | assert.Equal(t, StatusPrepareProcess, mp["table1"]["id3"].Status, "should be equal")
282 | assert.Equal(t, StatusDone, mp["table2"]["id1"].Status, "should be equal")
283 | assert.Equal(t, StatusWaitFather, mp["table2"]["id10"].Status, "should be equal")
284 | }
285 | }
286 |
--------------------------------------------------------------------------------
/nimo-shake/common/callback.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import "time"
4 |
5 | /*
6 | * retry the callback function until successfully or overpass the threshold.
7 | * @input:
8 | * times: retry times
9 | * sleep: sleep time by ms interval
10 | * cb: callback
11 | * the callback should return true means need retry.
12 | */
13 | func CallbackRetry(times int, sleep int, cb func() bool) bool {
14 | for i := 0; i < times; i++ {
15 | if cb() == false { // callback, true means retry
16 | return true
17 | }
18 | time.Sleep(time.Duration(sleep) * time.Millisecond)
19 | }
20 | return false
21 | }
--------------------------------------------------------------------------------
/nimo-shake/common/common.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "strings"
7 |
8 | LOG "github.com/vinllen/log4go"
9 | )
10 |
11 | const (
12 | GolangSecurityTime = "2006-01-02T15:04:05Z"
13 |
14 | ConvertTypeRaw = "raw"
15 | ConvertTypeChange = "change"
16 | ConvertMTypeChange = "mchange"
17 | ConvertTypeSame = "same" // used in dynamodb -> dynamo-proxy
18 |
19 | SyncModeAll = "all"
20 | SyncModeFull = "full"
21 | SyncModeIncr = "incr"
22 |
23 | TargetTypeMongo = "mongodb"
24 | TargetTypeAliyunDynamoProxy = "aliyun_dynamo_proxy"
25 |
26 | TargetMongoDBTypeReplica = "replica"
27 | TargetMongoDBTypeSharding = "sharding"
28 |
29 | TargetDBExistRename = "rename"
30 | TargetDBExistDrop = "drop"
31 |
32 | SIGNALPROFILE = 31
33 | SIGNALSTACK = 30
34 | )
35 |
36 | var (
37 | Version = "$"
38 | StartTime string
39 | )
40 |
41 | func InitialLogger(logFile string, level string, logBuffer bool) bool {
42 | logLevel := parseLogLevel(level)
43 | if len(logFile) != 0 {
44 | // create logs folder for log4go. because of its mistake that doesn't create !
45 | if err := os.MkdirAll("logs", os.ModeDir|os.ModePerm); err != nil {
46 | return false
47 | }
48 | if logBuffer {
49 | LOG.LogBufferLength = 32
50 | } else {
51 | LOG.LogBufferLength = 0
52 | }
53 | fileLogger := LOG.NewFileLogWriter(fmt.Sprintf("logs/%s", logFile), true)
54 | //fileLogger.SetRotateDaily(true)
55 | fileLogger.SetRotateSize(500 * 1024 * 1024)
56 | // fileLogger.SetFormat("[%D %T] [%L] [%s] %M")
57 | fileLogger.SetFormat("[%D %T] [%L] %M")
58 | fileLogger.SetRotateMaxBackup(100)
59 | LOG.AddFilter("file", logLevel, fileLogger)
60 | } else {
61 | LOG.AddFilter("console", logLevel, LOG.NewConsoleLogWriter())
62 | }
63 | return true
64 | }
65 |
66 | func parseLogLevel(level string) LOG.Level {
67 | switch strings.ToLower(level) {
68 | case "debug":
69 | return LOG.DEBUG
70 | case "info":
71 | return LOG.INFO
72 | case "warning":
73 | return LOG.WARNING
74 | case "error":
75 | return LOG.ERROR
76 | default:
77 | return LOG.DEBUG
78 | }
79 | }
80 |
81 | /**
82 | * block password in mongo_urls:
83 | * two kind mongo_urls:
84 | * 1. mongodb://username:password@address
85 | * 2. username:password@address
86 | */
87 | func BlockMongoUrlPassword(url, replace string) string {
88 | colon := strings.Index(url, ":")
89 | if colon == -1 || colon == len(url)-1 {
90 | return url
91 | } else if url[colon+1] == '/' {
92 | // find the second '/'
93 | for colon++; colon < len(url); colon++ {
94 | if url[colon] == ':' {
95 | break
96 | }
97 | }
98 |
99 | if colon == len(url) {
100 | return url
101 | }
102 | }
103 |
104 | at := strings.Index(url, "@")
105 | if at == -1 || at == len(url)-1 || at <= colon {
106 | return url
107 | }
108 |
109 | newUrl := make([]byte, 0, len(url))
110 | for i := 0; i < len(url); i++ {
111 | if i <= colon || i > at {
112 | newUrl = append(newUrl, byte(url[i]))
113 | } else if i == at {
114 | newUrl = append(newUrl, []byte(replace)...)
115 | newUrl = append(newUrl, byte(url[i]))
116 | }
117 | }
118 | return string(newUrl)
119 | }
120 |
--------------------------------------------------------------------------------
/nimo-shake/common/dynamodb.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "fmt"
5 | "net/http"
6 | "strings"
7 | "time"
8 |
9 | "github.com/aws/aws-sdk-go/aws"
10 | "github.com/aws/aws-sdk-go/aws/credentials"
11 | "github.com/aws/aws-sdk-go/aws/session"
12 | "github.com/aws/aws-sdk-go/service/dynamodb"
13 | "github.com/aws/aws-sdk-go/service/dynamodbstreams"
14 | )
15 |
16 | var (
17 | globalSession *session.Session
18 | )
19 |
20 | /*
21 | * all client share the same session.
22 | * Sessions can be shared across all service clients that share the same base configuration
23 | * refer: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/
24 | */
25 | func InitSession(accessKeyID, secretAccessKey, sessionToken, region, endpoint string, maxRetries, timeout uint) error {
26 | config := &aws.Config{
27 | Region: aws.String(region),
28 | Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken),
29 | MaxRetries: aws.Int(int(maxRetries)),
30 | HTTPClient: &http.Client{
31 | Timeout: time.Duration(timeout) * time.Millisecond,
32 | },
33 | }
34 |
35 | if endpoint != "" {
36 | config.Endpoint = aws.String(endpoint)
37 | }
38 |
39 | var err error
40 | globalSession, err = session.NewSession(config)
41 | if err != nil {
42 | return err
43 | }
44 |
45 | return nil
46 | }
47 |
48 | func CreateDynamoSession(logLevel string) (*dynamodb.DynamoDB, error) {
49 | if logLevel == "debug" {
50 | svc := dynamodb.New(globalSession, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody))
51 | return svc, nil
52 | }
53 | svc := dynamodb.New(globalSession)
54 | return svc, nil
55 | }
56 |
57 | func CreateDynamoStreamSession(logLevel string) (*dynamodbstreams.DynamoDBStreams, error) {
58 | if logLevel == "debug" {
59 | svc := dynamodbstreams.New(globalSession, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody))
60 | return svc, nil
61 | }
62 | svc := dynamodbstreams.New(globalSession)
63 | return svc, nil
64 | }
65 |
66 | func ParseIndexType(input []*dynamodb.AttributeDefinition) map[string]string {
67 | mp := make(map[string]string, len(input))
68 |
69 | for _, ele := range input {
70 | mp[*ele.AttributeName] = *ele.AttributeType
71 | }
72 |
73 | return mp
74 | }
75 |
76 | // fetch dynamodb table list
77 | func FetchTableList(dynamoSession *dynamodb.DynamoDB) ([]string, error) {
78 | ans := make([]string, 0)
79 | var lastEvaluatedTableName *string
80 |
81 | for {
82 | out, err := dynamoSession.ListTables(&dynamodb.ListTablesInput{
83 | ExclusiveStartTableName: lastEvaluatedTableName,
84 | })
85 |
86 | if err != nil {
87 | return nil, err
88 | }
89 |
90 | ans = AppendStringList(ans, out.TableNames)
91 | if out.LastEvaluatedTableName == nil {
92 | // finish
93 | break
94 | }
95 | lastEvaluatedTableName = out.LastEvaluatedTableName
96 | }
97 |
98 | return ans, nil
99 | }
100 |
101 | func ParsePrimaryAndSortKey(primaryIndexes []*dynamodb.KeySchemaElement, parseMap map[string]string) (string, string, error) {
102 | var primaryKey string
103 | var sortKey string
104 | for _, index := range primaryIndexes {
105 | if *(index.KeyType) == "HASH" {
106 | if primaryKey != "" {
107 | return "", "", fmt.Errorf("duplicate primary key type[%v]", *(index.AttributeName))
108 | }
109 | primaryKey = *(index.AttributeName)
110 | } else if *(index.KeyType) == "RANGE" {
111 | if sortKey != "" {
112 | return "", "", fmt.Errorf("duplicate sort key type[%v]", *(index.AttributeName))
113 | }
114 | sortKey = *(index.AttributeName)
115 | } else {
116 | return "", "", fmt.Errorf("unknonw key type[%v]", *(index.KeyType))
117 | }
118 | }
119 | return primaryKey, sortKey, nil
120 | }
121 |
122 | /*
123 | input:
124 |
125 | "begin```N```1646724207280~~~end```S```1646724207283"
126 |
127 | output:
128 |
129 | map[string]*dynamodb.AttributeValue{
130 | ":begin": &dynamodb.AttributeValue{N: aws.String("1646724207280")},
131 | ":end": &dynamodb.AttributeValue{S: aws.String("1646724207283")},
132 | }
133 | */
134 | func ParseAttributes(input string) map[string]*dynamodb.AttributeValue {
135 | result := make(map[string]*dynamodb.AttributeValue)
136 | pairs := strings.Split(input, "~~~")
137 |
138 | for _, pair := range pairs {
139 | parts := strings.Split(pair, "```")
140 | if len(parts) != 3 {
141 | continue
142 | }
143 |
144 | key := ":" + parts[0]
145 | dataType := parts[1]
146 | value := parts[2]
147 |
148 | switch dataType {
149 | case "N":
150 | result[key] = &dynamodb.AttributeValue{N: aws.String(value)}
151 | case "S":
152 | result[key] = &dynamodb.AttributeValue{S: aws.String(value)}
153 | }
154 | }
155 |
156 | return result
157 | }
158 |
--------------------------------------------------------------------------------
/nimo-shake/common/error.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "github.com/vinllen/mgo"
5 | )
6 |
7 | // true means error can be ignored
8 | // https://github.com/mongodb/mongo/blob/master/src/mongo/base/error_codes.yml
9 | func MongodbIgnoreError(err error, op string, isFullSyncStage bool) bool {
10 | if err == nil {
11 | return true
12 | }
13 |
14 | errorCode := mgo.ErrorCodeList(err)
15 | if err != nil && len(errorCode) == 0 {
16 | return false
17 | }
18 |
19 | for _, err := range errorCode {
20 | switch op {
21 | case "i":
22 | case "u":
23 | if isFullSyncStage {
24 | if err == 28 { // PathNotViable
25 | continue
26 | }
27 | }
28 | case "d":
29 | if err == 26 { // NamespaceNotFound
30 | continue
31 | }
32 | case "c":
33 | if err == 26 { // NamespaceNotFound
34 | continue
35 | }
36 | default:
37 | return false
38 | }
39 | return false
40 | }
41 |
42 | return true
43 | }
44 |
45 | func DynamoIgnoreError(err error, op string, isFullSyncStage bool) bool {
46 | if err == nil {
47 | return true
48 | }
49 |
50 | switch op {
51 | case "i":
52 | case "u":
53 | if isFullSyncStage {
54 | if err.Error() == "xxxx" { // PathNotViable
55 | return true
56 | }
57 | }
58 | case "d":
59 | if err.Error() == "xxxx" { // NamespaceNotFound
60 | return true
61 | }
62 | case "c":
63 | if err.Error() == "xxxx" { // NamespaceNotFound
64 | return true
65 | }
66 | default:
67 | return false
68 | }
69 | return false
70 | }
--------------------------------------------------------------------------------
/nimo-shake/common/fcv.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | var (
4 | FcvCheckpoint = Checkpoint{
5 | CurrentVersion: 0,
6 | FeatureCompatibleVersion: 0,
7 | }
8 | FcvConfiguration = Configuration{
9 | CurrentVersion: 4,
10 | FeatureCompatibleVersion: 1,
11 | }
12 |
13 | LowestCheckpointVersion = map[int]string{
14 | 0: "1.0.0",
15 | }
16 | LowestConfigurationVersion = map[int]string{
17 | 0: "1.0.0",
18 | 1: "1.0.6", // add full sync and incr sync http port
19 | 2: "1.0.8", // add full.read.concurrency
20 | 3: "1.0.9", // add full.document.write.batch
21 | 4: "1.0.11", // add source.endpoint_url
22 | }
23 | )
24 |
25 | type Fcv interface {
26 | IsCompatible(int) bool
27 | }
28 |
29 | // for checkpoint
30 | type Checkpoint struct {
31 | /*
32 | * version: 0(or set not), MongoShake < 2.4, fcv == 0
33 | * version: 1, MongoShake == 2.4, 0 < fcv <= 1
34 | */
35 | CurrentVersion int
36 | FeatureCompatibleVersion int
37 | }
38 |
39 | func (c Checkpoint) IsCompatible(v int) bool {
40 | return v >= c.FeatureCompatibleVersion && v <= c.CurrentVersion
41 | }
42 |
43 | // for configuration
44 | type Configuration struct {
45 | /*
46 | * version: 0(or set not), MongoShake < 2.4.0, fcv == 0
47 | * version: 1, MongoShake == 2.4.0, 0 <= fcv <= 1
48 | */
49 | CurrentVersion int
50 | FeatureCompatibleVersion int
51 | }
52 |
53 | func (c Configuration) IsCompatible(v int) bool {
54 | return v >= c.FeatureCompatibleVersion && v <= c.CurrentVersion
55 | }
56 |
--------------------------------------------------------------------------------
/nimo-shake/common/http.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | nimo "github.com/gugemichael/nimo4go"
5 | )
6 |
7 | var (
8 | FullSyncHttpApi *nimo.HttpRestProvider
9 | IncrSyncHttpApi *nimo.HttpRestProvider
10 | )
11 |
12 | func FullSyncInitHttpApi(port int) {
13 | FullSyncHttpApi = nimo.NewHttpRestProvider(port)
14 | }
15 |
16 | func IncrSyncInitHttpApi(port int) {
17 | IncrSyncHttpApi = nimo.NewHttpRestProvider(port)
18 | }
19 |
--------------------------------------------------------------------------------
/nimo-shake/common/math.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "crypto/md5"
5 | "encoding/binary"
6 | )
7 |
8 | /*
9 | // === FUNCTION ======================================================================
10 | // Name: Md5
11 | // Description: 128位md5
12 | // =====================================================================================
13 | */
14 | func Md5(data []byte) [16]byte {
15 | return md5.Sum(data)
16 | }
17 |
18 | /*
19 | // === FUNCTION ======================================================================
20 | // Name: Md5
21 | // Description: 64位md5
22 | // =====================================================================================
23 | */
24 | func Md5In64(data []byte) uint64 {
25 | var md5 = md5.Sum(data)
26 | var lowMd5 = md5[0:8]
27 | return binary.LittleEndian.Uint64(lowMd5)
28 | }
29 |
--------------------------------------------------------------------------------
/nimo-shake/common/metric.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "fmt"
5 | "sync/atomic"
6 | "time"
7 | "sync"
8 |
9 | LOG "github.com/vinllen/log4go"
10 | )
11 |
12 | const (
13 | TypeFull = "full"
14 | TypeIncr = "incr"
15 | )
16 |
17 | type Status string
18 |
19 | // full sync
20 | const (
21 | StatusWaitStart Status = "wait start"
22 | StatusProcessing Status = "in processing"
23 | StatusFinish Status = "finish"
24 | )
25 |
26 | type CollectionMetric struct {
27 | CollectionStatus Status
28 | TotalCount uint64
29 | FinishCount uint64
30 | }
31 |
32 | func NewCollectionMetric() *CollectionMetric {
33 | return &CollectionMetric{
34 | CollectionStatus: StatusWaitStart,
35 | }
36 | }
37 |
38 | func (cm *CollectionMetric) String() string {
39 | if cm.CollectionStatus == StatusWaitStart {
40 | return fmt.Sprintf("-")
41 | }
42 |
43 | if cm.TotalCount == 0 || cm.FinishCount >= cm.TotalCount {
44 | return fmt.Sprintf("100%% (%v/%v)", cm.FinishCount, cm.TotalCount)
45 | } else {
46 | return fmt.Sprintf("%.2f%% (%v/%v)", float64(cm.FinishCount) / float64(cm.TotalCount) * 100,
47 | cm.FinishCount, cm.TotalCount)
48 | }
49 | }
50 |
51 | /*************************************************************************/
52 | // incr sync
53 |
54 | const (
55 | FrequentInSeconds = 5
56 | TimeFormat string = "2006-01-02 15:04:05"
57 | )
58 |
59 | const (
60 | KB = 1024
61 | MB = 1024 * KB
62 | GB = 1024 * MB
63 | TB = 1024 * GB
64 | PB = 1024 * TB
65 | )
66 |
67 | // struct used to mark the metric delta.
68 | // Value: current value
69 | // Delta: the difference between current value and previous value
70 | // previous: store the previous value
71 | type MetricDelta struct {
72 | Value uint64
73 | Delta uint64
74 | previous uint64
75 | }
76 |
77 | func (o *MetricDelta) Update() {
78 | current := atomic.LoadUint64(&o.Value)
79 | o.Delta, o.previous = current-o.previous, current
80 | }
81 |
82 | type ReplicationStatus uint64
83 |
84 | const (
85 | METRIC_NONE = 0x0000000000000000
86 | METRIC_CKPT_TIMES = 0x0000000000000001
87 | METRIC_TUNNEL_TRAFFIC = 0x0000000000000010
88 | METRIC_LSN = 0x0000000000000100
89 | METRIC_TPS = 0x0000000000010000
90 | METRIC_SUCCESS = 0x0000000000100000
91 | METRIC_FULLSYNC_WRITE = 0x0000000010000000 // full sync writer
92 | METRIC_FILTER = 0x0000000100000000
93 | )
94 |
95 | type ReplicationMetric struct {
96 | // NAME string
97 | STAGE string
98 | SUBSCRIBE uint64
99 |
100 | OplogFilter MetricDelta
101 | OplogGet MetricDelta
102 | OplogConsume MetricDelta
103 | OplogApply MetricDelta
104 | OplogSuccess MetricDelta
105 | OplogFail MetricDelta
106 | OplogWriteFail MetricDelta // full: write failed. currently, only used in full sync stage.
107 | CheckpointTimes uint64
108 | Retransmission uint64
109 | TunnelTraffic uint64
110 | LSN int64
111 | LSNAck int64
112 | LSNCheckpoint int64
113 |
114 | OplogMaxSize int64
115 | OplogAvgSize int64
116 |
117 | TableOperations *TableOps
118 |
119 | // replication status
120 | ReplStatus ReplicationStatus
121 |
122 | isClosed bool
123 | }
124 |
125 | //var Metric *ReplicationMetric
126 |
127 | func NewMetric(stage string, subscribe uint64) *ReplicationMetric {
128 | metric := &ReplicationMetric{}
129 | // metric.NAME = name
130 | metric.STAGE = stage
131 | metric.SUBSCRIBE = subscribe
132 | metric.startup()
133 | return metric
134 | }
135 |
136 | func (metric *ReplicationMetric) init() {
137 | metric.TableOperations = NewTableOps()
138 | }
139 |
140 | func (metric *ReplicationMetric) Close() {
141 | metric.isClosed = true
142 | }
143 |
144 | func (metric *ReplicationMetric) String() string {
145 | // return fmt.Sprintf("name[%v] stage[%v]", metric.NAME, metric.STAGE)
146 | return fmt.Sprintf("stage[%v]", metric.STAGE)
147 | }
148 |
149 | func (metric *ReplicationMetric) resetEverySecond(items []*MetricDelta) {
150 | for _, item := range items {
151 | item.Update()
152 | }
153 | }
154 |
155 | func (metric *ReplicationMetric) startup() {
156 | metric.init()
157 | go func() {
158 | tick := 0
159 | // items that need be reset
160 | resetItems := []*MetricDelta{&metric.OplogSuccess}
161 | for range time.NewTicker(1 * time.Second).C {
162 | if metric.isClosed {
163 | break
164 | }
165 |
166 | tick++
167 | metric.resetEverySecond(resetItems)
168 | if tick%FrequentInSeconds != 0 {
169 | continue
170 | }
171 |
172 | ckpt := atomic.LoadUint64(&metric.CheckpointTimes)
173 | tps := atomic.LoadUint64(&metric.OplogSuccess.Delta)
174 | success := atomic.LoadUint64(&metric.OplogSuccess.Value)
175 |
176 | verbose := "[stage=%s, get=%d"
177 | if metric.SUBSCRIBE&METRIC_FILTER != 0 {
178 | verbose += fmt.Sprintf(", filter=%d", atomic.LoadUint64(&metric.OplogFilter.Value))
179 | }
180 | if metric.SUBSCRIBE&METRIC_SUCCESS != 0 {
181 | verbose += fmt.Sprintf(", write_success=%d", success)
182 | }
183 | if metric.SUBSCRIBE&METRIC_TPS != 0 {
184 | verbose += fmt.Sprintf(", tps=%d", tps)
185 | }
186 | if metric.SUBSCRIBE&METRIC_CKPT_TIMES != 0 {
187 | verbose += fmt.Sprintf(", ckpt_times=%d", ckpt)
188 | }
189 | if metric.SUBSCRIBE&METRIC_TUNNEL_TRAFFIC != 0 {
190 | verbose += fmt.Sprintf(", tunnel_traffic=%s", metric.getTunnelTraffic())
191 | }
192 | if metric.SUBSCRIBE&METRIC_FULLSYNC_WRITE != 0 {
193 | verbose += fmt.Sprintf(", fail=%d", atomic.LoadUint64(&metric.OplogWriteFail.Value))
194 | }
195 | verbose += "]"
196 |
197 | // LOG.Info(verbose, metric.NAME, metric.STAGE, atomic.LoadUint64(&metric.OplogGet.Value))
198 | LOG.Info(verbose, metric.STAGE, atomic.LoadUint64(&metric.OplogGet.Value))
199 | }
200 |
201 | LOG.Info("metric[%v] exit", metric)
202 | }()
203 | }
204 |
205 | func (metric *ReplicationMetric) getTunnelTraffic() string {
206 | traffic := atomic.LoadUint64(&metric.TunnelTraffic)
207 | return GetMetricWithSize(traffic)
208 | }
209 |
210 | func (metric *ReplicationMetric) Get() uint64 {
211 | return atomic.LoadUint64(&metric.OplogGet.Value)
212 | }
213 |
214 | func (metric *ReplicationMetric) Apply() uint64 {
215 | return atomic.LoadUint64(&metric.OplogApply.Value)
216 | }
217 |
218 | func (metric *ReplicationMetric) Success() uint64 {
219 | return atomic.LoadUint64(&metric.OplogSuccess.Value)
220 | }
221 |
222 | func (metric *ReplicationMetric) Tps() uint64 {
223 | return atomic.LoadUint64(&metric.OplogSuccess.Delta)
224 | }
225 |
226 | func (metric *ReplicationMetric) AddSuccess(incr uint64) {
227 | atomic.AddUint64(&metric.OplogSuccess.Value, incr)
228 | }
229 |
230 | func (metric *ReplicationMetric) AddGet(incr uint64) {
231 | atomic.AddUint64(&metric.OplogGet.Value, incr)
232 | }
233 |
234 | func (metric *ReplicationMetric) AddCheckpoint(number uint64) {
235 | atomic.AddUint64(&metric.CheckpointTimes, number)
236 | }
237 |
238 | func (metric *ReplicationMetric) AddRetransmission(number uint64) {
239 | atomic.AddUint64(&metric.Retransmission, number)
240 | }
241 |
242 | func (metric *ReplicationMetric) AddTunnelTraffic(number uint64) {
243 | atomic.AddUint64(&metric.TunnelTraffic, number)
244 | }
245 |
246 | func (metric *ReplicationMetric) AddFilter(incr uint64) {
247 | atomic.AddUint64(&metric.OplogFilter.Value, incr)
248 | }
249 |
250 | func (metric *ReplicationMetric) AddApply(incr uint64) {
251 | atomic.AddUint64(&metric.OplogApply.Value, incr)
252 | }
253 |
254 | func (metric *ReplicationMetric) AddFailed(incr uint64) {
255 | atomic.AddUint64(&metric.OplogFail.Value, incr)
256 | }
257 |
258 | func (metric *ReplicationMetric) AddConsume(incr uint64) {
259 | atomic.AddUint64(&metric.OplogConsume.Value, incr)
260 | }
261 |
262 | func (metric *ReplicationMetric) SetOplogMax(max int64) {
263 | forwardCas(&metric.OplogMaxSize, max)
264 | }
265 |
266 | func (metric *ReplicationMetric) SetOplogAvg(size int64) {
267 | // not atomic update ! acceptable
268 | avg := (atomic.LoadInt64(&metric.OplogAvgSize) + size) / 2
269 | atomic.StoreInt64(&metric.OplogAvgSize, avg)
270 | }
271 |
272 | func (metric *ReplicationMetric) SetLSNCheckpoint(ckpt int64) {
273 | forwardCas(&metric.LSNCheckpoint, ckpt)
274 | }
275 |
276 | func (metric *ReplicationMetric) SetLSN(lsn int64) {
277 | forwardCas(&metric.LSN, lsn)
278 | }
279 |
280 | func (metric *ReplicationMetric) SetLSNACK(ack int64) {
281 | forwardCas(&metric.LSNAck, ack)
282 | }
283 |
284 | func (metric *ReplicationMetric) AddTableOps(table string, n uint64) {
285 | metric.TableOperations.Incr(table, n)
286 | }
287 |
288 | func (metric *ReplicationMetric) TableOps() map[string]uint64 {
289 | return metric.TableOperations.MakeCopy()
290 | }
291 |
292 | func (metric *ReplicationMetric) AddWriteFailed(incr uint64) {
293 | atomic.AddUint64(&metric.OplogWriteFail.Value, incr)
294 | }
295 |
296 | /************************************************************/
297 |
298 | func forwardCas(v *int64, new int64) {
299 | var current int64
300 | for current = atomic.LoadInt64(v); new > current; {
301 | if atomic.CompareAndSwapInt64(v, current, new) {
302 | break
303 | }
304 | current = atomic.LoadInt64(v)
305 | }
306 | }
307 |
308 | func (status *ReplicationStatus) Update(s uint64) {
309 | atomic.StoreUint64((*uint64)(status), s)
310 | }
311 |
312 | // TableOps, count collection operations
313 | type TableOps struct {
314 | sync.Mutex
315 | ops map[string]uint64
316 | }
317 |
318 | func NewTableOps() *TableOps {
319 | return &TableOps{ops: make(map[string]uint64)}
320 | }
321 |
322 | func (t *TableOps) Incr(table string, n uint64) {
323 | t.Lock()
324 | defer t.Unlock()
325 | t.ops[table] += n
326 | }
327 |
328 | func (t *TableOps) MakeCopy() map[string]uint64 {
329 | t.Lock()
330 | defer t.Unlock()
331 | c := make(map[string]uint64, len(t.ops))
332 | for k, v := range t.ops {
333 | c[k] = v
334 | }
335 | return c
336 | }
337 |
338 | func GetMetricWithSize(input interface{}) string {
339 | var val float64
340 | switch v := input.(type) {
341 | case uint64:
342 | val = float64(v)
343 | case uint32:
344 | val = float64(v)
345 | case uint16:
346 | val = float64(v)
347 | case uint:
348 | val = float64(v)
349 | case int64:
350 | val = float64(v)
351 | case int32:
352 | val = float64(v)
353 | case int16:
354 | val = float64(v)
355 | case int:
356 | val = float64(v)
357 | default:
358 | return "unknown type"
359 | }
360 |
361 | switch {
362 | case val > PB:
363 | return fmt.Sprintf("%.2fPB", val/PB)
364 | case val > TB:
365 | return fmt.Sprintf("%.2fTB", val/TB)
366 | case val > GB:
367 | return fmt.Sprintf("%.2fGB", val/GB)
368 | case val > MB:
369 | return fmt.Sprintf("%.2fMB", val/MB)
370 | case val > KB:
371 | return fmt.Sprintf("%.2fKB", val/KB)
372 | default:
373 | return fmt.Sprintf("%.2fB", val)
374 | }
375 | }
--------------------------------------------------------------------------------
/nimo-shake/common/mix.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "os"
5 | "path/filepath"
6 |
7 | "github.com/nightlyone/lockfile"
8 | LOG "github.com/vinllen/log4go"
9 | )
10 |
11 | func WritePid(id string) (err error) {
12 | var lock lockfile.Lockfile
13 | lock, err = lockfile.New(id)
14 | if err != nil {
15 | return err
16 | }
17 | if err = lock.TryLock(); err != nil {
18 | return err
19 | }
20 |
21 | return nil
22 | }
23 |
24 | func WritePidById(id string, path string) error {
25 | var dir string
26 | var err error
27 | if path == "" {
28 | if dir, err = os.Getwd(); err != nil {
29 | return err
30 | }
31 | } else {
32 | dir = path
33 | if _, err := os.Stat(dir); os.IsNotExist(err) {
34 | os.Mkdir(dir, os.ModePerm)
35 | }
36 | }
37 |
38 | if dir, err = filepath.Abs(dir); err != nil {
39 | return err
40 | }
41 |
42 | pidfile := filepath.Join(dir, id) + ".pid"
43 | if err := WritePid(pidfile); err != nil {
44 | return err
45 | }
46 | return nil
47 | }
48 |
49 | func Welcome() {
50 | welcome :=
51 | `______________________________
52 | \ \ _ ______ |
53 | \ \ / \___-=O'/|O'/__|
54 | \ NimoShake, here we go !! \_______\ / | / )
55 | / / '/-==__ _/__|/__=-| -GM
56 | / Alibaba Cloud / * \ | |
57 | / zhuzhao / (o)
58 | ------------------------------
59 | `
60 | startMsg := "if you have any problem, call aliyun"
61 | LOG.Warn("\n%s%s\n\n", welcome, startMsg)
62 | }
--------------------------------------------------------------------------------
/nimo-shake/common/mongodb_community.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "context"
5 | "fmt"
6 |
7 | LOG "github.com/vinllen/log4go"
8 | "go.mongodb.org/mongo-driver/mongo"
9 | "go.mongodb.org/mongo-driver/mongo/options"
10 | "go.mongodb.org/mongo-driver/mongo/readconcern"
11 | "go.mongodb.org/mongo-driver/mongo/readpref"
12 | )
13 |
14 | type MongoCommunityConn struct {
15 | Client *mongo.Client
16 | URL string
17 | ctx context.Context
18 | }
19 |
20 | func NewMongoCommunityConn(url string, connectMode string, timeout bool) (*MongoCommunityConn, error) {
21 | clientOps := options.Client().ApplyURI(url)
22 |
23 | clientOps.SetReadConcern(readconcern.New(readconcern.Level("local")))
24 |
25 | // read pref
26 | if mode, err := readpref.ModeFromString(connectMode); err != nil {
27 | return nil, fmt.Errorf("create connectMode[%v] failed: %v", connectMode, err)
28 | } else if opts, err := readpref.New(mode); err != nil {
29 | return nil, fmt.Errorf("new mode with connectMode[%v] failed: %v", connectMode, err)
30 | } else {
31 | clientOps.SetReadPreference(opts)
32 | }
33 |
34 | // set timeout
35 | if !timeout {
36 | clientOps.SetConnectTimeout(0)
37 | }
38 |
39 | // create default context
40 | ctx := context.Background()
41 |
42 | // connect
43 | client, err := mongo.NewClient(clientOps)
44 | if err != nil {
45 | return nil, fmt.Errorf("new client failed: %v", err)
46 | }
47 | if err := client.Connect(ctx); err != nil {
48 | return nil, fmt.Errorf("connect to %s failed: %v", BlockMongoUrlPassword(url, "***"), err)
49 | }
50 |
51 | // ping
52 | if err = client.Ping(ctx, clientOps.ReadPreference); err != nil {
53 | return nil, fmt.Errorf("ping to %v failed: %v", BlockMongoUrlPassword(url, "***"), err)
54 | }
55 |
56 | LOG.Info("New session to %s successfully", BlockMongoUrlPassword(url, "***"))
57 | return &MongoCommunityConn{
58 | Client: client,
59 | URL: url,
60 | }, nil
61 | }
62 |
63 | func (conn *MongoCommunityConn) Close() {
64 | LOG.Info("Close client with %s", BlockMongoUrlPassword(conn.URL, "***"))
65 | conn.Client.Disconnect(conn.ctx)
66 | }
67 |
--------------------------------------------------------------------------------
/nimo-shake/common/mongodb_mgo.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "time"
5 | "fmt"
6 | "reflect"
7 |
8 | "github.com/vinllen/mgo"
9 | "github.com/vinllen/mgo/bson"
10 | LOG "github.com/vinllen/log4go"
11 | "github.com/jinzhu/copier"
12 | )
13 |
14 | const (
15 | ConnectModePrimary = "primary"
16 | ConnectModeSecondaryPreferred = "secondaryPreferred"
17 | ConnectModeStandalone = "standalone"
18 |
19 | OplogNS = "oplog.rs"
20 | )
21 |
22 | var (
23 | NotFountErr = "not found"
24 | NsNotFountErr = "ns not found"
25 | )
26 |
27 | type NS struct {
28 | Database string
29 | Collection string
30 | }
31 |
32 | func (ns NS) Str() string {
33 | return fmt.Sprintf("%s.%s", ns.Database, ns.Collection)
34 | }
35 |
36 | type MongoConn struct {
37 | Session *mgo.Session
38 | URL string
39 | }
40 |
41 | func NewMongoConn(url string, connectMode string, timeout bool) (*MongoConn, error) {
42 | if connectMode == ConnectModeStandalone {
43 | url += "?connect=direct"
44 | }
45 |
46 | session, err := mgo.Dial(url)
47 | if err != nil {
48 | LOG.Critical("Connect to [%s] failed. %v", url, err)
49 | return nil, err
50 | }
51 | // maximum pooled connections. the overall established sockets
52 | // should be lower than this value(will block otherwise)
53 | session.SetPoolLimit(256)
54 | if timeout {
55 | session.SetSocketTimeout(10 * time.Minute)
56 | } else {
57 | session.SetSocketTimeout(0)
58 | }
59 |
60 | // already ping in the session
61 | /*if err := session.Ping(); err != nil {
62 | LOG.Critical("Verify ping command to %s failed. %v", url, err)
63 | return nil, err
64 | }*/
65 |
66 | // Switch the session to a eventually behavior. In that case session
67 | // may read for any secondary node. default mode is mgo.Strong
68 | switch connectMode {
69 | case ConnectModePrimary:
70 | session.SetMode(mgo.Primary, true)
71 | case ConnectModeSecondaryPreferred:
72 | session.SetMode(mgo.SecondaryPreferred, true)
73 | case ConnectModeStandalone:
74 | session.SetMode(mgo.Monotonic, true)
75 | default:
76 | err = fmt.Errorf("unknown connect mode[%v]", connectMode)
77 | return nil, err
78 | }
79 |
80 | LOG.Info("New session to %s successfully", url)
81 | return &MongoConn{Session: session, URL: url}, nil
82 | }
83 |
84 | func (conn *MongoConn) Close() {
85 | LOG.Info("Close session with %s", conn.URL)
86 | conn.Session.Close()
87 | }
88 |
89 | func (conn *MongoConn) IsGood() bool {
90 | if err := conn.Session.Ping(); err != nil {
91 | return false
92 | }
93 |
94 | return true
95 | }
96 |
97 | func (conn *MongoConn) AcquireReplicaSetName() string {
98 | var replicaset struct {
99 | Id string `bson:"set"`
100 | }
101 | if err := conn.Session.DB("admin").Run(bson.M{"replSetGetStatus": 1}, &replicaset); err != nil {
102 | LOG.Warn("Replica set name not found in system.replset, %v", err)
103 | }
104 | return replicaset.Id
105 | }
106 |
107 | func (conn *MongoConn) HasOplogNs() bool {
108 | if ns, err := conn.Session.DB("local").CollectionNames(); err == nil {
109 | for _, table := range ns {
110 | if table == OplogNS {
111 | return true
112 | }
113 | }
114 | }
115 | return false
116 | }
117 |
118 | func (conn *MongoConn) HasUniqueIndex() bool {
119 | checkNs := make([]NS, 0, 128)
120 | var databases []string
121 | var err error
122 | if databases, err = conn.Session.DatabaseNames(); err != nil {
123 | LOG.Critical("Couldn't get databases from remote server %v", err)
124 | return false
125 | }
126 |
127 | for _, db := range databases {
128 | if db != "admin" && db != "local" {
129 | coll, _ := conn.Session.DB(db).CollectionNames()
130 | for _, c := range coll {
131 | if c != "system.profile" {
132 | // push all collections
133 | checkNs = append(checkNs, NS{Database: db, Collection: c})
134 | }
135 | }
136 | }
137 | }
138 |
139 | for _, ns := range checkNs {
140 | indexes, _ := conn.Session.DB(ns.Database).C(ns.Collection).Indexes()
141 | for _, idx := range indexes {
142 | // has unique index
143 | if idx.Unique {
144 | LOG.Info("Found unique index %s on %s.%s in auto shard mode", idx.Name, ns.Database, ns.Collection)
145 | return true
146 | }
147 | }
148 | }
149 |
150 | return false
151 | }
152 |
153 | // first is from dynamo, second is from mongo
154 | func CompareBson(first, second bson.M) (bool, error) {
155 | v2 := make(bson.M, 0)
156 | if err := copier.Copy(&v2, &second); err != nil {
157 | return false, fmt.Errorf("copy[%v] failed[%v]", second, err)
158 | }
159 |
160 | if _, ok := v2["_id"]; ok {
161 | delete(v2, "_id")
162 | }
163 |
164 | return reflect.DeepEqual(first, v2), nil
165 | // return DeepEqual(first, v2), nil
166 | }
167 |
--------------------------------------------------------------------------------
/nimo-shake/common/operator.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "strings"
5 | )
6 |
7 | func AppendStringList(input []string, list []*string) ([]string) {
8 | for _, ele := range list {
9 | input = append(input, *ele)
10 | }
11 | return input
12 | }
13 |
14 | func StringListToMap(input []string) map[string]struct{} {
15 | mp := make(map[string]struct{}, len(input))
16 | for _, ele := range input {
17 | mp[ele] = struct{}{}
18 | }
19 | return mp
20 | }
21 |
22 | // @vinllen. see BulkError.Error. -1 means not found
23 | func FindFirstErrorIndexAndMessage(error string) (int, string, bool) {
24 | subIndex := "index["
25 | subMsg := "msg["
26 | subDup := "dup["
27 | index := strings.Index(error, subIndex)
28 | if index == -1 {
29 | return index, "", false
30 | }
31 |
32 | indexVal := 0
33 | for i := index + len(subIndex); i < len(error) && error[i] != ']'; i++ {
34 | // fmt.Printf("%c %d\n", rune(error[i]), int(error[i] - '0'))
35 | indexVal = indexVal * 10 + int(error[i] - '0')
36 | }
37 |
38 | index = strings.Index(error, subMsg)
39 | if index == -1 {
40 | return indexVal, "", false
41 | }
42 |
43 | i := index + len(subMsg)
44 | stack := 0
45 | for ; i < len(error); i++ {
46 | if error[i] == ']' {
47 | if stack == 0 {
48 | break
49 | } else {
50 | stack -= 1
51 | }
52 | } else if error[i] == '[' {
53 | stack += 1
54 | }
55 | }
56 | msg := error[index + len(subMsg): i]
57 |
58 | index = strings.Index(error, subDup)
59 | if index == -1 {
60 | return indexVal, msg, false
61 | }
62 | i = index + len(subMsg)
63 | for ; i < len(error) && error[i] != ']'; i++ {}
64 | dupVal := error[index + len(subMsg):i]
65 |
66 | return indexVal, msg, dupVal == "true"
67 | }
--------------------------------------------------------------------------------
/nimo-shake/common/shard.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "fmt"
5 | "sort"
6 | "strings"
7 |
8 | "github.com/aws/aws-sdk-go/service/dynamodbstreams"
9 | )
10 |
11 | var (
12 | StopTraverseSonErr = fmt.Errorf("stop traverse")
13 | )
14 |
15 | type ShardNode struct {
16 | Shard *dynamodbstreams.Shard
17 | ShardArn string
18 | Sons map[string]*ShardNode
19 | Table string
20 | }
21 |
22 | // build shard tree base on the input shard list, return root node
23 | func BuildShardTree(shards []*dynamodbstreams.Shard, table string, ShardArn string) *ShardNode {
24 | pathMap := make(map[string]*ShardNode, len(shards)) // store the inheritance relationship
25 | inDegree := make(map[string]bool, len(shards))
26 | // initial
27 | for _, shard := range shards {
28 | pathMap[*shard.ShardId] = &ShardNode{
29 | Shard: shard,
30 | ShardArn: ShardArn,
31 | Sons: make(map[string]*ShardNode),
32 | Table: table,
33 | }
34 |
35 | inDegree[*shard.ShardId] = false
36 | }
37 |
38 | // build path
39 | for _, shard := range shards {
40 | node := pathMap[*shard.ShardId]
41 | father := shard.ParentShardId
42 | if father != nil {
43 | if _, ok := pathMap[*father]; !ok {
44 | // father node isn't exist on the pathMap, which means deleted
45 | continue
46 | }
47 | inDegree[*shard.ShardId] = true
48 | pathMap[*father].Sons[*shard.ShardId] = node
49 | }
50 | }
51 |
52 | // root is fake node
53 | rootNode := &ShardNode{
54 | Shard: nil,
55 | ShardArn: "",
56 | Sons: make(map[string]*ShardNode),
57 | Table: table,
58 | }
59 | for key, val := range inDegree {
60 | if val == false {
61 | rootNode.Sons[key] = pathMap[key]
62 | // fmt.Printf("root->%s\n", key)
63 | }
64 | }
65 | return rootNode
66 | }
67 |
68 | // dfs
69 | func TraverseShard(node *ShardNode, callback func(node *ShardNode) error) error {
70 | if node == nil {
71 | return nil
72 | }
73 |
74 | if node.Shard != nil {
75 | // skip root node
76 | if err := callback(node); err != nil {
77 | if err != StopTraverseSonErr {
78 | return err
79 | } else {
80 | // return if error == StopTraverseSonErr
81 | return nil
82 | }
83 | }
84 | // fmt.Printf("%s->%s\n", *node.Shard.ParentShardId, *node.Shard.ShardId)
85 | }
86 |
87 | for _, son := range node.Sons {
88 | if err := TraverseShard(son, callback); err != nil {
89 | return err
90 | }
91 | }
92 |
93 | return nil
94 | }
95 |
96 | // calculate md5. TODO: add UT
97 | func CalMd5(root *ShardNode) uint64 {
98 | if root == nil {
99 | return 0
100 | }
101 |
102 | list := make([]string, 0, len(root.Sons))
103 | for _, son := range root.Sons {
104 | ret := CalMd5(son)
105 | list = append(list, fmt.Sprintf("%s->%d", *son.Shard.ShardId, ret))
106 | }
107 |
108 | sort.Strings(list)
109 | concat := strings.Join(list, ";")
110 | // fmt.Println("concat: ", concat)
111 | return Md5In64(String2Bytes(concat))
112 | }
113 |
114 | func PrintShardTree(node *ShardNode) (string, error) {
115 | newSet := make([]string, 0)
116 | err := TraverseShard(node, func(node *ShardNode) error {
117 | var father string
118 | if node.Shard.ParentShardId != nil {
119 | father = *node.Shard.ParentShardId
120 | } else {
121 | father = "nil"
122 | }
123 | inheritance := strings.Join([]string{father, *node.Shard.ShardId}, "->")
124 | newSet = append(newSet, inheritance)
125 |
126 | return nil
127 | })
128 | if err != nil {
129 | return "", err
130 | }
131 |
132 | return strings.Join(newSet, "\n"), nil
133 | }
--------------------------------------------------------------------------------
/nimo-shake/common/unsafe.go:
--------------------------------------------------------------------------------
1 | /*
2 | // =====================================================================================
3 | //
4 | // Filename: BytesString.go
5 | //
6 | // Description: ref from fast http
7 | //
8 | // Version: 1.0
9 | // Created: 06/23/2018 02:34:41 PM
10 | // Revision: none
11 | // Compiler: go1.10.3
12 | //
13 | // Author: boyi.gw, boyi.gw@alibaba-inc.com
14 | // Company: Alibaba Group
15 | //
16 | // =====================================================================================
17 | */
18 |
19 | package utils
20 |
21 | import (
22 | "reflect"
23 | "unsafe"
24 | )
25 |
26 | /*
27 | // === FUNCTION ======================================================================
28 | // Name: String2Bytes
29 | // Description: return GoString's buffer slice(enable modify string)
30 | // =====================================================================================
31 | */
32 | func String2Bytes(s string) []byte {
33 | sh := (*reflect.StringHeader)(unsafe.Pointer(&s))
34 | bh := reflect.SliceHeader{
35 | Data: sh.Data,
36 | Len: sh.Len,
37 | Cap: sh.Len,
38 | }
39 | return *(*[]byte)(unsafe.Pointer(&bh))
40 | }
41 |
42 | /*
43 | // === FUNCTION ======================================================================
44 | // Name: Bytes2String
45 | // Description: convert b to string without copy
46 | // =====================================================================================
47 | */
48 | func Bytes2String(b []byte) string {
49 | return *(*string)(unsafe.Pointer(&b))
50 | }
51 |
52 | /*
53 | // === FUNCTION ======================================================================
54 | // Name: StringPointer
55 | // Description: returns &s[0]
56 | // =====================================================================================
57 | */
58 | func StringPointer(s string) unsafe.Pointer {
59 | p := (*reflect.StringHeader)(unsafe.Pointer(&s))
60 | return unsafe.Pointer(p.Data)
61 | }
62 |
63 | /*
64 | // === FUNCTION ======================================================================
65 | // Name: BytesPointer
66 | // Description: returns &b[0]
67 | // =====================================================================================
68 | */
69 | func BytesPointer(b []byte) unsafe.Pointer {
70 | p := (*reflect.SliceHeader)(unsafe.Pointer(&b))
71 | return unsafe.Pointer(p.Data)
72 | }
73 |
--------------------------------------------------------------------------------
/nimo-shake/conf/nimo-shake.conf:
--------------------------------------------------------------------------------
1 | # current configuration version, do not modify.
2 | # 当前配置文件的版本号，请不要修改该值。
3 | conf.version = 3
4 |
5 | # id
6 | id = nimo-shake
7 |
8 | # log file name，all log will be printed in stdout if log.file is empty
9 | # 日志文件，不配置将打印到stdout (e.g. dynamo-shake.log )
10 | log.file =
11 | # log level: "none", "error", "warn", "info", "debug". default is "info".
12 | log.level = info
13 | # log buffer，enabling it to make every log print but performance maybe decrease.
14 | # log buffer选项，不启用将会降低性能但保证退出时每条log都被打印，否则，退出时可能有最后几条log丢失
15 | log.buffer = true
16 |
17 | # pprof port.
18 | system_profile = 9330
19 | # restful port, not used currently. 查看metric
20 | # 全量和增量的restful监控端口，可以用curl查看内部监控metric统计情况。详见wiki。
21 | full_sync.http_port = 9341
22 | incr_sync.http_port = 9340
23 |
24 | # sync mode. currently, only support "full".
25 | # all: full sync and increase sync.
26 | # full: full sync only.
27 | # incr: increase sync only.
28 | # 同步的类型。
29 | # all: 全量+增量同步
30 | # full: 全量同步
31 | # incr: 增量同步
32 | sync_mode = all
33 |
34 | # incr sync parallel
35 | # 是否并行做增量同步: true为开启，false为关闭。开启后会消耗更多内存
36 | incr_sync_parallel = false
37 |
38 | # dynamodb configuration. leave empty if not set.
39 | # 源端dynamodb的账号信息配置，source.session_token和source.region没有可以留空
40 | source.access_key_id =
41 | source.secret_access_key =
42 | source.session_token =
43 | source.region =
44 | # 源端如果是endpoint类型，可以配置该参数，启用该参数则上述source参数失效
45 | # 例如：http://100.123.124.125:1010
46 | source.endpoint_url =
47 | # max_retries in session once failed
48 | source.session.max_retries = 3
49 | # session timeout, 0 means disable. unit: ms.
50 | source.session.timeout = 3000
51 |
52 | # filter collection split by semicolon(;). at most one of these two parameters can be given.
53 | # if the filter.collection.black is not empty, the given collection will be filtered while others collection passed.
54 | # if the filter.collection.white is not empty, the given collection will be passed while others collection filtered.
55 | # all the namespace will be passed if no condition given.
56 | # E.g., "filter.collection.white = c1;c2" means only c1 and c2 passed while the others filtered.
57 | # 表粒度黑白名单过滤，白名单表示通过，黑名单表示过滤，这两个参数不能同时指定，都指定表示全部通过。分号分隔不同表。
58 | # 举例："filter.collection.white = c1;c2"表示c1和c2表通过，剩下的都被过滤掉。
59 | filter.collection.white =
60 | filter.collection.black =
61 |
62 | # qps limit for each table.
63 | # 对表级别限速
64 | # the scan call(Scan) per second.
65 | # 全量阶段，我们调用的是scan命令，这个参数表示一秒钟最多调用的scan个数
66 | qps.full = 1000
67 | # the limit batch number in one query. default is 128.
68 | # 1次query内部的条数大小
69 | qps.full.batch_num = 128
70 | # the query call(GetRecords) per second.
71 | # 增量阶段，我们调用的是GetRecords命令，这个参数表示一秒钟最多调用的GetRecords个数
72 | qps.incr = 1000
73 | # the limit batch number in one query. default is 128.
74 | # 1次query内部的条数大小
75 | qps.incr.batch_num = 128
76 |
77 | # target mongodb configuration, currently, only supports sync to mongodb.
78 | # 目的端配置, 目前支持mongodb和aliyun_dynamo_proxy
79 | target.type = mongodb
80 | # target mongodb address, e.g., mongodb://username:password@10.1.1.1:3791,10.1.1.2:3792
81 | # 如果是mongodb，此处需要配置目的mongodb的连接串，否则请配置阿里云dynamodb的连接串
82 | # 例如：http://dds-xxxx:3717
83 | target.address =
84 | # target moongodb type, replica or sharding.
85 | # 目的mongodb类型, 副本集选择replica，分片集群请选择sharding
86 | target.mongodb.type = sharding
87 | # how to solve if target mongodb has the same name table.
88 | # "drop" means drop the table before syncing.
89 | # "rename" means rename current table which timestamp suffix, e.g., c1 -> c1.2019-07-01Z12:10:11
90 | # rename仅支持target.type=mongodb的情况
91 | # 如果目的端已经有重名的表，rename将会对原来的表进行重命名，添加
92 | # 时间戳后缀，比如c1变为c1.2019-07-01Z12:10:11；drop表示删除目的表；留空表示不处理。
93 | target.db.exist = drop
94 |
95 | # only sync schema without any data.
96 | # 仅同步数据结构，不同步任何数据。
97 | sync_schema_only = false
98 |
99 | # full sync configuration.
100 | # 全量同步参数
101 | # how many tables will be synced at the same time.
102 | # 表级别并发度，1次最多同步表的数目
103 | full.concurrency = 4
104 | # how many reading threads working in one table.
105 | # 表内文档的并发度，1个表最多有几个线程同时并发读取源端，对应Scan接口的TotalSegments
106 | full.read.concurrency = 1
107 | # how many writing threads working in one table.
108 | # 表内文档的并发度，1个表最多有几个线程同时并发写入目的端
109 | full.document.concurrency = 4
110 | # how many doc batched in one writing request
111 | # 一次聚合写入多少条数据，如果目的端是DynamoDB协议最大配置25
112 | full.document.write.batch = 25
113 | # the number of parsers which do parse dynamodb to mongodb.
114 | # 表内解析线程个数，用户转换dynamo协议到目的端对应协议
115 | full.document.parser = 2
116 | # enable sync user created indexes?
117 | # 主键索引primaryKey会默认创建索引，除此以外是否同步用户自建的索引，
118 | full.enable_index.user = true
119 | # change insert to update when duplicate key error found
120 | # 全量同步，目的端碰到相同的key，是否将insert改为update
121 | full.executor.insert_on_dup_update = true
122 |
123 | # increase sync configuration.
124 | # 增量同步参数
125 |
126 | # 增量同步并发参数，1个表最多有几个线程来读取这个表的shard数据，1个线程同时抓取一个shard
127 | increase.concurrency = 16
128 | # 增量同步，insert语句目的端碰到相同的key，是否将insert改为update
129 | increase.executor.insert_on_dup_update = true
130 | # 增量同步，update语句目的端不存在key，是否将update改为upsert
131 | increase.executor.upsert = true
132 |
133 | # checkpoint存储的类型，可以是mongodb（target.type必须是mongodb），也可以是
134 | # file，本地落盘
135 | checkpoint.type = mongodb
136 | # checkpoint存储的地址，如果是目的端是mongodb则存储目的端的mongodb的地址，不配置默认目的mongodb
137 | # 如果是file则填写相对路径（例如checkpoint），不配置默认名为checkpoint
138 | checkpoint.address =
139 | # checkpoint存储在目的mongodb中的db名，默认以"$id-checkpoint"存储
140 | checkpoint.db =
141 |
142 | # 给DynamoDB中的_id字段增加前缀，不和MongoDB的_id冲突
143 | convert._id = pre
144 |
145 | # :begin 和 :end，这两个冒号开头的是变量，实际的值在 filter_attributevalues 中
146 | # begin:N:1646724207280，begin 是变量名，N 为类型，1646724207280 为值内容
147 | # N 为 Number, S 为 String
148 | full.read.filter_expression = create_time > :begin AND create_time < :end
149 | full.read.filter_attributevalues = begin```N```1646724207280~~~end```N```1646724207283
--------------------------------------------------------------------------------
/nimo-shake/configure/check.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | import (
4 | "bufio"
5 | "strings"
6 | "strconv"
7 | "fmt"
8 | "os"
9 |
10 | "nimo-shake/common"
11 | )
12 |
13 | // read the given file and parse the fcv do comparison
14 | func CheckFcv(file string, fcv int) (int, error) {
15 | // read line by line and parse the version
16 |
17 | f, err := os.Open(file)
18 | if err != nil {
19 | return -1, err
20 | }
21 |
22 | scanner := bufio.NewScanner(f)
23 | versionName := "conf.version"
24 | version := 0
25 | for scanner.Scan() {
26 | field := strings.Split(scanner.Text(), "=")
27 | if len(field) >= 2 && strings.HasPrefix(field[0], versionName) {
28 | if value, err := strconv.Atoi(strings.Trim(field[1], " ")); err != nil {
29 | return 0, fmt.Errorf("illegal value[%v]", field[1])
30 | } else {
31 | version = value
32 | break
33 | }
34 | }
35 | }
36 |
37 | if version < fcv {
38 | return version, fmt.Errorf("current required configuration version[%v] > input[%v], please upgrade NimoShake to version >= %v",
39 | fcv, version, utils.LowestConfigurationVersion[fcv])
40 | }
41 | return version, nil
42 | }
--------------------------------------------------------------------------------
/nimo-shake/configure/conf.go:
--------------------------------------------------------------------------------
1 | package conf
2 |
3 | type Configuration struct {
4 | ConfVersion uint `config:"conf.version"` // do not modify the tag name
5 | Id string `config:"id"`
6 | LogFile string `config:"log.file"`
7 | LogLevel string `config:"log.level"`
8 | LogBuffer bool `config:"log.buffer"`
9 | PidPath string `config:"pid_path"`
10 | SystemProfile int `config:"system_profile"`
11 | FullSyncHTTPListenPort int `config:"full_sync.http_port"`
12 | IncrSyncHTTPListenPort int `config:"incr_sync.http_port"`
13 | SyncMode string `config:"sync_mode"`
14 | SourceAccessKeyID string `config:"source.access_key_id"`
15 | SourceSecretAccessKey string `config:"source.secret_access_key"`
16 | SourceSessionToken string `config:"source.session_token"`
17 | SourceRegion string `config:"source.region"`
18 | SourceEndpointUrl string `config:"source.endpoint_url"`
19 | SourceSessionMaxRetries uint `config:"source.session.max_retries"`
20 | SourceSessionTimeout uint `config:"source.session.timeout"`
21 | QpsFull uint `config:"qps.full"`
22 | QpsFullBatchNum int64 `config:"qps.full.batch_num"`
23 | QpsIncr uint `config:"qps.incr"`
24 | QpsIncrBatchNum int64 `config:"qps.incr.batch_num"`
25 | FilterCollectionWhite string `config:"filter.collection.white"`
26 | FilterCollectionBlack string `config:"filter.collection.black"`
27 | TargetType string `config:"target.type"`
28 | TargetAddress string `config:"target.address"`
29 | TargetMongoDBType string `config:"target.mongodb.type"`
30 | TargetDBExist string `config:"target.db.exist"`
31 | SyncSchemaOnly bool `config:"sync_schema_only"`
32 | FullConcurrency uint `config:"full.concurrency"`
33 | FullReadConcurrency uint `config:"full.read.concurrency"`
34 | FullDocumentConcurrency uint `config:"full.document.concurrency"`
35 | FullDocumentWriteBatch uint `config:"full.document.write.batch"`
36 | FullDocumentParser uint `config:"full.document.parser"`
37 | FullEnableIndexPrimary bool `config:"full.enable_index.primary"`
38 | FullEnableIndexUser bool `config:"full.enable_index.user"`
39 | FullExecutorInsertOnDupUpdate bool `config:"full.executor.insert_on_dup_update"`
40 | FullFilterExpression string `config:"full.read.filter_expression"`
41 | FullFilterAttributeValues string `config:"full.read.filter_attributevalues"`
42 | ConvertType string `config:"convert.type"`
43 | ConvertId string `config:"convert._id"`
44 | IncreaseConcurrency uint `config:"increase.concurrency"`
45 | IncreaseExecutorInsertOnDupUpdate bool `config:"increase.executor.insert_on_dup_update"`
46 | IncreaseExecutorUpsert bool `config:"increase.executor.upsert"`
47 | IncreasePersistDir string `config:"increase.persist.dir"`
48 | CheckpointType string `config:"checkpoint.type"`
49 | CheckpointAddress string `config:"checkpoint.address"`
50 | CheckpointDb string `config:"checkpoint.db"`
51 | IncrSyncParallel bool `config:"incr_sync_parallel"`
52 |
53 | /*---------------------------------------------------------*/
54 | // generated variables
55 | Version string // version
56 | }
57 |
58 | var Options Configuration
59 |
60 | func ConvertIdFunc(name string) string {
61 | if Options.ConvertId != "" && name == "_id" {
62 | return Options.ConvertId + name
63 | }
64 |
65 | return name
66 | }
67 |
--------------------------------------------------------------------------------
/nimo-shake/filter/filter.go:
--------------------------------------------------------------------------------
1 | package filter
2 |
3 | import (
4 | "strings"
5 | )
6 |
7 | const (
8 | Sep = ";"
9 | )
10 |
11 | var (
12 | f = &Filter{
13 | collectionWhiteMap: make(map[string]bool),
14 | collectionBlackMap: make(map[string]bool),
15 | }
16 | )
17 |
18 | type Filter struct {
19 | collectionWhiteMap map[string]bool
20 | collectionBlackMap map[string]bool
21 | }
22 |
23 | func Init(collectionWhite, collectBlack string) {
24 | var collectionWhiteList, collectionBlackList []string
25 | if collectionWhite != "" {
26 | collectionWhiteList = strings.Split(collectionWhite, Sep)
27 | }
28 | if collectBlack != "" {
29 | collectionBlackList = strings.Split(collectBlack, Sep)
30 | }
31 |
32 | for _, ele := range collectionWhiteList {
33 | f.collectionWhiteMap[ele] = true
34 | }
35 | for _, ele := range collectionBlackList {
36 | f.collectionBlackMap[ele] = true
37 | }
38 | }
39 |
40 | func IsFilter(collection string) bool {
41 | // fmt.Println(f.collectionWhiteMap, collection, f.collectionWhiteMap[collection])
42 | if len(f.collectionWhiteMap) != 0 {
43 | return !f.collectionWhiteMap[collection]
44 | } else if len(f.collectionBlackMap) != 0 {
45 | return f.collectionBlackMap[collection]
46 | }
47 | return false
48 | }
49 |
50 | func FilterList(collectionList []string) []string {
51 | ret := make([]string, 0, len(collectionList))
52 | for _, ele := range collectionList {
53 | if !IsFilter(ele) {
54 | ret = append(ret, ele)
55 | }
56 | }
57 | return ret
58 | }
--------------------------------------------------------------------------------
/nimo-shake/filter/filter_test.go:
--------------------------------------------------------------------------------
1 | package filter
2 |
3 | import (
4 | "testing"
5 | "fmt"
6 |
7 | "github.com/stretchr/testify/assert"
8 | )
9 |
10 | func TestFilter(t *testing.T) {
11 | // test InsertCkpt, UpdateCkpt, QueryCkpt, DropCheckpoint
12 |
13 | var nr int
14 | {
15 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr)
16 | nr++
17 |
18 | f = &Filter{
19 | collectionWhiteMap: make(map[string]bool),
20 | collectionBlackMap: make(map[string]bool),
21 | }
22 | Init("abc;efg;a", "")
23 |
24 | assert.Equal(t, true, IsFilter("anc"), "should be equal")
25 | assert.Equal(t, true, IsFilter("ab"), "should be equal")
26 | assert.Equal(t, false, IsFilter("abc"), "should be equal")
27 | assert.Equal(t, false, IsFilter("efg"), "should be equal")
28 | }
29 |
30 | {
31 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr)
32 | nr++
33 |
34 | f = &Filter{
35 | collectionWhiteMap: make(map[string]bool),
36 | collectionBlackMap: make(map[string]bool),
37 | }
38 | Init("","abc;efg;a")
39 |
40 | assert.Equal(t, false, IsFilter("anc"), "should be equal")
41 | assert.Equal(t, false, IsFilter("ab"), "should be equal")
42 | assert.Equal(t, true, IsFilter("abc"), "should be equal")
43 | assert.Equal(t, true, IsFilter("efg"), "should be equal")
44 | }
45 |
46 | {
47 | fmt.Printf("TestCheckpointCRUD case %d.\n", nr)
48 | nr++
49 |
50 | f = &Filter{
51 | collectionWhiteMap: make(map[string]bool),
52 | collectionBlackMap: make(map[string]bool),
53 | }
54 | Init("","")
55 |
56 | assert.Equal(t, false, IsFilter("anc"), "should be equal")
57 | assert.Equal(t, false, IsFilter("ab"), "should be equal")
58 | assert.Equal(t, false, IsFilter("abc"), "should be equal")
59 | assert.Equal(t, false, IsFilter("efg"), "should be equal")
60 | }
61 | }
--------------------------------------------------------------------------------
/nimo-shake/full-sync/document-syncer.go:
--------------------------------------------------------------------------------
1 | package full_sync
2 |
3 | import (
4 | "fmt"
5 | "time"
6 |
7 | "nimo-shake/common"
8 | "nimo-shake/configure"
9 | "nimo-shake/writer"
10 |
11 | "github.com/aws/aws-sdk-go/service/dynamodb"
12 | LOG "github.com/vinllen/log4go"
13 | "nimo-shake/protocal"
14 | "sync/atomic"
15 | )
16 |
17 | const (
18 | batchSize = 2 * utils.MB // mongodb limit: 16MB
19 | batchTimeout = 1 // seconds
20 | )
21 |
22 | var (
23 | UT_TestDocumentSyncer = false
24 | UT_TestDocumentSyncer_Chan chan []interface{}
25 | )
26 |
27 | /*------------------------------------------------------*/
28 | // one document link corresponding to one documentSyncer
29 | type documentSyncer struct {
30 | tableSyncerId int
31 | id int // documentSyncer id
32 | ns utils.NS
33 | inputChan chan interface{} // parserChan in table-syncer
34 | writer writer.Writer
35 | collectionMetric *utils.CollectionMetric
36 | }
37 |
38 | func NewDocumentSyncer(tableSyncerId int, table string, id int, inputChan chan interface{},
39 | tableDescribe *dynamodb.TableDescription, collectionMetric *utils.CollectionMetric) *documentSyncer {
40 | ns := utils.NS{
41 | Database: conf.Options.Id,
42 | Collection: table,
43 | }
44 |
45 | w := writer.NewWriter(conf.Options.TargetType, conf.Options.TargetAddress, ns, conf.Options.LogLevel)
46 | if w == nil {
47 | LOG.Crashf("tableSyncer[%v] documentSyncer[%v] create writer failed", tableSyncerId, table)
48 | }
49 |
50 | w.PassTableDesc(tableDescribe)
51 |
52 | return &documentSyncer{
53 | tableSyncerId: tableSyncerId,
54 | id: id,
55 | inputChan: inputChan,
56 | writer: w,
57 | ns: ns,
58 | collectionMetric: collectionMetric,
59 | }
60 | }
61 |
62 | func (ds *documentSyncer) String() string {
63 | return fmt.Sprintf("tableSyncer[%v] documentSyncer[%v] ns[%v]", ds.tableSyncerId, ds.id, ds.ns)
64 | }
65 |
66 | func (ds *documentSyncer) Close() {
67 | ds.writer.Close()
68 | }
69 |
70 | func (ds *documentSyncer) Run() {
71 | batchNumber := int(conf.Options.FullDocumentWriteBatch)
72 | LOG.Info("%s start with batchSize[%v]", ds.String(), batchNumber)
73 |
74 | var data interface{}
75 | var ok bool
76 | batchGroup := make([]interface{}, 0, batchNumber)
77 | timeout := false
78 | batchGroupSize := 0
79 | exit := false
80 | for {
81 | StartT := time.Now()
82 | select {
83 | case data, ok = <-ds.inputChan:
84 | if !ok {
85 | exit = true
86 | LOG.Info("%s channel already closed, flushing cache and exiting...", ds.String())
87 | }
88 | case <-time.After(time.Second * batchTimeout):
89 | // timeout
90 | timeout = true
91 | data = nil
92 | }
93 | readParserChanDuration := time.Since(StartT)
94 |
95 | LOG.Debug("exit[%v], timeout[%v], len(batchGroup)[%v], batchGroupSize[%v], data[%v]", exit, timeout,
96 | len(batchGroup), batchGroupSize, data)
97 |
98 | if data != nil {
99 | if UT_TestDocumentSyncer {
100 | batchGroup = append(batchGroup, data)
101 | } else {
102 | switch v := data.(type) {
103 | case protocal.RawData:
104 | if v.Size > 0 {
105 | batchGroup = append(batchGroup, v.Data)
106 | batchGroupSize += v.Size
107 | }
108 | case map[string]*dynamodb.AttributeValue:
109 | batchGroup = append(batchGroup, v)
110 | // meaningless batchGroupSize
111 | }
112 | }
113 | }
114 |
115 | if exit || timeout || len(batchGroup) >= batchNumber || batchGroupSize >= batchSize {
116 | StartT = time.Now()
117 | batchGroupLen := len(batchGroup)
118 | if len(batchGroup) != 0 {
119 | if err := ds.write(batchGroup); err != nil {
120 | LOG.Crashf("%s write data failed[%v]", ds.String(), err)
121 | }
122 |
123 | batchGroup = make([]interface{}, 0, batchNumber)
124 | batchGroupSize = 0
125 | }
126 | writeDestDBDuration := time.Since(StartT)
127 | LOG.Info("%s write db batch[%v] parserChan.len[%v] readParserChanTime[%v] writeDestDbTime[%v]",
128 | ds.String(), batchGroupLen, len(ds.inputChan), readParserChanDuration, writeDestDBDuration)
129 |
130 | if exit {
131 | break
132 | }
133 | timeout = false
134 | }
135 | }
136 |
137 | go func() {
138 | <-time.NewTimer(time.Minute * 5).C
139 | ds.writer.Close()
140 | LOG.Info("%s full-sync writer close", ds.String())
141 | }()
142 | LOG.Info("%s finish writing", ds.String())
143 | }
144 |
145 | func (ds *documentSyncer) write(input []interface{}) error {
146 | LOG.Debug("%s writing data with length[%v]", ds.String(), len(input))
147 | if len(input) == 0 {
148 | return nil
149 | }
150 |
151 | if UT_TestDocumentSyncer {
152 | UT_TestDocumentSyncer_Chan <- input
153 | return nil
154 | }
155 |
156 | defer atomic.AddUint64(&ds.collectionMetric.FinishCount, uint64(len(input)))
157 | return ds.writer.WriteBulk(input)
158 | }
159 |
--------------------------------------------------------------------------------
/nimo-shake/full-sync/document-syncer_test.go:
--------------------------------------------------------------------------------
1 | package full_sync
2 |
3 | import (
4 | "testing"
5 | "fmt"
6 | "github.com/stretchr/testify/assert"
7 | "time"
8 | "nimo-shake/configure"
9 | )
10 |
11 | func TestDocumentSyncer(t *testing.T) {
12 | // test documentSyncer main function
13 |
14 | conf.Options.FullDocumentWriteBatch = 25
15 | batchNumber := int(conf.Options.FullDocumentWriteBatch)
16 |
17 | var nr int
18 | {
19 | fmt.Printf("TestDocumentSyncer case %d.\n", nr)
20 | nr++
21 |
22 | UT_TestDocumentSyncer = true
23 | UT_TestDocumentSyncer_Chan = make(chan []interface{}, 1000)
24 |
25 | docSyncer := &documentSyncer{
26 | inputChan: make(chan interface{}, 10),
27 | }
28 | go docSyncer.Run()
29 |
30 | for i := 0; i < batchNumber - 5; i++ {
31 | docSyncer.inputChan<-i
32 | }
33 |
34 | out := <-UT_TestDocumentSyncer_Chan
35 | assert.Equal(t, batchNumber - 5, len(out), "should be equal")
36 | for i := 0; i < len(out); i++ {
37 | assert.Equal(t, i, out[i].(int), "should be equal")
38 | }
39 | }
40 |
41 | // output length > batchNumber
42 | {
43 | fmt.Printf("TestDocumentSyncer case %d.\n", nr)
44 | nr++
45 |
46 | UT_TestDocumentSyncer = true
47 | UT_TestDocumentSyncer_Chan = make(chan []interface{}, 1000)
48 |
49 | docSyncer := &documentSyncer{
50 | inputChan: make(chan interface{}, 10),
51 | }
52 | go docSyncer.Run()
53 |
54 | for i := 0; i < batchNumber + 5; i++ {
55 | docSyncer.inputChan<-i
56 | }
57 |
58 | out := make([]interface{}, 0)
59 | out1 := <-UT_TestDocumentSyncer_Chan
60 | assert.Equal(t, batchNumber, len(out1), "should be equal")
61 | out = append(out, out1...)
62 |
63 | out2 := <-UT_TestDocumentSyncer_Chan
64 | assert.Equal(t, 5, len(out2), "should be equal")
65 | out = append(out, out2...)
66 |
67 | for i := 0; i < len(out); i++ {
68 | assert.Equal(t, i, out[i].(int), "should be equal")
69 | }
70 | }
71 |
72 | // output timeout
73 | {
74 | fmt.Printf("TestDocumentSyncer case %d.\n", nr)
75 | nr++
76 |
77 | UT_TestDocumentSyncer = true
78 | UT_TestDocumentSyncer_Chan = make(chan []interface{}, 1000)
79 |
80 | docSyncer := &documentSyncer{
81 | inputChan: make(chan interface{}, 10),
82 | }
83 | go docSyncer.Run()
84 |
85 | for i := 0; i < batchNumber - 10; i++ {
86 | docSyncer.inputChan<-i
87 | }
88 | time.Sleep((batchTimeout + 2) * time.Second)
89 | for i := batchNumber - 10; i < batchNumber + 5; i++ {
90 | docSyncer.inputChan<-i
91 | }
92 |
93 | out := make([]interface{}, 0)
94 | out1 := <-UT_TestDocumentSyncer_Chan
95 | assert.Equal(t, batchNumber - 10, len(out1), "should be equal")
96 | out = append(out, out1...)
97 |
98 | out2 := <-UT_TestDocumentSyncer_Chan
99 | assert.Equal(t, 15, len(out2), "should be equal")
100 | out = append(out, out2...)
101 | fmt.Println(out)
102 |
103 | for i := 0; i < len(out); i++ {
104 | assert.Equal(t, i, out[i].(int), "should be equal")
105 | }
106 | }
107 | }
--------------------------------------------------------------------------------
/nimo-shake/full-sync/syncer.go:
--------------------------------------------------------------------------------
1 | package full_sync
2 |
3 | import (
4 | "fmt"
5 | "strings"
6 | "sync"
7 |
8 | utils "nimo-shake/common"
9 | conf "nimo-shake/configure"
10 | "nimo-shake/filter"
11 | "nimo-shake/writer"
12 |
13 | "time"
14 |
15 | "github.com/aws/aws-sdk-go/aws"
16 | "github.com/aws/aws-sdk-go/service/dynamodb"
17 | nimo "github.com/gugemichael/nimo4go"
18 | LOG "github.com/vinllen/log4go"
19 | bson2 "go.mongodb.org/mongo-driver/bson"
20 | "go.mongodb.org/mongo-driver/mongo"
21 | )
22 |
23 | var (
24 | metricNsMapLock sync.Mutex
25 | metricNsMap = make(map[string]*utils.CollectionMetric) // namespace map: db.collection -> collection metric
26 | )
27 |
28 | func Start(dynamoSession *dynamodb.DynamoDB, w writer.Writer) {
29 | // fetch all tables
30 | LOG.Info("start fetching table list")
31 | tableList, err := utils.FetchTableList(dynamoSession)
32 | if err != nil {
33 | LOG.Crashf("fetch table list failed[%v]", err)
34 | }
35 | LOG.Info("finish fetching table list: %v", tableList)
36 |
37 | tableList = filter.FilterList(tableList)
38 |
39 | if err := checkTableExists(tableList, w); err != nil {
40 | if !strings.Contains(err.Error(), "ResourceNotFoundException") {
41 | LOG.Crashf("check table exists failed[%v]", err)
42 | return
43 | }
44 | }
45 |
46 | LOG.Info("start syncing: %v", tableList)
47 |
48 | metricNsMapLock.Lock()
49 | for _, table := range tableList {
50 | metricNsMap[table] = utils.NewCollectionMetric()
51 | }
52 | metricNsMapLock.Unlock()
53 |
54 | fullChan := make(chan string, len(tableList))
55 | for _, table := range tableList {
56 | fullChan <- table
57 | }
58 |
59 | var wg sync.WaitGroup
60 | wg.Add(len(tableList))
61 | for i := 0; i < int(conf.Options.FullConcurrency); i++ {
62 | go func(id int) {
63 | for {
64 | table, ok := <-fullChan
65 | if !ok {
66 | // chan closed
67 | break
68 | }
69 |
70 | // no need to lock map because the map size won't change
71 | ts := NewTableSyncer(id, table, metricNsMap[table])
72 | if ts == nil {
73 | LOG.Crashf("tableSyncer[%v] create failed", id)
74 | }
75 |
76 | LOG.Info("tableSyncer[%v] starts sync table[%v]", id, table)
77 | ts.Sync()
78 | LOG.Info("tableSyncer[%v] finish sync table[%v]", id, table)
79 | ts.Close()
80 |
81 | wg.Done()
82 | }
83 | }(i)
84 | }
85 |
86 | wg.Wait()
87 | close(fullChan)
88 |
89 | LOG.Info("finish syncing all tables and indexes!")
90 | }
91 |
92 | func checkTableExists(tableList []string, w writer.Writer) error {
93 | LOG.Info("target.db.exist is set[%v]", conf.Options.TargetDBExist)
94 | switch conf.Options.TargetType {
95 | case utils.TargetTypeMongo:
96 |
97 | sess := w.GetSession().(*mongo.Client)
98 |
99 | now := time.Now().Format(utils.GolangSecurityTime)
100 | collections, err := sess.Database(conf.Options.Id).ListCollectionNames(nil, bson2.M{})
101 | if err != nil {
102 | return fmt.Errorf("get target collection names error[%v]", err)
103 | }
104 |
105 | collectionsMp := utils.StringListToMap(collections)
106 | for _, table := range tableList {
107 | // check exist on the target mongodb
108 | if _, ok := collectionsMp[table]; ok {
109 | // exist
110 | LOG.Info("table[%v] exists", table)
111 | if conf.Options.TargetDBExist == utils.TargetDBExistDrop {
112 | if err := sess.Database(conf.Options.Id).Collection(table).Drop(nil); err != nil {
113 | return fmt.Errorf("drop target collection[%v] failed[%v]", table, err)
114 | }
115 | } else if conf.Options.TargetDBExist == utils.TargetDBExistRename {
116 | fromCollection := fmt.Sprintf("%s.%s", conf.Options.Id, table)
117 | toCollection := fmt.Sprintf("%s.%s_%v", conf.Options.Id, table, now)
118 | res := sess.Database("admin").RunCommand(nil, bson2.D{
119 | {"renameCollection", fromCollection},
120 | {"to", toCollection},
121 | {"dropTarget", false},
122 | })
123 | if err := res.Err(); err != nil {
124 | return fmt.Errorf("rename target collection[%v] failed[%v]", table, err)
125 | }
126 | } else {
127 | //return fmt.Errorf("collection[%v] exists on the target", table)
128 | }
129 | }
130 | }
131 | case utils.TargetTypeAliyunDynamoProxy:
132 | sess := w.GetSession().(*dynamodb.DynamoDB)
133 |
134 | // query table list
135 | collections := make([]string, 0, 16)
136 |
137 | // dynamo-proxy is not support Limit and ExclusiveStartTableName
138 | /*lastTableName := aws.String("")
139 | var count int64 = 100
140 | for i := 0; ; i++ {
141 | LOG.Debug("list table round[%v]", i)
142 | var input *dynamodb.ListTablesInput
143 | if i == 0 {
144 | input = &dynamodb.ListTablesInput{
145 | Limit: aws.Int64(count),
146 | }
147 | } else {
148 | input = &dynamodb.ListTablesInput{
149 | ExclusiveStartTableName: lastTableName,
150 | Limit: aws.Int64(count),
151 | }
152 | }
153 | out, err := sess.ListTables(input)
154 | if err != nil {
155 | return fmt.Errorf("list table failed: %v", err)
156 | }
157 |
158 | for _, collection := range out.TableNames {
159 | collections = append(collections, *collection)
160 | }
161 |
162 | lastTableName = out.LastEvaluatedTableName
163 | if len(out.TableNames) < int(count) {
164 | break
165 | }
166 | }*/
167 | out, err := sess.ListTables(&dynamodb.ListTablesInput{})
168 | if err != nil {
169 | return fmt.Errorf("list table failed: %v", err)
170 | }
171 | for _, collection := range out.TableNames {
172 | collections = append(collections, *collection)
173 | }
174 |
175 | collectionsMp := utils.StringListToMap(collections)
176 | LOG.Info("target exit db list: %v", collections)
177 | for _, table := range tableList {
178 | // check exist on the target
179 | if _, ok := collectionsMp[table]; ok {
180 | // exist
181 | LOG.Info("table[%v] exists, try [%v]", table, conf.Options.TargetDBExist)
182 | if conf.Options.TargetDBExist == utils.TargetDBExistDrop {
183 | if _, err := sess.DeleteTable(&dynamodb.DeleteTableInput{
184 | TableName: aws.String(table),
185 | }); err != nil {
186 | return fmt.Errorf("drop target collection[%v] failed[%v]", table, err)
187 | }
188 | } else {
189 | return fmt.Errorf("collection[%v] exists on the target", table)
190 | }
191 | }
192 | }
193 | }
194 |
195 | LOG.Info("finish handling table exists")
196 |
197 | return nil
198 | }
199 |
200 | func RestAPI() {
201 | type FullSyncInfo struct {
202 | Progress string `json:"progress"` // synced_collection_number / total_collection_number
203 | TotalCollection int `json:"total_collection_number"` // total collection
204 | FinishedCollection int `json:"finished_collection_number"` // finished
205 | ProcessingCollection int `json:"processing_collection_number"` // in processing
206 | WaitCollection int `json:"wait_collection_number"` // wait start
207 | CollectionMetric map[string]string `json:"collection_metric"` // collection_name -> process
208 | }
209 |
210 | utils.FullSyncHttpApi.RegisterAPI("/progress", nimo.HttpGet, func([]byte) interface{} {
211 | ret := FullSyncInfo{
212 | CollectionMetric: make(map[string]string),
213 | }
214 |
215 | metricNsMapLock.Lock()
216 | defer metricNsMapLock.Unlock()
217 |
218 | ret.TotalCollection = len(metricNsMap)
219 | for ns, collectionMetric := range metricNsMap {
220 | ret.CollectionMetric[ns] = collectionMetric.String()
221 | switch collectionMetric.CollectionStatus {
222 | case utils.StatusWaitStart:
223 | ret.WaitCollection += 1
224 | case utils.StatusProcessing:
225 | ret.ProcessingCollection += 1
226 | case utils.StatusFinish:
227 | ret.FinishedCollection += 1
228 | }
229 | }
230 |
231 | if ret.TotalCollection == 0 {
232 | ret.Progress = "-%"
233 | } else {
234 | ret.Progress = fmt.Sprintf("%.2f%%", float64(ret.FinishedCollection)/float64(ret.TotalCollection)*100)
235 | }
236 |
237 | return ret
238 | })
239 |
240 | }
241 |
--------------------------------------------------------------------------------
/nimo-shake/full-sync/table-syncer.go:
--------------------------------------------------------------------------------
1 | package full_sync
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | "time"
7 |
8 | "github.com/aws/aws-sdk-go/aws/awserr"
9 | "github.com/aws/aws-sdk-go/aws/request"
10 |
11 | utils "nimo-shake/common"
12 | conf "nimo-shake/configure"
13 | "nimo-shake/protocal"
14 | "nimo-shake/qps"
15 | "nimo-shake/writer"
16 |
17 | "github.com/aws/aws-sdk-go/aws"
18 | "github.com/aws/aws-sdk-go/service/dynamodb"
19 | LOG "github.com/vinllen/log4go"
20 | )
21 |
22 | const (
23 | fetcherChanSize = 1024
24 | parserChanSize = 81920
25 | )
26 |
27 | type tableSyncer struct {
28 | id int
29 | ns utils.NS
30 | sourceConn *dynamodb.DynamoDB
31 | sourceTableDescribe *dynamodb.TableDescription
32 | fetcherChan chan *dynamodb.ScanOutput // chan between fetcher and parser
33 | parserChan chan interface{} // chan between parser and writer
34 | converter protocal.Converter // converter
35 | collectionMetric *utils.CollectionMetric
36 | }
37 |
38 | func NewTableSyncer(id int, table string, collectionMetric *utils.CollectionMetric) *tableSyncer {
39 | sourceConn, err := utils.CreateDynamoSession(conf.Options.LogLevel)
40 | if err != nil {
41 | LOG.Error("tableSyncer[%v] with table[%v] create dynamodb session error[%v]", id, table, err)
42 | return nil
43 | }
44 |
45 | // describe source table
46 | tableDescription, err := sourceConn.DescribeTable(&dynamodb.DescribeTableInput{
47 | TableName: aws.String(table),
48 | })
49 | if err != nil {
50 | LOG.Error("tableSyncer[%v] with table[%v] describe failed[%v]", id, table, err)
51 | return nil
52 | }
53 |
54 | converter := protocal.NewConverter(conf.Options.ConvertType)
55 | if converter == nil {
56 | LOG.Error("tableSyncer[%v] with table[%v] create converter failed", id, table)
57 | return nil
58 | }
59 |
60 | return &tableSyncer{
61 | id: id,
62 | sourceConn: sourceConn,
63 | sourceTableDescribe: tableDescription.Table,
64 | converter: converter,
65 | ns: utils.NS{
66 | Database: conf.Options.Id,
67 | Collection: table,
68 | },
69 | collectionMetric: collectionMetric,
70 | }
71 | }
72 |
73 | func (ts *tableSyncer) String() string {
74 | return fmt.Sprintf("tableSyncer[%v] with table[%v]", ts.id, ts.ns.Collection)
75 | }
76 |
77 | func (ts *tableSyncer) Sync() {
78 | ts.fetcherChan = make(chan *dynamodb.ScanOutput, fetcherChanSize)
79 | ts.parserChan = make(chan interface{}, parserChanSize)
80 |
81 | targetWriter := writer.NewWriter(conf.Options.TargetType, conf.Options.TargetAddress, ts.ns, conf.Options.LogLevel)
82 | if targetWriter == nil {
83 | LOG.Crashf("%s create writer failed", ts)
84 | return
85 | }
86 | // create table and index with description
87 | if err := targetWriter.CreateTable(ts.sourceTableDescribe); err != nil {
88 | LOG.Crashf("%s create table failed: %v", ts, err)
89 | return
90 | }
91 |
92 | // wait dynamo proxy to sync cache
93 | time.Sleep(10 * time.Second)
94 |
95 | if conf.Options.SyncSchemaOnly {
96 | LOG.Info("sync_schema_only enabled, %s exits", ts)
97 | return
98 | }
99 |
100 | // total table item count
101 | totalCount := ts.count()
102 |
103 | ts.collectionMetric.CollectionStatus = utils.StatusProcessing
104 | ts.collectionMetric.TotalCount = totalCount
105 |
106 | // start fetcher to fetch all data from DynamoDB
107 | go ts.fetcher()
108 |
109 | // start parser to get data from fetcher and write into writer.
110 | // we can also start several parsers to accelerate
111 | var wgParser sync.WaitGroup
112 | wgParser.Add(int(conf.Options.FullDocumentParser))
113 | for i := 0; i < int(conf.Options.FullDocumentParser); i++ {
114 | go func(id int) {
115 | ts.parser(id)
116 | wgParser.Done()
117 | }(i)
118 | }
119 |
120 | // start writer
121 | var wgWriter sync.WaitGroup
122 | wgWriter.Add(int(conf.Options.FullDocumentConcurrency))
123 | for i := 0; i < int(conf.Options.FullDocumentConcurrency); i++ {
124 | go func(id int) {
125 | LOG.Info("%s create document syncer with id[%v]", ts, id)
126 | ds := NewDocumentSyncer(ts.id, ts.ns.Collection, id, ts.parserChan, ts.sourceTableDescribe,
127 | ts.collectionMetric)
128 | ds.Run()
129 | LOG.Info("%s document syncer with id[%v] exit", ts, id)
130 | wgWriter.Done()
131 | }(i)
132 | }
133 |
134 | LOG.Info("%s wait all parsers exiting", ts.String())
135 | wgParser.Wait() // wait all parser exit
136 | close(ts.parserChan) // close parser channel
137 |
138 | LOG.Info("%s all parsers exited, wait all writers exiting", ts.String())
139 | wgWriter.Wait() // wait all writer exit
140 |
141 | ts.collectionMetric.CollectionStatus = utils.StatusFinish
142 | LOG.Info("%s finish syncing table", ts.String())
143 | }
144 |
145 | func (ts *tableSyncer) Close() {
146 | // TODO, dynamo-session doesn't have close function?
147 | }
148 |
149 | func (ts *tableSyncer) fetcher() {
150 | LOG.Info("%s start fetcher with %v reader", ts.String(), conf.Options.FullReadConcurrency)
151 |
152 | qos := qps.StartQoS(int(conf.Options.QpsFull))
153 | defer qos.Close()
154 |
155 | var wg sync.WaitGroup
156 | wg.Add(int(conf.Options.FullReadConcurrency))
157 | for i := 0; i < int(conf.Options.FullReadConcurrency); i++ {
158 | go func(segmentId int64) {
159 | LOG.Info("%s start reader[%v]", ts.String(), segmentId)
160 | defer LOG.Info("%s stop reader[%v]", ts.String(), segmentId)
161 |
162 | // init nil
163 | var previousKey map[string]*dynamodb.AttributeValue
164 | for {
165 | <-qos.Bucket
166 |
167 | startT := time.Now()
168 | scanInput := &dynamodb.ScanInput{
169 | TableName: aws.String(ts.ns.Collection),
170 | TotalSegments: aws.Int64(int64(conf.Options.FullReadConcurrency)),
171 | Segment: aws.Int64(segmentId),
172 | ExclusiveStartKey: previousKey,
173 | Limit: aws.Int64(conf.Options.QpsFullBatchNum),
174 | }
175 | if len(conf.Options.FullFilterExpression) > 0 {
176 | scanInput.FilterExpression = aws.String(conf.Options.FullFilterExpression)
177 | scanInput.ExpressionAttributeValues = utils.ParseAttributes(conf.Options.FullFilterAttributeValues)
178 | }
179 | out, err := ts.sourceConn.Scan(scanInput)
180 | if err != nil {
181 | // TODO check network error and retry
182 | if aerr, ok := err.(awserr.Error); ok {
183 |
184 | switch aerr.Code() {
185 | case dynamodb.ErrCodeProvisionedThroughputExceededException:
186 | LOG.Warn("%s fetcher reader[%v] recv ProvisionedThroughputExceededException continue",
187 | ts.String(), segmentId)
188 | time.Sleep(5 * time.Second)
189 | continue
190 |
191 | case request.ErrCodeSerialization:
192 | LOG.Warn("%s fetcher reader[%v] recv SerializationError[%v] continue",
193 | ts.String(), segmentId, err)
194 | time.Sleep(5 * time.Second)
195 | continue
196 |
197 | case request.ErrCodeRequestError, request.CanceledErrorCode,
198 | request.ErrCodeResponseTimeout, request.HandlerResponseTimeout,
199 | request.WaiterResourceNotReadyErrorCode, request.ErrCodeRead:
200 | LOG.Warn("%s fetcher reader[%v] recv Error[%v] continue",
201 | ts.String(), segmentId, err)
202 | time.Sleep(5 * time.Second)
203 | continue
204 |
205 | default:
206 | LOG.Crashf("%s fetcher scan failed[%v] errcode[%v]", ts.String(), err, aerr.Code())
207 | }
208 | } else {
209 | LOG.Crashf("%s fetcher scan failed[%v]", ts.String(), err)
210 | }
211 | }
212 | scanDuration := time.Since(startT)
213 |
214 | // LOG.Info(*out.Count)
215 |
216 | // pass result to parser
217 | startT = time.Now()
218 | ts.fetcherChan <- out
219 | writeFetcherChan := time.Since(startT)
220 |
221 | LOG.Info("%s fetcher reader[%v] ts.fetcherChan.len[%v] "+
222 | "scanTime[%v] scanCount[%v] writeFetcherChanTime[%v]",
223 | ts.String(), segmentId, len(ts.fetcherChan), scanDuration, *out.Count, writeFetcherChan)
224 |
225 | previousKey = out.LastEvaluatedKey
226 | if previousKey == nil {
227 | // complete
228 | break
229 | }
230 | }
231 | wg.Done()
232 | }(int64(i))
233 | }
234 | wg.Wait()
235 |
236 | LOG.Info("%s close fetcher", ts.String())
237 | close(ts.fetcherChan)
238 | }
239 |
240 | func (ts *tableSyncer) parser(id int) {
241 | LOG.Info("%s start parser[%v]", ts.String(), id)
242 |
243 | for {
244 | startT := time.Now()
245 | data, ok := <-ts.fetcherChan
246 | if !ok {
247 | break
248 | }
249 | readFetcherChanDuration := time.Since(startT)
250 |
251 | LOG.Debug("%s parser[%v] read data[%v]", ts.String(), id, data)
252 |
253 | var parserDuration, writeParseChanDuration time.Duration = 0, 0
254 |
255 | list := data.Items
256 | for _, ele := range list {
257 | startT = time.Now()
258 | out, err := ts.converter.Run(ele)
259 | parserDuration = parserDuration + time.Since(startT)
260 | if err != nil {
261 | LOG.Crashf("%s parser[%v] parse ele[%v] failed[%v]", ts.String(), id, ele, err)
262 | }
263 |
264 | startT = time.Now()
265 | ts.parserChan <- out
266 | writeParseChanDuration = writeParseChanDuration + time.Since(startT)
267 | }
268 |
269 | LOG.Info("%s parser parser[%v] readFetcherChanTime[%v] parserTime[%v]"+
270 | " writeParseChantime[%v] parserChan.len[%v]",
271 | ts.String(), id, readFetcherChanDuration, parserDuration, writeParseChanDuration, len(ts.parserChan))
272 |
273 | }
274 | LOG.Info("%s close parser", ts.String())
275 | }
276 |
277 | func (ts *tableSyncer) count() uint64 {
278 | return uint64(*ts.sourceTableDescribe.ItemCount)
279 | }
280 |
--------------------------------------------------------------------------------
/nimo-shake/go.mod:
--------------------------------------------------------------------------------
1 | module nimo-shake
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/aws/aws-sdk-go v1.44.61
7 | github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40
8 | github.com/jinzhu/copier v0.3.5
9 | github.com/nightlyone/lockfile v1.0.0
10 | github.com/stretchr/testify v1.6.1
11 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d
12 | github.com/vinllen/mgo v0.0.0-20220329061231-e5ecea62f194
13 | go.mongodb.org/mongo-driver v1.16.1
14 | )
15 |
--------------------------------------------------------------------------------
/nimo-shake/go.sum:
--------------------------------------------------------------------------------
1 | github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRco=
2 | github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
3 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
7 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
8 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
9 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
10 | github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40 h1:6TWAiHVyKs75ZHEn7XtVv7SO7M4rHwvY/5Tf7xdJBkc=
11 | github.com/gugemichael/nimo4go v0.0.0-20210413043712-ccb2ff0d7b40/go.mod h1:ibO7uKpO8fOH/bKD4trmwm5tHhHKiAjC0u288Rd+GnI=
12 | github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
13 | github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
14 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
15 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
16 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
17 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
18 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
19 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
20 | github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
21 | github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
22 | github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA=
23 | github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI=
24 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
27 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
28 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
29 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
30 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d h1:V+5NxRH9I8rVJyazHI25yXaszF4JA64Isr4ilU+nnck=
31 | github.com/vinllen/log4go v0.0.0-20180514124125-3848a366df9d/go.mod h1:F8d+yTmuUlynKo7Wn26s+Q8bCZBfcymR5bYSydymIlc=
32 | github.com/vinllen/mgo v0.0.0-20220329061231-e5ecea62f194 h1:UlCPbcl8pHR5vAFP3qPx7HItWNa7+wjxRVySHstMsec=
33 | github.com/vinllen/mgo v0.0.0-20220329061231-e5ecea62f194/go.mod h1:nwebO37lLx+dkv3Dzp24vuzrapu+JudOSYpue46unDk=
34 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
35 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
36 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
37 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
38 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
39 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
40 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
41 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
42 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
43 | go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8=
44 | go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
45 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
46 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
47 | golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
48 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
49 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
50 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
51 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
52 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
53 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
54 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
55 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
56 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
57 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
58 | golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
59 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
60 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
61 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
62 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
63 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
64 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
65 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
66 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
67 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
68 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
69 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
70 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
71 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
72 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
73 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
74 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
75 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
76 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
77 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
78 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
79 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
80 | golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
81 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
82 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
83 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
84 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
85 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
86 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
87 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
88 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
89 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
90 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
91 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
92 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
93 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
94 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
95 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
96 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
97 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
98 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
99 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
100 |
--------------------------------------------------------------------------------
/nimo-shake/incr-sync/fetcher.go:
--------------------------------------------------------------------------------
1 | package incr_sync
2 |
3 | import (
4 | "time"
5 |
6 | "nimo-shake/checkpoint"
7 | utils "nimo-shake/common"
8 | conf "nimo-shake/configure"
9 | "nimo-shake/qps"
10 |
11 | "fmt"
12 |
13 | "github.com/aws/aws-sdk-go/service/dynamodbstreams"
14 | LOG "github.com/vinllen/log4go"
15 | )
16 |
17 | const (
18 | FetcherInterval = 60 // seconds
19 | )
20 |
21 | type Fetcher struct {
22 | dynamoClient *dynamodbstreams.DynamoDBStreams
23 | table string
24 | stream *dynamodbstreams.Stream
25 | shardChan chan *utils.ShardNode
26 | ckptWriter checkpoint.Writer
27 | metric *utils.ReplicationMetric
28 | }
29 |
30 | func NewFetcher(table string, stream *dynamodbstreams.Stream, shardChan chan *utils.ShardNode, ckptWriter checkpoint.Writer, metric *utils.ReplicationMetric) *Fetcher {
31 | // create dynamo stream client
32 | dynamoStreamSession, err := utils.CreateDynamoStreamSession(conf.Options.LogLevel)
33 | if err != nil {
34 | LOG.Error("create dynamodb stream session failed[%v]", err)
35 | return nil
36 | }
37 |
38 | return &Fetcher{
39 | dynamoClient: dynamoStreamSession,
40 | table: table,
41 | stream: stream,
42 | shardChan: shardChan,
43 | ckptWriter: ckptWriter,
44 | metric: metric,
45 | }
46 | }
47 |
48 | func (f *Fetcher) Run() {
49 | md5Map := make(map[string]uint64)
50 | tableEpoch := make(map[string]int) // GlobalFetcherMoreFlag, restore previous epoch
51 |
52 | qos := qps.StartQoS(10)
53 | defer qos.Close()
54 |
55 | for range time.NewTicker(FetcherInterval * time.Second).C {
56 | shardList := make([]*utils.ShardNode, 0)
57 | // LOG.Debug("fetch table[%v] stream", table)
58 |
59 | preEpoch, ok := tableEpoch[f.table]
60 | if !ok {
61 | tableEpoch[f.table] = 0
62 | }
63 |
64 | var allShards []*dynamodbstreams.Shard
65 | var lastShardIdString *string = nil
66 | for {
67 | var describeStreamInput *dynamodbstreams.DescribeStreamInput
68 | if lastShardIdString != nil {
69 | describeStreamInput = &dynamodbstreams.DescribeStreamInput{
70 | StreamArn: f.stream.StreamArn,
71 | ExclusiveStartShardId: lastShardIdString,
72 | }
73 | } else {
74 | describeStreamInput = &dynamodbstreams.DescribeStreamInput{
75 | StreamArn: f.stream.StreamArn,
76 | }
77 | }
78 |
79 | // limit qos of api DescribeStreamInput
80 | <-qos.Bucket
81 |
82 | desStream, err := f.dynamoClient.DescribeStream(describeStreamInput)
83 | if err != nil {
84 | LOG.Crashf("describe table[%v] with stream[%v] failed[%v]", f.table, *f.stream.StreamArn, err)
85 | }
86 | if *desStream.StreamDescription.StreamStatus == "DISABLED" {
87 | LOG.Crashf("table[%v] with stream[%v] has already been disabled", f.table, *f.stream.StreamArn)
88 | }
89 |
90 | allShards = append(allShards, desStream.StreamDescription.Shards...)
91 |
92 | if desStream.StreamDescription.LastEvaluatedShardId == nil {
93 | break
94 | } else {
95 | lastShardIdString = desStream.StreamDescription.LastEvaluatedShardId
96 | LOG.Info("table[%v] have next shardId,LastEvaluatedShardId[%v]",
97 | f.table, *desStream.StreamDescription.LastEvaluatedShardId)
98 | }
99 | }
100 | LOG.Info("fetch.Run table[%v] allShards(len:%d)[%v]", f.table, len(allShards), allShards)
101 |
102 | rootNode := utils.BuildShardTree(allShards, f.table, *f.stream.StreamArn)
103 | md5 := utils.CalMd5(rootNode)
104 |
105 | GlobalFetcherLock.Lock()
106 | curEpoch := GlobalFetcherMoreFlag[f.table]
107 | GlobalFetcherLock.Unlock()
108 |
109 | if val, ok := md5Map[f.table]; !ok || val != md5 {
110 | // shards is changed
111 | LOG.Info("table[%v] md5 changed from old[%v] to new[%v], need fetch shard", f.table, val, md5)
112 | md5Map[f.table] = md5
113 | } else if preEpoch != curEpoch {
114 | // old shard has already been finished
115 | LOG.Info("table[%v] curEpoch[%v] != preEpoch[%v]", f.table, curEpoch, preEpoch)
116 | tableEpoch[f.table] = curEpoch
117 | } else {
118 | LOG.Info("table[%v] md5-old[%v] md5-new[%v]", f.table, val, md5)
119 |
120 | continue
121 | }
122 |
123 | // extract checkpoint from mongodb
124 | ckptSingleMap, err := f.ckptWriter.ExtractSingleCheckpoint(f.table)
125 | if err != nil {
126 | LOG.Crashf("extract checkpoint failed[%v]", err)
127 | } else {
128 | LOG.Info("table:[%v] ckptSingleMap:[%v]", f.table, ckptSingleMap)
129 | }
130 |
131 | if tree, err := utils.PrintShardTree(rootNode); err != nil {
132 | LOG.Info("table[%v] traverse to print tree failed[%v]", f.table, err)
133 | } else {
134 | LOG.Info("traverse stream tree for table[%v](father->child): \n-----\n%v\n-----", f.table, tree)
135 | }
136 |
137 | // traverse shards
138 | err = utils.TraverseShard(rootNode, func(node *utils.ShardNode) error {
139 | LOG.Info("traverse shard[%v]", *node.Shard.ShardId)
140 | id := *node.Shard.ShardId
141 | var father string
142 | if node.Shard.ParentShardId != nil {
143 | father = *node.Shard.ParentShardId
144 | }
145 |
146 | ckpt, ok := ckptSingleMap[id]
147 | if !ok {
148 | // insert checkpoint
149 | newCkpt := &checkpoint.Checkpoint{
150 | ShardId: id,
151 | SequenceNumber: *node.Shard.SequenceNumberRange.StartingSequenceNumber,
152 | Status: checkpoint.StatusPrepareProcess,
153 | WorkerId: "unknown",
154 | FatherId: father,
155 | IteratorType: checkpoint.IteratorTypeTrimHorizon,
156 | UpdateDate: "", // empty at first
157 | }
158 | f.ckptWriter.Insert(newCkpt, f.table)
159 | shardList = append(shardList, node)
160 | LOG.Info("insert new checkpoint: %v ckptSingleMap[id]:%v", *newCkpt, ckptSingleMap[id])
161 | return utils.StopTraverseSonErr
162 | }
163 | switch ckpt.Status {
164 | case checkpoint.StatusNoNeedProcess:
165 | LOG.Info("no need to process: %v", *ckpt)
166 | return nil
167 | case checkpoint.StatusPrepareProcess:
168 | LOG.Info("status already in prepare: %v", *ckpt)
169 | shardList = append(shardList, node)
170 | return utils.StopTraverseSonErr
171 | case checkpoint.StatusInProcessing:
172 | LOG.Info("status already in processing: %v", *ckpt)
173 | shardList = append(shardList, node)
174 | return utils.StopTraverseSonErr
175 | case checkpoint.StatusNotProcess:
176 | fallthrough
177 | case checkpoint.StatusWaitFather:
178 | LOG.Info("status need to process: %v", *ckpt)
179 | ckpt.SequenceNumber = *node.Shard.SequenceNumberRange.StartingSequenceNumber
180 | ckpt.Status = checkpoint.StatusPrepareProcess
181 | ckpt.IteratorType = checkpoint.IteratorTypeTrimHorizon
182 | f.ckptWriter.Update(ckpt.ShardId, ckpt, f.table)
183 | shardList = append(shardList, node)
184 | return utils.StopTraverseSonErr
185 | case checkpoint.StatusDone:
186 | LOG.Info("already done: %v", *ckpt)
187 | return nil
188 | default:
189 | return fmt.Errorf("unknown checkpoint status[%v]", ckpt.Status)
190 | }
191 |
192 | return nil
193 | })
194 | if err != nil {
195 | LOG.Crashf("traverse shard tree failed[%v]", err)
196 | }
197 |
198 | // dispatch shard list
199 | for _, shard := range shardList {
200 | LOG.Info("need to dispatch shard[%v]", *shard.Shard.ShardId)
201 | f.shardChan <- shard
202 | }
203 | }
204 | LOG.Crashf("can't see me!")
205 | }
206 |
--------------------------------------------------------------------------------
/nimo-shake/main/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "flag"
6 | "fmt"
7 | _ "net/http/pprof"
8 | "os"
9 | "runtime"
10 | "syscall"
11 | "time"
12 |
13 | "nimo-shake/checkpoint"
14 | "nimo-shake/common"
15 | "nimo-shake/configure"
16 | "nimo-shake/run"
17 |
18 | "github.com/gugemichael/nimo4go"
19 | LOG "github.com/vinllen/log4go"
20 | )
21 |
22 | type Exit struct{ Code int }
23 |
24 | func main() {
25 | defer LOG.Close()
26 |
27 | //http.DefaultServeMux.Handle("/debug/fgprof", fgprof.Handler())
28 | //go func() {
29 | // http.ListenAndServe(":6060", nil)
30 | //}()
31 |
32 | runtime.GOMAXPROCS(256)
33 | fmt.Println("max process:", runtime.GOMAXPROCS(0))
34 |
35 | var err error
36 | // argument options
37 | configuration := flag.String("conf", "", "configuration path")
38 | version := flag.Bool("version", false, "show version")
39 | flag.Parse()
40 |
41 | if *version {
42 | fmt.Println(utils.Version)
43 | return
44 | }
45 |
46 | if *configuration == "" {
47 | fmt.Println(utils.Version)
48 | flag.PrintDefaults()
49 | return
50 | }
51 |
52 | conf.Options.Version = utils.Version
53 |
54 | var file *os.File
55 | if file, err = os.Open(*configuration); err != nil {
56 | crash(fmt.Sprintf("Configure file open failed. %v", err), -1)
57 | }
58 |
59 | configure := nimo.NewConfigLoader(file)
60 | configure.SetDateFormat(utils.GolangSecurityTime)
61 | if err := configure.Load(&conf.Options); err != nil {
62 | crash(fmt.Sprintf("Configure file %s parse failed. %v", *configuration, err), -2)
63 | }
64 |
65 | utils.InitialLogger(conf.Options.LogFile, conf.Options.LogLevel, conf.Options.LogBuffer)
66 |
67 | // sanitize options
68 | if err := sanitizeOptions(); err != nil {
69 | crash(fmt.Sprintf("Conf.Options check failed: %s", err.Error()), -4)
70 | }
71 |
72 | // read fcv and do comparison
73 | if _, err := conf.CheckFcv(*configuration, utils.FcvConfiguration.FeatureCompatibleVersion); err != nil {
74 | crash(err.Error(), -5)
75 | }
76 |
77 | utils.Welcome()
78 | utils.StartTime = fmt.Sprintf("%v", time.Now().Format(utils.GolangSecurityTime))
79 |
80 | // write pid
81 | if err = utils.WritePidById(conf.Options.Id, "."); err != nil {
82 | crash(fmt.Sprintf("write pid failed. %v", err), -5)
83 | }
84 |
85 | // print configuration
86 | if opts, err := json.Marshal(conf.Options); err != nil {
87 | crash(fmt.Sprintf("marshal configuration failed[%v]", err), -6)
88 | } else {
89 | LOG.Info("%v configuration: %s", conf.Options.Id, string(opts))
90 | }
91 |
92 | nimo.Profiling(int(conf.Options.SystemProfile))
93 | nimo.RegisterSignalForProfiling(syscall.Signal(utils.SIGNALPROFILE)) // syscall.SIGUSR2
94 | nimo.RegisterSignalForPrintStack(syscall.Signal(utils.SIGNALSTACK), func(bytes []byte) { // syscall.SIGUSR1
95 | LOG.Info(string(bytes))
96 | })
97 |
98 | run.Start()
99 |
100 | LOG.Info("sync complete!")
101 | }
102 |
103 | func sanitizeOptions() error {
104 | if len(conf.Options.Id) == 0 {
105 | return fmt.Errorf("id[%v] shouldn't be empty", conf.Options.Id)
106 | }
107 |
108 | if conf.Options.SyncMode != utils.SyncModeAll && conf.Options.SyncMode != utils.SyncModeFull {
109 | return fmt.Errorf("sync_mode[%v] illegal, should in {all, full}", conf.Options.SyncMode)
110 | }
111 |
112 | if conf.Options.IncrSyncParallel != true {
113 | conf.Options.IncrSyncParallel = false
114 | } else {
115 | if conf.Options.SyncMode != utils.SyncModeAll {
116 | return fmt.Errorf("sync_mode must be all when incr_sync_parallel is true")
117 | }
118 | }
119 | if conf.Options.IncreasePersistDir == "" {
120 | conf.Options.IncreasePersistDir = "/tmp/"
121 | }
122 |
123 | if conf.Options.SourceAccessKeyID == "" {
124 | return fmt.Errorf("source.access_key_id shouldn't be empty")
125 | }
126 |
127 | if conf.Options.SourceSecretAccessKey == "" {
128 | return fmt.Errorf("source.secret_access_key shouldn't be empty")
129 | }
130 |
131 | if conf.Options.FilterCollectionBlack != "" && conf.Options.FilterCollectionWhite != "" {
132 | return fmt.Errorf("filter.collection.white and filter.collection.black can't both be given")
133 | }
134 |
135 | if conf.Options.QpsFull <= 0 || conf.Options.QpsIncr <= 0 {
136 | return fmt.Errorf("qps should > 0")
137 | }
138 |
139 | if conf.Options.QpsFullBatchNum <= 0 {
140 | conf.Options.QpsFullBatchNum = 128
141 | }
142 | if conf.Options.QpsIncrBatchNum <= 0 {
143 | conf.Options.QpsIncrBatchNum = 128
144 | }
145 |
146 | if conf.Options.TargetType != utils.TargetTypeMongo && conf.Options.TargetType != utils.TargetTypeAliyunDynamoProxy {
147 | return fmt.Errorf("conf.Options.TargetType[%v] supports {mongodb, aliyun_dynamo_proxy} currently", conf.Options.TargetType)
148 | }
149 |
150 | if len(conf.Options.TargetAddress) == 0 {
151 | return fmt.Errorf("target.address[%v] illegal", conf.Options.TargetAddress)
152 | }
153 |
154 | if conf.Options.FullConcurrency > 4096 || conf.Options.FullConcurrency == 0 {
155 | return fmt.Errorf("full.concurrency[%v] should in (0, 4096]", conf.Options.FullConcurrency)
156 | }
157 |
158 | if conf.Options.FullDocumentConcurrency > 4096 || conf.Options.FullDocumentConcurrency == 0 {
159 | return fmt.Errorf("full.document.concurrency[%v] should in (0, 4096]", conf.Options.FullDocumentConcurrency)
160 | }
161 |
162 | if conf.Options.FullDocumentWriteBatch <= 0 {
163 | if conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy {
164 | conf.Options.FullDocumentWriteBatch = 25
165 | } else {
166 | conf.Options.FullDocumentWriteBatch = 128
167 | }
168 | } else if conf.Options.FullDocumentWriteBatch > 25 && conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy {
169 | conf.Options.FullDocumentWriteBatch = 25
170 | }
171 |
172 | if conf.Options.FullReadConcurrency <= 0 {
173 | conf.Options.FullReadConcurrency = 1
174 | } else if conf.Options.FullReadConcurrency > 8192 {
175 | return fmt.Errorf("full.read.concurrency[%v] should in (0, 8192]", conf.Options.FullReadConcurrency)
176 | }
177 |
178 | if conf.Options.FullDocumentParser > 4096 || conf.Options.FullDocumentParser == 0 {
179 | return fmt.Errorf("full.document.parser[%v] should in (0, 4096]", conf.Options.FullDocumentParser)
180 | }
181 |
182 | // always enable
183 | conf.Options.FullEnableIndexPrimary = true
184 |
185 | if conf.Options.ConvertType == "" {
186 | conf.Options.ConvertType = utils.ConvertMTypeChange
187 | }
188 | if conf.Options.ConvertType != utils.ConvertTypeRaw &&
189 | conf.Options.ConvertType != utils.ConvertTypeChange &&
190 | conf.Options.ConvertType != utils.ConvertMTypeChange {
191 | return fmt.Errorf("convert.type[%v] illegal", conf.Options.ConvertType)
192 | }
193 |
194 | if conf.Options.IncreaseConcurrency == 0 {
195 | return fmt.Errorf("increase.concurrency should > 0")
196 | }
197 |
198 | if conf.Options.TargetMongoDBType != "" && conf.Options.TargetMongoDBType != utils.TargetMongoDBTypeReplica &&
199 | conf.Options.TargetMongoDBType != utils.TargetMongoDBTypeSharding {
200 | return fmt.Errorf("illegal target.mongodb.type[%v]", conf.Options.TargetMongoDBType)
201 | }
202 |
203 | if conf.Options.TargetType == utils.TargetTypeMongo && conf.Options.TargetDBExist != "" &&
204 | conf.Options.TargetDBExist != utils.TargetDBExistRename &&
205 | conf.Options.TargetDBExist != utils.TargetDBExistDrop ||
206 | conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy && conf.Options.TargetDBExist != "" &&
207 | conf.Options.TargetDBExist != utils.TargetDBExistDrop {
208 | return fmt.Errorf("target.mongodb.exist[%v] should be 'drop' when target.type=%v",
209 | conf.Options.TargetDBExist, conf.Options.TargetType)
210 | }
211 | // set ConvertType
212 | if conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy {
213 | conf.Options.ConvertType = utils.ConvertTypeSame
214 | }
215 |
216 | // checkpoint
217 | if conf.Options.CheckpointType == "" {
218 | conf.Options.CheckpointType = checkpoint.CheckpointWriterTypeFile
219 | }
220 | if conf.Options.CheckpointType == checkpoint.CheckpointWriterTypeMongo &&
221 | conf.Options.CheckpointAddress == "" &&
222 | conf.Options.TargetType != utils.TargetTypeMongo {
223 | return fmt.Errorf("checkpoint.type should == file when checkpoint.address is empty and target.type != mongodb")
224 | }
225 |
226 | if conf.Options.CheckpointAddress == "" {
227 | if conf.Options.TargetType == utils.TargetTypeMongo {
228 | conf.Options.CheckpointAddress = conf.Options.TargetAddress
229 | } else {
230 | conf.Options.CheckpointAddress = "checkpoint"
231 | }
232 | }
233 | if conf.Options.CheckpointDb == "" {
234 | conf.Options.CheckpointDb = fmt.Sprintf("%s-%s", conf.Options.Id, "checkpoint")
235 | }
236 |
237 | if conf.Options.TargetType == utils.TargetTypeAliyunDynamoProxy &&
238 | (!conf.Options.IncreaseExecutorUpsert || !conf.Options.IncreaseExecutorInsertOnDupUpdate) {
239 | return fmt.Errorf("increase.executor.upsert and increase.executor.insert_on_dup_update should be "+
240 | "enable when target type is %v", utils.TargetTypeAliyunDynamoProxy)
241 | }
242 |
243 | if conf.Options.SourceEndpointUrl != "" && conf.Options.SyncMode != "full" {
244 | return fmt.Errorf("only support sync.mode=full when source.endpoint_url is set")
245 | }
246 |
247 | return nil
248 | }
249 |
250 | func crash(msg string, errCode int) {
251 | fmt.Println(msg)
252 | panic(Exit{errCode})
253 | }
254 |
--------------------------------------------------------------------------------
/nimo-shake/protocal/converter_test.go:
--------------------------------------------------------------------------------
1 | package protocal
2 |
3 | import (
4 | "testing"
5 | "fmt"
6 |
7 | "github.com/aws/aws-sdk-go/service/dynamodb"
8 | "github.com/aws/aws-sdk-go/aws"
9 | "github.com/stretchr/testify/assert"
10 | "github.com/vinllen/mgo/bson"
11 | "strconv"
12 | )
13 |
14 | func TestRawConverter(t *testing.T) {
15 | // test RawConverter
16 |
17 | var nr int
18 | {
19 | fmt.Printf("TestRawConverter case %d.\n", nr)
20 | nr++
21 |
22 | src := map[string]*dynamodb.AttributeValue {
23 | "test": {
24 | N: aws.String("12345"),
25 | },
26 | }
27 |
28 | rc := new(RawConverter)
29 | out, err := rc.Run(src)
30 | assert.Equal(t, nil, err, "should be equal")
31 | assert.Equal(t, bson.M{
32 | "test": bson.M{
33 | "N": "12345",
34 | },
35 | }, out.(RawData).Data, "should be equal")
36 | }
37 |
38 | {
39 | fmt.Printf("TestRawConverter case %d.\n", nr)
40 | nr++
41 |
42 | src := map[string]*dynamodb.AttributeValue {
43 | "test": {
44 | N: aws.String("12345"),
45 | },
46 | "fuck": {
47 | S: aws.String("hello"),
48 | },
49 | }
50 |
51 | rc := new(RawConverter)
52 | out, err := rc.Run(src)
53 | assert.Equal(t, nil, err, "should be equal")
54 | assert.Equal(t, bson.M{
55 | "test": bson.M {
56 | "N": "12345",
57 | },
58 | "fuck": bson.M {
59 | "S": "hello",
60 | },
61 | }, out.(RawData).Data, "should be equal")
62 | }
63 |
64 | {
65 | fmt.Printf("TestRawConverter case %d.\n", nr)
66 | nr++
67 |
68 | src := map[string]*dynamodb.AttributeValue {
69 | "test": {
70 | N: aws.String("12345"),
71 | },
72 | "fuck": {
73 | S: aws.String("hello"),
74 | },
75 | "test-string-list": {
76 | SS: []*string{aws.String("z1"), aws.String("z2"), aws.String("z3")},
77 | },
78 | "test-number-list": {
79 | NS: []*string{aws.String("123"), aws.String("456"), aws.String("78999999999999999999999999999")},
80 | },
81 | "test-bool": {
82 | BOOL: aws.Bool(true),
83 | },
84 | "test-byte": {
85 | B: []byte{123, 45, 78, 0, 12},
86 | },
87 | "test-byte-list": {
88 | BS: [][]byte{
89 | {123, 33, 44, 0, 55},
90 | {0, 1, 2, 0, 5},
91 | },
92 | },
93 | }
94 |
95 | rc := new(RawConverter)
96 | out, err := rc.Run(src)
97 | assert.Equal(t, nil, err, "should be equal")
98 | assert.Equal(t, bson.M{
99 | "test": bson.M {
100 | "N": "12345",
101 | },
102 | "fuck": bson.M {
103 | "S": "hello",
104 | },
105 | "test-string-list": bson.M {
106 | "SS": []interface{}{"z1", "z2", "z3"},
107 | },
108 | "test-number-list": bson.M {
109 | "NS": []interface{}{"123", "456", "78999999999999999999999999999"},
110 | },
111 | "test-bool": bson.M {
112 | "BOOL": true,
113 | },
114 | "test-byte": bson.M {
115 | "B": []byte{123, 45, 78, 0, 12},
116 | },
117 | "test-byte-list": bson.M{
118 | "BS": [][]byte{
119 | {123, 33, 44, 0, 55},
120 | {0, 1, 2, 0, 5},
121 | },
122 | },
123 | }, out.(RawData).Data, "should be equal")
124 | }
125 |
126 | {
127 | fmt.Printf("TestRawConverter case %d.\n", nr)
128 | nr++
129 |
130 | src := map[string]*dynamodb.AttributeValue {
131 | "test": {
132 | N: aws.String("12345"),
133 | },
134 | "test-inner-struct": {
135 | L: []*dynamodb.AttributeValue {
136 | {
137 | S: aws.String("hello-inner"),
138 | N: aws.String("12345"),
139 | },
140 | {
141 | SS: []*string{aws.String("zi1"), aws.String("zi2"), aws.String("zi3")},
142 | },
143 | },
144 | },
145 | "test-inner-map": {
146 | M: map[string]*dynamodb.AttributeValue{
147 | "test": {
148 | N: aws.String("12345000"),
149 | },
150 | },
151 | },
152 | "test-NULL": {
153 | NULL: aws.Bool(false),
154 | },
155 | }
156 |
157 | rc := new(RawConverter)
158 | out, err := rc.Run(src)
159 | assert.Equal(t, nil, err, "should be equal")
160 | assert.Equal(t, bson.M{
161 | "test": bson.M {
162 | "N": "12345",
163 | },
164 | "test-inner-struct": bson.M {
165 | "L": []interface{} {
166 | bson.M{
167 | "S": "hello-inner",
168 | "N": "12345",
169 | },
170 | bson.M{
171 | "SS": []interface{}{"zi1", "zi2", "zi3"},
172 | },
173 | },
174 | },
175 | "test-inner-map": bson.M {
176 | "M": bson.M {
177 | "test": bson.M{
178 | "N": "12345000",
179 | },
180 | },
181 | },
182 | "test-NULL": bson.M {
183 | "NULL": false,
184 | },
185 | }, out.(RawData).Data, "should be equal")
186 | }
187 | }
188 |
189 | func TestTypeConverter(t *testing.T) {
190 | // test TypeConverter
191 |
192 | var nr int
193 | {
194 | fmt.Printf("TestTypeConverter case %d.\n", nr)
195 | nr++
196 |
197 | src := map[string]*dynamodb.AttributeValue{
198 | "test": {
199 | N: aws.String("12345"),
200 | },
201 | }
202 |
203 | rc := new(TypeConverter)
204 | out, err := rc.Run(src)
205 | assert.Equal(t, nil, err, "should be equal")
206 | val, err := bson.ParseDecimal128("12345")
207 | assert.Equal(t, nil, err, "should be equal")
208 | assert.Equal(t, bson.M{
209 | "test": val,
210 | }, out.(RawData).Data, "should be equal")
211 | }
212 |
213 | {
214 | fmt.Printf("TestTypeConverter case %d.\n", nr)
215 | nr++
216 |
217 | src := map[string]*dynamodb.AttributeValue{
218 | "test": {
219 | N: aws.String("123456789101112131415161718192021"),
220 | },
221 | "test2": {
222 | N: aws.String("3.141592653589793238462643383279"),
223 | },
224 | "test3": {
225 | N: aws.String("3.1415926535897932384626433832795012345"),
226 | },
227 | }
228 |
229 | rc := new(TypeConverter)
230 | out, err := rc.Run(src)
231 | assert.Equal(t, nil, err, "should be equal")
232 | val, err := bson.ParseDecimal128("123456789101112131415161718192021")
233 | assert.Equal(t, nil, err, "should be equal")
234 | val2, err := bson.ParseDecimal128("3.141592653589793238462643383279")
235 | assert.Equal(t, nil, err, "should be equal")
236 | val3, err := strconv.ParseFloat("3.1415926535897932384626433832795012345", 64)
237 | assert.Equal(t, nil, err, "should be equal")
238 | val3_2, err := bson.ParseDecimal128(fmt.Sprintf("%v", val3))
239 | assert.Equal(t, nil, err, "should be equal")
240 | assert.Equal(t, bson.M{
241 | "test": val,
242 | "test2": val2,
243 | "test3": val3_2,
244 | }, out.(RawData).Data, "should be equal")
245 | }
246 |
247 | {
248 | fmt.Printf("TestTypeConverter case %d.\n", nr)
249 | nr++
250 |
251 | src := map[string]*dynamodb.AttributeValue {
252 | "test": {
253 | N: aws.String("12345"),
254 | },
255 | "fuck": {
256 | S: aws.String("hello"),
257 | },
258 | }
259 |
260 | rc := new(TypeConverter)
261 | out, err := rc.Run(src)
262 | assert.Equal(t, nil, err, "should be equal")
263 | val, err := bson.ParseDecimal128("12345")
264 | assert.Equal(t, nil, err, "should be equal")
265 | assert.Equal(t, bson.M{
266 | "test": val,
267 | "fuck": "hello",
268 | }, out.(RawData).Data, "should be equal")
269 | }
270 |
271 | {
272 | fmt.Printf("TestTypeConverter case %d.\n", nr)
273 | nr++
274 |
275 | src := map[string]*dynamodb.AttributeValue {
276 | "test": {
277 | N: aws.String("12345"),
278 | },
279 | "fuck": {
280 | S: aws.String("hello"),
281 | },
282 | "test-string-list": {
283 | SS: []*string{aws.String("z1"), aws.String("z2"), aws.String("z3")},
284 | },
285 | "test-number-list": {
286 | NS: []*string{aws.String("123"), aws.String("456"), aws.String("789999999999")},
287 | },
288 | "test-bool": {
289 | BOOL: aws.Bool(true),
290 | },
291 | "test-byte": {
292 | B: []byte{123, 45, 78, 0, 12},
293 | },
294 | "test-byte-list": {
295 | BS: [][]byte{
296 | {123, 33, 44, 0, 55},
297 | {0, 1, 2, 0, 5},
298 | },
299 | },
300 | }
301 |
302 | rc := new(TypeConverter)
303 | out, err := rc.Run(src)
304 | assert.Equal(t, nil, err, "should be equal")
305 | val, err := bson.ParseDecimal128("12345")
306 | assert.Equal(t, nil, err, "should be equal")
307 | val2, err := bson.ParseDecimal128("123")
308 | assert.Equal(t, nil, err, "should be equal")
309 | val3, err := bson.ParseDecimal128("456")
310 | assert.Equal(t, nil, err, "should be equal")
311 | val4, err := bson.ParseDecimal128("789999999999")
312 | assert.Equal(t, nil, err, "should be equal")
313 | assert.Equal(t, bson.M{
314 | "test": val,
315 | "fuck": "hello",
316 | "test-string-list": []string{"z1", "z2", "z3"},
317 | "test-number-list": []bson.Decimal128{val2, val3, val4},
318 | "test-bool": true,
319 | "test-byte": []byte{123, 45, 78, 0, 12},
320 | "test-byte-list": [][]byte{
321 | {123, 33, 44, 0, 55},
322 | {0, 1, 2, 0, 5},
323 | },
324 | }, out.(RawData).Data, "should be equal")
325 | }
326 |
327 | {
328 | fmt.Printf("TestTypeConverter case %d.\n", nr)
329 | nr++
330 |
331 | src := map[string]*dynamodb.AttributeValue {
332 | "test": {
333 | N: aws.String("12345"),
334 | },
335 | "test-inner-struct": {
336 | L: []*dynamodb.AttributeValue {
337 | {
338 | S: aws.String("hello-inner"),
339 | // N: aws.String("12345"),
340 | },
341 | {
342 | SS: []*string{aws.String("zi1"), aws.String("zi2"), aws.String("zi3")},
343 | },
344 | },
345 | },
346 | "test-inner-map": {
347 | M: map[string]*dynamodb.AttributeValue{
348 | "test": {
349 | N: aws.String("12345000"),
350 | },
351 | },
352 | },
353 | "test-NULL": {
354 | NULL: aws.Bool(false),
355 | },
356 | "N": {
357 | M:map[string]*dynamodb.AttributeValue{
358 | "NN": {
359 | N: aws.String("567"),
360 | },
361 | "M": {
362 | S: aws.String("899"),
363 | },
364 | },
365 | },
366 | }
367 |
368 | rc := new(TypeConverter)
369 | out, err := rc.Run(src)
370 | assert.Equal(t, nil, err, "should be equal")
371 | val, err := bson.ParseDecimal128("12345")
372 | assert.Equal(t, nil, err, "should be equal")
373 | val2, err := bson.ParseDecimal128("12345000")
374 | assert.Equal(t, nil, err, "should be equal")
375 | val3, err := bson.ParseDecimal128("567")
376 | assert.Equal(t, nil, err, "should be equal")
377 | assert.Equal(t, bson.M{
378 | "test": val,
379 | "test-inner-struct": []interface{} {
380 | "hello-inner",
381 | []string{"zi1", "zi2", "zi3"},
382 | },
383 | "test-inner-map": bson.M {
384 | "test": val2,
385 | },
386 | "test-NULL": false,
387 | "N": bson.M {
388 | "NN": val3,
389 | "M": "899",
390 | },
391 | }, out.(RawData).Data, "should be equal")
392 | }
393 | }
--------------------------------------------------------------------------------
/nimo-shake/protocal/mtype_converter.go:
--------------------------------------------------------------------------------
1 | package protocal
2 |
3 | import (
4 | "fmt"
5 | "github.com/aws/aws-sdk-go/service/dynamodb"
6 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
7 | LOG "github.com/vinllen/log4go"
8 | conf "nimo-shake/configure"
9 | "time"
10 | )
11 |
12 | type MTypeConverter struct {
13 | }
14 |
15 |
16 | func (tc *MTypeConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) {
17 | funcStartT := time.Now()
18 |
19 | outLen := 0
20 | for key, value := range input {
21 | outLen = outLen + len(key) + len(value.String())
22 | }
23 |
24 | out := new(interface{})
25 | if err := dynamodbattribute.UnmarshalMap(input, out); err == nil {
26 |
27 | for key, value := range (*out).(map[string]interface {}) {
28 | if key == "_id" {
29 | delete((*out).(map[string]interface {}), key)
30 | ((*out).(map[string]interface {}))[conf.ConvertIdFunc(key)] = value
31 | }
32 | }
33 |
34 | LOG.Debug("Run_func input[%v] out[%v] out_len[%v] duration[%v]",
35 | input, *out, outLen, time.Since(funcStartT))
36 |
37 | return RawData{outLen, *out}, nil
38 | } else {
39 | LOG.Debug("Run_func input[%v] out[%v] err[%v]", input, *out, err)
40 | }
41 |
42 | return RawData{}, fmt.Errorf("parse failed, return nil")
43 | }
--------------------------------------------------------------------------------
/nimo-shake/protocal/protocal.go:
--------------------------------------------------------------------------------
1 | package protocal
2 |
3 | import (
4 | "nimo-shake/common"
5 |
6 | "github.com/aws/aws-sdk-go/service/dynamodb"
7 | )
8 |
9 | // convert DynamoDB attributeValue to bson
10 | type Converter interface {
11 | // run
12 | Run(input map[string]*dynamodb.AttributeValue) (interface{}, error)
13 | }
14 |
15 | func NewConverter(tp string) Converter {
16 | switch tp {
17 | case utils.ConvertTypeRaw:
18 | return new(RawConverter)
19 | case utils.ConvertTypeChange:
20 | return new(TypeConverter)
21 | case utils.ConvertMTypeChange:
22 | return new(MTypeConverter)
23 | case utils.ConvertTypeSame:
24 | return new(SameConverter)
25 | default:
26 | return nil
27 | }
28 | }
29 |
30 | type RawData struct {
31 | Size int // fake size, only calculate real data
32 | Data interface{} // real data
33 | }
34 |
--------------------------------------------------------------------------------
/nimo-shake/protocal/raw_converter.go:
--------------------------------------------------------------------------------
1 | package protocal
2 |
3 | import (
4 | "fmt"
5 | conf "nimo-shake/configure"
6 | "reflect"
7 |
8 | bson2 "go.mongodb.org/mongo-driver/bson"
9 |
10 | "github.com/aws/aws-sdk-go/service/dynamodb"
11 | LOG "github.com/vinllen/log4go"
12 | )
13 |
14 | type RawConverter struct {
15 | }
16 |
17 | // use dfs to convert to bson.M
18 | func (rc *RawConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) {
19 | v := reflect.ValueOf(input)
20 | if output := rc.dfs(v); output == nil {
21 | return RawData{}, fmt.Errorf("parse failed, return nil")
22 | } else if out, ok := output.(RawData); !ok {
23 | return RawData{}, fmt.Errorf("parse failed, return type isn't RawData")
24 | } else if _, ok := out.Data.(bson2.M); !ok {
25 | return RawData{}, fmt.Errorf("parse failed, return data isn't bson.M")
26 | } else {
27 | return out, nil
28 | }
29 | }
30 |
31 | func (rc *RawConverter) dfs(v reflect.Value) interface{} {
32 | switch v.Kind() {
33 | case reflect.Invalid:
34 | return nil
35 | case reflect.Slice, reflect.Array:
36 | if v.Len() == 0 {
37 | return nil
38 | }
39 |
40 | size := 0
41 | ret := make([]interface{}, 0, v.Len())
42 | for i := 0; i < v.Len(); i++ {
43 | out := rc.dfs(v.Index(i))
44 | md := out.(RawData)
45 | size += md.Size
46 | ret = append(ret, md.Data)
47 | }
48 | return RawData{size, ret}
49 | case reflect.Struct:
50 | if v.NumField() == 0 {
51 | return nil
52 | }
53 |
54 | size := 0
55 | ret := make(bson2.M)
56 | for i := 0; i < v.NumField(); i++ {
57 | name := v.Type().Field(i).Name
58 | if out := rc.dfs(v.Field(i)); out != nil {
59 | md := out.(RawData)
60 | size += md.Size
61 | size += len(name)
62 | if _, ok := md.Data.([]interface{}); ok {
63 | // is type array
64 | md.Data = rc.convertListToDetailList(name, md.Data)
65 | }
66 | ret[name] = md.Data
67 | }
68 | }
69 | return RawData{size, ret}
70 | case reflect.Map:
71 | if len(v.MapKeys()) == 0 {
72 | return nil
73 | }
74 |
75 | size := 0
76 | ret := make(bson2.M)
77 | for _, key := range v.MapKeys() {
78 | name := key.String()
79 | name = conf.ConvertIdFunc(name)
80 | if out := rc.dfs(v.MapIndex(key)); out != nil {
81 | md := out.(RawData)
82 | size += md.Size
83 | size += len(name)
84 | if _, ok := md.Data.([]interface{}); ok {
85 | // is type array
86 | out = rc.convertListToDetailList(name, md.Data)
87 | }
88 | ret[name] = md.Data
89 | }
90 | }
91 | return RawData{size, ret}
92 | case reflect.Ptr:
93 | if v.IsNil() {
94 | return nil
95 | } else {
96 | return rc.dfs(v.Elem())
97 | }
98 | case reflect.Interface:
99 | if v.IsNil() {
100 | return nil
101 | } else {
102 | return rc.dfs(v.Elem())
103 | }
104 | case reflect.String:
105 | out := v.String()
106 | return RawData{len(out), out}
107 | case reflect.Int:
108 | fallthrough
109 | case reflect.Int64:
110 | return RawData{8, v.Int()}
111 | case reflect.Int8:
112 | return RawData{1, int8(v.Int())}
113 | case reflect.Int16:
114 | return RawData{2, int16(v.Int())}
115 | case reflect.Int32:
116 | return RawData{4, int32(v.Int())}
117 | case reflect.Uint:
118 | fallthrough
119 | case reflect.Uint64:
120 | return RawData{8, v.Uint()}
121 | case reflect.Uint8:
122 | return RawData{1, uint8(v.Uint())}
123 | case reflect.Uint16:
124 | return RawData{2, uint16(v.Uint())}
125 | case reflect.Uint32:
126 | return RawData{4, uint32(v.Uint())}
127 | case reflect.Bool:
128 | // fake size
129 | return RawData{1, v.Bool()}
130 | default:
131 | // not support
132 | LOG.Error("unknown type[%v]", v.Kind())
133 | return nil
134 | }
135 | }
136 |
137 | func (rc *RawConverter) convertListToDetailList(name string, input interface{}) interface{} {
138 | list := input.([]interface{})
139 | switch name {
140 | case "B":
141 | output := make([]byte, 0, len(list))
142 | for _, ele := range list {
143 | output = append(output, ele.(byte))
144 | }
145 | return output
146 | case "BS":
147 | output := make([][]byte, 0, len(list))
148 | for _, ele := range list {
149 | inner := rc.convertListToDetailList("B", ele)
150 | output = append(output, inner.([]byte))
151 | }
152 | return output
153 | case "NS":
154 | fallthrough
155 | case "SS":
156 | output := make([]interface{}, 0, len(list))
157 | for _, ele := range list {
158 | output = append(output, ele.(string))
159 | }
160 | return output
161 | case "L":
162 | output := make([]interface{}, 0, len(list))
163 | for _, ele := range list {
164 | output = append(output, ele.(bson2.M))
165 | }
166 | return output
167 | }
168 | return list
169 | }
170 |
--------------------------------------------------------------------------------
/nimo-shake/protocal/same_converter.go:
--------------------------------------------------------------------------------
1 | package protocal
2 |
3 | import "github.com/aws/aws-sdk-go/service/dynamodb"
4 |
5 | type SameConverter struct {
6 | }
7 |
8 | func (sc *SameConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) {
9 | return input, nil
10 | }
--------------------------------------------------------------------------------
/nimo-shake/protocal/type_converter.go:
--------------------------------------------------------------------------------
1 | package protocal
2 |
3 | import (
4 | "fmt"
5 | conf "nimo-shake/configure"
6 | "reflect"
7 | "strconv"
8 | "time"
9 |
10 | "github.com/aws/aws-sdk-go/service/dynamodb"
11 | LOG "github.com/vinllen/log4go"
12 | bson2 "go.mongodb.org/mongo-driver/bson"
13 | "go.mongodb.org/mongo-driver/bson/primitive"
14 | )
15 |
16 | type TypeConverter struct {
17 | }
18 |
19 | // use dfs to convert to bson.M
20 | func (tc *TypeConverter) Run(input map[string]*dynamodb.AttributeValue) (interface{}, error) {
21 | v := reflect.ValueOf(input)
22 | if output := tc.dfs(v); output == nil {
23 | return RawData{}, fmt.Errorf("parse failed, return nil")
24 | } else if out, ok := output.(RawData); !ok {
25 | return RawData{}, fmt.Errorf("parse failed, return type isn't RawData")
26 | } else if _, ok := out.Data.(bson2.M); !ok {
27 | return RawData{}, fmt.Errorf("parse failed, return data isn't bson.M")
28 | } else {
29 | return out, nil
30 | }
31 | }
32 |
33 | func (tc *TypeConverter) dfs(v reflect.Value) interface{} {
34 |
35 | funcStartT := time.Now()
36 | defer LOG.Debug("dfs_func kind[%v] value[%v] duration[%v]",
37 | v.Kind().String(), v, time.Since(funcStartT))
38 |
39 | switch v.Kind() {
40 | case reflect.Invalid:
41 | return nil
42 | case reflect.Slice, reflect.Array:
43 | if v.Len() == 0 {
44 | return nil
45 | }
46 |
47 | size := 0
48 | ret := make([]interface{}, 0, v.Len())
49 | for i := 0; i < v.Len(); i++ {
50 | out := tc.dfs(v.Index(i))
51 | md := out.(RawData)
52 | size += md.Size
53 | ret = append(ret, md.Data)
54 | }
55 | return RawData{size, ret}
56 | case reflect.Struct:
57 | if v.NumField() == 0 {
58 | return nil
59 | }
60 | if v.Type().Name() == "bson.Decimal128" {
61 | return RawData{16, v}
62 | }
63 |
64 | size := 0
65 | var ret interface{}
66 | cnt := 0
67 | // at most one field in AttributeValue
68 | for i := 0; i < v.NumField(); i++ {
69 | name := v.Type().Field(i).Name
70 | if out := tc.dfs(v.Field(i)); out != nil {
71 | cnt++
72 | if cnt > 2 {
73 | LOG.Crashf("illegal struct field number")
74 | }
75 |
76 | md := out.(RawData)
77 | size += md.Size
78 | md.Data = tc.convertToDetail(name, md.Data)
79 | ret = md.Data
80 | }
81 | }
82 | return RawData{size, ret}
83 | case reflect.Map:
84 | if len(v.MapKeys()) == 0 {
85 | return nil
86 | }
87 |
88 | size := 0
89 | ret := make(bson2.M)
90 | for _, key := range v.MapKeys() {
91 | name := key.String()
92 | name = conf.ConvertIdFunc(name)
93 | if out := tc.dfs(v.MapIndex(key)); out != nil {
94 | md := out.(RawData)
95 | size += md.Size
96 | size += len(name)
97 | // out = tc.convertToDetail(name, md.Data, false)
98 | ret[name] = md.Data
99 | }
100 | }
101 | return RawData{size, ret}
102 | case reflect.Ptr:
103 | if v.IsNil() {
104 | return nil
105 | } else {
106 | return tc.dfs(v.Elem())
107 | }
108 | case reflect.Interface:
109 | if v.IsNil() {
110 | return nil
111 | } else {
112 | return tc.dfs(v.Elem())
113 | }
114 | case reflect.String:
115 | out := v.String()
116 | return RawData{len(out), out}
117 | case reflect.Int:
118 | fallthrough
119 | case reflect.Int64:
120 | return RawData{8, v.Int()}
121 | case reflect.Int8:
122 | return RawData{1, int8(v.Int())}
123 | case reflect.Int16:
124 | return RawData{2, int16(v.Int())}
125 | case reflect.Int32:
126 | return RawData{4, int32(v.Int())}
127 | case reflect.Uint:
128 | fallthrough
129 | case reflect.Uint64:
130 | return RawData{8, v.Uint()}
131 | case reflect.Uint8:
132 | return RawData{1, uint8(v.Uint())}
133 | case reflect.Uint16:
134 | return RawData{2, uint16(v.Uint())}
135 | case reflect.Uint32:
136 | return RawData{4, uint32(v.Uint())}
137 | case reflect.Bool:
138 | // fake size
139 | return RawData{1, v.Bool()}
140 | default:
141 | // not support
142 | LOG.Error("unknown type[%v]", v.Kind())
143 | return nil
144 | }
145 | }
146 |
147 | func (tc *TypeConverter) convertToDetail(name string, input interface{}) interface{} {
148 |
149 | funcStartT := time.Now()
150 | defer LOG.Debug("convertToDetail_func name[%v] input[%v] duration[%v]", name, input, time.Since(funcStartT))
151 |
152 | switch name {
153 | case "B":
154 | list := input.([]interface{})
155 | output := make([]byte, 0, len(list))
156 | for _, ele := range list {
157 | output = append(output, ele.(byte))
158 | }
159 | return output
160 | case "BS":
161 | list := input.([]interface{})
162 | output := make([][]byte, 0, len(list))
163 | for _, ele := range list {
164 | inner := tc.convertToDetail("B", ele)
165 | output = append(output, inner.([]byte))
166 | }
167 | return output
168 | case "NS":
169 | list := input.([]interface{})
170 |
171 | var nType reflect.Type
172 | for _, ele := range list {
173 | inner := tc.convertToDetail("N", ele)
174 | nType = reflect.TypeOf(inner)
175 | break
176 | }
177 |
178 | if nType.Name() == "int" {
179 |
180 | output := make([]int, 0, len(list))
181 | for _, ele := range list {
182 | inner := tc.convertToDetail("N", ele)
183 | output = append(output, inner.(int))
184 | }
185 |
186 | return output
187 | } else {
188 | output := make([]primitive.Decimal128, 0, len(list))
189 | for _, ele := range list {
190 | inner := tc.convertToDetail("N", ele)
191 | output = append(output, inner.(primitive.Decimal128))
192 | }
193 |
194 | return output
195 | }
196 |
197 | case "SS":
198 | list := input.([]interface{})
199 | output := make([]string, 0, len(list))
200 | for _, ele := range list {
201 | inner := tc.convertToDetail("S", ele)
202 | output = append(output, inner.(string))
203 | }
204 | return output
205 | case "L":
206 | list := input.([]interface{})
207 | output := make([]interface{}, 0, len(list))
208 | for _, ele := range list {
209 | output = append(output, ele)
210 | }
211 | return output
212 | case "BOOL":
213 | fallthrough
214 | case "NULL":
215 | return input.(bool)
216 | case "N":
217 | v := input.(string)
218 |
219 | val_int, err := strconv.Atoi(v)
220 | if err == nil {
221 | return val_int
222 | }
223 |
224 | val, err := primitive.ParseDecimal128(v)
225 | if err != nil {
226 | LOG.Error("convert N to decimal128 failed[%v]", err)
227 | val2, err := strconv.ParseFloat(v, 64)
228 | if err != nil {
229 | LOG.Crashf("convert N to decimal128 and float64 both failed[%v]", err)
230 | }
231 |
232 | val, _ = primitive.ParseDecimal128(fmt.Sprintf("%v", val2))
233 | return val
234 | }
235 | return val
236 | case "S":
237 | return input.(string)
238 | }
239 |
240 | // "M"
241 | return input
242 | }
243 |
--------------------------------------------------------------------------------
/nimo-shake/qps/qps.go:
--------------------------------------------------------------------------------
1 | package qps
2 |
3 | import "time"
4 |
5 | type Qos struct {
6 | Bucket chan struct{}
7 |
8 | limit int // qps
9 | close bool
10 | }
11 |
12 | func StartQoS(limit int) *Qos {
13 | q := new(Qos)
14 | q.limit = limit
15 | q.Bucket = make(chan struct{}, limit)
16 |
17 | go q.timer()
18 | return q
19 | }
20 |
21 | func (q *Qos) timer() {
22 | for range time.NewTicker(1 * time.Second).C {
23 | if q.close {
24 | return
25 | }
26 | for i := 0; i < q.limit; i++ {
27 | select {
28 | case q.Bucket <- struct{}{}:
29 | default:
30 | // break if bucket if full
31 | break
32 | }
33 | }
34 | }
35 | }
36 |
37 | func (q *Qos) Close() {
38 | q.close = true
39 | }
40 |
--------------------------------------------------------------------------------
/nimo-shake/run/run.go:
--------------------------------------------------------------------------------
1 | package run
2 |
3 | import (
4 | "nimo-shake/checkpoint"
5 | "nimo-shake/common"
6 | "nimo-shake/configure"
7 | "nimo-shake/filter"
8 | "nimo-shake/full-sync"
9 | "nimo-shake/incr-sync"
10 | "nimo-shake/writer"
11 |
12 | "github.com/aws/aws-sdk-go/service/dynamodbstreams"
13 | "github.com/gugemichael/nimo4go"
14 | LOG "github.com/vinllen/log4go"
15 | )
16 |
17 | func incrStart(streamMap map[string]*dynamodbstreams.Stream, ckptWriter checkpoint.Writer) {
18 | LOG.Info("start increase sync")
19 |
20 | // register restful api
21 | incr_sync.RestAPI()
22 |
23 | // start http server.
24 | nimo.GoRoutine(func() {
25 | // before starting, we must register all interface
26 | if err := utils.IncrSyncHttpApi.Listen(); err != nil {
27 | LOG.Critical("start incr sync server with port[%v] failed: %v", conf.Options.IncrSyncHTTPListenPort,
28 | err)
29 | }
30 | })
31 |
32 | LOG.Info("------------------------start incr sync------------------------")
33 | incr_sync.Start(streamMap, ckptWriter)
34 | LOG.Info("------------------------end incr sync------------------------")
35 | }
36 |
37 | func Start() {
38 | LOG.Info("check connections")
39 |
40 | utils.FullSyncInitHttpApi(conf.Options.FullSyncHTTPListenPort)
41 | utils.IncrSyncInitHttpApi(conf.Options.IncrSyncHTTPListenPort)
42 |
43 | // init filter
44 | filter.Init(conf.Options.FilterCollectionWhite, conf.Options.FilterCollectionBlack)
45 |
46 | if err := utils.InitSession(conf.Options.SourceAccessKeyID, conf.Options.SourceSecretAccessKey,
47 | conf.Options.SourceSessionToken, conf.Options.SourceRegion, conf.Options.SourceEndpointUrl,
48 | conf.Options.SourceSessionMaxRetries, conf.Options.SourceSessionTimeout); err != nil {
49 | LOG.Crashf("init global session failed[%v]", err)
50 | }
51 |
52 | // check writer connection
53 | w := writer.NewWriter(conf.Options.TargetType, conf.Options.TargetAddress,
54 | utils.NS{"nimo-shake", "shake_writer_test"}, conf.Options.LogLevel)
55 | if w == nil {
56 | LOG.Crashf("connect type[%v] address[%v] failed[%v]",
57 | conf.Options.TargetType, conf.Options.TargetAddress)
58 | }
59 |
60 | // create dynamo session
61 | dynamoSession, err := utils.CreateDynamoSession(conf.Options.LogLevel)
62 | if err != nil {
63 | LOG.Crashf("create dynamodb session failed[%v]", err)
64 | }
65 |
66 | // create dynamo stream client
67 | dynamoStreamSession, err := utils.CreateDynamoStreamSession(conf.Options.LogLevel)
68 | if err != nil {
69 | LOG.Crashf("create dynamodb stream session failed[%v]", err)
70 | }
71 |
72 | LOG.Info("create checkpoint writer: type=%v", conf.Options.CheckpointType)
73 | ckptWriter := checkpoint.NewWriter(conf.Options.CheckpointType, conf.Options.CheckpointAddress,
74 | conf.Options.CheckpointDb)
75 |
76 | var skipFull bool
77 | var streamMap map[string]*dynamodbstreams.Stream
78 | if conf.Options.SyncMode == utils.SyncModeAll {
79 | LOG.Info("------------------------check checkpoint------------------------")
80 | skipFull, streamMap, err = checkpoint.CheckCkpt(ckptWriter, dynamoStreamSession)
81 | if err != nil {
82 | LOG.Crashf("check checkpoint failed[%v]", err)
83 | }
84 | LOG.Info("------------------------end check checkpoint------------------------")
85 | }
86 |
87 | // full sync
88 | skipIncrSync := false
89 | if skipFull == false {
90 | // register restful api
91 | full_sync.RestAPI()
92 |
93 | // start http server.
94 | nimo.GoRoutine(func() {
95 | // before starting, we must register all interface
96 | if err := utils.FullSyncHttpApi.Listen(); err != nil {
97 | LOG.Critical("start full sync server with port[%v] failed: %v", conf.Options.FullSyncHTTPListenPort,
98 | err)
99 | }
100 | })
101 |
102 | if conf.Options.SyncMode == utils.SyncModeAll {
103 | LOG.Info("------------------------drop old checkpoint------------------------")
104 | if err := ckptWriter.DropAll(); err != nil && err.Error() != utils.NotFountErr {
105 | LOG.Crashf("drop checkpoint failed[%v]", err)
106 | }
107 |
108 | LOG.Info("------------------------prepare checkpoint start------------------------")
109 | streamMap, err = checkpoint.PrepareFullSyncCkpt(ckptWriter, dynamoSession, dynamoStreamSession)
110 | if err != nil {
111 | LOG.Crashf("prepare checkpoint failed[%v]", err)
112 | }
113 | LOG.Info("------------------------prepare checkpoint done------------------------")
114 |
115 | // select{}
116 | } else {
117 | LOG.Info("sync.mode is 'full', no need to check checkpoint")
118 | }
119 |
120 | if conf.Options.IncrSyncParallel == true {
121 | skipIncrSync = true
122 | go incrStart(streamMap, ckptWriter)
123 | }
124 |
125 | // update checkpoint
126 | if err := ckptWriter.UpdateStatus(checkpoint.CheckpointStatusValueFullSync); err != nil {
127 | LOG.Crashf("set checkpoint to [%v] failed[%v]", checkpoint.CheckpointStatusValueFullSync, err)
128 | }
129 |
130 | LOG.Info("------------------------start full sync------------------------")
131 | full_sync.Start(dynamoSession, w)
132 | LOG.Info("------------------------full sync done!------------------------")
133 | }
134 |
135 | if conf.Options.SyncMode == utils.SyncModeFull {
136 | LOG.Info("sync.mode is 'full', finish")
137 | return
138 | }
139 |
140 | if conf.Options.SyncSchemaOnly {
141 | LOG.Info("sync_schema_only enabled, finish")
142 | return
143 | }
144 |
145 | // update checkpoint
146 | if err := ckptWriter.UpdateStatus(checkpoint.CheckpointStatusValueIncrSync); err != nil {
147 | LOG.Crashf("set checkpoint to [%v] failed[%v]", checkpoint.CheckpointStatusValueIncrSync, err)
148 | }
149 |
150 | if skipIncrSync == false {
151 | go incrStart(streamMap, ckptWriter)
152 | }
153 |
154 | select {}
155 | }
156 |
--------------------------------------------------------------------------------
/nimo-shake/unit_test_common/include.go:
--------------------------------------------------------------------------------
1 | package unit_test_common
2 |
3 | const (
4 | TestUrl = "mongodb://100.81.164.186:31881,100.81.164.186:31882,100.81.164.186:31883"
5 | TestUrlServerless = "mongodb://100.81.164.181:32155"
6 | TestUrlServerlessTenant = "mongodb://100.81.164.181:36106" // sharding mongos with tenant
7 | TestUrlSharding = "mongodb://100.81.164.181:33010"
8 | )
9 |
--------------------------------------------------------------------------------
/nimo-shake/writer/dynamo_proxy.go:
--------------------------------------------------------------------------------
1 | package writer
2 |
3 | import (
4 | "fmt"
5 | "bytes"
6 | "net/http"
7 | "time"
8 |
9 | "nimo-shake/common"
10 | "nimo-shake/configure"
11 |
12 | "github.com/aws/aws-sdk-go/aws"
13 | "github.com/aws/aws-sdk-go/aws/session"
14 | LOG "github.com/vinllen/log4go"
15 | "github.com/aws/aws-sdk-go/service/dynamodb"
16 | )
17 |
18 | type DynamoProxyWriter struct {
19 | Name string
20 | svc *dynamodb.DynamoDB
21 | ns utils.NS
22 | }
23 |
24 | func NewDynamoProxyWriter(name, address string, ns utils.NS, logLevel string) *DynamoProxyWriter {
25 | config := &aws.Config{
26 | Region: aws.String("us-east-2"), // meaningless
27 | Endpoint: aws.String(address),
28 | MaxRetries: aws.Int(3),
29 | DisableSSL: aws.Bool(true),
30 | HTTPClient: &http.Client{
31 | Timeout: time.Duration(5000) * time.Millisecond,
32 | },
33 | }
34 |
35 | var err error
36 | sess, err := session.NewSession(config)
37 | if err != nil {
38 | LOG.Crashf("create dynamo connection error[%v]", err)
39 | return nil
40 | }
41 |
42 | var svc *dynamodb.DynamoDB
43 | if logLevel == "debug" {
44 | svc = dynamodb.New(sess, aws.NewConfig().WithLogLevel(aws.LogDebugWithHTTPBody))
45 | } else {
46 | svc = dynamodb.New(sess)
47 | }
48 |
49 | return &DynamoProxyWriter{
50 | Name: name,
51 | svc: svc,
52 | ns: ns,
53 | }
54 | }
55 |
56 | func (dpw *DynamoProxyWriter) String() string {
57 | return dpw.Name
58 | }
59 |
60 | func (dpw *DynamoProxyWriter) GetSession() interface{} {
61 | return dpw.svc
62 | }
63 |
64 | func (dpw *DynamoProxyWriter) PassTableDesc(tableDescribe *dynamodb.TableDescription) {
65 | }
66 |
67 | func (dpw *DynamoProxyWriter) CreateTable(tableDescribe *dynamodb.TableDescription) error {
68 | createTableInput := &dynamodb.CreateTableInput{
69 | AttributeDefinitions: tableDescribe.AttributeDefinitions,
70 | KeySchema: tableDescribe.KeySchema,
71 | TableName: tableDescribe.TableName,
72 | }
73 |
74 | LOG.Info("try create table: %v", *tableDescribe)
75 |
76 | if conf.Options.FullEnableIndexUser {
77 | // convert []*GlobalSecondaryIndexDescription => []*GlobalSecondaryIndex
78 | gsiList := make([]*dynamodb.GlobalSecondaryIndex, 0, len(tableDescribe.GlobalSecondaryIndexes))
79 | for _, gsiDesc := range tableDescribe.GlobalSecondaryIndexes {
80 | gsiIndex := &dynamodb.GlobalSecondaryIndex{
81 | IndexName: gsiDesc.IndexName,
82 | KeySchema: gsiDesc.KeySchema,
83 | Projection: gsiDesc.Projection,
84 | // ProvisionedThroughput: gsiDesc.ProvisionedThroughput,
85 | }
86 |
87 | // meaningless, support aliyun_dynamodb
88 | if gsiDesc.Projection == nil {
89 | gsiIndex.Projection = &dynamodb.Projection{}
90 | }
91 | gsiList = append(gsiList, gsiIndex)
92 | }
93 | createTableInput.SetGlobalSecondaryIndexes(gsiList)
94 |
95 | // convert []*LocalSecondaryIndexDescription => []*LocalSecondaryIndex
96 | lsiList := make([]*dynamodb.LocalSecondaryIndex, 0, len(tableDescribe.LocalSecondaryIndexes))
97 | for _, lsiDesc := range tableDescribe.LocalSecondaryIndexes {
98 | lsiIndex := &dynamodb.LocalSecondaryIndex{
99 | IndexName: lsiDesc.IndexName,
100 | KeySchema: lsiDesc.KeySchema,
101 | Projection: lsiDesc.Projection,
102 | }
103 |
104 | // meaningless, support aliyun_dynamodb
105 | if lsiDesc.Projection == nil {
106 | lsiIndex.Projection = &dynamodb.Projection{}
107 | }
108 | lsiList = append(lsiList, lsiIndex)
109 | }
110 | createTableInput.SetLocalSecondaryIndexes(lsiList)
111 | }
112 |
113 | _, err := dpw.svc.CreateTable(createTableInput)
114 | if err != nil {
115 | LOG.Error("create table[%v] fail: %v", *tableDescribe.TableName, err)
116 | return err
117 | }
118 |
119 | checkReady := func() bool {
120 | // check table is ready
121 | out, err := dpw.svc.DescribeTable(&dynamodb.DescribeTableInput{
122 | TableName: tableDescribe.TableName,
123 | })
124 | if err != nil {
125 | LOG.Warn("create table[%v] ok but describe failed: %v", *tableDescribe.TableName, err)
126 | return true
127 | }
128 |
129 | if *out.Table.TableStatus != "ACTIVE" {
130 | LOG.Warn("create table[%v] ok but describe not ready: %v", *tableDescribe.TableName, *out.Table.TableStatus)
131 | return true
132 | }
133 |
134 | return false
135 | }
136 |
137 | // check with retry 5 times and 1s gap
138 | ok := utils.CallbackRetry(5, 1000, checkReady)
139 | if !ok {
140 | return fmt.Errorf("create table[%v] fail: check ready fail", dpw.ns.Collection)
141 | }
142 |
143 | return nil
144 | }
145 |
146 | func (dpw *DynamoProxyWriter) DropTable() error {
147 | _, err := dpw.svc.DeleteTable(&dynamodb.DeleteTableInput{
148 | TableName: aws.String(dpw.ns.Collection),
149 | })
150 | return err
151 | }
152 |
153 | func (dpw *DynamoProxyWriter) WriteBulk(input []interface{}) error {
154 | if len(input) == 0 {
155 | return nil
156 | }
157 |
158 | // convert to WriteRequest
159 | request := make([]*dynamodb.WriteRequest, len(input))
160 | for i, ele := range input {
161 | request[i] = &dynamodb.WriteRequest{
162 | PutRequest: &dynamodb.PutRequest{
163 | Item: ele.(map[string]*dynamodb.AttributeValue),
164 | },
165 | }
166 | }
167 |
168 | _, err := dpw.svc.BatchWriteItem(&dynamodb.BatchWriteItemInput{
169 | RequestItems: map[string][]*dynamodb.WriteRequest{
170 | dpw.ns.Collection: request,
171 | },
172 | })
173 | return err
174 | }
175 |
176 | func (dpw *DynamoProxyWriter) Close() {
177 |
178 | }
179 |
180 | // input type is map[string]*dynamodb.AttributeValue
181 | func (dpw *DynamoProxyWriter) Insert(input []interface{}, index []interface{}) error {
182 | if len(input) == 0 {
183 | return nil
184 | }
185 |
186 | request := make([]*dynamodb.WriteRequest, len(index))
187 | for i, ele := range input {
188 | request[i] = &dynamodb.WriteRequest{
189 | PutRequest: &dynamodb.PutRequest{
190 | Item: ele.(map[string]*dynamodb.AttributeValue),
191 | },
192 | }
193 | }
194 |
195 | _, err := dpw.svc.BatchWriteItem(&dynamodb.BatchWriteItemInput{
196 | RequestItems: map[string][]*dynamodb.WriteRequest{
197 | dpw.ns.Collection: request,
198 | },
199 | })
200 |
201 | if err != nil && utils.DynamoIgnoreError(err, "i", true) {
202 | LOG.Warn("%s ignore error[%v] when insert", dpw, err)
203 | return nil
204 | }
205 |
206 | return err
207 | }
208 |
209 | func (dpw *DynamoProxyWriter) Delete(index []interface{}) error {
210 | if len(index) == 0 {
211 | return nil
212 | }
213 |
214 | request := make([]*dynamodb.WriteRequest, len(index))
215 | for i, ele := range index {
216 | request[i] = &dynamodb.WriteRequest{
217 | DeleteRequest: &dynamodb.DeleteRequest{
218 | Key: ele.(map[string]*dynamodb.AttributeValue),
219 | },
220 | }
221 | }
222 |
223 | _, err := dpw.svc.BatchWriteItem(&dynamodb.BatchWriteItemInput{
224 | RequestItems: map[string][]*dynamodb.WriteRequest{
225 | dpw.ns.Collection: request,
226 | },
227 | })
228 |
229 | if utils.DynamoIgnoreError(err, "d", true) {
230 | LOG.Warn("%s ignore error[%v] when delete", dpw, err)
231 | return nil
232 | }
233 |
234 | return err
235 | }
236 |
237 | // upsert
238 | func (dpw *DynamoProxyWriter) Update(input []interface{}, index []interface{}) error {
239 | if len(input) == 0 {
240 | return nil
241 | }
242 |
243 | // fmt.Println(input, index)
244 |
245 | for i := range input {
246 | val := input[i].(map[string]*dynamodb.AttributeValue)
247 | key := index[i].(map[string]*dynamodb.AttributeValue)
248 |
249 | // why no update interface like BatchWriteItem !!!!
250 | // generate new map(expression-attribute-values) and expression(update-expression)
251 | newMap := make(map[string]*dynamodb.AttributeValue, len(val))
252 | expressionBuffer := new(bytes.Buffer)
253 | expressionBuffer.WriteString("SET")
254 | cnt := 1
255 | for k, v := range val {
256 | newKey := fmt.Sprintf(":v%d", cnt)
257 | newMap[newKey] = v
258 |
259 | if cnt == 1 {
260 | expressionBuffer.WriteString(fmt.Sprintf(" %s=%s", k, newKey))
261 | } else {
262 | expressionBuffer.WriteString(fmt.Sprintf(",%s=%s", k, newKey))
263 | }
264 |
265 | cnt++
266 | }
267 |
268 | // fmt.Println(newMap)
269 | _, err := dpw.svc.UpdateItem(&dynamodb.UpdateItemInput{
270 | TableName: aws.String(dpw.ns.Collection),
271 | Key: key,
272 | UpdateExpression: aws.String(expressionBuffer.String()),
273 | ExpressionAttributeValues: newMap,
274 | })
275 | if err != nil && utils.DynamoIgnoreError(err, "u", true) {
276 | LOG.Warn("%s ignore error[%v] when insert", dpw, err)
277 | return nil
278 | }
279 | }
280 |
281 | return nil
282 | }
283 |
--------------------------------------------------------------------------------
/nimo-shake/writer/mongodb_mgo_driver.go:
--------------------------------------------------------------------------------
1 | package writer
2 |
3 | import (
4 | "nimo-shake/common"
5 | "fmt"
6 |
7 | LOG "github.com/vinllen/log4go"
8 | "github.com/aws/aws-sdk-go/service/dynamodb"
9 | "github.com/vinllen/mgo"
10 | "github.com/vinllen/mgo/bson"
11 | "nimo-shake/configure"
12 | "strings"
13 | )
14 |
15 | // deprecated
16 | type MongoWriter struct {
17 | Name string
18 | ns utils.NS
19 | conn *utils.MongoConn
20 | primaryIndexes []*dynamodb.KeySchemaElement
21 | }
22 |
23 | func NewMongoWriter(name, address string, ns utils.NS) *MongoWriter {
24 | targetConn, err := utils.NewMongoConn(address, utils.ConnectModePrimary, true)
25 | if err != nil {
26 | LOG.Error("create mongodb connection error[%v]", err)
27 | return nil
28 | }
29 |
30 | return &MongoWriter{
31 | Name: name,
32 | ns: ns,
33 | conn: targetConn,
34 | }
35 | }
36 |
37 | func (mw *MongoWriter) String() string {
38 | return mw.Name
39 | }
40 |
41 | func (mw *MongoWriter) GetSession() interface{} {
42 | return mw.conn.Session
43 | }
44 |
45 | func (mw *MongoWriter) PassTableDesc(tableDescribe *dynamodb.TableDescription) {
46 | mw.primaryIndexes = tableDescribe.KeySchema
47 | }
48 |
49 | func (mw *MongoWriter) CreateTable(tableDescribe *dynamodb.TableDescription) error {
50 | // parse primary key with sort key
51 | allIndexes := tableDescribe.AttributeDefinitions
52 | primaryIndexes := tableDescribe.KeySchema
53 | globalSecondaryIndexes := tableDescribe.GlobalSecondaryIndexes
54 |
55 | mw.primaryIndexes = primaryIndexes
56 | LOG.Info("%s table[%s] primary index length: %v", mw.String(), *tableDescribe.TableName, len(mw.primaryIndexes))
57 |
58 | // parse index type
59 | parseMap := utils.ParseIndexType(allIndexes)
60 |
61 | // create primary key if has
62 | if len(primaryIndexes) == 0 {
63 | LOG.Info("%s no index found", mw)
64 | return nil
65 | }
66 |
67 | // check if legal
68 | if len(primaryIndexes) > 2 {
69 | return fmt.Errorf("%s illegal primary index[%v] number, should <= 2", mw, len(primaryIndexes))
70 | }
71 |
72 | if conf.Options.FullEnableIndexPrimary {
73 | LOG.Info("%s try create primary index", mw)
74 | // create primary index
75 | if err := mw.createPrimaryIndex(primaryIndexes, parseMap); err != nil {
76 | return err
77 | }
78 |
79 | // create user index
80 | if conf.Options.FullEnableIndexUser {
81 | LOG.Info("%s try create user index", mw)
82 | // create user index
83 | if err := mw.createUserIndex(globalSecondaryIndexes, parseMap); err != nil {
84 | return err
85 | }
86 | }
87 | }
88 |
89 | return nil
90 | }
91 |
92 | func (mw *MongoWriter) DropTable() error {
93 | err := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).DropCollection()
94 | if err != nil && err.Error() == utils.NsNotFountErr {
95 | return nil
96 | }
97 | return err
98 | }
99 |
100 | func (mw *MongoWriter) WriteBulk(input []interface{}) error {
101 | if len(input) == 0 {
102 | return nil
103 | }
104 |
105 | bulk := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).Bulk()
106 | bulk.Unordered()
107 | bulk.Insert(input...)
108 | if _, err := bulk.Run(); err != nil {
109 | if mgo.IsDup(err) {
110 | LOG.Warn("%s duplicated document found[%v]. reinsert or update", err, mw)
111 | if !conf.Options.FullExecutorInsertOnDupUpdate || len(mw.primaryIndexes) == 0 {
112 | LOG.Error("full.executor.insert_on_dup_update==[%v], primaryIndexes length[%v]", conf.Options.FullExecutorInsertOnDupUpdate,
113 | len(mw.primaryIndexes))
114 | return err
115 | }
116 |
117 | // 1. generate index list
118 | indexList := make([]interface{}, len(input))
119 | for i, ele := range input {
120 | inputData := ele.(bson.M)
121 | index := make(bson.M, len(mw.primaryIndexes))
122 | for _, primaryIndex := range mw.primaryIndexes {
123 | // currently, we only support convert type == 'convert', so there is no type inside
124 | key := *primaryIndex.AttributeName
125 | if _, ok := inputData[key]; !ok {
126 | LOG.Error("primary key[%v] is not exists on input data[%v]",
127 | *primaryIndex.AttributeName, inputData)
128 | } else {
129 | index[key] = inputData[key]
130 | }
131 | }
132 | indexList[i] = index
133 | }
134 |
135 | LOG.Debug(indexList)
136 |
137 | return mw.updateOnInsert(input, indexList)
138 | }
139 | return fmt.Errorf("%s insert docs with length[%v] into ns[%s] of dest mongo failed[%v]. first doc: %v",
140 | mw, len(input), mw.ns, err, input[0])
141 | }
142 | return nil
143 | }
144 |
145 | func (mw *MongoWriter) Close() {
146 | mw.conn.Close()
147 | }
148 |
149 | func (mw *MongoWriter) Insert(input []interface{}, index []interface{}) error {
150 | bulk := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).Bulk()
151 | bulk.Unordered()
152 | bulk.Insert(input...)
153 |
154 | if _, err := bulk.Run(); err != nil {
155 | if utils.MongodbIgnoreError(err, "i", false) {
156 | LOG.Warn("%s ignore error[%v] when insert", mw, err)
157 | return nil
158 | }
159 |
160 | // duplicate key
161 | if mgo.IsDup(err) {
162 | if conf.Options.IncreaseExecutorInsertOnDupUpdate {
163 | LOG.Warn("%s duplicated document found. reinsert or update", mw)
164 | return mw.updateOnInsert(input, index)
165 | }
166 | }
167 | return err
168 | }
169 | return nil
170 | }
171 |
172 | func (mw *MongoWriter) updateOnInsert(input []interface{}, index []interface{}) error {
173 | // upsert one by one
174 | for i := range input {
175 | LOG.Debug("upsert: selector[%v] update[%v]", index[i], input[i])
176 | _, err := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).Upsert(index[i], input[i])
177 | if err != nil {
178 | if utils.MongodbIgnoreError(err, "u", true) {
179 | LOG.Warn("%s ignore error[%v] when upsert", mw, err)
180 | return nil
181 | }
182 |
183 | return err
184 | }
185 | }
186 | return nil
187 | }
188 |
189 | func (mw *MongoWriter) Delete(index []interface{}) error {
190 | bulk := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).Bulk()
191 | bulk.Unordered()
192 | bulk.Remove(index...)
193 |
194 | if _, err := bulk.Run(); err != nil {
195 | LOG.Warn(err)
196 | // always ignore ns not found error
197 | if utils.MongodbIgnoreError(err, "d", true) {
198 | LOG.Warn("%s ignore error[%v] when delete", mw, err)
199 | return nil
200 | }
201 |
202 | return err
203 | }
204 |
205 | return nil
206 | }
207 |
208 | func (mw *MongoWriter) Update(input []interface{}, index []interface{}) error {
209 | updates := make([]interface{}, 0, len(input)*2)
210 | for i := range input {
211 | updates = append(updates, index[i])
212 | updates = append(updates, input[i])
213 | }
214 |
215 | bulk := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).Bulk()
216 | if conf.Options.IncreaseExecutorUpsert {
217 | bulk.Upsert(updates...)
218 | } else {
219 | bulk.Update(updates...)
220 | }
221 |
222 | if _, err := bulk.Run(); err != nil {
223 | LOG.Warn(err)
224 | // parse error
225 | idx, _, _ := utils.FindFirstErrorIndexAndMessage(err.Error())
226 | if idx == -1 {
227 | return err
228 | }
229 |
230 | // always upsert data
231 | if utils.MongodbIgnoreError(err, "u", true) {
232 | return mw.updateOnInsert(input[idx:], index[idx:])
233 | }
234 |
235 | if mgo.IsDup(err) {
236 | LOG.Info("error[%v] is dup, ignore", err)
237 | return mw.updateOnInsert(input[idx+1:], index[idx+1:])
238 | }
239 | return err
240 | }
241 |
242 | return nil
243 | }
244 |
245 | func (mw *MongoWriter) createPrimaryIndex(primaryIndexes []*dynamodb.KeySchemaElement, parseMap map[string]string) error {
246 | primaryKeyWithType, err := mw.createSingleIndex(primaryIndexes, parseMap, true)
247 | if err != nil {
248 | return err
249 | }
250 |
251 | // write shard key if target mongodb is sharding
252 | if conf.Options.TargetMongoDBType == utils.TargetMongoDBTypeSharding {
253 | err := mw.conn.Session.DB("admin").Run(bson.D{
254 | {Name: "enablesharding", Value: mw.ns.Database},
255 | }, nil)
256 | if err != nil {
257 | if strings.Contains(err.Error(), "sharding already enabled") == false {
258 | return fmt.Errorf("enable sharding failed[%v]", err)
259 | }
260 | LOG.Warn("ns[%s] sharding already enabled: %v", mw.ns, err)
261 | }
262 |
263 | err = mw.conn.Session.DB("admin").Run(bson.D{
264 | {Name: "shardCollection", Value: mw.ns.Str()},
265 | {Name: "key", Value: bson.M{primaryKeyWithType: "hashed"}},
266 | {Name: "options", Value: bson.M{"numInitialChunks": NumInitialChunks}},
267 | }, nil)
268 | if err != nil {
269 | return fmt.Errorf("shard collection[%s] failed[%v]", mw.ns, err)
270 | }
271 | }
272 |
273 | return nil
274 | }
275 |
276 | func (mw *MongoWriter) createUserIndex(globalSecondaryIndexes []*dynamodb.GlobalSecondaryIndexDescription, parseMap map[string]string) error {
277 | for _, gsi := range globalSecondaryIndexes {
278 | primaryIndexes := gsi.KeySchema
279 | // duplicate index will be ignored by MongoDB
280 | if _, err := mw.createSingleIndex(primaryIndexes, parseMap, false); err != nil {
281 | LOG.Error("ns[%s] create users' single index failed[%v]", mw.ns, err)
282 | return err
283 | }
284 | }
285 | return nil
286 | }
287 |
288 | func (mw *MongoWriter) createSingleIndex(primaryIndexes []*dynamodb.KeySchemaElement, parseMap map[string]string,
289 | isPrimaryKey bool) (string, error) {
290 | primaryKey, sortKey, err := utils.ParsePrimaryAndSortKey(primaryIndexes, parseMap)
291 | if err != nil {
292 | return "", fmt.Errorf("parse primary and sort key failed[%v]", err)
293 | }
294 |
295 | primaryKeyWithType := mw.fetchKey(primaryKey, parseMap[primaryKey])
296 | indexList := make([]string, 0, 2)
297 | indexList = append(indexList, primaryKeyWithType)
298 | if sortKey != "" {
299 | indexList = append(indexList, mw.fetchKey(sortKey, parseMap[sortKey]))
300 | }
301 |
302 | LOG.Info("ns[%s] single index[%v] list[%v]", mw.ns, primaryKeyWithType, indexList)
303 |
304 | // primary key should be unique
305 | unique := isPrimaryKey
306 |
307 | // create union unique index
308 | if len(indexList) >= 2 {
309 | // write index
310 | index := mgo.Index{
311 | Key: indexList,
312 | Background: true,
313 | Unique: unique,
314 | }
315 |
316 | LOG.Info("create union-index isPrimary[%v]: %v", isPrimaryKey, index.Key)
317 |
318 | if err := mw.conn.Session.DB(mw.ns.Database).C(mw.ns.Collection).EnsureIndex(index); err != nil {
319 | return "", fmt.Errorf("create primary union unique[%v] index failed[%v]", unique, err)
320 | }
321 | }
322 |
323 | var indexType interface{}
324 | indexType = "hashed"
325 | if conf.Options.TargetMongoDBType == utils.TargetMongoDBTypeReplica {
326 | indexType = 1
327 | }
328 | if len(indexList) >= 2 {
329 | // unique has already be set on the above index
330 | unique = false
331 | } else if unique {
332 | // must be range if only has 1 key
333 | indexType = 1
334 | }
335 |
336 | doc := bson.D{
337 | {Name: "createIndexes", Value: mw.ns.Collection},
338 | {Name: "indexes", Value: []bson.M{
339 | {
340 | "key": bson.M{
341 | primaryKeyWithType: indexType,
342 | },
343 | "name": fmt.Sprintf("%s_%v", primaryKeyWithType, indexType),
344 | "background": true,
345 | "unique": unique,
346 | },
347 | }},
348 | }
349 | LOG.Info("create index isPrimary[%v]: %v", isPrimaryKey, doc)
350 | // create hash key only
351 | if err := mw.conn.Session.DB(mw.ns.Database).Run(doc, nil); err != nil {
352 | return "", fmt.Errorf("create primary[%v] %v index failed[%v]", isPrimaryKey, indexType, err)
353 | }
354 |
355 | return primaryKeyWithType, nil
356 | }
357 |
358 | func (mw *MongoWriter) fetchKey(key, tp string) string {
359 | switch conf.Options.ConvertType {
360 | case utils.ConvertTypeChange:
361 | fallthrough
362 | case utils.ConvertTypeSame:
363 | return key
364 | case utils.ConvertTypeRaw:
365 | return fmt.Sprintf("%s.%s", key, tp)
366 | }
367 | return ""
368 | }
369 |
--------------------------------------------------------------------------------
/nimo-shake/writer/writer.go:
--------------------------------------------------------------------------------
1 | package writer
2 |
3 | import (
4 | "nimo-shake/common"
5 |
6 | LOG "github.com/vinllen/log4go"
7 | "github.com/aws/aws-sdk-go/service/dynamodb"
8 | )
9 |
10 | type Writer interface{
11 | // create table
12 | CreateTable(tableDescribe *dynamodb.TableDescription) error
13 | // pass table description
14 | PassTableDesc(tableDescribe *dynamodb.TableDescription)
15 | // drop table
16 | DropTable() error
17 | // write bulk data, used in full sync
18 | WriteBulk(input []interface{}) error
19 | // insert
20 | Insert(input []interface{}, index []interface{}) error
21 | // delete
22 | Delete(input []interface{}) error
23 | // update
24 | Update(input []interface{}, index []interface{}) error
25 | // close
26 | Close()
27 | // get session
28 | GetSession() interface{}
29 | }
30 |
31 | func NewWriter(name, address string, ns utils.NS, logLevel string) Writer {
32 | switch name {
33 | case utils.TargetTypeMongo:
34 | // return NewMongoWriter(name, address, ns)
35 | return NewMongoCommunityWriter(name, address, ns)
36 | case utils.TargetTypeAliyunDynamoProxy:
37 | return NewDynamoProxyWriter(name, address, ns, logLevel)
38 | default:
39 | LOG.Crashf("unknown writer[%v]", name)
40 | }
41 | return nil
42 | }
--------------------------------------------------------------------------------
/scripts/hypervisor.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Compilation:
3 | * gcc -Wall -O3 hypervisor.c -o hypervisor
4 | *
5 | * */
6 |
7 | #include
8 | #include
9 | #include
10 | #include
11 |
12 | #ifdef __linux__
13 | #include
14 | #else
15 | #include
16 | #endif
17 |
18 | #include
19 | #include
20 | #include
21 | #include
22 | #include
23 | #include
24 |
25 |
26 | #define MAXOPT 256
27 | #define INTERVAL 5
28 | #define MAXINTERVAL 180
29 |
30 | #define USAGE() do{ \
31 | fprintf(stderr, "Usage : %s [--daemon] --exec=\"command arg1 arg2 arg3 ...\"\n", argv[0]); \
32 | }while(0)
33 |
34 | static char *cmd, *cmdopt[MAXOPT + 1];
35 | static int daemonize;
36 |
37 | static int parseopt(int argc, char *argv[]);
38 | static int set_nonblock(int fd);
39 | static int getstatus(char *buf, int size, int status);
40 |
41 | static int parseopt(int argc, char *argv[])
42 | {
43 | int ch, i;
44 | char *token, *tmpptr, *cmdstr;
45 |
46 | cmdstr = cmd = NULL;
47 | daemonize = 0;
48 | for(i = 0; i < MAXOPT + 1; i++){
49 | cmdopt[i] = NULL;
50 | }
51 |
52 | struct option long_options[] = {
53 | {"daemon",optional_argument,NULL,'d'},
54 | {"exec",required_argument,NULL,'e'},
55 | {0,0,0,0},
56 | };
57 |
58 | while((ch=getopt_long(argc, argv, "dec:", long_options, NULL)) != -1) {
59 | switch(ch)
60 | {
61 | case 'e':
62 | if((cmdstr = strdup(optarg)) == NULL )
63 | return -1;
64 | break;
65 | case 'd':
66 | daemonize = 1;
67 | break;
68 | default:
69 | USAGE();
70 | return -1;
71 | }
72 | }
73 |
74 | if(cmdstr == NULL){
75 | USAGE();
76 | return -1;
77 | }
78 |
79 | for(i = 0;i < MAXOPT + 1;cmdstr = NULL, i++){
80 | token = strtok_r(cmdstr, " \t\n", &tmpptr);
81 | if(token == NULL){
82 | break;
83 | } else {
84 | cmdopt[i] = strdup(token);
85 |
86 | if(i == 0){
87 | cmd = strdup(token);
88 | }
89 | }
90 | }
91 |
92 | if( (cmd == NULL) || (strlen(cmd) == 0) ){
93 | fprintf(stderr, "Error, cmd should not be empty.\n");
94 | return -1;
95 | }
96 |
97 | if(i == MAXOPT + 1){
98 | fprintf(stderr, "Argument too long\n");
99 | return -1;
100 | }
101 |
102 | cmdopt[i] = NULL;
103 |
104 | return 0;
105 | }
106 |
107 | static int set_nonblock(int fd)
108 | {
109 | int flags = fcntl(fd, F_GETFL, 0);
110 | if (flags == -1) {
111 | return -1;
112 | }
113 | return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
114 | }
115 |
116 | static int getstatus(char *buf, int size, int status)
117 | {
118 | int n, len;
119 |
120 | len = size;
121 |
122 | if(WIFEXITED(status)){
123 | n = snprintf(buf, len, "- normal termination, exit status = %d\n", WEXITSTATUS(status));
124 | } else if(WIFSIGNALED(status)) {
125 | n = snprintf(buf, len, "- abnormal termination, signal number = %d%s",
126 | WTERMSIG(status),
127 | #ifdef WCOREDUMP
128 | WCOREDUMP(status) ? "-> core file generated" : "");
129 | #else
130 | "");
131 | #endif
132 | } else if(WIFSTOPPED(status)) {
133 | n = snprintf(buf, len, "child stopped, signal number = %d\n", WSTOPSIG(status));
134 | }
135 |
136 | return n;
137 | }
138 |
139 |
140 |
141 | void go_daemon() {
142 | int fd;
143 |
144 | if (fork() != 0) exit(0); /* parent exits */
145 | setsid(); /* create a new session */
146 |
147 | /* Every output goes to /dev/null. If Redis is daemonized but
148 | * * the 'logfile' is set to 'stdout' in the configuration file
149 | * * it will not log at all. */
150 | if ((fd = open("/tmp/mongo4bls.output", O_RDWR, 0)) != -1) {
151 | dup2(fd, STDIN_FILENO);
152 | dup2(fd, STDOUT_FILENO);
153 | dup2(fd, STDERR_FILENO);
154 | if (fd > STDERR_FILENO) close(fd);
155 | }
156 | }
157 |
158 |
159 | int main(int argc, char *argv[])
160 | {
161 | int ssec = INTERVAL, ret, status;
162 | int first_start = 1;
163 | int pipefd[2], waited, alive, isdaemon;
164 | char buf[1024], info[4096];
165 | pid_t pid;
166 | time_t now, last = time(NULL);
167 |
168 | if((ret = parseopt(argc, argv)) < 0 )
169 | exit(ret);
170 |
171 | daemonize ? go_daemon() : 0;
172 |
173 | while(1){
174 | if(pipe(pipefd) < 0){
175 | fprintf(stderr, "- make pipe error : %s\n", strerror(errno));
176 | exit(-1);
177 | }
178 |
179 | if( (ret = set_nonblock(pipefd[0])) < 0 ){
180 | fprintf(stderr, "- set read nonblock error : %s\n", strerror(errno));
181 | exit(-1);
182 | }
183 |
184 | if((pid = fork()) < 0){
185 | fprintf(stderr, "- call fork() error : %s\n", strerror(errno));
186 | exit(-1);
187 | } else if (pid > 0){
188 | close(pipefd[1]);
189 | alive = waited = 1;
190 | isdaemon = 0;
191 | while(alive){
192 | if(waited){
193 | if(pid != waitpid(pid, &status, 0)){
194 | sleep(INTERVAL);
195 | continue;
196 | } else {
197 | fprintf(stderr, "- child process[%d] terminated .\n",pid);
198 | if (first_start && (time(NULL)-last)<=5) {
199 | fprintf(stderr,"- child process killed in %ld seconds , may wrong ! exit !\n",(time(NULL)-last));
200 | exit(-1);
201 | } else
202 | first_start = 0;
203 | waited = 0;
204 | }
205 | }
206 |
207 | ret = read(pipefd[0], buf, sizeof(buf));
208 | if(ret < 0){
209 | if(errno == EAGAIN){
210 | if(isdaemon == 0){
211 | fprintf(stderr, "- this daemon process has no output !.\n");
212 | isdaemon = 1;
213 | }
214 | sleep(INTERVAL);
215 | continue;
216 | } else {
217 | fprintf(stderr, "- read pipe error : %s\n", strerror(errno));
218 | exit(-1);
219 | }
220 | } else if(ret == 0) {
221 | alive = 0;
222 | close(pipefd[0]);
223 | fprintf(stderr, "- read zero from pipe of children.\n");
224 | if(isdaemon == 0){
225 | getstatus(info, sizeof(info), status);
226 | fprintf(stderr, "- extra info: %s\n", info);
227 | } else {
228 | strcpy(info, "");
229 | }
230 | continue;
231 | } else {
232 | fprintf(stderr, " - read pipe return: %d bytes\n", ret);
233 | exit(-1);
234 | }
235 | }
236 |
237 | fprintf(stderr, "- process: \"%s\" exit, restart it\n", cmd);
238 |
239 | sleep(ssec);
240 |
241 | now = time(NULL);
242 | if(now - last > 3600){
243 | ssec = INTERVAL;
244 | last = now;
245 | } else {
246 | ssec = (ssec << 1) < MAXINTERVAL ? (ssec << 1) : MAXINTERVAL;
247 | }
248 | } else {
249 | close(pipefd[0]);
250 | fprintf(stderr, "- execute: \"%s\"\n", cmd);
251 | if(execvp(cmd, cmdopt) < 0){
252 | fprintf(stderr, "- execute: \"%s\" error, %s\n", cmd, strerror(errno));
253 | exit(-1);
254 | }
255 |
256 | }
257 | }
258 |
259 | return 0;
260 | }
261 |
--------------------------------------------------------------------------------
/scripts/run_ut_test.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | # -*- coding:utf-8 -*-
4 |
5 | import os
6 | from os import listdir
7 | from os.path import isfile, join, isdir
8 | import subprocess
9 |
10 | go_path="."
11 | """
12 | run unit test in recursive
13 | """
14 | def run_ut(cur_path):
15 | print(os.path.abspath(os.path.curdir), cur_path)
16 |
17 | only_files = [f for f in listdir(".") if isfile(join(".", f))]
18 | only_dirs = [f for f in listdir(".") if isdir(join(".", f))]
19 | ut_files = [f for f in only_files if "_test.go" in f]
20 | print(only_files, only_dirs, ut_files)
21 |
22 | if len(ut_files) != 0:
23 | # with ut file, need run ut test
24 | print("----------- run ut test on dir[%s] -----------" % os.path.abspath(os.path.curdir))
25 | ret = subprocess.call(["go", "test"])
26 | # subprocess.check_output(["/bin/sh", "-c", "go", "test"])
27 |
28 | print("********************************** %s *************************************" % ("OK" if ret == 0 else "FAIL"))
29 | if ret != 0:
30 | print("run failed")
31 | exit(ret)
32 |
33 | for dir in only_dirs:
34 | print("cd dir[%s]" % dir)
35 |
36 | # dfs
37 | os.chdir(dir)
38 | run_ut(dir)
39 |
40 | # backtracking
41 | os.chdir("..")
42 |
43 | if __name__ == "__main__":
44 | root_path = os.path.join("..", "src/nimo-shake")
45 | os.chdir(root_path)
46 | go_path=os.path.abspath("../..")
47 | print("GOPATH=%s" % go_path)
48 | #subprocess.call(["export GOPATH=%s" % go_path])
49 | os.environ['GOPATH'] = go_path
50 | run_ut(".")
51 |
52 | print("-----------------------------------")
53 | print("all is well ^_^")
54 |
--------------------------------------------------------------------------------
/scripts/start.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | catalog=$(dirname "$0")
4 |
5 | cd "${catalog}" || exit 1
6 |
7 | if [ $# != 2 ] ; then
8 | echo "USAGE: $0 [conf] [mode]"
9 | exit 0
10 | fi
11 |
12 | name="dynamo-shake"
13 |
14 | if [ "Darwin" == "$(uname -s)" ];then
15 | printf "\\nWARNING !!! MacOs doesn't supply to use this script, please use \"./%s -conf=config_file_name\" manual command to run\\n" "$name"
16 | exit 1
17 | fi
18 |
19 | ./hypervisor --daemon --exec="./$name -conf=$1 -type=$2 1>>$name.output 2>&1" 1>>hypervisor.output 2>&1
20 |
--------------------------------------------------------------------------------
/scripts/stop.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | #kill -9 "$(cat "$1")"
3 | if [ $# != 1 ] ; then
4 | echo "USAGE: $0 [pid filename which by default is 'dynamo-shake.pid']"
5 | exit 0
6 | fi
7 | ppid=$(ps -ef | awk '{if ($2=='`cat $1`') print $3}')
8 | [ -z $ppid ] && echo "[Fail] No process number for $(cat "$1")." && exit 1
9 | if [ $ppid -eq 1 ];then
10 | kill -9 "$(cat "$1")"
11 | else
12 | kill -9 "$ppid" "$(cat "$1")"
13 | fi
14 |
--------------------------------------------------------------------------------