├── version ├── assets └── email-image.png ├── client ├── pkg │ ├── constants │ │ ├── version.go │ │ ├── constant.go │ │ └── variables.go │ ├── model │ │ ├── constants.go │ │ ├── operation_id_test.go │ │ ├── client.go │ │ ├── message_header.go │ │ ├── operation.go │ │ ├── checkpoint.go │ │ ├── messages.go │ │ ├── timestamp.go │ │ └── operation_id.go │ ├── iface │ │ ├── snapshot.go │ │ ├── operation.go │ │ ├── wired.go │ │ ├── context.go │ │ └── datatype.go │ ├── errors │ │ ├── pushpull.go │ │ ├── errors_test.go │ │ ├── rpc.go │ │ ├── multiple_errors.go │ │ ├── error.go │ │ └── errorcode.go │ ├── utils │ │ ├── strings.go │ │ ├── struct.go │ │ └── prints.go │ ├── log │ │ └── strings.go │ ├── orda │ │ ├── config.go │ │ ├── local_only_client_test.go │ │ ├── encoding_test.go │ │ ├── timed.go │ │ ├── handlers.go │ │ ├── ordered.go │ │ ├── json_element.go │ │ ├── document_marshal.go │ │ ├── marshal_test.go │ │ ├── datatype.go │ │ └── counter_test.go │ ├── context │ │ ├── orda_context_test.go │ │ ├── client_datatype.go │ │ └── orda_context.go │ ├── operations │ │ ├── counter.go │ │ ├── encoding_decoding_test.go │ │ ├── map.go │ │ ├── base.go │ │ ├── list.go │ │ ├── meta.go │ │ └── converter.go │ ├── types │ │ ├── uid.go │ │ ├── uid_test.go │ │ ├── json_values_test.go │ │ └── json_values.go │ ├── testonly │ │ ├── test_helper.go │ │ └── test_wire.go │ └── internal │ │ ├── datatypes │ │ └── snapshot.go │ │ └── managers │ │ ├── notify.go │ │ └── sync.go └── go.mod ├── server ├── testonly │ ├── constants.go │ └── service.go ├── schema │ ├── common.go │ ├── counter.go │ ├── collections.go │ ├── snapshots.go │ ├── schema.go │ ├── operations.go │ └── clients.go ├── redis │ ├── cofig.go │ └── client.go ├── constants │ └── constants.go ├── utils │ ├── lock.go │ ├── local_lock.go │ └── redis_lock.go ├── admin │ └── admin.go ├── server.go ├── service │ ├── service_orda.go │ ├── service_collections.go │ ├── service_client.go │ ├── service_pushpull_client.go │ ├── service_patch_document.go │ └── service_test.go ├── mongodb │ ├── collection_col_num_generator.go │ ├── config.go │ ├── collection_snapshots.go │ ├── collection_real_collection.go │ ├── collection_clients.go │ ├── mongo_collection.go │ ├── collection_operations.go │ ├── collection_collections.go │ └── mongo_collection_test.go ├── notification │ ├── mqtt_test.go │ └── notifier.go ├── wrapper │ └── datatype.go ├── managers │ ├── managers.go │ └── config.go ├── go.mod └── snapshot │ └── manager.go ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── ISSUE_TEMPLATE │ ├── doc_issue.md │ ├── disc_issue.md │ ├── bug_issue.md │ └── feat_issue.md └── workflows │ └── cla.yml ├── deployments ├── config │ ├── local-config-no-redis.json │ ├── local-config.json │ └── docker-config.json ├── Dockerfile └── docker-compose.yaml ├── .gitignore ├── Dockerfile ├── test ├── list_test.go ├── map_test.go ├── datatypes_test.go ├── protocol_test.go ├── integration_test.go ├── lock_test.go ├── test_helper.go └── realtime_test.go ├── examples └── document │ └── document.go ├── proto ├── orda.enum.proto ├── orda.grpc.proto └── orda.proto ├── CONTRIBUTING.md └── go.mod /version: -------------------------------------------------------------------------------- 1 | 0.2.0 -------------------------------------------------------------------------------- /assets/email-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/orda-io/orda/HEAD/assets/email-image.png -------------------------------------------------------------------------------- /client/pkg/constants/version.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | // Version is a version string which is injected by Makefile 4 | const Version = "0.2.0" 5 | -------------------------------------------------------------------------------- /server/testonly/constants.go: -------------------------------------------------------------------------------- 1 | package testonly 2 | 3 | // TestDBName is the name of mongodb for testing orda server package 4 | const TestDBName = "server_unit_test" 5 | -------------------------------------------------------------------------------- /client/pkg/model/constants.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | // ProtocolVersion is the version of the protocol. 4 | const ProtocolVersion = "v1" 5 | 6 | // Agent is the sdk name 7 | const Agent = "orda-go" 8 | -------------------------------------------------------------------------------- /client/pkg/iface/snapshot.go: -------------------------------------------------------------------------------- 1 | package iface 2 | 3 | // Snapshot defines the interfaces for snapshot used in a datatype. Snapshot contains metadata 4 | type Snapshot interface { 5 | ToJSON() interface{} 6 | } 7 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Relevant issues 2 | 3 | - Resolved: # 4 | - ... 5 | 6 | ### Checklists 7 | 8 | - [ ] New tests created? 9 | - [ ] Related docs updated? 10 | 11 | ### Description 12 | 13 | -------------------------------------------------------------------------------- /server/schema/common.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import "go.mongodb.org/mongo-driver/mongo" 4 | 5 | // MongoDBDoc is an interface for documents stored in MongoDB 6 | type MongoDBDoc interface { 7 | GetIndexModel() []mongo.IndexModel 8 | } 9 | -------------------------------------------------------------------------------- /server/redis/cofig.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | // Config describes the configuration of redis 4 | type Config struct { 5 | Addrs []string `json:"Addrs"` 6 | Username string `json:"Username"` 7 | Password string `json:"Password"` 8 | } 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/doc_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue for documentation about: Documentation title: "📜doc: " 3 | labels: documentation 4 | --- 5 | 6 | ### What to do? 7 | ---- 8 | > describe briefly what feature you would like to add 9 | 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/disc_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue for discussion or proposal about: Suggest a subject to discuss or propose title: "🗣disc: " 3 | labels: proposal assignees: '' 4 | 5 | --- 6 | 7 | ### What to do? 8 | ---- 9 | > describe the subject to discuss or propose 10 | 11 | 12 | -------------------------------------------------------------------------------- /client/pkg/errors/pushpull.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // PushPullError defines the errors used in datatype. 8 | type PushPullError struct { 9 | Code ErrorCode 10 | Msg string 11 | } 12 | 13 | func (its *PushPullError) Error() string { 14 | return fmt.Sprintf("[PushPullErr: %d] %s", its.Code, its.Msg) 15 | } 16 | -------------------------------------------------------------------------------- /deployments/config/local-config-no-redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "RPCServerPort": 29062, 3 | "RestfulPort": 29862, 4 | "SwaggerJSON": "./resources/orda.grpc.swagger.json", 5 | "Notification": "tcp://127.0.0.1:18181", 6 | "Mongo": { 7 | "MongoHost": "127.0.0.1:27017", 8 | "OrdaDB": "ordadb", 9 | "User": "root", 10 | "Password": "orda-test" 11 | } 12 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue for bug about: Report or want to fix a bug title: "🦠bug: " 3 | labels: bug 4 | 5 | --- 6 | 7 | ### What to do? 8 | ---- 9 | > describe briefly bugs. 10 | 11 | 12 | ### How to reproduce? 13 | ---- 14 | > describe briefly how to produce bug. 15 | 16 | ### How to fix? 17 | ---- 18 | > describe briefly how to fix. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | .idea/ 14 | docker_volumes/ 15 | deployments/docker_volumes 16 | build/ 17 | bin/ 18 | proto/thirdparty 19 | .vscode -------------------------------------------------------------------------------- /client/pkg/constants/constant.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | // OperationBufferSize denotes the size of operation buffer 5 | OperationBufferSize int = 1024 6 | ) 7 | 8 | const ( 9 | // TagSdkClient is the emoji tag for clients in orda sdk 10 | TagSdkClient = "🎪" 11 | // TagSdkDatatype is the emoji tag for datatypes of orda sdk 12 | TagSdkDatatype = "🐎" 13 | ) 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feat_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue for feature about: Suggest an idea for this project title: "✨feat: " 3 | labels: feat 4 | --- 5 | 6 | ### What to do? 7 | ---- 8 | > describe briefly what feature you would like to add 9 | 10 | 11 | ### Details 12 | ---- 13 | > describe briefly why you would like to add 14 | 15 | ### Checklist 16 | 17 | - [ ] Did you assign project? 18 | -------------------------------------------------------------------------------- /deployments/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | MAINTAINER Hyun-Gul Roh 3 | 4 | RUN apt-get update 5 | RUN apt-get -y upgrade 6 | RUN mkdir /app /config /app/resources 7 | COPY ./tmp/resources /app/resources 8 | COPY ./tmp/server /app/orda_server 9 | 10 | WORKDIR /app 11 | 12 | CMD ["/app/orda_server", "--conf", "/config/docker-config.json"] 13 | 14 | EXPOSE 19061 15 | EXPOSE 19861 -------------------------------------------------------------------------------- /client/pkg/iface/operation.go: -------------------------------------------------------------------------------- 1 | package iface 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | // Operation defines the interfaces of any operation 8 | type Operation interface { 9 | GetType() model.TypeOfOperation 10 | String() string 11 | GetID() *model.OperationID 12 | SetID(opID *model.OperationID) 13 | ToJSON() interface{} 14 | ToModelOperation() *model.Operation 15 | } 16 | -------------------------------------------------------------------------------- /client/pkg/model/operation_id_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestClientId(t *testing.T) { 10 | t.Run("Can compare OperationIDs", func(t *testing.T) { 11 | opID1 := NewOperationID() 12 | opID2 := NewOperationID() 13 | assert.Assert(t, opID1.Compare(opID2) == 0) 14 | opID1.Next() 15 | assert.Assert(t, opID1.Compare(opID2) > 0) 16 | }) 17 | } 18 | -------------------------------------------------------------------------------- /server/schema/counter.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | // CounterDoc defines a document for counter stored in MongoDB. Counter is used to assign number to a collection 4 | type CounterDoc struct { 5 | ID string `bson:"_id"` 6 | Num int32 `bson:"num"` 7 | } 8 | 9 | // CounterDocFields defines the fields of CounterDoc 10 | var CounterDocFields = struct { 11 | ID string 12 | Num string 13 | }{ 14 | ID: "_id", 15 | Num: "num", 16 | } 17 | -------------------------------------------------------------------------------- /client/pkg/utils/strings.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "fmt" 7 | ) 8 | 9 | // HashSum returns the hash sum of arguments 10 | func HashSum(args ...interface{}) string { 11 | hash := md5.New() 12 | for _, arg := range args { 13 | if stringer, ok := arg.(fmt.Stringer); ok { 14 | hash.Write([]byte(stringer.String())) 15 | } 16 | } 17 | return hex.EncodeToString(hash.Sum(nil)) 18 | } 19 | -------------------------------------------------------------------------------- /deployments/config/local-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "RPCServerPort": 29062, 3 | "RestfulPort": 29862, 4 | "SwaggerJSON": "./resources/orda.grpc.swagger.json", 5 | "Notification": "tcp://127.0.0.1:18181", 6 | "Mongo": { 7 | "MongoHost": "127.0.0.1:27017", 8 | "OrdaDB": "ordadb", 9 | "User": "root", 10 | "Password": "orda-test" 11 | }, 12 | "Redis": { 13 | "Addrs": [ 14 | "127.0.0.1:16379" 15 | ] 16 | } 17 | } -------------------------------------------------------------------------------- /deployments/config/docker-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "RPCServerPort": 19061, 3 | "RestfulPort": 19861, 4 | "Notification": "tcp://orda-emqx:1883", 5 | "SwaggerJSON": "./resources/orda.grpc.swagger.json", 6 | "Mongo": { 7 | "MongoHost": "orda-mongodb:27017", 8 | "OrdaDB": "ordadb", 9 | "User": "root", 10 | "Password": "orda-test" 11 | }, 12 | "Redis": { 13 | "Addrs": [ 14 | "orda-redis:6379" 15 | ] 16 | } 17 | } -------------------------------------------------------------------------------- /client/pkg/iface/wired.go: -------------------------------------------------------------------------------- 1 | package iface 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | ) 7 | 8 | // Wire defines the interfaces related to delivering operations. This is called when a datatype needs to send messages 9 | type Wire interface { 10 | DeliverTransaction(wired WiredDatatype) 11 | OnChangeDatatypeState(dt Datatype, state model.StateOfDatatype) errors.OrdaError 12 | } 13 | -------------------------------------------------------------------------------- /client/pkg/errors/errors_test.go: -------------------------------------------------------------------------------- 1 | package errors_test 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "testing" 7 | ) 8 | 9 | func TestOrdaError(t *testing.T) { 10 | _ = call1() 11 | } 12 | 13 | func call1() errors.OrdaError { 14 | return call2() 15 | } 16 | 17 | func call2() errors.OrdaError { 18 | return errors.DatatypeNoOp.New(log.Logger, "This is a generated sample error") 19 | } 20 | -------------------------------------------------------------------------------- /server/constants/constants.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import "math" 4 | 5 | // InfinitySseq is infinite number of sseq 6 | const InfinitySseq uint64 = math.MaxUint64 7 | 8 | // TagXXX are emoji tags of logs 9 | const ( 10 | TagServer = "👽" 11 | TagReset = "🕊" 12 | TagCreate = "✨" 13 | TagClient = "🚖" 14 | TagPushPull = "🐙" 15 | TagPostPushPull = "🧽" 16 | TagTest = "🦠" 17 | TagPatch = "🧵" 18 | ) 19 | -------------------------------------------------------------------------------- /client/pkg/log/strings.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import "fmt" 4 | 5 | const maxLength = 20 6 | 7 | // MakeShort returns a short tag 8 | func MakeShort(tag string, maxLength int) string { 9 | l := len(tag) 10 | if l > maxLength { 11 | tag = fmt.Sprintf("%s♻%s", tag[:(maxLength/2)-1], tag[l-(maxLength/2)+1:]) 12 | } 13 | return fmt.Sprintf("%.*s", maxLength, tag) 14 | } 15 | 16 | // MakeDefaultShort makes short 17 | func MakeDefaultShort(tag string) string { 18 | return MakeShort(tag, maxLength) 19 | } 20 | -------------------------------------------------------------------------------- /client/pkg/model/client.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | ) 7 | 8 | // ToString returns customized string 9 | func (its *Client) ToString() string { 10 | return fmt.Sprintf("%s(%s)|%s|%s", its.Alias, its.CUID, its.SyncType.String(), its.Collection) 11 | } 12 | 13 | // GetSummary returns the summary of client 14 | func (its *Client) GetSummary() string { 15 | return fmt.Sprintf("%s|%s(%s)", its.Collection, log.MakeDefaultShort(its.Alias), its.CUID) 16 | } 17 | -------------------------------------------------------------------------------- /client/pkg/utils/struct.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "github.com/mitchellh/mapstructure" 4 | 5 | // StructToMap is used to transform struct to map type 6 | func StructToMap(in interface{}) (map[string]interface{}, error) { 7 | toMap := make(map[string]interface{}) 8 | msDecoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ 9 | TagName: "json", 10 | Result: &toMap, 11 | }) 12 | if err != nil { 13 | return nil, err 14 | } 15 | if err := msDecoder.Decode(in); err != nil { 16 | return nil, err 17 | } 18 | return toMap, nil 19 | } 20 | -------------------------------------------------------------------------------- /server/utils/lock.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | const ( 9 | prefix = "orda:__lock__:" 10 | defaultLeaseTime = 5 * time.Second 11 | defaultExpireTime = 10 * time.Second 12 | ) 13 | 14 | // Lock defines the lock interfaces 15 | type Lock interface { 16 | TryLock() bool 17 | Unlock() bool 18 | } 19 | 20 | // GetLockName returns a lock name with the prefix 21 | func GetLockName(prefix string, collectionNum int32, key string) string { 22 | return fmt.Sprintf("%s:%d:%s", prefix, collectionNum, key) 23 | } 24 | -------------------------------------------------------------------------------- /client/pkg/model/message_header.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/constants" 6 | ) 7 | 8 | // NewMessageHeader generates a message header. 9 | func NewMessageHeader(typeOf RequestType) *Header { 10 | return &Header{ 11 | Version: ProtocolVersion, 12 | Agent: fmt.Sprintf("%s-%v", constants.SDKType, constants.Version), 13 | Type: typeOf, 14 | } 15 | } 16 | 17 | // ToString returns customized string 18 | func (its *Header) ToString() string { 19 | return fmt.Sprintf("%s|%s|%s", its.Version, its.Type, its.Agent) 20 | } 21 | -------------------------------------------------------------------------------- /server/testonly/service.go: -------------------------------------------------------------------------------- 1 | package testonly 2 | 3 | import ( 4 | ctx "context" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | "github.com/orda-io/orda/server/service" 8 | "github.com/stretchr/testify/require" 9 | "testing" 10 | ) 11 | 12 | // RegisterClient is used to test client 13 | func RegisterClient(t *testing.T, service *service.OrdaService, client *model.Client) { 14 | 15 | req := model.NewClientMessage(client) 16 | res, err := service.ProcessClient(ctx.TODO(), req) 17 | require.NoError(t, err) 18 | log.Logger.Infof("ProcessClient result:%v", res) 19 | } 20 | -------------------------------------------------------------------------------- /client/pkg/iface/context.go: -------------------------------------------------------------------------------- 1 | package iface 2 | 3 | import ( 4 | "context" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | ) 7 | 8 | // OrdaContext is used to pass over the context of clients or datatypes 9 | type OrdaContext interface { 10 | context.Context 11 | L() *log.OrdaLog 12 | Ctx() context.Context 13 | SetLogger(l *log.OrdaLog) 14 | UpdateAllTags(emoji, collection, colnum, client, cuid, datatype, duid string) 15 | UpdateCollectionTags(collection string, colNum int32) OrdaContext 16 | UpdateClientTags(client, cuid string) OrdaContext 17 | UpdateDatatypeTags(datatype, duid string) OrdaContext 18 | CloneWithNewEmoji(emoji string) OrdaContext 19 | } 20 | -------------------------------------------------------------------------------- /client/pkg/orda/config.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | // ClientConfig is a configuration for OrdaClient 8 | type ClientConfig struct { 9 | ServerAddr string 10 | NotificationAddr string 11 | CollectionName string 12 | SyncType model.SyncType 13 | } 14 | 15 | // NewLocalClientConfig makes a new local client which do not synchronize with OrdaServer 16 | func NewLocalClientConfig(collectionName string) *ClientConfig { 17 | return &ClientConfig{ 18 | ServerAddr: "", 19 | NotificationAddr: "", 20 | CollectionName: collectionName, 21 | SyncType: model.SyncType_LOCAL_ONLY, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/pkg/utils/prints.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | 7 | "github.com/TylerBrock/colorjson" 8 | ) 9 | 10 | // PrintMarshalDoc logs out doc without error 11 | func PrintMarshalDoc(l *log.OrdaLog, doc interface{}) { 12 | f := colorjson.NewFormatter() 13 | f.Indent = 2 14 | f.DisabledColor = true 15 | m, _ := json.Marshal(doc) 16 | var obj map[string]interface{} 17 | _ = json.Unmarshal(m, &obj) 18 | s, _ := f.Marshal(obj) 19 | l.Infof("%v", string(s)) 20 | } 21 | 22 | // ToStringMarshalDoc returns marshaled string without error 23 | func ToStringMarshalDoc(doc interface{}) string { 24 | m, _ := json.Marshal(doc) 25 | return string(m) 26 | } 27 | -------------------------------------------------------------------------------- /client/pkg/context/orda_context_test.go: -------------------------------------------------------------------------------- 1 | package context_test 2 | 3 | import ( 4 | gocontext "context" 5 | "github.com/orda-io/orda/client/pkg/context" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "testing" 8 | ) 9 | 10 | func TestOrdaContext(t *testing.T) { 11 | t.Run("Can work with OrdaConext", func(t *testing.T) { 12 | log.Logger.Infof("print well?") 13 | ctx1 := context.NewOrdaContext(gocontext.TODO(), "🎪") 14 | ctx1.UpdateCollectionTags("Hello_Collection", 1) 15 | ctx1.UpdateClientTags("Hello_Client", "abcdefghijk") 16 | ctx1.UpdateDatatypeTags("Hello_Datatype", "123456789") 17 | ctx1.L().Infof("print well?") 18 | 19 | ctx2 := ctx1.CloneWithNewEmoji("👽") 20 | ctx2.L().Infof("clone print well?") 21 | }) 22 | } 23 | -------------------------------------------------------------------------------- /client/pkg/operations/counter.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | type increaseBody struct { 8 | Delta int32 9 | } 10 | 11 | // NewIncreaseOperation creates an IncreaseOperation. 12 | func NewIncreaseOperation(delta int32) *IncreaseOperation { 13 | return &IncreaseOperation{ 14 | baseOperation: newBaseOperation( 15 | model.TypeOfOperation_COUNTER_INCREASE, 16 | nil, 17 | &increaseBody{ 18 | Delta: delta, 19 | }, 20 | ), 21 | } 22 | } 23 | 24 | // IncreaseOperation is used to increase value to IntCounter. 25 | type IncreaseOperation struct { 26 | baseOperation 27 | } 28 | 29 | // GetBody returns the body 30 | func (its *IncreaseOperation) GetBody() int32 { 31 | return its.Body.(*increaseBody).Delta 32 | } 33 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 golang:latest 2 | RUN apt -y update 3 | RUN apt -y upgrade 4 | RUN apt install -y protobuf-compiler 5 | RUN apt install -y vim 6 | RUN apt install -y unzip 7 | 8 | WORKDIR /root 9 | ENV PROTOC_VERSION 21.6 10 | RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip 11 | RUN unzip -o protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local bin/protoc 12 | RUN unzip -o protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local 'include/*' 13 | RUN rm -f protoc-${PROTOC_VERSION}-linux-x86_64.zip 14 | RUN git clone https://github.com/googleapis/googleapis.git 15 | RUN git clone https://github.com/grpc-ecosystem/grpc-gateway.git 16 | COPY Makefile /root 17 | RUN make install-golibs 18 | ENV TERM=xterm-256color -------------------------------------------------------------------------------- /client/pkg/constants/variables.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime/debug" 7 | ) 8 | 9 | // BuildInfo is a git commit hash which is injected by Makefile 10 | var BuildInfo = func() string { 11 | var goVer, hash, os, arch = "", "", "", "" 12 | if info, ok := debug.ReadBuildInfo(); ok { 13 | goVer = info.GoVersion 14 | for _, setting := range info.Settings { 15 | if setting.Key == "vcs.revision" { 16 | hash = setting.Value[:7] 17 | } else if setting.Key == "GOOS" { 18 | os = setting.Value 19 | } else if setting.Key == "GOARCH" { 20 | arch = setting.Value 21 | } 22 | } 23 | return fmt.Sprintf("%s-%s-%s-%s", goVer, os, arch, hash) 24 | } 25 | return "" 26 | }() 27 | 28 | // SDKType is a version string which is injected by Makefile 29 | var SDKType = func() string { 30 | if env, ok := os.LookupEnv("ORDA_SDK_TYPE"); ok { 31 | return env 32 | } 33 | return "go" 34 | }() 35 | -------------------------------------------------------------------------------- /client/pkg/orda/local_only_client_test.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/iface" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestLocalOnlyClientTest(t *testing.T) { 11 | t.Run("Can make local client", func(t *testing.T) { 12 | client1 := NewClient(NewLocalClientConfig("testCollection"), "localOnly1") 13 | client2 := NewClient(NewLocalClientConfig("testCollection"), "localOnly2") 14 | 15 | intCounter1 := client1.CreateCounter("key", nil) 16 | _, _ = intCounter1.IncreaseBy(2) 17 | _, _ = intCounter1.IncreaseBy(3) 18 | meta, snap, err := intCounter1.(iface.Datatype).GetMetaAndSnapshot() 19 | require.NoError(t, err) 20 | intCounter2 := client2.CreateCounter("key", nil) 21 | err = intCounter2.(iface.Datatype).SetMetaAndSnapshot(meta, snap) 22 | require.NoError(t, err) 23 | require.Equal(t, intCounter1.Get(), intCounter2.Get()) 24 | }) 25 | 26 | } 27 | -------------------------------------------------------------------------------- /client/pkg/model/operation.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // ToString returns a string of the operation with its body. 9 | func (its *Operation) ToString() string { 10 | return fmt.Sprintf("%s|%s|%v", its.OpType.String(), its.ID.ToString(), string(its.Body)) 11 | } 12 | 13 | // ToShortString returns a string of the operation without its body 14 | func (its *Operation) ToShortString() string { 15 | return fmt.Sprintf("%s|%s", its.OpType.String(), its.ID.ToString()) 16 | } 17 | 18 | // OpList is a list of *Operation 19 | type OpList []*Operation 20 | 21 | // ToString returns string 22 | func (its OpList) ToString(isFull bool) string { 23 | sb := strings.Builder{} 24 | 25 | sb.WriteString("[ ") 26 | for _, op := range its { 27 | if isFull { 28 | sb.WriteString(op.ToString()) 29 | } else { 30 | sb.WriteString(op.ToShortString()) 31 | } 32 | sb.WriteString(" ") 33 | } 34 | sb.WriteString("]") 35 | return sb.String() 36 | } 37 | -------------------------------------------------------------------------------- /client/pkg/types/uid.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | 7 | gonanoid "github.com/matoous/go-nanoid/v2" 8 | ) 9 | 10 | const ( 11 | defaultUIDLength = 16 12 | defaultIDCharacters = "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 13 | ) 14 | 15 | func newUniqueID() string { 16 | return gonanoid.Must(defaultUIDLength) 17 | } 18 | 19 | // NewUID creates a new DUID. 20 | func NewUID() string { 21 | return newUniqueID() 22 | } 23 | 24 | // ValidateUID validate UID from string. 25 | func ValidateUID(uidStr string) bool { 26 | if len(uidStr) != defaultUIDLength { 27 | return false 28 | } 29 | for _, c := range uidStr { 30 | if !strings.Contains(defaultIDCharacters, string(c)) { 31 | return false 32 | } 33 | } 34 | return true 35 | } 36 | 37 | // NewNilUID creates an instance of Nil UID. 38 | func NewNilUID() string { 39 | var b bytes.Buffer 40 | for i := 0; i < defaultUIDLength; i++ { 41 | b.WriteString("0") 42 | } 43 | return b.String() 44 | } 45 | -------------------------------------------------------------------------------- /server/admin/admin.go: -------------------------------------------------------------------------------- 1 | package admin 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | "github.com/orda-io/orda/server/schema" 6 | "time" 7 | ) 8 | 9 | const ordaPatchAPICUID string = "!@#$OrdaPatchAPI" 10 | 11 | var administrators = map[string]string{ 12 | ordaPatchAPICUID: "ordaPatchAPI", 13 | } 14 | 15 | // NewPatchClient creates a new patch client for each collection 16 | func NewPatchClient(collectionDoc *schema.CollectionDoc) *schema.ClientDoc { 17 | 18 | alias := administrators[ordaPatchAPICUID] 19 | return &schema.ClientDoc{ 20 | CUID: ordaPatchAPICUID, 21 | Alias: alias, 22 | CollectionNum: collectionDoc.Num, 23 | Type: int8(model.ClientType_VOLATILE), 24 | SyncType: 0, 25 | CreatedAt: time.Now(), 26 | UpdatedAt: time.Now(), 27 | } 28 | } 29 | 30 | // IsAdminCUID returns true if the client is admin 31 | func IsAdminCUID(cuid string) bool { 32 | if _, ok := administrators[cuid]; ok { 33 | return true 34 | } 35 | return false 36 | } 37 | -------------------------------------------------------------------------------- /client/pkg/operations/encoding_decoding_test.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | "testing" 8 | ) 9 | 10 | func TestConvertingOperations(t *testing.T) { 11 | t.Run("Can convert operations", func(t *testing.T) { 12 | snapBody := &struct { 13 | A string 14 | B int32 15 | }{A: "abc", B: 1234} 16 | snapb, _ := json.Marshal(snapBody) 17 | 18 | sOp := NewSnapshotOperation(model.TypeOfDatatype_DOCUMENT, snapb) 19 | sOpModel := sOp.ToModelOperation() 20 | log.Logger.Infof("%v", sOp) 21 | log.Logger.Infof("%v", sOpModel) 22 | // sOpModel.Body = []byte("{\"Type\":3,\"Snapshot\":\"{\\\"nm\\\":[{\\\"t\\\":\\\"O\\\",\\\"c\\\":{\\\"c\\\":\\\"0000000000000000\\\"},\\\"o\\\":{\\\"m\\\":{},\\\"s\\\":0}}]}\"}") 23 | // log.Logger.Infof("%v", sOpModel) 24 | sOp2 := ModelToOperation(sOpModel) 25 | // 26 | // log.Logger.Infof("%v", sOpModel) 27 | log.Logger.Infof("%v", sOp2) 28 | 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/orda-io/orda/server/managers" 5 | "os" 6 | 7 | "github.com/orda-io/orda/server/server" 8 | 9 | "github.com/urfave/cli/v2" 10 | ) 11 | 12 | func main() { 13 | app := &cli.App{ 14 | Flags: []cli.Flag{ 15 | &cli.StringFlag{ 16 | Name: "conf", 17 | Usage: "server configuration file in JSON format", 18 | Required: true, 19 | }, 20 | }, 21 | 22 | Action: func(c *cli.Context) error { 23 | confFile := c.String("conf") 24 | 25 | conf, err := managers.LoadOrdaServerConfig(confFile) 26 | if err != nil { 27 | os.Exit(1) 28 | } 29 | svr, err := server.NewOrdaServer(c.Context, conf) 30 | if err != nil { 31 | os.Exit(1) 32 | } 33 | go func() { 34 | if err := svr.Start(); err != nil { 35 | os.Exit(1) 36 | } 37 | }() 38 | os.Exit(svr.HandleSignals()) 39 | return nil 40 | }, 41 | } 42 | if err := app.Run(os.Args); err != nil { 43 | // TODO: should close services and resources 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /server/service/service_orda.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/managers" 7 | "github.com/orda-io/orda/server/schema" 8 | ) 9 | 10 | // OrdaService is a rpc service of Orda 11 | type OrdaService struct { 12 | managers *managers.Managers 13 | } 14 | 15 | // NewOrdaService creates a new OrdaService 16 | func NewOrdaService(managers *managers.Managers) *OrdaService { 17 | return &OrdaService{ 18 | managers: managers, 19 | } 20 | } 21 | 22 | func (its *OrdaService) getCollectionDocWithRPCError( 23 | ctx iface.OrdaContext, 24 | collection string, 25 | ) (*schema.CollectionDoc, error) { 26 | collectionDoc, err := its.managers.Mongo.GetCollection(ctx, collection) 27 | if err != nil { 28 | return nil, errors.NewRPCError(err) 29 | } 30 | if collectionDoc == nil { 31 | return nil, errors.NewRPCError(errors.ServerNoResource.New(ctx.L(), "collection "+collection)) 32 | } 33 | return collectionDoc, nil 34 | } 35 | -------------------------------------------------------------------------------- /server/schema/collections.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "go.mongodb.org/mongo-driver/mongo" 8 | "go.mongodb.org/mongo-driver/x/bsonx" 9 | ) 10 | 11 | // CollectionDoc defines the document of Collections collection, stored in MongoDB. More specifically, it stores a number associated to the collection. 12 | type CollectionDoc struct { 13 | Name string `bson:"_id"` 14 | Num int32 `bson:"num"` 15 | CreatedAt time.Time `bson:"createdAt"` 16 | } 17 | 18 | // CollectionDocFields defines the fields of CollectionDoc 19 | var CollectionDocFields = struct { 20 | Name string 21 | Num string 22 | CreatedAt string 23 | }{ 24 | Name: "_id", 25 | Num: "num", 26 | CreatedAt: "createdAt", 27 | } 28 | 29 | // GetIndexModel returns the index models of CollectionDoc 30 | func (its *CollectionDoc) GetIndexModel() []mongo.IndexModel { 31 | return []mongo.IndexModel{{ 32 | Keys: bsonx.Doc{{Key: CollectionDocFields.Num, Value: bsonx.Int32(1)}}, 33 | }} 34 | } 35 | 36 | // GetSummary returns the summary of CollectionDoc 37 | func (its *CollectionDoc) GetSummary() string { 38 | return fmt.Sprintf("%s(%d)", its.Name, its.Num) 39 | } 40 | -------------------------------------------------------------------------------- /test/list_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "github.com/orda-io/orda/client/pkg/orda" 7 | "github.com/stretchr/testify/require" 8 | "time" 9 | ) 10 | 11 | func (its *IntegrationTestSuite) TestList() { 12 | key := GetFunctionName() 13 | 14 | its.Run("Can update snapshot for list", func() { 15 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_MANUALLY) 16 | client1 := orda.NewClient(config, "listClient") 17 | 18 | err := client1.Connect() 19 | require.NoError(its.T(), err) 20 | its.ctx.L().Infof("end") 21 | defer func() { 22 | require.NoError(its.T(), client1.Close()) 23 | }() 24 | // time.Sleep(3 * time.Second) 25 | list1 := client1.CreateList(key, orda.NewHandlers( 26 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 27 | 28 | }, 29 | func(dt orda.Datatype, opList []interface{}) { 30 | 31 | }, 32 | func(dt orda.Datatype, errs ...errors.OrdaError) { 33 | 34 | })) 35 | _, _ = list1.InsertMany(0, "a", 2, 3.141592, time.Now()) 36 | require.NoError(its.T(), client1.Sync()) 37 | time.Sleep(2 * time.Second) 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /server/mongodb/collection_col_num_generator.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/schema" 7 | "go.mongodb.org/mongo-driver/bson" 8 | "go.mongodb.org/mongo-driver/mongo" 9 | "go.mongodb.org/mongo-driver/mongo/options" 10 | ) 11 | 12 | const ( 13 | idForCollection = "collectionID" 14 | ) 15 | 16 | // GetNextCollectionNum gets a collection number that is assigned to a collection. 17 | func (its *MongoCollections) GetNextCollectionNum(ctx iface.OrdaContext) (int32, errors.OrdaError) { 18 | opts := options.FindOneAndUpdate() 19 | opts.SetUpsert(true) 20 | var update = bson.M{ 21 | "$inc": bson.M{schema.CounterDocFields.Num: 1}, 22 | } 23 | 24 | result := its.counters.FindOneAndUpdate(ctx, schema.FilterByID(idForCollection), update, opts) 25 | if err := result.Err(); err != nil { 26 | if err == mongo.ErrNoDocuments { 27 | return 1, nil 28 | } 29 | return 0, errors.ServerDBQuery.New(ctx.L(), err.Error()) 30 | } 31 | var counterDoc = schema.CounterDoc{} 32 | if err := result.Decode(&counterDoc); err != nil { 33 | return 0, errors.ServerDBDecode.New(ctx.L(), err.Error()) 34 | } 35 | return counterDoc.Num, nil 36 | } 37 | -------------------------------------------------------------------------------- /client/pkg/model/checkpoint.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "fmt" 4 | 5 | // NewCheckPoint creates a new checkpoint 6 | func NewCheckPoint() *CheckPoint { 7 | return &CheckPoint{ 8 | Sseq: 0, 9 | Cseq: 0, 10 | } 11 | } 12 | 13 | // NewSetCheckPoint creates a new checkpoint with set values 14 | func NewSetCheckPoint(sseq, cseq uint64) *CheckPoint { 15 | return &CheckPoint{ 16 | Sseq: sseq, 17 | Cseq: cseq, 18 | } 19 | } 20 | 21 | // Set sets the values of checkpoint 22 | func (its *CheckPoint) Set(sseq, cseq uint64) *CheckPoint { 23 | its.Sseq = sseq 24 | its.Cseq = cseq 25 | return its 26 | } 27 | 28 | // SyncCseq syncs Cseq 29 | func (its *CheckPoint) SyncCseq(cseq uint64) *CheckPoint { 30 | if its.Cseq < cseq { 31 | its.Cseq = cseq 32 | } 33 | return its 34 | } 35 | 36 | // Clone makes a carbon copy of this one. 37 | func (its *CheckPoint) Clone() *CheckPoint { 38 | return NewCheckPoint().Set(its.Sseq, its.Cseq) 39 | } 40 | 41 | // Compare returns true if this CheckPoint is equal to other; otherwise, false. 42 | func (its *CheckPoint) Compare(other *CheckPoint) bool { 43 | return its.Cseq == other.Cseq && its.Sseq == other.Sseq 44 | } 45 | 46 | // ToString returns a short string of CheckPoint 47 | func (its *CheckPoint) ToString() string { 48 | if its == nil { 49 | return "nil" 50 | } 51 | return fmt.Sprintf("(s:%d c:%d)", its.Sseq, its.Cseq) 52 | } 53 | -------------------------------------------------------------------------------- /client/pkg/orda/encoding_test.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/iface" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | "github.com/orda-io/orda/client/pkg/operations" 8 | "github.com/orda-io/orda/client/pkg/testonly" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestOperationEncoding(t *testing.T) { 15 | t.Run("Test encoding counter operations", func(t *testing.T) { 16 | base := testonly.NewBase("key1", model.TypeOfDatatype_COUNTER) 17 | counter1, _ := newCounter(base, nil, nil) 18 | counter1.IncreaseBy(1024) 19 | 20 | gOp1, err := counter1.(iface.Datatype).CreateSnapshotOperation() 21 | 22 | require.NoError(t, err) 23 | log.Logger.Infof("%v", testonly.Marshal(t, gOp1)) 24 | 25 | mOp1 := gOp1.ToModelOperation() 26 | log.Logger.Infof("%v", mOp1) 27 | gmOp1 := operations.ModelToOperation(mOp1) 28 | 29 | require.Equal(t, testonly.Marshal(t, gOp1), testonly.Marshal(t, gmOp1)) 30 | 31 | counter2, _ := newCounter(base, nil, nil) 32 | counter2.(iface.Datatype).ExecuteRemote(gOp1) 33 | 34 | gOp2, err := counter2.(iface.Datatype).CreateSnapshotOperation() 35 | require.NoError(t, err) 36 | log.Logger.Infof("%v", testonly.Marshal(t, gOp2)) 37 | require.Equal(t, testonly.Marshal(t, gOp1), testonly.Marshal(t, gOp2)) 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /client/pkg/types/uid_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/log" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestUID(t *testing.T) { 13 | t.Run("Can validate UIDs", func(t *testing.T) { 14 | uid := NewUID() 15 | log.Logger.Infof("%s", uid) 16 | assert.True(t, ValidateUID(uid)) 17 | 18 | invalidUID1 := "X" 19 | assert.False(t, ValidateUID(invalidUID1)) 20 | 21 | invalidUID2 := "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" 22 | assert.False(t, ValidateUID(invalidUID2)) 23 | 24 | var cnt int32 = 0 25 | for i := 0; i < 100; i++ { 26 | uid1 := NewUID() 27 | uid2 := NewUID() 28 | require.NotEqual(t, uid1, uid2) 29 | log.Logger.Infof("%v vs %v : %v", uid1, uid2, uid1 < uid2) 30 | if uid1 < uid2 { 31 | cnt++ 32 | } 33 | } 34 | 35 | log.Logger.Info(cnt) 36 | }) 37 | 38 | t.Run("Can compare strings consistently", func(t *testing.T) { 39 | // In the all SDKs, these comparison should be consistent 40 | uid1 := "123abc" 41 | uid2 := "abc123" 42 | uid3 := "ABC123" 43 | uid4 := "abc1234" 44 | require.True(t, strings.Compare(uid1, uid1) == 0) 45 | require.True(t, strings.Compare(uid1, uid2) == -1) 46 | require.True(t, strings.Compare(uid2, uid3) == 1) 47 | require.True(t, strings.Compare(uid2, uid4) == -1) 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /server/notification/mqtt_test.go: -------------------------------------------------------------------------------- 1 | package notification_test 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | 8 | mqtt "github.com/eclipse/paho.mqtt.golang" 9 | ) 10 | 11 | func TestMqttPubSub(t *testing.T) { 12 | 13 | const TOPIC = "mytopic/test" 14 | 15 | opts := mqtt.NewClientOptions().AddBroker("tcp://127.0.0.1:18181") 16 | opts.SetClientID("client1") 17 | client1 := mqtt.NewClient(opts) 18 | if token := client1.Connect(); token.Wait() && token.Error() != nil { 19 | t.Fatal(token.Error()) 20 | } 21 | opts.SetClientID("client2") 22 | client2 := mqtt.NewClient(opts) 23 | if token2 := client2.Connect(); token2.Wait() && token2.Error() != nil { 24 | t.Fatal(token2.Error()) 25 | } 26 | 27 | var wg sync.WaitGroup 28 | wg.Add(2) 29 | 30 | var subscribeFn = func(client mqtt.Client, msg mqtt.Message) { 31 | reader := client.OptionsReader() 32 | fmt.Printf("at %v: %s\n", reader.ClientID(), string(msg.Payload())) 33 | wg.Done() 34 | } 35 | 36 | if token := client1.Subscribe(TOPIC, 0, subscribeFn); token.Wait() && token.Error() != nil { 37 | t.Fatal(token.Error()) 38 | } 39 | 40 | if token2 := client2.Subscribe(TOPIC, 0, subscribeFn); token2.Wait() && token2.Error() != nil { 41 | t.Fatal(token2.Error()) 42 | } 43 | 44 | if token := client1.Publish(TOPIC, 0, false, "mymessage"); token.Wait() && token.Error() != nil { 45 | t.Fatal(token.Error()) 46 | } 47 | wg.Wait() 48 | } 49 | -------------------------------------------------------------------------------- /client/pkg/testonly/test_helper.go: -------------------------------------------------------------------------------- 1 | package testonly 2 | 3 | import ( 4 | gocontext "context" 5 | "encoding/json" 6 | "github.com/orda-io/orda/client/pkg/context" 7 | "github.com/orda-io/orda/client/pkg/internal/datatypes" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | "github.com/orda-io/orda/client/pkg/types" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | // OperationsToString returns a string of an array of operations 17 | func OperationsToString(ops []*model.Operation) string { 18 | sb := strings.Builder{} 19 | sb.WriteString("[ ") 20 | for i, op := range ops { 21 | sb.WriteString(op.ToString()) 22 | if len(ops)-1 != i { 23 | sb.WriteString(", ") 24 | } 25 | } 26 | sb.WriteString(" ]") 27 | return sb.String() 28 | } 29 | 30 | // Marshal marshals an interface with checking error 31 | func Marshal(t *testing.T, j interface{}) string { 32 | data, err := json.Marshal(j) 33 | require.NoError(t, err) 34 | return string(data) 35 | } 36 | 37 | // NewBase returns a new BaseDatatype 38 | func NewBase(key string, t model.TypeOfDatatype) *datatypes.BaseDatatype { 39 | cm := &model.Client{ 40 | CUID: types.NewUID(), 41 | Alias: "", 42 | Collection: "", 43 | SyncType: 0, 44 | } 45 | ctx := context.NewClientContext(gocontext.TODO(), cm) 46 | return datatypes.NewBaseDatatype(key, t, ctx, model.StateOfDatatype_DUE_TO_CREATE) 47 | } 48 | -------------------------------------------------------------------------------- /server/mongodb/config.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "net/url" 8 | ) 9 | 10 | const connectionFormat = "mongodb://%s:%s@%s/" 11 | 12 | // Config is a configuration for MongoDB 13 | type Config struct { 14 | Host string `json:"MongoHost"` 15 | OrdaDB string `json:"OrdaDB"` 16 | User string `json:"User"` 17 | Password string `json:"Password"` 18 | CertFile string `json:"CertFile"` 19 | Options string `json:"Options"` 20 | } 21 | 22 | func (its *Config) getConnectionString() string { 23 | ret := fmt.Sprintf(connectionFormat, its.User, url.QueryEscape(its.Password), its.Host) 24 | if its.Options == "" { 25 | return ret 26 | } 27 | return ret + "?" + its.Options 28 | } 29 | 30 | func (its *Config) String() string { 31 | clone := struct { 32 | Config 33 | }{ 34 | *its, 35 | } 36 | clone.Password = "" 37 | log.Logger.Infof("%v", clone) 38 | b, _ := json.Marshal(clone) 39 | return string(b) 40 | } 41 | 42 | // MarshalJSON returns marshaled JSON 43 | func (its *Config) MarshalJSON() ([]byte, error) { 44 | return json.Marshal(&struct { 45 | Host string 46 | OrdaDB string 47 | User string 48 | CertFile string 49 | Options string 50 | }{ 51 | Host: its.Host, 52 | OrdaDB: its.OrdaDB, 53 | User: its.User, 54 | CertFile: its.CertFile, 55 | Options: its.Options, 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /server/service/service_collections.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | goctx "context" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/context" 7 | "github.com/orda-io/orda/client/pkg/errors" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | 10 | "github.com/orda-io/orda/server/constants" 11 | "github.com/orda-io/orda/server/mongodb" 12 | ) 13 | 14 | // CreateCollection creates a collection 15 | func (its *OrdaService) CreateCollection(goCtx goctx.Context, in *model.CollectionMessage) (*model.CollectionMessage, error) { 16 | ctx := context.NewOrdaContext(goCtx, constants.TagCreate). 17 | UpdateCollectionTags(in.Collection, 0) 18 | num, err := mongodb.MakeCollection(ctx, its.managers.Mongo, in.Collection) 19 | var msg string 20 | if err != nil { 21 | return nil, errors.NewRPCError(err) 22 | } 23 | 24 | msg = fmt.Sprintf("create collection '%s(%d)'", in.Collection, num) 25 | 26 | ctx.L().Infof("%s", msg) 27 | return in, nil 28 | } 29 | 30 | // ResetCollection resets a collection 31 | func (its *OrdaService) ResetCollection(goCtx goctx.Context, in *model.CollectionMessage) (*model.CollectionMessage, error) { 32 | ctx := context.NewOrdaContext(goCtx, constants.TagReset). 33 | UpdateCollectionTags(in.Collection, 0) 34 | if err := its.managers.Mongo.PurgeCollection(ctx, in.Collection); err != nil { 35 | return nil, errors.NewRPCError(err) 36 | } 37 | ctx.L().Infof("reset %s collection", in.Collection) 38 | return its.CreateCollection(goCtx, in) 39 | } 40 | -------------------------------------------------------------------------------- /client/pkg/errors/rpc.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "google.golang.org/grpc/codes" 5 | "google.golang.org/grpc/status" 6 | ) 7 | 8 | type errorCodeRPC uint32 9 | 10 | const ( 11 | // RPCErrMongoDB is the error related to MongoDB 12 | RPCErrMongoDB errorCodeRPC = iota 13 | // RPCErrClientInconsistentCollection is the error when a client has different collection with the previously 14 | RPCErrClientInconsistentCollection 15 | // RPCErrNoClient is the error when the specified client does not exist. 16 | RPCErrNoClient 17 | ) 18 | 19 | var formatMap = map[errorCodeRPC]string{ 20 | RPCErrMongoDB: "work no MongoDB", 21 | RPCErrClientInconsistentCollection: "invalid collections: %s (server) vs. %s (client)", 22 | RPCErrNoClient: "exist no client in the server", 23 | } 24 | 25 | // NewRPCError creates a RPC error 26 | func NewRPCError(oErr OrdaError) error { 27 | var c codes.Code 28 | code := oErr.GetCode() 29 | switch code { 30 | case ServerDBQuery: 31 | c = codes.Unavailable // temporally unavailable 32 | case ServerDBInit: 33 | c = codes.Internal 34 | case ServerDBDecode: 35 | c = codes.Internal // something is broken 36 | case ServerNoResource: 37 | c = codes.NotFound 38 | case ServerNoPermission: 39 | c = codes.Unauthenticated 40 | case ServerInit: 41 | c = codes.Internal 42 | case ServerNotify: 43 | c = codes.Internal 44 | case ServerBadRequest: 45 | c = codes.InvalidArgument 46 | } 47 | return status.Error(c, oErr.Error()) 48 | } 49 | -------------------------------------------------------------------------------- /server/wrapper/datatype.go: -------------------------------------------------------------------------------- 1 | package wrapper 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/context" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | "github.com/orda-io/orda/client/pkg/orda" 8 | ) 9 | 10 | // DatatypeWrapper is used to manipulate datatype with internal APIs in orda server 11 | type DatatypeWrapper struct { 12 | iface.Datatype 13 | reqNum uint32 14 | } 15 | 16 | // NewDatatypeWrapper creates a new DatatypeWrapper 17 | func NewDatatypeWrapper(dt orda.Datatype) *DatatypeWrapper { 18 | return &DatatypeWrapper{ 19 | Datatype: dt.(iface.Datatype), 20 | reqNum: 0, 21 | } 22 | } 23 | 24 | // GetClientModel returns model.Client from its context 25 | func (its *DatatypeWrapper) GetClientModel() *model.Client { 26 | ctx := its.Datatype.(iface.BaseDatatype).GetCtx().(*context.DatatypeContext) 27 | return ctx.ClientContext.Client 28 | } 29 | 30 | // CreatePushPullPack creates PushPullPack 31 | func (its *DatatypeWrapper) CreatePushPullPack() *model.PushPullPack { 32 | return its.Datatype.(iface.WiredDatatype).CreatePushPullPack() 33 | } 34 | 35 | // CreatePushPullMessage creates PushPullPackMessage 36 | func (its *DatatypeWrapper) CreatePushPullMessage() *model.PushPullMessage { 37 | its.reqNum++ 38 | return model.NewPushPullMessage(its.reqNum, its.GetClientModel(), its.CreatePushPullPack()) 39 | } 40 | 41 | // ApplyPushPullPack applies PushPullPack 42 | func (its *DatatypeWrapper) ApplyPushPullPack(ppp *model.PushPullPack) { 43 | its.Datatype.ApplyPushPullPack(ppp) 44 | } 45 | -------------------------------------------------------------------------------- /examples/document/document.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | "github.com/orda-io/orda/client/pkg/orda" 8 | "sync" 9 | ) 10 | 11 | func main() { 12 | conf := &orda.ClientConfig{ 13 | ServerAddr: "localhost:19061", 14 | NotificationAddr: "tcp://localhost:18181", 15 | CollectionName: "hello_world", 16 | SyncType: model.SyncType_REALTIME, 17 | } 18 | 19 | client1 := orda.NewClient(conf, "client1") 20 | client2 := orda.NewClient(conf, "client2") 21 | 22 | if err := client1.Connect(); err != nil { 23 | panic("fail to connect client1 to an Orda server:" + err.Error()) 24 | } 25 | if err := client2.Connect(); err != nil { 26 | panic("fail to connect client2 to an Orda server" + err.Error()) 27 | } 28 | defer func() { 29 | if err := client1.Close(); err != nil { 30 | _ = fmt.Errorf("fail to close client1: %v", err.Error()) 31 | } 32 | if err := client2.Close(); err != nil { 33 | _ = fmt.Errorf("fail to close client2: %v", err.Error()) 34 | } 35 | }() 36 | wg := &sync.WaitGroup{} 37 | wg.Add(1) 38 | client1.SubscribeOrCreateDocument("sampleDoc", orda.NewHandlers( 39 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 40 | if new == model.StateOfDatatype_SUBSCRIBED { 41 | wg.Done() 42 | } 43 | }, 44 | func(dt orda.Datatype, opList []interface{}) { 45 | 46 | }, 47 | func(dt orda.Datatype, errs ...errors.OrdaError) { 48 | 49 | }, 50 | )) 51 | wg.Done() 52 | } 53 | -------------------------------------------------------------------------------- /client/pkg/context/client_datatype.go: -------------------------------------------------------------------------------- 1 | package context 2 | 3 | import ( 4 | "context" 5 | "github.com/orda-io/orda/client/pkg/constants" 6 | "github.com/orda-io/orda/client/pkg/iface" 7 | "github.com/orda-io/orda/client/pkg/log" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | ) 10 | 11 | // ClientContext is used to pass over the context of clients 12 | type ClientContext struct { 13 | iface.OrdaContext 14 | Client *model.Client 15 | } 16 | 17 | // NewClientContext creates a new ClientContext 18 | func NewClientContext(ctx context.Context, client *model.Client) *ClientContext { 19 | return &ClientContext{ 20 | OrdaContext: NewOrdaContextWithAllTags(ctx, constants.TagSdkClient, client.Collection, "", client.Alias, client.CUID, "", ""), 21 | Client: client, 22 | } 23 | } 24 | 25 | // DatatypeContext is used to pass over the context of datatypes 26 | type DatatypeContext struct { 27 | *ClientContext 28 | Data iface.BaseDatatype 29 | } 30 | 31 | // NewDatatypeContext creates a new DatatypeContext 32 | func NewDatatypeContext(clientContext *ClientContext, baseDatatype iface.BaseDatatype) *DatatypeContext { 33 | logger := log.NewWithTags(constants.TagSdkDatatype, 34 | clientContext.Client.Collection, "", 35 | clientContext.Client.Alias, clientContext.Client.CUID, 36 | baseDatatype.GetKey(), baseDatatype.GetCUID()) 37 | return &DatatypeContext{ 38 | ClientContext: &ClientContext{ 39 | OrdaContext: &ordaContext{ 40 | Context: clientContext.Ctx(), 41 | logger: logger, 42 | }, 43 | Client: clientContext.Client, 44 | }, 45 | Data: baseDatatype, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /client/pkg/operations/map.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | // PutBody is the body of PutOperation 8 | type PutBody struct { 9 | Key string 10 | Value interface{} 11 | } 12 | 13 | // NewPutOperation creates a PutOperation of hash map. 14 | func NewPutOperation(key string, value interface{}) *PutOperation { 15 | return &PutOperation{ 16 | baseOperation: newBaseOperation( 17 | model.TypeOfOperation_MAP_PUT, 18 | nil, 19 | &PutBody{ 20 | Key: key, 21 | Value: value, 22 | }, 23 | ), 24 | } 25 | } 26 | 27 | // PutOperation is used to put something in the hash map. 28 | type PutOperation struct { 29 | baseOperation 30 | } 31 | 32 | // GetBody returns the body 33 | func (its *PutOperation) GetBody() *PutBody { 34 | return its.Body.(*PutBody) 35 | } 36 | 37 | // ////////////////// RemoveOperation //////////////////// 38 | 39 | // RemoveBody is the body of RemoveOperation 40 | type RemoveBody struct { 41 | Key string 42 | } 43 | 44 | // NewRemoveOperation creates a RemoveOperation of hash map. 45 | func NewRemoveOperation(key string) *RemoveOperation { 46 | return &RemoveOperation{ 47 | baseOperation: newBaseOperation( 48 | model.TypeOfOperation_MAP_REMOVE, 49 | nil, 50 | &RemoveBody{ 51 | Key: key, 52 | }, 53 | ), 54 | } 55 | } 56 | 57 | // RemoveOperation is used to remove something in the hash map. 58 | type RemoveOperation struct { 59 | baseOperation 60 | } 61 | 62 | // GetBody returns the body 63 | func (its *RemoveOperation) GetBody() *RemoveBody { 64 | return its.Body.(*RemoveBody) 65 | } 66 | -------------------------------------------------------------------------------- /client/pkg/orda/timed.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "github.com/orda-io/orda/client/pkg/types" 7 | ) 8 | 9 | // timedType is the most primitive type which allows to store a timestamp. 10 | type timedType interface { 11 | getValue() types.JSONValue 12 | setValue(v types.JSONValue) 13 | // getTime and setTime are used when the timestamp is used to resolve conflict. 14 | // It can be overridden. 15 | getTime() *model.Timestamp 16 | setTime(ts *model.Timestamp) 17 | makeTomb(ts *model.Timestamp) 18 | isTomb() bool 19 | String() string 20 | } 21 | 22 | type timedNode struct { 23 | V types.JSONValue `json:"v"` 24 | T *model.Timestamp `json:"t"` 25 | } 26 | 27 | func newTimedNode(v types.JSONValue, t *model.Timestamp) timedType { 28 | return &timedNode{ 29 | V: v, 30 | T: t, 31 | } 32 | } 33 | 34 | func (its *timedNode) getValue() types.JSONValue { 35 | return its.V 36 | } 37 | 38 | func (its *timedNode) setValue(v types.JSONValue) { 39 | its.V = v 40 | } 41 | 42 | func (its *timedNode) getTime() *model.Timestamp { 43 | return its.T 44 | } 45 | 46 | func (its *timedNode) setTime(ts *model.Timestamp) { 47 | its.T = ts 48 | } 49 | 50 | // this is for ordaMap and list 51 | func (its *timedNode) makeTomb(ts *model.Timestamp) { 52 | its.T = ts 53 | its.V = nil 54 | } 55 | 56 | func (its *timedNode) isTomb() bool { 57 | return its.V == nil 58 | } 59 | 60 | func (its *timedNode) String() string { 61 | if its.V == nil { 62 | return fmt.Sprintf("Φ|%s", its.T.ToString()) 63 | } 64 | return fmt.Sprintf("TV[%v|C%s]", its.V, its.T.ToString()) 65 | } 66 | -------------------------------------------------------------------------------- /server/managers/managers.go: -------------------------------------------------------------------------------- 1 | package managers 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/mongodb" 7 | "github.com/orda-io/orda/server/notification" 8 | "github.com/orda-io/orda/server/redis" 9 | "github.com/orda-io/orda/server/utils" 10 | ) 11 | 12 | // Managers are a bundle of infra 13 | type Managers struct { 14 | Mongo *mongodb.RepositoryMongo 15 | Notifier *notification.Notifier 16 | Redis *redis.Client 17 | } 18 | 19 | // New creates Managers with context and config 20 | func New(ctx iface.OrdaContext, conf *OrdaServerConfig) (*Managers, errors.OrdaError) { 21 | var oErr errors.OrdaError 22 | clients := &Managers{} 23 | if clients.Mongo, oErr = mongodb.New(ctx, conf.Mongo); oErr != nil { 24 | return clients, oErr 25 | } 26 | 27 | if clients.Notifier, oErr = notification.NewNotifier(ctx, conf.Notification); oErr != nil { 28 | return clients, oErr 29 | } 30 | 31 | if clients.Redis, oErr = redis.New(ctx, conf.Redis); oErr != nil { 32 | return clients, oErr 33 | } 34 | return clients, nil 35 | } 36 | 37 | // GetLock returns either a local or redis lock 38 | func (its *Managers) GetLock(ctx iface.OrdaContext, lockName string) utils.Lock { 39 | return its.Redis.GetLock(ctx, lockName) 40 | } 41 | 42 | // Close closes this Managers 43 | func (its *Managers) Close(ctx iface.OrdaContext) { 44 | if err := its.Redis.Close(); err != nil { 45 | ctx.L().Errorf("fail to close redis: %v", err) 46 | } 47 | if err := its.Mongo.Close(ctx); err != nil { 48 | ctx.L().Errorf("fail to close mongo") 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /proto/orda.enum.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "client/pkg/model"; 4 | package orda; 5 | 6 | enum SyncType { 7 | LOCAL_ONLY = 0; 8 | MANUALLY = 1; 9 | REALTIME = 2; 10 | } 11 | 12 | enum ClientType { 13 | PERSISTENT = 0; // persistent clients can generate operations; even if they do not sync, their checkpoints remains for offline support 14 | EPHEMERAL = 1; // ephemeral clients can generate operations; if they do not sync for predetermined time, their information is removed 15 | VOLATILE = 2; // volatile clients can generate operations on the latest datatype, but no operations are pulled 16 | } 17 | 18 | enum TypeOfOperation { 19 | NO_OP = 0; 20 | ERROR = 1; 21 | TRANSACTION = 2; 22 | COUNTER_SNAPSHOT = 10; 23 | COUNTER_INCREASE = 11; 24 | MAP_SNAPSHOT = 20; 25 | MAP_PUT = 21; 26 | MAP_REMOVE = 22; 27 | LIST_SNAPSHOT = 30; 28 | LIST_INSERT = 31; 29 | LIST_DELETE = 32; 30 | LIST_UPDATE = 33; 31 | DOC_SNAPSHOT = 40; 32 | DOC_OBJ_PUT = 41; 33 | DOC_OBJ_RMV = 42; 34 | DOC_ARR_INS = 43; 35 | DOC_ARR_DEL = 44; 36 | DOC_ARR_UPD = 45; 37 | } 38 | 39 | 40 | enum StateOfDatatype { 41 | DUE_TO_CREATE = 0; 42 | DUE_TO_SUBSCRIBE = 1; 43 | DUE_TO_SUBSCRIBE_CREATE = 2; 44 | SUBSCRIBED = 3; 45 | DUE_TO_UNSUBSCRIBE = 4; 46 | CLOSED = 5; 47 | DELETED = 6; 48 | } 49 | 50 | enum StateOfResponse { 51 | OK = 0; 52 | ERR_CLIENT_INVALID_COLLECTION = 101; 53 | ERR_CLIENT_INVALID_SYNC_TYPE = 102; 54 | } 55 | 56 | enum RequestType { 57 | CLIENTS = 0; 58 | PUSHPULLS = 1; 59 | } 60 | 61 | enum TypeOfDatatype { 62 | COUNTER = 0; 63 | MAP = 1; 64 | LIST = 2; 65 | DOCUMENT = 3; 66 | } -------------------------------------------------------------------------------- /client/pkg/orda/handlers.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | ) 7 | 8 | // Handlers defines a set of handlers which can handles the events related to Datatype 9 | type Handlers struct { 10 | stateChangeHandler func(dt Datatype, old model.StateOfDatatype, new model.StateOfDatatype) 11 | remoteOperationHandler func(dt Datatype, opList []interface{}) 12 | errorHandler func(dt Datatype, errs ...errors.OrdaError) 13 | } 14 | 15 | // NewHandlers creates a set of handlers for a datatype. 16 | func NewHandlers( 17 | stateChangeHandler func(dt Datatype, old model.StateOfDatatype, new model.StateOfDatatype), 18 | remoteOperationHandler func(dt Datatype, opList []interface{}), 19 | errorHandler func(dt Datatype, errs ...errors.OrdaError)) *Handlers { 20 | return &Handlers{ 21 | stateChangeHandler: stateChangeHandler, 22 | remoteOperationHandler: remoteOperationHandler, 23 | errorHandler: errorHandler, 24 | } 25 | } 26 | 27 | // SetHandlers sets the handlers if a given handler is not nil. 28 | func (its *Handlers) SetHandlers( 29 | stateChangeHandler func(dt Datatype, old model.StateOfDatatype, new model.StateOfDatatype), 30 | remoteOperationHandler func(dt Datatype, opList []interface{}), 31 | errorHandler func(dt Datatype, errs ...errors.OrdaError)) { 32 | if stateChangeHandler != nil { 33 | its.stateChangeHandler = stateChangeHandler 34 | } 35 | if remoteOperationHandler != nil { 36 | its.remoteOperationHandler = remoteOperationHandler 37 | } 38 | if errorHandler != nil { 39 | its.errorHandler = errorHandler 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/map_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | "github.com/orda-io/orda/client/pkg/orda" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func (its *IntegrationTestSuite) TestMap() { 13 | key := GetFunctionName() 14 | 15 | its.Run("Can update snapshot for hash map", func() { 16 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_MANUALLY) 17 | client1 := orda.NewClient(config, "client1") 18 | 19 | err := client1.Connect() 20 | require.NoError(its.T(), err) 21 | defer func() { 22 | _ = client1.Close() 23 | }() 24 | 25 | map1 := client1.CreateMap(key, orda.NewHandlers( 26 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 27 | 28 | }, 29 | func(dt orda.Datatype, opList []interface{}) { 30 | 31 | }, 32 | func(dt orda.Datatype, errs ...errors.OrdaError) { 33 | 34 | })) 35 | _, _ = map1.Put("hello", "world") 36 | _, _ = map1.Put("num", 1234) 37 | _, _ = map1.Put("float", 3.141592) 38 | _, _ = map1.Put("struct", struct { 39 | ID string 40 | Age uint 41 | }{ 42 | ID: "hello", 43 | Age: 10, 44 | }) 45 | _, _ = map1.Put("list", []string{"x", "y", "z"}) 46 | _, _ = map1.Put("Removed", "deleted") 47 | _, _ = map1.Remove("Removed") 48 | require.Nil(its.T(), map1.Get("Removed")) 49 | require.NoError(its.T(), client1.Sync()) 50 | sop, err := map1.(iface.Datatype).CreateSnapshotOperation() 51 | log.Logger.Infof("%v", sop.String()) 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /client/pkg/operations/base.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | ) 7 | 8 | // ////////////////// baseOperation //////////////////// 9 | 10 | func newBaseOperation(typeOf model.TypeOfOperation, opID *model.OperationID, content interface{}) baseOperation { 11 | return baseOperation{ 12 | Type: typeOf, 13 | ID: opID, 14 | Body: content, 15 | } 16 | } 17 | 18 | type baseOperation struct { 19 | Type model.TypeOfOperation 20 | ID *model.OperationID 21 | Body interface{} 22 | } 23 | 24 | func (its *baseOperation) SetID(opID *model.OperationID) { 25 | its.ID = opID 26 | } 27 | 28 | func (its *baseOperation) GetID() *model.OperationID { 29 | return its.ID 30 | } 31 | 32 | func (its *baseOperation) GetTimestamp() *model.Timestamp { 33 | return its.ID.GetTimestamp() 34 | } 35 | 36 | func (its *baseOperation) GetType() model.TypeOfOperation { 37 | return its.Type 38 | } 39 | 40 | // ToJSON returns the operation in the format of JSON compatible struct. 41 | func (its *baseOperation) ToJSON() interface{} { 42 | return struct { 43 | ID interface{} 44 | Type string 45 | Body interface{} 46 | }{ 47 | ID: its.ID.ToJSON(), 48 | Type: its.Type.String(), 49 | Body: its.Body, 50 | } 51 | } 52 | 53 | func (its *baseOperation) String() string { 54 | body := its.Body 55 | switch body.(type) { 56 | case []byte: 57 | body = string(its.Body.([]byte)) 58 | } 59 | return fmt.Sprintf("%s(%s|%+v)", its.Type, its.ID.ToString(), body) 60 | } 61 | 62 | func (its *baseOperation) ToModelOperation() *model.Operation { 63 | return &model.Operation{ 64 | ID: its.ID, 65 | OpType: its.Type, 66 | Body: marshalBody(its.Body), 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /server/utils/local_lock.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | ctx "context" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | golock "github.com/viney-shih/go-lock" 7 | "sync" 8 | ) 9 | 10 | var localLockMap sync.Map 11 | 12 | // LocalLock is used for local locking in a single server 13 | type LocalLock struct { 14 | ctx iface.OrdaContext 15 | mutex *golock.CASMutex 16 | lockName string 17 | } 18 | 19 | // GetLocalLock returns a LocalLock with the specified name 20 | func GetLocalLock(ctx iface.OrdaContext, lockName string) *LocalLock { 21 | 22 | value, loaded := localLockMap.LoadOrStore(lockName, &LocalLock{ 23 | ctx: ctx, 24 | mutex: golock.NewCASMutex(), 25 | lockName: lockName, 26 | }) 27 | if loaded { 28 | ctx.L().Infof("[🔒] load lock '%v'", lockName) 29 | } else { 30 | ctx.L().Infof("[🔒] create lock '%v'", lockName) 31 | } 32 | return value.(*LocalLock) 33 | } 34 | 35 | // TryLock tries to a local lock, and returns true if it succeeds; otherwise false 36 | func (its *LocalLock) TryLock() bool { 37 | timeCtx, cancel := ctx.WithTimeout(its.ctx, defaultLeaseTime) 38 | defer cancel() 39 | if !its.mutex.TryLockWithContext(timeCtx) { 40 | if err := timeCtx.Err(); err != nil { 41 | its.ctx.L().Warnf("[🔒] fail to lock '%v':%v", its.lockName, err.Error()) 42 | return false 43 | } 44 | its.ctx.L().Warnf("[🔒] fail to lock '%v'", its.lockName) 45 | return false 46 | } 47 | 48 | ts, _ := timeCtx.Deadline() 49 | its.ctx.L().Infof("[🔒] lock '%v': %v", its.lockName, ts) 50 | return true 51 | } 52 | 53 | // Unlock unlocks the local lock, and returns true if it succeeds; otherwise false 54 | func (its *LocalLock) Unlock() bool { 55 | its.mutex.Unlock() 56 | its.ctx.L().Infof("[🔒] unlock '%v'", its.lockName) 57 | return true 58 | } 59 | -------------------------------------------------------------------------------- /client/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/orda-io/orda/client 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 7 | github.com/eclipse/paho.mqtt.golang v1.4.1 8 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.0 9 | github.com/matoous/go-nanoid/v2 v2.0.0 10 | github.com/mitchellh/mapstructure v1.5.0 11 | github.com/sirupsen/logrus v1.9.0 12 | github.com/stretchr/testify v1.8.0 13 | github.com/wI2L/jsondiff v0.2.0 14 | github.com/ztrue/tracerr v0.3.0 15 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 16 | google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252 17 | google.golang.org/grpc v1.48.0 18 | google.golang.org/protobuf v1.28.0 19 | gotest.tools v2.2.0+incompatible 20 | ) 21 | 22 | require ( 23 | github.com/davecgh/go-spew v1.1.1 // indirect 24 | github.com/fatih/color v1.13.0 // indirect 25 | github.com/golang/protobuf v1.5.2 // indirect 26 | github.com/google/go-cmp v0.5.8 // indirect 27 | github.com/gorilla/websocket v1.4.2 // indirect 28 | github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect 29 | github.com/kr/text v0.2.0 // indirect 30 | github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e // indirect 31 | github.com/mattn/go-colorable v0.1.9 // indirect 32 | github.com/mattn/go-isatty v0.0.14 // indirect 33 | github.com/pkg/errors v0.9.1 // indirect 34 | github.com/pmezard/go-difflib v1.0.0 // indirect 35 | github.com/tidwall/gjson v1.14.0 // indirect 36 | github.com/tidwall/match v1.1.1 // indirect 37 | github.com/tidwall/pretty v1.2.0 // indirect 38 | golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect 39 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect 40 | golang.org/x/text v0.3.7 // indirect 41 | gopkg.in/yaml.v3 v3.0.1 // indirect 42 | ) 43 | -------------------------------------------------------------------------------- /server/utils/redis_lock.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | ctx "context" 5 | "github.com/go-redsync/redsync/v4" 6 | "github.com/orda-io/orda/client/pkg/iface" 7 | ) 8 | 9 | // RedisLock is used for distributed locking among multiple servers 10 | type RedisLock struct { 11 | ctx iface.OrdaContext 12 | lockName string 13 | mutex *redsync.Mutex 14 | } 15 | 16 | // GetRedisLock returns a RedisLock with the specified name 17 | func GetRedisLock(ctx iface.OrdaContext, lockName string, rs *redsync.Redsync) *RedisLock { 18 | mutex := rs.NewMutex(prefix+lockName, redsync.WithTries(100), redsync.WithExpiry(defaultExpireTime)) 19 | return &RedisLock{ 20 | ctx: ctx, 21 | lockName: lockName, 22 | mutex: mutex, 23 | } 24 | } 25 | 26 | // TryLock tries to lock by redis, and returns true if it succeeds; otherwise false 27 | func (its *RedisLock) TryLock() bool { 28 | timeCtx, cancel := ctx.WithTimeout(its.ctx, defaultLeaseTime) 29 | defer cancel() 30 | if err := its.mutex.LockContext(timeCtx); err != nil { 31 | its.ctx.L().Warnf("[🔒] fail to lock '%v': %v", its.lockName, err.Error()) 32 | return false 33 | } 34 | if err := timeCtx.Err(); err != nil { 35 | its.ctx.L().Warnf("[🔒] fail to lock '%v':%v", its.lockName, err.Error()) 36 | return false 37 | } 38 | its.ctx.L().Infof("[🔒] lock '%v': %v", its.lockName, its.mutex.Until()) 39 | return true 40 | } 41 | 42 | // Unlock unlocks the redis lock, and returns true if it succeeds; otherwise false 43 | func (its *RedisLock) Unlock() bool { 44 | success, err := its.mutex.Unlock() 45 | if err != nil { 46 | its.ctx.L().Errorf("[🔒] fail to unlock '%v': %v", its.lockName, err.Error()) 47 | return false 48 | } 49 | if success { 50 | its.ctx.L().Infof("[🔒] unlock '%v'", its.lockName) 51 | return true 52 | } 53 | its.ctx.L().Warnf("[🔒] something wrong with lock '%v'", its.lockName) 54 | return false 55 | } 56 | -------------------------------------------------------------------------------- /proto/orda.grpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "client/pkg/model"; 4 | package orda; 5 | 6 | import "orda.proto"; 7 | import "orda.enum.proto"; 8 | import "thirdparty/google/api/annotations.proto"; 9 | import "thirdparty/protoc-gen-openapiv2/options/annotations.proto"; 10 | 11 | option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { 12 | info: { 13 | title: "Orda gRPC gateway APIs", 14 | version: "v1", 15 | } 16 | }; 17 | 18 | message PatchMessage { 19 | string key = 1; 20 | string collection = 2; 21 | string json = 3; 22 | } 23 | 24 | service OrdaService { 25 | rpc ProcessPushPull (PushPullMessage) returns (PushPullMessage) { 26 | option (google.api.http) = { 27 | post: "/api/v1/collections/{collection}/pushpulls/{cuid}" 28 | body: "*" 29 | }; 30 | } 31 | rpc ProcessClient (ClientMessage) returns (ClientMessage) { 32 | option (google.api.http) = { 33 | post: "/api/v1/collections/{collection}/clients/{cuid}" 34 | body: "*" 35 | }; 36 | } 37 | rpc PatchDocument (PatchMessage) returns (PatchMessage) { 38 | option (google.api.http) = { 39 | post: "/api/v1/collections/{collection}/documents/{key}" 40 | body: "*" 41 | }; 42 | } 43 | rpc CreateCollection (CollectionMessage) returns (CollectionMessage) { 44 | option (google.api.http) = { 45 | put: "/api/v1/collections/{collection}" 46 | }; 47 | } 48 | 49 | rpc ResetCollection (CollectionMessage) returns (CollectionMessage) { 50 | option (google.api.http) = { 51 | put: "/api/v1/collections/{collection}/reset" 52 | }; 53 | } 54 | 55 | rpc TestEncodingOperation (EncodingMessage) returns (EncodingMessage) { 56 | option (google.api.http) = { 57 | post: "/api/v1/samples/operation" 58 | body: "*" 59 | }; 60 | } 61 | } 62 | 63 | message EncodingMessage { 64 | TypeOfDatatype type = 1; 65 | Operation op = 2; 66 | } 67 | -------------------------------------------------------------------------------- /server/notification/notifier.go: -------------------------------------------------------------------------------- 1 | package notification 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/constants" 7 | "github.com/orda-io/orda/client/pkg/errors" 8 | "github.com/orda-io/orda/client/pkg/iface" 9 | "github.com/orda-io/orda/client/pkg/model" 10 | "github.com/orda-io/orda/server/schema" 11 | 12 | mqtt "github.com/eclipse/paho.mqtt.golang" 13 | ) 14 | 15 | // Notifier is a struct that takes responsibility for notification 16 | type Notifier struct { 17 | mqttClient mqtt.Client 18 | } 19 | 20 | // NewNotifier creates an instance of Notifier 21 | func NewNotifier(ctx iface.OrdaContext, pubSubAddr string) (*Notifier, errors.OrdaError) { 22 | serverName := fmt.Sprintf("Orda-Server-%s(%s)", constants.Version, constants.BuildInfo) 23 | opts := mqtt.NewClientOptions().AddBroker(pubSubAddr).SetUsername(serverName) 24 | client := mqtt.NewClient(opts) 25 | if token := client.Connect(); token.Wait() && token.Error() != nil { 26 | return nil, errors.ServerInit.New(ctx.L(), token.Error()) 27 | } 28 | return &Notifier{mqttClient: client}, nil 29 | } 30 | 31 | // NotifyAfterPushPull enables server to send a notification to MQTT server 32 | func (n *Notifier) NotifyAfterPushPull( 33 | ctx iface.OrdaContext, 34 | collectionName string, 35 | cuid string, 36 | datatype *schema.DatatypeDoc, 37 | sseq uint64, 38 | ) errors.OrdaError { 39 | topic := fmt.Sprintf("%s/%s", collectionName, datatype.Key) 40 | msg := model.Notification{ 41 | CUID: cuid, 42 | DUID: datatype.DUID, 43 | Sseq: sseq, 44 | } 45 | bMsg, err := json.Marshal(&msg) 46 | if err != nil { 47 | return errors.ServerNotify.New(ctx.L(), err.Error()) 48 | } 49 | ctx.L().Infof("notify datatype topic '%s': %s", topic, bMsg) 50 | if token := n.mqttClient.Publish(topic, 0, false, bMsg); token.Wait() && token.Error() != nil { 51 | return errors.ServerNotify.New(ctx.L(), token.Error()) 52 | } 53 | 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /client/pkg/types/json_values_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/log" 5 | "math" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestConvertToJSONSupportedType(t *testing.T) { 13 | t.Run("Can convert JSON supported types", func(t *testing.T) { 14 | i64Max := ConvertToJSONSupportedValue(math.MaxInt64) // 9223372036854775807 15 | log.Logger.Infof("%v", reflect.TypeOf(i64Max)) 16 | require.Equal(t, float64(math.MaxInt64), i64Max) 17 | log.Logger.Infof("[%v vs. %v]", i64Max, math.MaxInt64) 18 | 19 | i64Min := ConvertToJSONSupportedValue(math.MinInt64) // -9223372036854775808 20 | require.Equal(t, float64(math.MinInt64), i64Min) 21 | log.Logger.Infof("[%v vs. %v]", i64Min, math.MinInt64) 22 | 23 | u64Max := ConvertToJSONSupportedValue(uint64(math.MaxUint64)) // 18446744073709551615 24 | require.Equal(t, float64(math.MaxUint64), u64Max) 25 | 26 | log.Logger.Infof("%v %v", reflect.TypeOf(u64Max), u64Max) 27 | 28 | f64Max := ConvertToJSONSupportedValue(math.MaxFloat64) // 1.7976931348623157e+308 29 | require.Equal(t, math.MaxFloat64, f64Max) 30 | log.Logger.Infof("[%v vs. %v]", f64Max, math.MaxFloat64) 31 | 32 | str := ConvertToJSONSupportedValue("hello, world") 33 | require.Equal(t, "hello, world", str) 34 | log.Logger.Infof("[%v]", str) 35 | 36 | b := ConvertToJSONSupportedValue(true) 37 | require.Equal(t, true, b) 38 | log.Logger.Infof("[%v]", b) 39 | 40 | strt := &struct { 41 | B bool 42 | S string 43 | A []int 44 | }{ 45 | true, 46 | "hello world", 47 | []int{1, 2, 3}, 48 | } 49 | strtStr := ConvertToJSONSupportedValue(strt) 50 | require.Equal(t, strt, strtStr) 51 | log.Logger.Infof("[%v]", strtStr) 52 | }) 53 | 54 | t.Run("Can convert JSON supported pointer types", func(t *testing.T) { 55 | var intt = 1234 56 | cintt := ConvertToJSONSupportedValue(&intt) 57 | require.Equal(t, cintt, float64(intt)) 58 | }) 59 | 60 | } 61 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to orda 2 | 3 | Thank you for getting involved with Orda! 4 | In this guide, you will get an overflow of the contribution workflow. 5 | 6 | Briefly speaking, you can contribute to our Orda project by opening an issue, creating a PR, signing CLA, reviewing, and 7 | merging the PR. 8 | 9 | ## Getting started 10 | 11 | Our codebase is introduced in this page(TBD). 12 | 13 | ### Opening a new issue 14 | 15 | If you find a problem, please search first if any issue already exists. 16 | If no related issue exists, you can open a new issue. 17 | When some related issues remain unsolved without being assigned to anyone, you can take the issue by commenting and 18 | assigning yourself. 19 | 20 | ### Making changes 21 | 22 | You can resolve your own issues by cloning or forking our repositories( 23 | see [docs](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/adding-and-cloning-repositories/cloning-and-forking-repositories-from-github-desktop) 24 | of github). 25 | If possible, please add some tests that can verify new features and the correctness of fixing bugs. 26 | 27 | ### Creating a pull request (PR) 28 | 29 | Pull requests are compulsory with a few exceptions. Before submitting any pull request, please make sure if all the 30 | existing tests are passed. 31 | 32 | #### Signing the CLA 33 | 34 | If you are going to be contributing to Orda for the first time, please take a second to read and sign [our CLA](CLA.md), 35 | which is based on [The Fiduciary License Agreement (FLA)](https://contributoragreements.org/about.html) provided 36 | by [FSFE](https://fsfe.org/). 37 | You can sign our CLA by your first submitting any pull request to any repositories of Orda project as follows (TBD). 38 | 39 | Your signature, containing your Github id, is stored in our private repository of Orda project. 40 | 41 | ### Reviewing and Merging your PR 42 | 43 | We, some authorized reviewers, are going to review your PRs, and to merge them. 44 | 45 | 46 | -------------------------------------------------------------------------------- /server/schema/snapshots.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "time" 5 | 6 | "go.mongodb.org/mongo-driver/bson" 7 | "go.mongodb.org/mongo-driver/mongo" 8 | "go.mongodb.org/mongo-driver/x/bsonx" 9 | ) 10 | 11 | // SnapshotDoc defines the document for snapshot, stored in MongoDB 12 | type SnapshotDoc struct { 13 | ID string `bson:"_id"` 14 | CollectionNum int32 `bson:"colNum"` 15 | DUID string `bson:"duid"` 16 | Sseq uint64 `bson:"sseq"` 17 | Meta string `bson:"meta"` 18 | Snapshot []byte `bson:"snapshot"` 19 | CreatedAt time.Time `bson:"createdAt"` 20 | } 21 | 22 | // SnapshotDocFields defines the fields of SnapshotDoc 23 | var SnapshotDocFields = struct { 24 | ID string 25 | CollectionNum string 26 | DUID string 27 | Sseq string 28 | Meta string 29 | Snapshot string 30 | CreatedAt string 31 | }{ 32 | ID: "_id", 33 | CollectionNum: "colNum", 34 | DUID: "duid", 35 | Sseq: "sseq", 36 | Meta: "meta", 37 | Snapshot: "snapshot", 38 | CreatedAt: "createdAt", 39 | } 40 | 41 | // GetIndexModel returns the index models of the collection of SnapshotDoc 42 | func (c *SnapshotDoc) GetIndexModel() []mongo.IndexModel { 43 | return []mongo.IndexModel{{ 44 | Keys: bsonx.Doc{ 45 | {SnapshotDocFields.CollectionNum, bsonx.Int32(1)}, 46 | {SnapshotDocFields.DUID, bsonx.Int32(1)}, 47 | {SnapshotDocFields.Sseq, bsonx.Int64(1)}, 48 | }, 49 | }} 50 | } 51 | 52 | // ToInsertBSON transforms SnapshotDoc to BSON type 53 | func (c *SnapshotDoc) ToInsertBSON() bson.M { 54 | return bson.M{ 55 | SnapshotDocFields.ID: c.ID, 56 | SnapshotDocFields.CollectionNum: c.CollectionNum, 57 | SnapshotDocFields.DUID: c.DUID, 58 | SnapshotDocFields.Sseq: c.Sseq, 59 | SnapshotDocFields.Meta: c.Meta, 60 | SnapshotDocFields.Snapshot: c.Snapshot, 61 | SnapshotDocFields.CreatedAt: c.CreatedAt, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /test/datatypes_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "github.com/orda-io/orda/client/pkg/orda" 7 | "sync" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func (its *IntegrationTestSuite) TestClientServer() { 13 | key := GetFunctionName() 14 | 15 | its.Run("Can create a client and a datatype with server", func() { 16 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_MANUALLY) 17 | client1 := orda.NewClient(config, "client1") 18 | err := client1.Connect() 19 | require.NoError(its.T(), err) 20 | defer client1.Close() 21 | wg := sync.WaitGroup{} 22 | wg.Add(1) 23 | client1.CreateCounter(key, orda.NewHandlers( 24 | func(dt orda.Datatype, oldState, newState model.StateOfDatatype) { 25 | intCounter := dt.(orda.Counter) 26 | _, _ = intCounter.Increase() 27 | _, _ = intCounter.Increase() 28 | _, _ = intCounter.Increase() 29 | require.NoError(its.T(), client1.Sync()) 30 | wg.Done() 31 | }, nil, 32 | func(dt orda.Datatype, errs ...errors.OrdaError) { 33 | its.T().Fatal(errs[0]) 34 | })) 35 | require.NoError(its.T(), client1.Sync()) 36 | wg.Wait() 37 | }) 38 | 39 | its.Run("Can subscribe not existing datatype", func() { 40 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_MANUALLY) 41 | client2 := orda.NewClient(config, "client2") 42 | err := client2.Connect() 43 | require.NoError(its.T(), err) 44 | defer func() { 45 | _ = client2.Close() 46 | }() 47 | wg := sync.WaitGroup{} 48 | wg.Add(1) 49 | client2.SubscribeCounter("NOT_EXISTING", orda.NewHandlers( 50 | nil, nil, 51 | func(dt orda.Datatype, errs ...errors.OrdaError) { 52 | for _, ordaError := range errs { 53 | if ordaError.GetCode() == errors.DatatypeSubscribe { 54 | wg.Done() 55 | return 56 | } 57 | } 58 | its.T().Fatal(errs[0]) 59 | })) 60 | require.NoError(its.T(), client2.Sync()) 61 | wg.Wait() 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /server/managers/config.go: -------------------------------------------------------------------------------- 1 | package managers 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/errors" 7 | "github.com/orda-io/orda/client/pkg/log" 8 | "github.com/orda-io/orda/server/redis" 9 | "io/ioutil" 10 | 11 | "github.com/orda-io/orda/server/mongodb" 12 | ) 13 | 14 | // OrdaServerConfig is a configuration of OrdaServer 15 | type OrdaServerConfig struct { 16 | RPCServerPort int `json:"RPCServerPort"` 17 | RestfulPort int `json:"RestfulPort"` 18 | SwaggerBasePath string `json:"SwaggerBasePath"` 19 | SwaggerJSON string `json:"SwaggerJSON"` 20 | Notification string `json:"Notification"` 21 | Mongo *mongodb.Config `json:"Mongo"` 22 | Redis *redis.Config `json:"Redis,omitempty"` 23 | } 24 | 25 | // LoadOrdaServerConfig loads config from file. 26 | func LoadOrdaServerConfig(filePath string) (*OrdaServerConfig, errors.OrdaError) { 27 | conf := &OrdaServerConfig{} 28 | if err := conf.loadConfig(filePath); err != nil { 29 | return nil, err 30 | } 31 | return conf, nil 32 | } 33 | 34 | func (its *OrdaServerConfig) loadConfig(filepath string) errors.OrdaError { 35 | data, err := ioutil.ReadFile(filepath) 36 | if err != nil { 37 | return errors.ServerInit.New(log.Logger, fmt.Sprintf("cannot read config file: %v", err.Error())) 38 | } 39 | if err := json.Unmarshal(data, its); err != nil { 40 | return errors.ServerInit.New(log.Logger, "cannot unmarshal server config file:"+err.Error()) 41 | } 42 | return nil 43 | } 44 | 45 | // GetRPCServerAddr returns RPC Server Address 46 | func (its *OrdaServerConfig) GetRPCServerAddr() string { 47 | return fmt.Sprintf(":%d", its.RPCServerPort) 48 | } 49 | 50 | // GetRestfulAddr returns Restful Server Address 51 | func (its *OrdaServerConfig) GetRestfulAddr() string { 52 | return fmt.Sprintf(":%d", its.RestfulPort) 53 | } 54 | 55 | // String returns a marshaled string 56 | func (its *OrdaServerConfig) String() string { 57 | b, _ := json.Marshal(its) 58 | return string(b) 59 | } 60 | -------------------------------------------------------------------------------- /test/protocol_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "github.com/orda-io/orda/client/pkg/orda" 7 | "sync" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func (its *IntegrationTestSuite) TestProtocol() { 13 | its.Run("Can produce an error when key is duplicated", func() { 14 | key := GetFunctionName() 15 | 16 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_MANUALLY) 17 | 18 | client1 := orda.NewClient(config, "client1") 19 | 20 | err := client1.Connect() 21 | require.NoError(its.T(), err) 22 | defer func() { 23 | _ = client1.Close() 24 | }() 25 | 26 | client2 := orda.NewClient(config, "client2") 27 | err = client2.Connect() 28 | require.NoError(its.T(), err) 29 | defer func() { 30 | _ = client2.Close() 31 | }() 32 | 33 | _ = client1.CreateCounter(key, orda.NewHandlers( 34 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 35 | require.Equal(its.T(), model.StateOfDatatype_DUE_TO_CREATE, old) 36 | require.Equal(its.T(), model.StateOfDatatype_SUBSCRIBED, new) 37 | }, nil, 38 | func(dt orda.Datatype, errs ...errors.OrdaError) { 39 | require.NoError(its.T(), errs[0]) 40 | })) 41 | require.NoError(its.T(), client1.Sync()) 42 | wg := &sync.WaitGroup{} 43 | wg.Add(1) 44 | _ = client2.CreateCounter(key, orda.NewHandlers( 45 | nil, nil, 46 | func(dt orda.Datatype, errs ...errors.OrdaError) { 47 | its.ctx.L().Infof("should be duplicate error:%v", errs[0]) 48 | require.Error(its.T(), errs[0]) 49 | wg.Done() 50 | })) 51 | require.NoError(its.T(), client2.Sync()) 52 | wg.Wait() 53 | }) 54 | 55 | its.Run("Can produce RPC error when connect", func() { 56 | config := NewTestOrdaClientConfig("NOT_EXISTING", model.SyncType_MANUALLY) 57 | client1 := orda.NewClient(config, its.getTestName()) 58 | err := client1.Connect() 59 | require.Error(its.T(), err) 60 | defer func() { 61 | _ = client1.Close() 62 | }() 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /client/pkg/orda/ordered.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | type orderedType interface { 8 | timedType 9 | getOrderTime() *model.Timestamp 10 | setOrderTime(ts *model.Timestamp) 11 | getPrev() orderedType 12 | setPrev(n orderedType) 13 | getNext() orderedType 14 | setNext(n orderedType) 15 | insertNext(n orderedType) 16 | getNextLive() orderedType 17 | getTimedType() timedType 18 | setTimedType(tt timedType) 19 | hash() string 20 | marshal() *marshaledNode 21 | } 22 | 23 | type orderedNode struct { 24 | timedType 25 | O *model.Timestamp 26 | prev orderedType 27 | next orderedType 28 | } 29 | 30 | func newHead() *orderedNode { 31 | return &orderedNode{ 32 | timedType: newTimedNode(nil, nil), 33 | O: model.OldestTimestamp(), 34 | prev: nil, 35 | next: nil, 36 | } 37 | } 38 | 39 | func (its *orderedNode) getOrderTime() *model.Timestamp { 40 | return its.O 41 | } 42 | 43 | func (its *orderedNode) setOrderTime(ts *model.Timestamp) { 44 | its.O = ts 45 | } 46 | 47 | func (its *orderedNode) hash() string { 48 | return its.O.Hash() 49 | } 50 | 51 | func (its *orderedNode) getPrev() orderedType { 52 | return its.prev 53 | } 54 | 55 | func (its *orderedNode) setPrev(n orderedType) { 56 | its.prev = n 57 | } 58 | 59 | func (its *orderedNode) getNext() orderedType { 60 | return its.next 61 | } 62 | 63 | func (its *orderedNode) setNext(n orderedType) { 64 | its.next = n 65 | } 66 | 67 | func (its *orderedNode) insertNext(n orderedType) { 68 | oldNext := its.next 69 | its.next = n 70 | n.setPrev(its) 71 | n.setNext(oldNext) 72 | if oldNext != nil { 73 | oldNext.setPrev(n) 74 | } 75 | } 76 | 77 | func (its *orderedNode) getNextLive() orderedType { 78 | ret := its.next 79 | for ret != nil { 80 | if !ret.isTomb() { 81 | return ret 82 | } 83 | ret = ret.getNext() 84 | } 85 | return nil 86 | } 87 | 88 | func (its *orderedNode) getTimedType() timedType { 89 | return its.timedType 90 | } 91 | 92 | func (its *orderedNode) setTimedType(tt timedType) { 93 | its.timedType = tt 94 | } 95 | -------------------------------------------------------------------------------- /client/pkg/testonly/test_wire.go: -------------------------------------------------------------------------------- 1 | package testonly 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/internal/datatypes" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | ) 9 | 10 | // TestWire ... 11 | type TestWire struct { 12 | datatypeList []*datatypes.WiredDatatype 13 | sseqMap map[string]int 14 | deliveredList []iface.WiredDatatype 15 | notifiable bool 16 | } 17 | 18 | // NewTestWire ... 19 | func NewTestWire(notifiable bool) *TestWire { 20 | return &TestWire{ 21 | datatypeList: make([]*datatypes.WiredDatatype, 0), 22 | sseqMap: make(map[string]int), 23 | notifiable: notifiable, 24 | } 25 | } 26 | 27 | // DeliverTransaction ... 28 | func (its *TestWire) DeliverTransaction(wired iface.WiredDatatype) { 29 | its.deliveredList = append(its.deliveredList, wired) 30 | if its.notifiable { 31 | its.Sync() 32 | } 33 | } 34 | 35 | // OnChangeDatatypeState ... 36 | func (its *TestWire) OnChangeDatatypeState(dt iface.Datatype, state model.StateOfDatatype) errors.OrdaError { 37 | return nil 38 | } 39 | 40 | // SetDatatypes ... 41 | func (its *TestWire) SetDatatypes(datatypeList ...*datatypes.WiredDatatype) { 42 | for _, v := range datatypeList { 43 | its.datatypeList = append(its.datatypeList, v) 44 | its.sseqMap[v.GetCUID()] = 0 45 | } 46 | } 47 | 48 | // Sync can sync 49 | func (its *TestWire) Sync() { 50 | for len(its.deliveredList) > 0 { 51 | var wired iface.WiredDatatype 52 | wired, its.deliveredList = its.deliveredList[0], its.deliveredList[1:] 53 | 54 | pushPullPack := wired.CreatePushPullPack() 55 | sseq := its.sseqMap[wired.GetCUID()] 56 | operations := pushPullPack.Operations[sseq:] 57 | 58 | // log.Logger.Infof("deliver transaction:%v", OperationsToString(operations)) 59 | its.sseqMap[wired.GetCUID()] = len(pushPullPack.Operations) 60 | for _, w := range its.datatypeList { 61 | if wired != w { 62 | // log.Logger.Info(wired.GetCUID(), " => ", w.GetCUID()) 63 | w.ReceiveRemoteModelOperations(operations, false) 64 | } 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /client/pkg/orda/json_element.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "github.com/orda-io/orda/client/pkg/types" 7 | ) 8 | 9 | // //////////////////////////////////// 10 | // jsonElement 11 | // //////////////////////////////////// 12 | 13 | type jsonElement struct { 14 | jsonType 15 | V types.JSONValue 16 | } 17 | 18 | func newJSONElement(parent jsonType, value interface{}, ts *model.Timestamp) *jsonElement { 19 | return &jsonElement{ 20 | jsonType: &jsonPrimitive{ 21 | parent: parent, 22 | common: parent.getCommon(), 23 | C: ts, 24 | }, 25 | V: value, 26 | } 27 | } 28 | 29 | func (its *jsonElement) getValue() types.JSONValue { 30 | return its.V 31 | } 32 | 33 | func (its *jsonElement) getType() TypeOfJSON { 34 | return TypeJSONElement 35 | } 36 | 37 | func (its *jsonElement) setValue(v types.JSONValue) { 38 | panic("not used yet") 39 | } 40 | 41 | func (its *jsonElement) String() string { 42 | parent := its.getParent() 43 | parentTS := "nil" 44 | if parent != nil { 45 | parentTS = parent.getCreateTime().ToString() 46 | } 47 | value := its.V 48 | if its.isTomb() { 49 | value = "#!DELETED" 50 | } 51 | return fmt.Sprintf("JE(P%v)[C%v|%v]", parentTS, its.getCreateTime().ToString(), value) 52 | } 53 | 54 | func (its *jsonElement) equal(o jsonType) bool { 55 | if its.getType() != o.getType() { 56 | return false 57 | } 58 | je := o.(*jsonElement) 59 | if !its.jsonType.equal(je.jsonType) { 60 | return false 61 | } 62 | 63 | if its.V != je.V { 64 | return false 65 | } 66 | return true 67 | } 68 | 69 | // ///////////////////// methods of iface.Snapshot /////////////////////////////////////// 70 | 71 | func (its *jsonElement) marshal() *marshaledJSONType { 72 | forMarshal := its.jsonType.marshal() 73 | forMarshal.T = marshalKeyJSONElement 74 | forMarshal.E = its.getValue() 75 | return forMarshal 76 | } 77 | 78 | func (its *jsonElement) unmarshal(marshaled *marshaledJSONType, assistant *unmarshalAssistant) { 79 | its.V = marshaled.E 80 | } 81 | 82 | func (its *jsonElement) ToJSON() interface{} { 83 | return its.getValue() 84 | } 85 | -------------------------------------------------------------------------------- /server/redis/client.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | goredislib "github.com/go-redis/redis/v8" 5 | "github.com/go-redsync/redsync/v4" 6 | "github.com/go-redsync/redsync/v4/redis/goredis/v8" 7 | "github.com/orda-io/orda/client/pkg/errors" 8 | "github.com/orda-io/orda/client/pkg/iface" 9 | "github.com/orda-io/orda/server/utils" 10 | ) 11 | 12 | // Client is used to manage redis client 13 | type Client struct { 14 | rs *redsync.Redsync 15 | ctx iface.OrdaContext 16 | client goredislib.UniversalClient 17 | mutexMap map[string]*redsync.Mutex 18 | } 19 | 20 | // New creates a redis client 21 | func New(ctx iface.OrdaContext, conf *Config) (*Client, errors.OrdaError) { 22 | if conf == nil || conf.Addrs == nil { 23 | ctx.L().Infof("redis is NOT initialized") 24 | return &Client{ 25 | ctx: ctx, 26 | }, nil 27 | } 28 | options := &goredislib.UniversalOptions{ 29 | Addrs: conf.Addrs, 30 | Username: conf.Username, 31 | Password: conf.Password, 32 | } 33 | client := goredislib.NewUniversalClient(options) 34 | pool := goredis.NewPool(client) 35 | rs := redsync.New(pool) 36 | mutexMap := make(map[string]*redsync.Mutex) 37 | ctx.L().Infof("redis is initialized") 38 | return &Client{ 39 | rs: rs, 40 | ctx: ctx, 41 | client: client, 42 | mutexMap: mutexMap, 43 | }, nil 44 | } 45 | 46 | // GetLock gets a lock of redis lock. If redis is not available, a local lock is gotten 47 | func (its *Client) GetLock(ctx iface.OrdaContext, lockName string) utils.Lock { 48 | if its.rs == nil { 49 | return utils.GetLocalLock(ctx, lockName) 50 | } 51 | return utils.GetRedisLock(ctx, lockName, its.rs) 52 | } 53 | 54 | // Close closes redis Client 55 | func (its *Client) Close() errors.OrdaError { 56 | if its.client == nil { 57 | return nil 58 | } 59 | for s, mutex := range its.mutexMap { 60 | if unlock, err := mutex.Unlock(); err != nil { 61 | its.ctx.L().Warnf("[🔒] fail to unlock '%v'", s) 62 | } else { 63 | if !unlock { 64 | 65 | } 66 | } 67 | } 68 | if err := its.client.Close(); err != nil { 69 | return errors.ServerInit.New(its.ctx.L(), "[🔒] fail to close redis") 70 | } 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /server/mongodb/collection_snapshots.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/iface" 7 | "github.com/orda-io/orda/server/schema" 8 | "time" 9 | 10 | "go.mongodb.org/mongo-driver/bson" 11 | "go.mongodb.org/mongo-driver/mongo" 12 | "go.mongodb.org/mongo-driver/mongo/options" 13 | ) 14 | 15 | // GetLatestSnapshot gets the latest snapshot for the specified datatype. 16 | func (its *MongoCollections) GetLatestSnapshot( 17 | ctx iface.OrdaContext, 18 | collectionNum int32, 19 | duid string, 20 | ) (*schema.SnapshotDoc, errors.OrdaError) { 21 | f := schema.GetFilter(). 22 | AddFilterEQ(schema.SnapshotDocFields.CollectionNum, collectionNum). 23 | AddFilterEQ(schema.SnapshotDocFields.DUID, duid) 24 | opt := options.FindOne() 25 | opt.SetSort(bson.D{{ 26 | Key: schema.SnapshotDocFields.Sseq, 27 | Value: -1, 28 | }}) 29 | result := its.snapshots.FindOne(ctx, f, opt) 30 | if err := result.Err(); err != nil { 31 | if err == mongo.ErrNoDocuments { 32 | return nil, nil 33 | } 34 | return nil, errors.ServerDBQuery.New(ctx.L(), err.Error()) 35 | } 36 | var snapshot schema.SnapshotDoc 37 | if err := result.Decode(&snapshot); err != nil { 38 | return nil, errors.ServerDBDecode.New(ctx.L(), err.Error()) 39 | } 40 | return &snapshot, nil 41 | } 42 | 43 | // InsertSnapshot inserts a snapshot for the specified datatype. 44 | func (its *MongoCollections) InsertSnapshot( 45 | ctx iface.OrdaContext, 46 | collectionNum int32, 47 | duid string, 48 | sseq uint64, 49 | meta []byte, 50 | snapshot []byte, 51 | ) errors.OrdaError { 52 | snap := schema.SnapshotDoc{ 53 | ID: fmt.Sprintf("%s:%d", duid, sseq), 54 | CollectionNum: collectionNum, 55 | DUID: duid, 56 | Sseq: sseq, 57 | Meta: string(meta), 58 | Snapshot: snapshot, 59 | CreatedAt: time.Now(), 60 | } 61 | result, err := its.snapshots.InsertOne(ctx, snap.ToInsertBSON()) 62 | if err != nil { 63 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 64 | } 65 | if result.InsertedID == snap.ID { 66 | ctx.L().Infof("insert snapshot: %s", result.InsertedID) 67 | } 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /client/pkg/errors/multiple_errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "strings" 7 | ) 8 | 9 | // MultipleOrdaErrors is used to manage multiple OrdaErrors 10 | type MultipleOrdaErrors struct { 11 | tError 12 | errs []*singleOrdaError 13 | } 14 | 15 | // Size returns the size of MultipleOrdaErrors 16 | func (its *MultipleOrdaErrors) Size() int { 17 | return len(its.errs) 18 | } 19 | 20 | // ToArray returns MultipleOrdaErrors to the array of OrdaError 21 | func (its *MultipleOrdaErrors) ToArray() []OrdaError { 22 | var errs []OrdaError 23 | for _, e := range its.errs { 24 | errs = append(errs, e) 25 | } 26 | return errs 27 | } 28 | 29 | // Have returns the number of errors having the specified error code. 30 | func (its *MultipleOrdaErrors) Have(code ErrorCode) int { 31 | cnt := 0 32 | for _, e := range its.errs { 33 | if e.Code == code { 34 | cnt++ 35 | } 36 | } 37 | return cnt 38 | } 39 | 40 | // Error returns the string error message 41 | func (its *MultipleOrdaErrors) Error() string { 42 | var ret []string 43 | for _, err := range its.errs { 44 | ret = append(ret, err.Error()) 45 | } 46 | return fmt.Sprintf("%+q", ret) 47 | } 48 | 49 | // GetCode returns the code 50 | func (its *MultipleOrdaErrors) GetCode() ErrorCode { 51 | return MultipleErrors 52 | } 53 | 54 | // Append adds a new OrdaError to MultipleOrdaErrors 55 | func (its *MultipleOrdaErrors) Append(e OrdaError) OrdaError { 56 | if e == nil { 57 | return its 58 | } 59 | switch cast := e.(type) { 60 | case *singleOrdaError: 61 | its.errs = append(its.errs, cast) 62 | case *MultipleOrdaErrors: 63 | its.errs = append(its.errs, cast.errs...) 64 | } 65 | return its 66 | } 67 | 68 | // Return returns itself as OrdaError 69 | func (its *MultipleOrdaErrors) Return() OrdaError { 70 | if len(its.errs) > 0 { 71 | return its 72 | } 73 | return nil 74 | } 75 | 76 | // Print prints out the concatenated errors 77 | func (its *MultipleOrdaErrors) Print(l *log.OrdaLog, skip int) { 78 | var sb strings.Builder 79 | sb.WriteString(its.tError.Error()) 80 | for _, frame := range its.StackTrace()[skip:] { 81 | sb.WriteString("\n\t") 82 | sb.WriteString(frame.String()) 83 | } 84 | l.Error(sb.String()) 85 | } 86 | -------------------------------------------------------------------------------- /.github/workflows/cla.yml: -------------------------------------------------------------------------------- 1 | name: "CLA Assistant" 2 | on: 3 | issue_comment: 4 | types: [ created ] 5 | pull_request: 6 | types: [ opened,closed,synchronize ] 7 | 8 | # explicitly configure permissions, in case your GITHUB_TOKEN workflow permissions are set to read-only in repository settings 9 | permissions: 10 | actions: write 11 | contents: write 12 | pull-requests: write 13 | statuses: write 14 | 15 | 16 | jobs: 17 | CLAssistant: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: "CLA Assistant" 21 | if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' || github.event_name == 'pull_request' 22 | # Beta Release 23 | uses: contributor-assistant/github-action@v2.3.0 24 | env: 25 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 26 | # the below token should have repo scope and must be manually added by you in the repository's secret 27 | PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_SECRET_TOKEN }} 28 | with: 29 | remote-organization-name: 'orda-io' 30 | remote-repository-name: 'clas' 31 | path-to-signatures: 'signatures/version1/cla.json' 32 | path-to-document: 'CLA.md' # e.g. a CLA or a DCO document 33 | signed-commit-message: '$contributorName has signed the CLA in #$pullRequestNo' 34 | # branch should not be protected 35 | branch: 'main' 36 | allowlist: bot* 37 | 38 | #below are the optional inputs - If the optional inputs are not given, then default values will be taken 39 | 40 | #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' 41 | #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' 42 | #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' 43 | #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' 44 | #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) 45 | #use-dco-flag: true - If you are using DCO instead of CLA 46 | -------------------------------------------------------------------------------- /client/pkg/model/messages.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | const ( 9 | clientHeadFormat = "[%s|%s|%s]" 10 | ) 11 | 12 | // NewPushPullMessage creates a new PushPullRequest 13 | func NewPushPullMessage(seq uint32, client *Client, pushPullPackList ...*PushPullPack) *PushPullMessage { 14 | return &PushPullMessage{ 15 | Header: NewMessageHeader(RequestType_PUSHPULLS), 16 | Collection: client.Collection, 17 | Cuid: client.CUID, 18 | PushPullPacks: pushPullPackList, 19 | } 20 | } 21 | 22 | // ToString returns customized string 23 | func (its *PushPullMessage) ToString(isFull bool) string { 24 | var b strings.Builder 25 | _, _ = fmt.Fprintf(&b, "Head[%s] PushPullPack[%d]{", its.Header.ToString(), len(its.PushPullPacks)) 26 | for _, ppp := range its.PushPullPacks { 27 | b.WriteString(" ") 28 | b.WriteString(ppp.ToString(isFull)) 29 | } 30 | b.WriteString("}") 31 | return b.String() 32 | } 33 | 34 | // GetClient returns the model of the client 35 | func (its *PushPullMessage) GetClient() *Client { 36 | return &Client{ 37 | CUID: its.Cuid, 38 | Alias: "", 39 | Collection: its.Collection, 40 | SyncType: SyncType_LOCAL_ONLY, 41 | } 42 | } 43 | 44 | // NewClientMessage creates a new ClientRequest 45 | func NewClientMessage(client *Client) *ClientMessage { 46 | return &ClientMessage{ 47 | Header: NewMessageHeader(RequestType_CLIENTS), 48 | Collection: client.Collection, 49 | Cuid: client.CUID, 50 | ClientAlias: client.Alias, 51 | SyncType: client.SyncType, 52 | } 53 | } 54 | 55 | // ToString returns customized string 56 | func (its *ClientMessage) ToString() string { 57 | var b strings.Builder 58 | _, _ = fmt.Fprintf(&b, clientHeadFormat, its.Header.ToString(), its.Collection, its.Cuid) 59 | b.WriteString(" SyncType:") 60 | b.WriteString(its.SyncType.String()) 61 | return b.String() 62 | } 63 | 64 | // GetClient returns the model of client 65 | func (its *ClientMessage) GetClient() *Client { 66 | return &Client{ 67 | CUID: its.Cuid, 68 | Alias: its.ClientAlias, 69 | Collection: its.Collection, 70 | Type: its.ClientType, 71 | SyncType: its.SyncType, 72 | } 73 | } 74 | 75 | // GetClientSummary returns the summary of client 76 | func (its *ClientMessage) GetClientSummary() string { 77 | return fmt.Sprintf("%s(%s)", its.ClientAlias, its.Cuid) 78 | } 79 | -------------------------------------------------------------------------------- /server/mongodb/collection_real_collection.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/schema" 7 | "go.mongodb.org/mongo-driver/bson" 8 | "go.mongodb.org/mongo-driver/mongo" 9 | "go.mongodb.org/mongo-driver/mongo/options" 10 | ) 11 | 12 | const ( 13 | // Ver is the field name that notes the version. 14 | Ver = "_orda_ver_" 15 | ) 16 | 17 | // InsertRealSnapshot inserts a snapshot for real collection. 18 | func (its *RepositoryMongo) InsertRealSnapshot( 19 | ctx iface.OrdaContext, 20 | collectionName string, 21 | id string, 22 | data interface{}, 23 | sseq uint64, 24 | ) errors.OrdaError { 25 | collection := its.db.Collection(collectionName) 26 | 27 | // interface{} is currently transformed to bson.M through two phases: interface{} -> bytes{} -> bson.M 28 | // TODO: need to develop a direct transformation method. 29 | marshaled, err := bson.Marshal(data) 30 | if err != nil { 31 | return errors.ServerDBQuery.New(ctx.L(), err.Error(), data) 32 | } 33 | var bsonM = bson.M{} 34 | if err := bson.Unmarshal(marshaled, &bsonM); err != nil { 35 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 36 | } 37 | 38 | bsonM[Ver] = sseq 39 | option := &options.ReplaceOptions{} 40 | option.SetUpsert(true) 41 | res, err := collection.ReplaceOne(ctx, schema.FilterByID(id), bsonM, option) 42 | if err != nil { 43 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 44 | } 45 | if res.ModifiedCount == 1 { 46 | ctx.L().Infof("update snapshot for 'key': %s (_orda_ver_:%d): in '%s'", id, sseq, collectionName) 47 | } 48 | return nil 49 | } 50 | 51 | // GetRealSnapshot returns a real snapshot 52 | func (its *RepositoryMongo) GetRealSnapshot( 53 | ctx iface.OrdaContext, 54 | collectionName string, 55 | id string, 56 | ) (map[string]interface{}, errors.OrdaError) { 57 | collection := its.db.Collection(collectionName) 58 | f := schema.FilterByID(id) 59 | result := collection.FindOne(ctx, f) 60 | if result.Err() != nil { 61 | if result.Err() == mongo.ErrNoDocuments { 62 | return nil, nil 63 | } 64 | return nil, errors.ServerDBQuery.New(ctx.L(), result.Err().Error()) 65 | } 66 | var snap map[string]interface{} 67 | if err := result.Decode(&snap); err != nil { 68 | return nil, errors.ServerDBDecode.New(ctx.L(), result.Err().Error()) 69 | } 70 | return snap, nil 71 | } 72 | -------------------------------------------------------------------------------- /deployments/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.4" 2 | services: 3 | orda-mongodb: 4 | container_name: orda-mongodb 5 | image: mongo:latest 6 | restart: always 7 | volumes: 8 | - mongodb-configdb:/data/configdb 9 | - mongodb-db:/data/db 10 | ports: 11 | - 0.0.0.0:27017:27017 12 | environment: 13 | MONGO_INITDB_ROOT_USERNAME: root 14 | MONGO_INITDB_ROOT_PASSWORD: orda-test 15 | 16 | orda-emqx: 17 | container_name: orda-emqx 18 | image: emqx/emqx:latest 19 | restart: always 20 | ports: 21 | - 0.0.0.0:18181:1883 # port for MQTT 22 | - 0.0.0.0:18881:8083 # for WebSocket/HTTP 23 | - 0.0.0.0:18081:18083 # for dashboard 24 | # - "8883" # port for MQTT(SSL) 25 | # - "8084" # for WSS / HTTPS 26 | # - "11883" # port for internal MQTT/TCP 27 | # - "4369" # for port mapping (epmd) 28 | # - "4370" # for port mapping 29 | # - "5369" # for gen_rpc port mapping 30 | # - "6369" # for distributed node 31 | # - "8081" # for mgmt API 32 | environment: 33 | EMQX_NAME: orda-emqx 34 | EMQX_HOST: 0.0.0.0 35 | EMQX_LOADED_PLUGINS: "emqx_recon,emqx_retainer,emqx_management,emqx_dashboard" 36 | EMQX_LOG__LEVEL: "debug" 37 | volumes: 38 | - emqx-data:/opt/emqx/data 39 | - emqx-etc:/opt/emqx/etc 40 | - emqx-log:/opt/emqx/log 41 | 42 | orda-redis: 43 | container_name: orda-redis 44 | image: redis:alpine 45 | hostname: orda-redis 46 | restart: always 47 | command: redis-server --port 6379 48 | ports: 49 | - 0.0.0.0:16379:6379 50 | 51 | orda-server: 52 | platform: linux/amd64 53 | container_name: orda-server 54 | image: orda-io/orda:${VERSION} 55 | restart: always 56 | command: sh -c "sleep 5 && /app/orda_server --conf /config/docker-config.json" 57 | ports: 58 | - 0.0.0.0:19061:19061 # for port gRPC protocol 59 | - 0.0.0.0:19861:19861 # for port HTTP control 60 | volumes: 61 | - type: bind 62 | source: ./config/docker-config.json 63 | target: /config/docker-config.json 64 | read_only: true 65 | depends_on: 66 | - orda-mongodb 67 | - orda-emqx 68 | - orda-redis 69 | 70 | volumes: 71 | mongodb-configdb: { } 72 | mongodb-db: { } 73 | emqx-data: { } 74 | emqx-etc: { } 75 | emqx-log: { } 76 | etcd-data: { } 77 | 78 | -------------------------------------------------------------------------------- /proto/orda.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | option go_package = "client/pkg/model"; 4 | package orda; 5 | 6 | import "orda.enum.proto"; 7 | 8 | message Client { 9 | string CUID = 1; 10 | string alias = 2; 11 | string collection = 3; 12 | ClientType type = 4; 13 | SyncType syncType = 5; 14 | } 15 | 16 | message Timestamp { 17 | // @inject_tag: json:"e,omitempty" 18 | uint32 era = 1; 19 | // @inject_tag: json:"l,omitempty" 20 | uint64 lamport = 2 [jstype = JS_STRING]; 21 | // @inject_tag: json:"c,omitempty" 22 | string CUID = 3; 23 | // @inject_tag: json:"d,omitempty" 24 | uint32 delimiter = 4; 25 | } 26 | 27 | message OperationID { 28 | // @inject_tag: json:"e,omitempty" 29 | uint32 era = 1; 30 | // @inject_tag: json:"l,omitempty" 31 | uint64 lamport = 2 [jstype = JS_STRING]; 32 | // @inject_tag: json:"c,omitempty" 33 | string CUID = 3; 34 | // @inject_tag: json:"s,omitempty" 35 | uint64 seq = 4 [jstype = JS_STRING]; 36 | } 37 | 38 | message Operation { 39 | OperationID ID = 1; 40 | TypeOfOperation opType = 2; 41 | bytes body = 3; 42 | } 43 | 44 | 45 | message PushPullPack { 46 | string DUID = 1; 47 | string key = 2; 48 | fixed32 option = 3; 49 | CheckPoint checkPoint = 4; 50 | uint32 era = 5; 51 | TypeOfDatatype type = 6; 52 | repeated Operation operations = 7; 53 | } 54 | 55 | message CheckPoint { 56 | // @inject_tag: bson:"s",json:"s" 57 | uint64 sseq = 1 [jstype = JS_STRING]; 58 | // @inject_tag: bson:"c",json:"c" 59 | uint64 cseq = 2 [jstype = JS_STRING]; 60 | } 61 | 62 | message Notification { 63 | string CUID = 1; 64 | string DUID = 2; 65 | uint64 sseq = 3 [jstype = JS_STRING]; 66 | } 67 | 68 | message DatatypeMeta { 69 | string key = 1; 70 | string DUID = 2; 71 | OperationID opID = 3; 72 | TypeOfDatatype typeOf = 4; 73 | } 74 | 75 | message Header { 76 | string version = 1; 77 | string agent = 2; 78 | RequestType type = 3; 79 | } 80 | 81 | message ClientMessage { 82 | Header header = 1; 83 | string collection = 2; 84 | string cuid = 3; 85 | string clientAlias = 4; 86 | ClientType clientType = 5; 87 | SyncType syncType = 6; 88 | } 89 | 90 | message PushPullMessage { 91 | Header header = 1; 92 | string collection = 2; 93 | string cuid = 3; 94 | repeated PushPullPack PushPullPacks = 4; 95 | } 96 | 97 | message CollectionMessage { 98 | string collection = 1; 99 | } 100 | -------------------------------------------------------------------------------- /server/service/service_client.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | gocontext "context" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/context" 7 | "github.com/orda-io/orda/client/pkg/errors" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | "github.com/orda-io/orda/server/admin" 10 | "time" 11 | 12 | "github.com/orda-io/orda/server/schema" 13 | 14 | "github.com/orda-io/orda/server/constants" 15 | ) 16 | 17 | // ProcessClient processes ClientRequest and returns ClientResponse 18 | func (its *OrdaService) ProcessClient( 19 | goCtx gocontext.Context, 20 | req *model.ClientMessage, 21 | ) (*model.ClientMessage, error) { 22 | ctx := context.NewOrdaContext(goCtx, constants.TagClient). 23 | UpdateCollectionTags(req.Collection, 0). 24 | UpdateClientTags(req.GetClientAlias(), req.GetCuid()) 25 | if admin.IsAdminCUID(req.GetCuid()) { 26 | return nil, errors.NewRPCError( 27 | errors.ServerNoPermission.New(ctx.L(), 28 | fmt.Sprintf("not allowed CUID '%s'", req.GetCuid()))) 29 | } 30 | 31 | collectionDoc, rpcErr := its.getCollectionDocWithRPCError(ctx, req.Collection) 32 | if rpcErr != nil { 33 | return nil, rpcErr 34 | } 35 | ctx.UpdateCollectionTags(collectionDoc.Name, collectionDoc.Num) 36 | 37 | clientDocFromReq := schema.ClientModelToBson(req.GetClient(), collectionDoc.Num) 38 | 39 | ctx.L().Infof("REQ[CLIE] %s %v %v", req.ToString(), len(req.Cuid), req.Cuid) 40 | 41 | clientDocFromDB, err := its.managers.Mongo.GetClient(ctx, clientDocFromReq.CUID) 42 | if err != nil { 43 | return nil, errors.NewRPCError(err) 44 | } 45 | if clientDocFromDB == nil { 46 | clientDocFromReq.CreatedAt = time.Now() 47 | ctx.L().Infof("create a new client:%+v", clientDocFromReq) 48 | if err := its.managers.Mongo.GetOrCreateRealCollection(ctx, req.Collection); err != nil { 49 | return nil, errors.NewRPCError(err) 50 | } 51 | } else { 52 | 53 | if clientDocFromDB.CollectionNum != clientDocFromReq.CollectionNum { 54 | msg := fmt.Sprintf("client '%s' accesses collection(%d)", 55 | clientDocFromDB.ToString(), clientDocFromReq.CollectionNum) 56 | return nil, errors.NewRPCError(errors.ServerNoPermission.New(ctx.L(), msg)) 57 | } 58 | ctx.L().Infof("Client will be updated:%+v", clientDocFromReq) 59 | } 60 | clientDocFromReq.CreatedAt = time.Now() 61 | if err = its.managers.Mongo.UpdateClient(ctx, clientDocFromReq); err != nil { 62 | return nil, errors.NewRPCError(err) 63 | } 64 | ctx.L().Infof("RES[CLIE] %s", req.ToString()) 65 | return req, nil 66 | } 67 | -------------------------------------------------------------------------------- /server/service/service_pushpull_client.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | gocontext "context" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/context" 7 | "github.com/orda-io/orda/client/pkg/errors" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | "reflect" 10 | 11 | "github.com/orda-io/orda/server/constants" 12 | ) 13 | 14 | // ProcessPushPull processes a GRPC for Push-Pull 15 | func (its *OrdaService) ProcessPushPull(goCtx gocontext.Context, in *model.PushPullMessage) (*model.PushPullMessage, error) { 16 | ctx := context.NewOrdaContext(goCtx, constants.TagPushPull). 17 | UpdateClientTags("", in.Cuid) 18 | collectionDoc, rpcErr := its.getCollectionDocWithRPCError(ctx, in.Collection) 19 | if rpcErr != nil { 20 | return nil, rpcErr 21 | } 22 | ctx.UpdateCollectionTags(collectionDoc.Name, collectionDoc.Num) 23 | 24 | clientDoc, err := its.managers.Mongo.GetClient(ctx, in.Cuid) 25 | if err != nil { 26 | return nil, errors.NewRPCError(err) 27 | } 28 | if clientDoc == nil { 29 | msg := fmt.Sprintf("no client '%s:%s'", in.Collection, in.Cuid) 30 | return nil, errors.NewRPCError(errors.ServerNoResource.New(ctx.L(), msg)) 31 | } 32 | ctx.UpdateClientTags(clientDoc.Alias, clientDoc.CUID) 33 | ctx.L().Infof("↪[PUPU] %v", in.ToString(false)) 34 | if clientDoc.CollectionNum != collectionDoc.Num { 35 | msg := fmt.Sprintf("client '%s' accesses collection(%d)", clientDoc.ToString(), collectionDoc.Num) 36 | return nil, errors.NewRPCError(errors.ServerNoPermission.New(ctx.L(), msg)) 37 | } 38 | 39 | response := &model.PushPullMessage{ 40 | Header: in.Header, 41 | Collection: in.Collection, 42 | Cuid: in.Cuid, 43 | } 44 | 45 | var chanList []<-chan *model.PushPullPack 46 | 47 | for _, ppp := range in.PushPullPacks { 48 | handler := newPushPullHandler(ctx, ppp, clientDoc, collectionDoc, its.managers) 49 | chanList = append(chanList, handler.Start()) 50 | } 51 | remainingChan := len(chanList) 52 | cases := make([]reflect.SelectCase, remainingChan) 53 | for i, ch := range chanList { 54 | cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch)} 55 | } 56 | 57 | for remainingChan > 0 && chanList != nil { 58 | _, value, ok := reflect.Select(cases) 59 | remainingChan-- 60 | if !ok { 61 | continue 62 | } else { 63 | ppp := value.Interface().(*model.PushPullPack) 64 | response.PushPullPacks = append(response.PushPullPacks, ppp) 65 | } 66 | } 67 | ctx.L().Infof("↩[PUPU] %v", response.ToString(false)) 68 | return response, nil 69 | } 70 | -------------------------------------------------------------------------------- /client/pkg/model/timestamp.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/types" 6 | "strings" 7 | ) 8 | 9 | // OldestTimestamp returns the oldest timestamp. 10 | func OldestTimestamp() *Timestamp { 11 | return &Timestamp{ 12 | Era: 0, 13 | Lamport: 0, 14 | CUID: types.NewNilUID(), 15 | Delimiter: 0, 16 | } 17 | } 18 | 19 | // NewTimestamp creates a new timestamp 20 | func NewTimestamp(era uint32, lamport uint64, cuid string, delimiter uint32) *Timestamp { 21 | return &Timestamp{ 22 | Era: era, 23 | Lamport: lamport, 24 | CUID: cuid, 25 | Delimiter: delimiter, 26 | } 27 | } 28 | 29 | // Compare is used to compared with another Timestamp. 30 | func (its *Timestamp) Compare(o *Timestamp) int { 31 | retEra := int32(its.Era - o.Era) 32 | if retEra > 0 { 33 | return 1 34 | } else if retEra < 0 { 35 | return -1 36 | } 37 | var diff = int64(its.Lamport - o.Lamport) 38 | if diff > 0 { 39 | return 1 40 | } else if diff < 0 { 41 | return -1 42 | } 43 | return strings.Compare(its.CUID, o.CUID) 44 | } 45 | 46 | // ToString is used to get string for Timestamp 47 | func (its *Timestamp) ToString() string { 48 | var b strings.Builder 49 | _, _ = fmt.Fprintf(&b, "[%d:%d:%s:%d]", its.Era, its.Lamport, 50 | its.CUID, its.Delimiter) 51 | return b.String() 52 | } 53 | 54 | // Hash returns the string hash of timestamp. 55 | // DON'T change this because protocol can be broken : TODO: this can be improved. 56 | func (its *Timestamp) Hash() string { 57 | var b strings.Builder 58 | _, _ = fmt.Fprintf(&b, "%d%d%d%s", its.Era, its.Lamport, its.Delimiter, its.CUID) 59 | return b.String() 60 | } 61 | 62 | // Next returns a next Timestamp having increased Lamport. 63 | func (its *Timestamp) Next() *Timestamp { 64 | return &Timestamp{ 65 | Era: its.Era, 66 | Lamport: its.Lamport + 1, 67 | CUID: its.CUID, 68 | Delimiter: 0, 69 | } 70 | } 71 | 72 | // GetAndNextDelimiter returns a next Timestamp having increased deliminator. 73 | func (its *Timestamp) GetAndNextDelimiter() *Timestamp { 74 | 75 | ts := &Timestamp{ 76 | Era: its.Era, 77 | Lamport: its.Lamport, 78 | CUID: its.CUID, 79 | Delimiter: its.Delimiter, 80 | } 81 | its.Delimiter++ 82 | return ts 83 | } 84 | 85 | // Clone returns clone of this Timestamp. 86 | func (its *Timestamp) Clone() *Timestamp { 87 | return &Timestamp{ 88 | Era: its.Era, 89 | Lamport: its.Lamport, 90 | CUID: its.CUID, 91 | Delimiter: its.Delimiter, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /client/pkg/internal/datatypes/snapshot.go: -------------------------------------------------------------------------------- 1 | package datatypes 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/iface" 7 | "github.com/orda-io/orda/client/pkg/operations" 8 | ) 9 | 10 | // SnapshotDatatype is responsible for managing the snapshot 11 | type SnapshotDatatype struct { 12 | *BaseDatatype 13 | Snapshot iface.Snapshot 14 | } 15 | 16 | // ToJSON returns the snapshot in the format of JSON 17 | func (its *SnapshotDatatype) ToJSON() interface{} { 18 | return its.Snapshot.ToJSON() 19 | } 20 | 21 | // NewSnapshotDatatype creates a new SnapshotDatatype 22 | func NewSnapshotDatatype(b *BaseDatatype, snap iface.Snapshot) *SnapshotDatatype { 23 | return &SnapshotDatatype{ 24 | BaseDatatype: b, 25 | Snapshot: snap, 26 | } 27 | } 28 | 29 | // ApplySnapshot applies for a snapshot 30 | func (its *SnapshotDatatype) ApplySnapshot(snapBody []byte) errors.OrdaError { 31 | its.L().Infof("apply SnapshotOperation: %v", string(snapBody)) 32 | if err := json.Unmarshal(snapBody, its.Snapshot); err != nil { 33 | return errors.DatatypeSnapshot.New(its.L(), err.Error()) 34 | } 35 | return nil 36 | } 37 | 38 | // GetSnapshot returns the snapshot 39 | func (its *SnapshotDatatype) GetSnapshot() iface.Snapshot { 40 | return its.Snapshot 41 | } 42 | 43 | // GetMetaAndSnapshot returns the meta and snapshot 44 | func (its *SnapshotDatatype) GetMetaAndSnapshot() ([]byte, []byte, errors.OrdaError) { 45 | meta, oErr := its.GetMeta() 46 | if oErr != nil { 47 | return nil, nil, oErr 48 | } 49 | snap, err := json.Marshal(its.GetSnapshot()) 50 | if err != nil { 51 | return nil, nil, errors.DatatypeMarshal.New(its.L(), err.Error()) 52 | } 53 | return meta, snap, nil 54 | } 55 | 56 | // SetMetaAndSnapshot sets the meta and snapshot 57 | func (its *SnapshotDatatype) SetMetaAndSnapshot(meta, snap []byte) errors.OrdaError { 58 | if err := its.SetMeta(meta); err != nil { 59 | return err 60 | } 61 | if err := json.Unmarshal(snap, its.GetSnapshot()); err != nil { 62 | return errors.DatatypeMarshal.New(its.L(), err.Error()) 63 | } 64 | return nil 65 | } 66 | 67 | // CreateSnapshotOperation returns the SnapshotOperation from the snapshot and meta 68 | func (its *SnapshotDatatype) CreateSnapshotOperation() (iface.Operation, errors.OrdaError) { 69 | snap, err := json.Marshal(its.Snapshot) 70 | if err != nil { 71 | return nil, errors.DatatypeMarshal.New(its.L(), err.Error()) 72 | } 73 | snapOp := operations.NewSnapshotOperation(its.TypeOf, snap) 74 | return snapOp, nil 75 | } 76 | -------------------------------------------------------------------------------- /server/schema/schema.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | 6 | "go.mongodb.org/mongo-driver/bson" 7 | "go.mongodb.org/mongo-driver/mongo/options" 8 | ) 9 | 10 | // CollectionNameXXXX is the name of the collection for XXXX 11 | const ( 12 | CollectionNameColNumGenerator = "-_-ColNumGenerator" 13 | CollectionNameClients = "-_-Clients" 14 | CollectionNameCollections = "-_-Collections" 15 | CollectionNameDatatypes = "-_-Datatypes" 16 | CollectionNameOperations = "-_-Operations" 17 | CollectionNameSnapshot = "-_-Snapshots" 18 | ) 19 | 20 | const ( 21 | // ID is an identifier of MongoDB 22 | ID = "_id" 23 | ) 24 | 25 | // Filter is the type used to filter 26 | type Filter bson.D 27 | 28 | // AddFilterEQ is a function to add EQ to Filter 29 | func (b Filter) AddFilterEQ(key string, value interface{}) Filter { 30 | return append(b, bson.E{Key: key, Value: value}) 31 | } 32 | 33 | // AddFilterGTE is a function to add GTE to Filter 34 | func (b Filter) AddFilterGTE(key string, from interface{}) Filter { 35 | return append(b, bson.E{Key: key, Value: bson.D{{Key: "$gte", Value: from}}}) 36 | } 37 | 38 | // AddFilterLTE is a function to add LTE to Filter 39 | func (b Filter) AddFilterLTE(key string, to interface{}) Filter { 40 | return append(b, bson.E{Key: key, Value: bson.D{{Key: "$lte", Value: to}}}) 41 | } 42 | 43 | // ToCheckPointBSON is a function to change a checkpoint to BSON 44 | func ToCheckPointBSON(checkPoint *model.CheckPoint) bson.M { 45 | return bson.M{"s": checkPoint.Sseq, "c": checkPoint.Cseq} 46 | } 47 | 48 | // GetFilter returns an instance of Filter 49 | func GetFilter() Filter { 50 | return Filter{} 51 | } 52 | 53 | // FilterByID returns an instance of Filter of ID 54 | func FilterByID(id interface{}) Filter { 55 | return Filter{bson.E{Key: ID, Value: id}} 56 | } 57 | 58 | // FilterByName returns an instance of Filter of Name 59 | func FilterByName(name string) Filter { 60 | return Filter{bson.E{Key: "name", Value: name}} 61 | } 62 | 63 | // AddSnapshot adds a snapshot value 64 | func (b Filter) AddSnapshot(data bson.M) Filter { 65 | return append(b, bson.E{Key: "$set", Value: data}) 66 | } 67 | 68 | // AddExists adds the Filter which examines the existence of the key 69 | func (b Filter) AddExists(key string) Filter { 70 | return append(b, bson.E{Key: key, Value: bson.D{ 71 | {Key: "$exists", Value: true}, 72 | }}) 73 | } 74 | 75 | // options 76 | var ( 77 | upsert = true 78 | UpsertOption = &options.UpdateOptions{ 79 | Upsert: &upsert, 80 | } 81 | ) 82 | -------------------------------------------------------------------------------- /client/pkg/context/orda_context.go: -------------------------------------------------------------------------------- 1 | package context 2 | 3 | import ( 4 | "context" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "strconv" 8 | ) 9 | 10 | type ordaContext struct { 11 | context.Context 12 | logger *log.OrdaLog 13 | } 14 | 15 | // UpdateAllTags updates all tags; if the specified tags are empty, previous tags are kept. 16 | func (its *ordaContext) UpdateAllTags(emoji, collection, colNum, client, cuid, datatype, duid string) { 17 | its.logger.SetTags(emoji, collection, colNum, client, cuid, datatype, duid) 18 | } 19 | 20 | // L returns OrdaLog 21 | func (its *ordaContext) L() *log.OrdaLog { 22 | if its.logger == nil { 23 | return log.Logger 24 | } 25 | return its.logger 26 | } 27 | 28 | // Ctx returns the context.Context 29 | func (its *ordaContext) Ctx() context.Context { 30 | return its.Context 31 | } 32 | 33 | // SetLogger sets the logger 34 | func (its *ordaContext) SetLogger(l *log.OrdaLog) { 35 | its.logger = l 36 | } 37 | 38 | // UpdateCollectionTags updates the collection tag 39 | func (its *ordaContext) UpdateCollectionTags(collection string, colNum int32) iface.OrdaContext { 40 | cn := "" 41 | if colNum > 0 { 42 | cn = strconv.Itoa(int(colNum)) 43 | } 44 | its.UpdateAllTags("", collection, cn, "", "", "", "") 45 | return its 46 | } 47 | 48 | // UpdateClientTags updates the client tag 49 | func (its *ordaContext) UpdateClientTags(client, cuid string) iface.OrdaContext { 50 | its.UpdateAllTags("", "", "", client, cuid, "", "") 51 | return its 52 | } 53 | 54 | // UpdateDatatypeTags updates the datatype tag 55 | func (its *ordaContext) UpdateDatatypeTags(datatype, duid string) iface.OrdaContext { 56 | its.UpdateAllTags("", "", "", "", "", datatype, duid) 57 | return its 58 | } 59 | 60 | func (its *ordaContext) CloneWithNewEmoji(emoji string) iface.OrdaContext { 61 | newLogger := its.logger.Clone() 62 | newLogger.SetTags(emoji, "", "", "", "", "", "") 63 | return &ordaContext{ 64 | Context: context.TODO(), 65 | logger: newLogger, 66 | } 67 | } 68 | 69 | func NewOrdaContext(ctx context.Context, emoji string) iface.OrdaContext { 70 | return NewOrdaContextWithAllTags(ctx, emoji, "", "", "", "", "", "") 71 | } 72 | 73 | // NewOrdaContextWithAllTags creates a new OrdaContext 74 | func NewOrdaContextWithAllTags(ctx context.Context, emoji, collection, colNum, client, cuid, datatype, duid string) iface.OrdaContext { 75 | logger := log.NewWithTags(emoji, collection, colNum, client, cuid, datatype, duid) 76 | return &ordaContext{ 77 | Context: ctx, 78 | logger: logger, 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /server/mongodb/collection_clients.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/schema" 7 | 8 | "go.mongodb.org/mongo-driver/mongo" 9 | ) 10 | 11 | // UpdateClient updates a clientDoc; if not exists, a new clientDoc is inserted. 12 | func (its *MongoCollections) UpdateClient( 13 | ctx iface.OrdaContext, 14 | client *schema.ClientDoc, 15 | ) errors.OrdaError { 16 | result, err := its.clients.UpdateOne(ctx, schema.FilterByID(client.CUID), client.ToUpdateBSON(), schema.UpsertOption) 17 | if err != nil { 18 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 19 | } 20 | 21 | if result.ModifiedCount == 1 || result.UpsertedCount == 1 { 22 | return nil 23 | } 24 | return errors.ServerDBQuery.New(ctx.L(), "fail to update client") 25 | } 26 | 27 | // DeleteClient deletes the specified client. 28 | func (its *MongoCollections) DeleteClient(ctx iface.OrdaContext, cuid string) errors.OrdaError { 29 | result, err := its.clients.DeleteOne(ctx, schema.FilterByID(cuid)) 30 | if err != nil { 31 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 32 | } 33 | if result.DeletedCount == 1 { 34 | return nil 35 | } 36 | ctx.L().Warnf("fail to find a client to delete: `%s`", cuid) 37 | return nil 38 | } 39 | 40 | // GetClient returns a ClientDoc for the specified CUID. 41 | func (its *MongoCollections) GetClient( 42 | ctx iface.OrdaContext, 43 | cuid string, 44 | ) (*schema.ClientDoc, errors.OrdaError) { 45 | // opts := options.FindOne() 46 | // if !withCheckPoint { 47 | // opts.SetProjection(bson.M{schema.ClientDocFields.CheckPoints: 0}) 48 | // } 49 | sr := its.clients.FindOne(ctx, schema.FilterByID(cuid)) 50 | if err := sr.Err(); err != nil { 51 | if err == mongo.ErrNoDocuments { 52 | return nil, nil 53 | } 54 | return nil, errors.ServerDBQuery.New(ctx.L(), err.Error()) 55 | } 56 | 57 | var client schema.ClientDoc 58 | if err := sr.Decode(&client); err != nil { 59 | return nil, errors.ServerDBDecode.New(ctx.L(), err.Error()) 60 | } 61 | return &client, nil 62 | } 63 | 64 | func (its *MongoCollections) purgeAllCollectionClients( 65 | ctx iface.OrdaContext, 66 | collectionNum int32, 67 | ) errors.OrdaError { 68 | filter := schema.GetFilter().AddFilterEQ(schema.ClientDocFields.CollectionNum, collectionNum) 69 | r1, err := its.clients.DeleteMany(ctx, filter) 70 | if err != nil { 71 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 72 | } 73 | ctx.L().Infof("delete %d clients in collection#%d", r1.DeletedCount, collectionNum) 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /client/pkg/orda/document_marshal.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | type marshaledDocument struct { 8 | NodeMap []*marshaledJSONType `json:"nm"` 9 | } 10 | 11 | type unmarshalAssistant struct { 12 | tsMap map[string]*model.Timestamp 13 | common *jsonCommon 14 | } 15 | 16 | type marshalKeyJSONType string 17 | 18 | const ( 19 | marshalKeyJSONElement marshalKeyJSONType = "E" 20 | marshalKeyJSONObject marshalKeyJSONType = "O" 21 | marshalKeyJSONArray marshalKeyJSONType = "A" 22 | ) 23 | 24 | type marshaledJSONType struct { 25 | C *model.Timestamp `json:"c,omitempty"` // jsonPrimitive.C 26 | T marshalKeyJSONType `json:"t,omitempty"` // type; "E": jsonElement, "O": jsonObject, "A": jsonArray 27 | P *model.Timestamp `json:"p,omitempty"` // jsonPrimitive.parent's C 28 | D *model.Timestamp `json:"d,omitempty"` // jsonPrimitive.D 29 | E interface{} `json:"e,omitempty"` // for jsonElement 30 | A *marshaledJSONArray `json:"a,omitempty"` // for jsonArray 31 | O *marshaledJSONObject `json:"o,omitempty"` // for jsonObject 32 | } 33 | 34 | type marshaledOrderedType [2]*model.Timestamp 35 | 36 | type marshaledJSONObject struct { 37 | M map[string]*model.Timestamp `json:"m"` // hashmapSnapshot.Map 38 | S int `json:"s"` // hashmapSnapshot.Size 39 | } 40 | 41 | type marshaledJSONArray struct { 42 | N []marshaledOrderedType `json:"n"` // an array of node 43 | S int `json:"s"` // size of a list 44 | } 45 | 46 | // unifyTimestamp is used to unify timestamps. This must be called when timestamps of any jsonType are set. 47 | func (its *unmarshalAssistant) unifyTimestamp(ts *model.Timestamp) *model.Timestamp { 48 | if ts == nil { 49 | return nil 50 | } 51 | if existing, ok := its.tsMap[ts.Hash()]; ok { 52 | return existing 53 | } 54 | its.tsMap[ts.Hash()] = ts 55 | return ts 56 | } 57 | 58 | func newMarshaledDocument() *marshaledDocument { 59 | return &marshaledDocument{ 60 | NodeMap: nil, 61 | } 62 | } 63 | 64 | func (its *marshaledJSONType) unmarshalAsJSONType(assistant *unmarshalAssistant) jsonType { 65 | jsonType := &jsonPrimitive{ 66 | common: assistant.common, 67 | C: assistant.unifyTimestamp(its.C), 68 | D: assistant.unifyTimestamp(its.D), 69 | } 70 | switch its.T { 71 | case marshalKeyJSONElement: 72 | return &jsonElement{ 73 | jsonType: jsonType, 74 | } 75 | case marshalKeyJSONObject: 76 | return &jsonObject{ 77 | jsonType: jsonType, 78 | } 79 | case marshalKeyJSONArray: 80 | return &jsonArray{ 81 | jsonType: jsonType, 82 | } 83 | } 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /client/pkg/errors/error.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "strings" 7 | 8 | "github.com/ztrue/tracerr" 9 | ) 10 | 11 | type tError tracerr.Error 12 | 13 | // OrdaError defines an OrdaError 14 | type OrdaError interface { 15 | tError 16 | GetCode() ErrorCode 17 | Append(e OrdaError) OrdaError 18 | Return() OrdaError 19 | Have(code ErrorCode) int 20 | ToArray() []OrdaError 21 | Size() int 22 | Print(l *log.OrdaLog, skip int) 23 | } 24 | 25 | // singleOrdaError implements an error related to Orda 26 | type singleOrdaError struct { 27 | tError 28 | Code ErrorCode 29 | } 30 | 31 | func (its *singleOrdaError) Size() int { 32 | return 1 33 | } 34 | 35 | func (its *singleOrdaError) ToArray() []OrdaError { 36 | return []OrdaError{its} 37 | } 38 | 39 | func (its *singleOrdaError) Have(code ErrorCode) int { 40 | if its.Code == code { 41 | return 1 42 | } 43 | return 0 44 | } 45 | 46 | func (its *singleOrdaError) Error() string { 47 | return its.tError.Error() 48 | } 49 | 50 | // GetCode returns ErrorCode 51 | func (its *singleOrdaError) GetCode() ErrorCode { 52 | return its.Code 53 | } 54 | 55 | func (its *singleOrdaError) Return() OrdaError { 56 | return its 57 | } 58 | 59 | func (its *singleOrdaError) Append(e OrdaError) OrdaError { 60 | var errs []*singleOrdaError 61 | switch cast := e.(type) { 62 | case *singleOrdaError: 63 | errs = append(errs, its, cast) 64 | case *MultipleOrdaErrors: 65 | errs = append(errs, its) 66 | errs = append(errs, cast.errs...) 67 | } 68 | return &MultipleOrdaErrors{ 69 | tError: tracerr.New("Multiple OrdaErrors"), 70 | errs: errs, 71 | } 72 | } 73 | 74 | func (its *singleOrdaError) Print(l *log.OrdaLog, skip int) { 75 | if l == nil { 76 | return 77 | } 78 | var sb strings.Builder 79 | sb.WriteString(its.tError.Error()) 80 | for _, frame := range its.StackTrace()[skip:] { 81 | sb.WriteString("\n\t") 82 | sb.WriteString(frame.String()) 83 | } 84 | l.Error(sb.String()) 85 | } 86 | 87 | // ToOrdaError casts an error to OrdaError if it is a OrdaError type 88 | func ToOrdaError(err error) OrdaError { 89 | if dErr, ok := err.(OrdaError); ok { 90 | return dErr 91 | } 92 | return nil 93 | } 94 | 95 | func newSingleOrdaError(l *log.OrdaLog, code ErrorCode, name string, msgFormat string, args ...interface{}) OrdaError { 96 | format := fmt.Sprintf("[%s: %d] %s", name, code, msgFormat) 97 | err := &singleOrdaError{ 98 | tError: tracerr.New(fmt.Sprintf(format, args...)), 99 | Code: code, 100 | } 101 | err.Print(l, 2) 102 | return err 103 | } 104 | -------------------------------------------------------------------------------- /client/pkg/operations/list.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/model" 5 | ) 6 | 7 | // NewInsertOperation creates a new InsertOperation 8 | func NewInsertOperation(pos int, values []interface{}) *InsertOperation { 9 | return &InsertOperation{ 10 | baseOperation: newBaseOperation( 11 | model.TypeOfOperation_LIST_INSERT, 12 | nil, 13 | &InsertBody{ 14 | V: values, 15 | }, 16 | ), 17 | Pos: pos, 18 | } 19 | } 20 | 21 | // InsertBody is the body of InsertOperation 22 | type InsertBody struct { 23 | T *model.Timestamp 24 | V []interface{} 25 | } 26 | 27 | // InsertOperation is used to insert a value to a list 28 | type InsertOperation struct { 29 | baseOperation 30 | Pos int // for local 31 | } 32 | 33 | // GetBody returns the body 34 | func (its *InsertOperation) GetBody() *InsertBody { 35 | return its.Body.(*InsertBody) 36 | } 37 | 38 | // ////////////////// DeleteOperation //////////////////// 39 | 40 | // NewDeleteOperation creates a new DeleteOperation. 41 | func NewDeleteOperation(pos int, numOfNodes int) *DeleteOperation { 42 | return &DeleteOperation{ 43 | baseOperation: newBaseOperation( 44 | model.TypeOfOperation_LIST_DELETE, 45 | nil, 46 | &DeleteBody{}, 47 | ), 48 | Pos: pos, 49 | NumOfNodes: numOfNodes, 50 | } 51 | } 52 | 53 | // DeleteBody is the body of DeleteOperation 54 | type DeleteBody struct { 55 | T []*model.Timestamp 56 | } 57 | 58 | // DeleteOperation is used to delete a value from a list. 59 | type DeleteOperation struct { 60 | baseOperation 61 | Pos int 62 | NumOfNodes int 63 | } 64 | 65 | // GetBody returns the body 66 | func (its *DeleteOperation) GetBody() *DeleteBody { 67 | return its.Body.(*DeleteBody) 68 | } 69 | 70 | // ////////////////// UpdateOperation //////////////////// 71 | 72 | // NewUpdateOperation creates a new UpdateOperation. 73 | func NewUpdateOperation(pos int, values []interface{}) *UpdateOperation { 74 | return &UpdateOperation{ 75 | baseOperation: newBaseOperation( 76 | model.TypeOfOperation_LIST_UPDATE, 77 | nil, 78 | &UpdateBody{ 79 | V: values, 80 | }, 81 | ), 82 | Pos: pos, 83 | } 84 | } 85 | 86 | // UpdateBody is the body of UpdateOperation 87 | type UpdateBody struct { 88 | T []*model.Timestamp 89 | V []interface{} 90 | } 91 | 92 | // UpdateOperation is used to update a value in a list. 93 | type UpdateOperation struct { 94 | baseOperation 95 | Pos int 96 | } 97 | 98 | // GetBody returns the body 99 | func (its *UpdateOperation) GetBody() *UpdateBody { 100 | return its.Body.(*UpdateBody) 101 | } 102 | -------------------------------------------------------------------------------- /server/mongodb/mongo_collection.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/schema" 7 | "reflect" 8 | 9 | "go.mongodb.org/mongo-driver/bson" 10 | "go.mongodb.org/mongo-driver/mongo" 11 | ) 12 | 13 | // MongoCollections is a bunch of collections used to provide 14 | type MongoCollections struct { 15 | mongoClient *mongo.Client 16 | clients *mongo.Collection 17 | counters *mongo.Collection 18 | snapshots *mongo.Collection 19 | datatypes *mongo.Collection 20 | operations *mongo.Collection 21 | collections *mongo.Collection 22 | } 23 | 24 | // Create creates an empty collection by inserting a document and immediately deleting it. 25 | func (its *MongoCollections) createCollection( 26 | ctx iface.OrdaContext, 27 | collection *mongo.Collection, 28 | docModel schema.MongoDBDoc, 29 | ) errors.OrdaError { 30 | result, err := collection.InsertOne(ctx, bson.D{}) 31 | if err != nil { 32 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 33 | } 34 | if _, err = collection.DeleteOne(ctx, schema.FilterByID(result.InsertedID)); err != nil { 35 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 36 | } 37 | ctx.L().Infof("create collection:%s", collection.Name()) 38 | if docModel != nil { 39 | return its.createIndex(ctx, collection, docModel) 40 | } 41 | return nil 42 | } 43 | 44 | func (its *MongoCollections) createIndex( 45 | ctx iface.OrdaContext, 46 | collection *mongo.Collection, 47 | docModel schema.MongoDBDoc, 48 | ) errors.OrdaError { 49 | if docModel != nil { 50 | indexModel := docModel.GetIndexModel() 51 | if len(indexModel) > 0 { 52 | indexes, err := collection.Indexes().CreateMany(ctx, indexModel) 53 | if err != nil { 54 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 55 | } 56 | ctx.L().Infof("index is created: %v in %s", indexes, reflect.TypeOf(docModel)) 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | func (its *MongoCollections) doTransaction( 63 | ctx iface.OrdaContext, 64 | transactions func() errors.OrdaError, 65 | ) errors.OrdaError { 66 | session, err := its.mongoClient.StartSession() 67 | if err != nil { 68 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 69 | } 70 | 71 | if err := session.StartTransaction(); err != nil { 72 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 73 | } 74 | if err = mongo.WithSession(ctx, session, func(sc mongo.SessionContext) error { 75 | if err := transactions(); err != nil { 76 | return err 77 | } 78 | if err := session.CommitTransaction(sc); err != nil { 79 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 80 | } 81 | return nil 82 | }); err != nil { 83 | return err.(errors.OrdaError) 84 | } 85 | session.EndSession(ctx) 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /server/schema/operations.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "time" 7 | 8 | "go.mongodb.org/mongo-driver/mongo" 9 | "go.mongodb.org/mongo-driver/x/bsonx" 10 | ) 11 | 12 | // NewOperationDoc creates a new OperationDoc with the given parameters 13 | func NewOperationDoc(op *model.Operation, duid string, sseq uint64, colNum int32) *OperationDoc { 14 | 15 | return &OperationDoc{ 16 | ID: fmt.Sprintf("%s:%d", duid, sseq), 17 | DUID: duid, 18 | CollectionNum: colNum, 19 | OpType: op.OpType.String(), 20 | OpID: OpID{ 21 | Era: op.ID.Era, 22 | Lamport: op.ID.Lamport, 23 | CUID: op.ID.CUID, 24 | Seq: op.ID.Seq, 25 | }, 26 | Sseq: sseq, 27 | Body: op.Body, 28 | CreatedAt: time.Now(), 29 | } 30 | 31 | } 32 | 33 | // OpID is used to denote the operation ID in MongoDB 34 | type OpID struct { 35 | Era uint32 `bson:"era"` 36 | Lamport uint64 `bson:"lamport"` 37 | CUID string `bson:"cuid"` 38 | Seq uint64 `bson:"seq"` 39 | } 40 | 41 | // OperationDoc defines a document for operation, stored in MongoDB 42 | type OperationDoc struct { 43 | ID string `bson:"_id"` 44 | DUID string `bson:"duid"` 45 | CollectionNum int32 `bson:"colNum"` 46 | OpType string `bson:"type"` 47 | OpID OpID `bson:"id"` 48 | Sseq uint64 `bson:"sseq"` 49 | Body []byte `bson:"body"` 50 | CreatedAt time.Time `bson:"createdAt"` 51 | } 52 | 53 | // GetOperation returns a model.Operation by composing parameters of OperationDoc. 54 | func (its *OperationDoc) GetOperation() *model.Operation { 55 | opID := &model.OperationID{ 56 | Era: its.OpID.Era, 57 | Lamport: its.OpID.Lamport, 58 | CUID: its.OpID.CUID, 59 | Seq: its.OpID.Seq, 60 | } 61 | return &model.Operation{ 62 | ID: opID, 63 | OpType: model.TypeOfOperation(model.TypeOfOperation_value[its.OpType]), 64 | Body: its.Body, 65 | } 66 | } 67 | 68 | // OperationDocFields defines the fields of OperationDoc 69 | var OperationDocFields = struct { 70 | ID string 71 | DUID string 72 | CollectionNum string 73 | OpType string 74 | Sseq string 75 | Operation string 76 | CreatedAt string 77 | }{ 78 | ID: "_id", 79 | DUID: "duid", 80 | CollectionNum: "colNum", 81 | OpType: "type", 82 | Sseq: "sseq", 83 | Operation: "op", 84 | CreatedAt: "createdAt", 85 | } 86 | 87 | // GetIndexModel returns the index models of the collection of OperationDoc 88 | func (its *OperationDoc) GetIndexModel() []mongo.IndexModel { 89 | return []mongo.IndexModel{{ 90 | Keys: bsonx.Doc{ 91 | {OperationDocFields.DUID, bsonx.Int32(1)}, 92 | {OperationDocFields.Sseq, bsonx.Int32(-1)}, 93 | }, 94 | }} 95 | } 96 | -------------------------------------------------------------------------------- /client/pkg/iface/datatype.go: -------------------------------------------------------------------------------- 1 | package iface 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/log" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | ) 8 | 9 | // BaseDatatype defines a base operations for datatype 10 | type BaseDatatype interface { 11 | GetType() model.TypeOfDatatype 12 | GetState() model.StateOfDatatype 13 | GetKey() string // @baseDatatype 14 | GetDUID() string // @baseDatatype 15 | SetDUID(duid string) // @baseDatatype 16 | GetCUID() string // @baseDatatype 17 | SetState(state model.StateOfDatatype) // @baseDatatype 18 | SetLogger(l *log.OrdaLog) // @baseDatatype 19 | GetCtx() OrdaContext // @baseDatatype 20 | GetMeta() ([]byte, errors.OrdaError) 21 | SetMeta(meta []byte) errors.OrdaError 22 | GetSummary() string 23 | L() *log.OrdaLog 24 | } 25 | 26 | // SnapshotDatatype defines the interfaces of the datatype that enables the snapshot. 27 | type SnapshotDatatype interface { 28 | ResetSnapshot() 29 | GetSnapshot() Snapshot 30 | GetMetaAndSnapshot() ([]byte, []byte, errors.OrdaError) 31 | SetMetaAndSnapshot(meta []byte, snap []byte) errors.OrdaError 32 | CreateSnapshotOperation() (Operation, errors.OrdaError) 33 | ToJSON() interface{} 34 | } 35 | 36 | // WiredDatatype defines the internal interface related to the synchronization with Orda server 37 | type WiredDatatype interface { 38 | BaseDatatype 39 | SetCheckPoint(sseq uint64, cseq uint64) 40 | ReceiveRemoteModelOperations(ops []*model.Operation, obtainList bool) ([]interface{}, errors.OrdaError) 41 | ApplyPushPullPack(*model.PushPullPack) 42 | CreatePushPullPack() *model.PushPullPack 43 | DeliverTransaction(transaction []Operation) 44 | NeedPull(sseq uint64) bool 45 | NeedPush() bool 46 | SubscribeOrCreate(state model.StateOfDatatype) errors.OrdaError 47 | ResetWired() 48 | } 49 | 50 | // OperationalDatatype defines interfaces related to executing operations. 51 | type OperationalDatatype interface { 52 | ExecuteLocal(op interface{}) (interface{}, errors.OrdaError) // @Real datatype 53 | ExecuteRemote(op interface{}) (interface{}, errors.OrdaError) // @Real datatype 54 | ExecuteRemoteTransaction(transaction []*model.Operation, obtainList bool) ([]interface{}, errors.OrdaError) 55 | } 56 | 57 | // HandleableDatatype defines handlers for Orda datatype 58 | type HandleableDatatype interface { 59 | HandleStateChange(oldState, newState model.StateOfDatatype) 60 | HandleErrors(err ...errors.OrdaError) 61 | HandleRemoteOperations(operations []interface{}) 62 | } 63 | 64 | // Datatype defines the interface of executing operations, which is implemented by every datatype. 65 | type Datatype interface { 66 | SnapshotDatatype 67 | WiredDatatype 68 | OperationalDatatype 69 | HandleableDatatype 70 | } 71 | -------------------------------------------------------------------------------- /test/integration_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | gocontext "context" 5 | "github.com/orda-io/orda/client/pkg/context" 6 | "github.com/orda-io/orda/client/pkg/errors" 7 | "github.com/orda-io/orda/client/pkg/iface" 8 | "github.com/orda-io/orda/server/managers" 9 | "github.com/orda-io/orda/server/redis" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | "github.com/stretchr/testify/suite" 16 | 17 | "github.com/orda-io/orda/server/mongodb" 18 | "github.com/orda-io/orda/server/server" 19 | ) 20 | 21 | const ( 22 | dbName = "test" 23 | TagTest = "TEST" 24 | ) 25 | 26 | // IntegrationTestSuite is the base test suite for integration test. 27 | type IntegrationTestSuite struct { 28 | suite.Suite 29 | collectionName string 30 | collectionNum int32 31 | conf *managers.OrdaServerConfig 32 | server *server.OrdaServer 33 | mongo *mongodb.RepositoryMongo 34 | redis *redis.Client 35 | ctx iface.OrdaContext 36 | } 37 | 38 | // SetupTest builds some prerequisite for testing. 39 | func (its *IntegrationTestSuite) SetupSuite() { 40 | its.ctx = context.NewOrdaContext(gocontext.TODO(), TagTest). 41 | UpdateCollectionTags(its.T().Name(), 0) 42 | var err errors.OrdaError 43 | its.conf = NewTestOrdaServerConfig(dbName) 44 | 45 | if its.server, err = server.NewOrdaServer(gocontext.TODO(), its.conf); err != nil { 46 | its.Fail("fail to setup") 47 | } 48 | if err = its.setupClients(); err != nil { 49 | its.Fail("fail to setup client") 50 | } 51 | 52 | go func() { 53 | require.NoError(its.T(), its.server.Start()) 54 | }() 55 | time.Sleep(1 * time.Second) 56 | } 57 | 58 | func (its *IntegrationTestSuite) setupClients() errors.OrdaError { 59 | var err errors.OrdaError 60 | if its.mongo, err = mongodb.New(its.ctx, its.conf.Mongo); err != nil { 61 | return err 62 | } 63 | 64 | if its.redis, err = redis.New(its.ctx, its.conf.Redis); err != nil { 65 | return err 66 | } 67 | return nil 68 | } 69 | 70 | func (its *IntegrationTestSuite) SetupTest() { 71 | its.collectionName = strings.Split(its.T().Name(), "/")[1] 72 | its.ctx.L().Infof("set collection: %s", its.collectionName) 73 | var err error 74 | require.NoError(its.T(), its.mongo.PurgeCollection(its.ctx, its.collectionName)) 75 | its.collectionNum, err = mongodb.MakeCollection(its.ctx, its.mongo, its.collectionName) 76 | require.NoError(its.T(), err) 77 | } 78 | 79 | // TearDownTest tears down IntegrationTestSuite. 80 | func (its *IntegrationTestSuite) TearDownSuite() { 81 | its.T().Log("TearDown IntegrationTestSuite") 82 | its.server.Close(true) 83 | } 84 | 85 | func (its *IntegrationTestSuite) getTestName() string { 86 | sp := strings.Split(its.T().Name(), "/") 87 | return sp[len(sp)-1] 88 | } 89 | 90 | func TestIntegration(t *testing.T) { 91 | suite.Run(t, new(IntegrationTestSuite)) 92 | } 93 | -------------------------------------------------------------------------------- /client/pkg/orda/marshal_test.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | type HelloIface interface { 13 | json.Marshaler 14 | json.Unmarshaler 15 | } 16 | 17 | type Hello1 struct { 18 | json string 19 | } 20 | 21 | type Hello2 struct { 22 | json int 23 | } 24 | 25 | type World struct { 26 | Hellos map[string]HelloIface 27 | Size int 28 | } 29 | 30 | type World2 struct { 31 | Hellos map[string]interface{} 32 | Size int 33 | } 34 | 35 | func (w World) UnmarshalJSON(bytes []byte) error { 36 | fakeWorld := &struct { 37 | Hellos map[string]string 38 | Size int 39 | }{} 40 | if err := json.Unmarshal(bytes, fakeWorld); err != nil { 41 | return err 42 | } 43 | w.Size = fakeWorld.Size 44 | // w.Hellos 45 | return nil 46 | } 47 | 48 | func (h Hello1) UnmarshalJSON(bytes []byte) error { 49 | fmt.Printf("unmarshal:%s", string(bytes)) 50 | return json.Unmarshal(bytes, &h) 51 | } 52 | 53 | func (h Hello1) MarshalJSON() ([]byte, error) { 54 | return []byte(fmt.Sprintf("{ \"json\":\"%s\" }", h.json)), nil 55 | } 56 | 57 | func (h Hello2) UnmarshalJSON(bytes []byte) error { 58 | fmt.Printf("unmarshal:%s", string(bytes)) 59 | return json.Unmarshal(bytes, &h) 60 | } 61 | 62 | func (h Hello2) MarshalJSON() ([]byte, error) { 63 | return []byte(fmt.Sprintf("{ \"json\":%d }", h.json)), nil 64 | } 65 | 66 | func TestMarshaling(t *testing.T) { 67 | t.Run("Can test marshalling and unmarshalling", func(t *testing.T) { 68 | w := &World{ 69 | Hellos: make(map[string]HelloIface), 70 | Size: 0, 71 | } 72 | w.Hellos["a"] = &Hello1{ 73 | json: "hello1", 74 | } 75 | w.Hellos["b"] = &Hello2{ 76 | json: 1234, 77 | } 78 | j, err := json.Marshal(w) 79 | require.NoError(t, err) 80 | log.Logger.Infof("Marshaled: %v", string(j)) 81 | 82 | m := &World2{} 83 | err = json.Unmarshal(j, m) 84 | require.NoError(t, err) 85 | log.Logger.Infof("Unmarshaled: %+v", m) 86 | // h:= Hello1{ 87 | // json: "eeee", 88 | // } 89 | // j, err := json.Marshal(h) 90 | // require.NoError(t, err) 91 | // log.Logger.Infof("%v", string(j)) 92 | // 93 | // m1 := make(map[string]string) 94 | // m1["a"]="b" 95 | // m1["b"]="c" 96 | // j2, err := json.Marshal(m1) 97 | // require.NoError(t, err) 98 | // log.Logger.Infof("%v", string(j2)) 99 | // 100 | // m2 := make(map[string]HelloIface) 101 | // m2["a"]= h 102 | // m2["b"]= h 103 | // j3, err := json.Marshal(m2) 104 | // require.NoError(t, err) 105 | // log.Logger.Infof("%v", string(j3)) 106 | // 107 | // m2_copy := make(map[string]HelloIface) 108 | // 109 | // err = json.Unmarshal(j3, &m2_copy) 110 | // require.NoError(t, err) 111 | }) 112 | } 113 | -------------------------------------------------------------------------------- /client/pkg/orda/datatype.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/internal/datatypes" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | ) 9 | 10 | // Datatype is an Orda Datatype which provides common interfaces. 11 | type Datatype interface { 12 | GetType() model.TypeOfDatatype 13 | GetState() model.StateOfDatatype 14 | GetKey() string // @baseDatatype 15 | ToJSON() interface{} 16 | } 17 | 18 | type datatype struct { 19 | *datatypes.WiredDatatype 20 | TxCtx *datatypes.TransactionContext 21 | handlers *Handlers 22 | } 23 | 24 | func newDatatype( 25 | base *datatypes.BaseDatatype, 26 | wire iface.Wire, 27 | handlers *Handlers, 28 | ) *datatype { 29 | t := datatypes.NewTransactionDatatype(base) 30 | w := datatypes.NewWiredDatatype(wire, t) 31 | return &datatype{ 32 | WiredDatatype: w, 33 | TxCtx: nil, 34 | handlers: handlers, 35 | } 36 | } 37 | 38 | func (its *datatype) init(data iface.Datatype) errors.OrdaError { 39 | its.Datatype = data 40 | its.ResetWired() 41 | its.ResetSnapshot() 42 | return its.ResetTransaction() 43 | } 44 | 45 | func (its *datatype) cloneDatatype(txCtx *datatypes.TransactionContext) *datatype { 46 | return &datatype{ 47 | WiredDatatype: its.WiredDatatype, 48 | TxCtx: txCtx, 49 | handlers: its.handlers, 50 | } 51 | } 52 | 53 | func (its *datatype) HandleStateChange(old, new model.StateOfDatatype) { 54 | if its.handlers != nil && its.handlers.stateChangeHandler != nil { 55 | its.handlers.stateChangeHandler(its.Datatype, old, new) 56 | } 57 | } 58 | 59 | func (its *datatype) HandleErrors(errs ...errors.OrdaError) { 60 | if its.handlers != nil && its.handlers.errorHandler != nil { 61 | its.handlers.errorHandler(its.Datatype, errs...) 62 | } 63 | } 64 | 65 | func (its *datatype) HandleRemoteOperations(operations []interface{}) { 66 | if its.handlers != nil && its.handlers.remoteOperationHandler != nil { 67 | its.handlers.remoteOperationHandler(its.Datatype, operations) 68 | } 69 | } 70 | 71 | // SubscribeOrCreate enables a datatype to subscribe and create itself. 72 | func (its *datatype) SubscribeOrCreate(state model.StateOfDatatype) errors.OrdaError { 73 | if state == model.StateOfDatatype_DUE_TO_SUBSCRIBE { 74 | its.DeliverTransaction(nil) 75 | return nil 76 | } 77 | snapOp, err := its.CreateSnapshotOperation() 78 | if err != nil { 79 | return errors.DatatypeSubscribe.New(its.L(), err.Error()) 80 | } 81 | _, err = its.SentenceInTx(its.TxCtx, snapOp, true) 82 | if err != nil { 83 | return errors.DatatypeSubscribe.New(its.L(), err.Error()) 84 | } 85 | return nil 86 | } 87 | 88 | func (its *datatype) ExecuteRemoteTransaction( 89 | transaction []*model.Operation, 90 | obtainList bool, 91 | ) ([]interface{}, errors.OrdaError) { 92 | return its.ExecuteRemoteTransactionWithCtx(transaction, its.TxCtx, obtainList) 93 | } 94 | -------------------------------------------------------------------------------- /client/pkg/model/operation_id.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/types" 6 | "strings" 7 | ) 8 | 9 | // NewOperationID creates a new OperationID. 10 | func NewOperationID() *OperationID { 11 | return &OperationID{ 12 | Era: 0, 13 | Lamport: 0, 14 | CUID: types.NewNilUID(), 15 | Seq: 0, 16 | } 17 | } 18 | 19 | // NewOperationIDWithCUID creates a new OperationID with CUID. 20 | func NewOperationIDWithCUID(cuid string) *OperationID { 21 | return &OperationID{ 22 | Era: 0, 23 | Lamport: 0, 24 | CUID: cuid, 25 | Seq: 0, 26 | } 27 | } 28 | 29 | // GetTimestamp returns Timestamp from OperationID 30 | func (its *OperationID) GetTimestamp() *Timestamp { 31 | return &Timestamp{ 32 | Era: its.Era, 33 | Lamport: its.Lamport, 34 | CUID: its.CUID, 35 | Delimiter: 0, 36 | } 37 | } 38 | 39 | // SetOperationID sets the values of OperationID. 40 | func (its *OperationID) SetOperationID(other *OperationID) { 41 | its.Era = other.Era 42 | its.Lamport = other.Lamport 43 | its.CUID = other.CUID 44 | its.Seq = other.Seq 45 | 46 | } 47 | 48 | // Next increments an OperationID 49 | func (its *OperationID) Next() *OperationID { 50 | its.Lamport++ 51 | its.Seq++ 52 | return &OperationID{ 53 | Era: its.Era, 54 | Lamport: its.Lamport, 55 | CUID: its.CUID, 56 | Seq: its.Seq, 57 | } 58 | } 59 | 60 | // RollBack rollbacks the OperationID 61 | func (its *OperationID) RollBack() { 62 | its.Lamport-- 63 | its.Seq-- 64 | } 65 | 66 | // SyncLamport synchronizes the value of Lamport. 67 | func (its *OperationID) SyncLamport(other uint64) uint64 { 68 | if its.Lamport < other { 69 | its.Lamport = other 70 | } else { 71 | its.Lamport++ 72 | } 73 | return its.Lamport 74 | } 75 | 76 | // Clone ... 77 | func (its *OperationID) Clone() *OperationID { 78 | return &OperationID{ 79 | Era: its.Era, 80 | Lamport: its.Lamport, 81 | CUID: its.CUID, 82 | Seq: its.Seq, 83 | } 84 | } 85 | 86 | // ToString returns customized string 87 | func (its *OperationID) ToString() string { 88 | if its == nil { 89 | return "" 90 | } 91 | var b strings.Builder 92 | _, _ = fmt.Fprintf(&b, "[%d:%d:%s:%d]", 93 | its.Era, its.Lamport, its.CUID, its.Seq) 94 | return b.String() 95 | } 96 | 97 | // ToJSON returns an OperationID as JSON 98 | func (its *OperationID) ToJSON() interface{} { 99 | return struct { 100 | Era uint32 101 | Lamport uint64 102 | CUID string 103 | Seq uint64 104 | }{ 105 | Era: its.Era, 106 | Lamport: its.Lamport, 107 | CUID: its.CUID, 108 | Seq: its.Seq, 109 | } 110 | } 111 | 112 | // Compare compares two operationIDs. 113 | func (its *OperationID) Compare(other *OperationID) int { 114 | retEra := int32(its.Era - other.Era) 115 | if retEra > 0 { 116 | return 1 117 | } else if retEra < 0 { 118 | return -1 119 | } 120 | var diff = int64(its.Lamport - other.Lamport) 121 | if diff > 0 { 122 | return 1 123 | } else if diff < 0 { 124 | return -1 125 | } 126 | return strings.Compare(its.CUID, other.CUID) 127 | } 128 | -------------------------------------------------------------------------------- /server/service/service_patch_document.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | gocontext "context" 5 | "github.com/orda-io/orda/client/pkg/context" 6 | "github.com/orda-io/orda/client/pkg/errors" 7 | "github.com/orda-io/orda/client/pkg/iface" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | "github.com/orda-io/orda/client/pkg/orda" 10 | "github.com/orda-io/orda/server/admin" 11 | "github.com/orda-io/orda/server/constants" 12 | "github.com/orda-io/orda/server/schema" 13 | "github.com/orda-io/orda/server/snapshot" 14 | "github.com/orda-io/orda/server/utils" 15 | ) 16 | 17 | // PatchDocument patches document datatype 18 | func (its *OrdaService) PatchDocument(goCtx gocontext.Context, req *model.PatchMessage) (*model.PatchMessage, error) { 19 | ctx := context.NewOrdaContext(goCtx, constants.TagPatch). 20 | UpdateCollectionTags(req.Collection, 0) 21 | collectionDoc, rpcErr := its.getCollectionDocWithRPCError(ctx, req.Collection) 22 | if rpcErr != nil { 23 | return nil, rpcErr 24 | } 25 | clientDoc := admin.NewPatchClient(collectionDoc) 26 | ctx.UpdateCollectionTags(collectionDoc.Name, collectionDoc.Num). 27 | UpdateClientTags(clientDoc.Alias, clientDoc.CUID). 28 | UpdateDatatypeTags(req.Key, "") 29 | 30 | ctx.L().Infof("BEGIN PatchDocument: '%v' %#v", req.Key, req.GetJson()) 31 | defer ctx.L().Infof("END PatchDocument: '%v'", req.Key) 32 | 33 | lock := its.managers.GetLock(ctx, utils.GetLockName("PD", collectionDoc.Num, req.Key)) 34 | if !lock.TryLock() { 35 | return nil, errors.NewRPCError(errors.ServerInit.New(ctx.L(), "fail to lock")) 36 | } 37 | defer lock.Unlock() 38 | 39 | datatypeDoc, rpcErr := its.managers.Mongo.GetDatatypeByKey(ctx, collectionDoc.Num, req.Key) 40 | if rpcErr != nil { 41 | return nil, rpcErr 42 | } 43 | 44 | if datatypeDoc == nil { 45 | datatypeDoc = schema.NewDatatypeDoc("", req.Key, collectionDoc.Num, model.TypeOfDatatype_DOCUMENT.String()) 46 | } 47 | if datatypeDoc.Type != model.TypeOfDatatype_DOCUMENT.String() { 48 | return nil, errors.NewRPCError(errors.ServerBadRequest.New(ctx.L(), "not document type: "+datatypeDoc.Type)) 49 | } 50 | 51 | snapshotManager := snapshot.NewManager(ctx, its.managers, datatypeDoc, collectionDoc) 52 | data, lastSseq, err := snapshotManager.GetLatestDatatype() 53 | if err != nil { 54 | return nil, errors.NewRPCError(err) 55 | } 56 | ctx.UpdateDatatypeTags(data.GetKey(), data.GetDUID()) 57 | 58 | if lastSseq > 0 { 59 | data.SetState(model.StateOfDatatype_SUBSCRIBED) 60 | data.SetCheckPoint(lastSseq, 0) 61 | } 62 | doc := data.(orda.Document) 63 | patches, err := doc.(orda.Document).PatchByJSON(req.Json) 64 | if err != nil { 65 | return nil, errors.NewRPCError(errors.ServerBadRequest.New(ctx.L(), err.Error())) 66 | } 67 | 68 | if len(patches) > 0 { 69 | ppp := doc.(iface.Datatype).CreatePushPullPack() 70 | ctx.L().Infof("%v", ppp.ToString(true)) 71 | 72 | pushPullHandler := newPushPullHandler(ctx, ppp, clientDoc, collectionDoc, its.managers) 73 | pppCh := pushPullHandler.Start() 74 | _ = <-pppCh 75 | } 76 | 77 | return &model.PatchMessage{ 78 | Key: req.Key, 79 | Collection: req.Collection, 80 | Json: string(doc.ToJSONBytes()), 81 | }, nil 82 | 83 | } 84 | -------------------------------------------------------------------------------- /client/pkg/types/json_values.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // JSONValue is an internal type used in storing various types, for converting any type to JSON supported type. 8 | type JSONValue interface{} 9 | 10 | // ConvertValueList converts an array of values to JSONSupportedValue 11 | func ConvertValueList(values []interface{}) ([]interface{}, error) { 12 | var jsonValues []interface{} 13 | for _, val := range values { 14 | if val == nil { 15 | return nil, fmt.Errorf("null value cannot be inserted") 16 | } 17 | jsonValues = append(jsonValues, ConvertToJSONSupportedValue(val)) 18 | } 19 | return jsonValues, nil 20 | } 21 | 22 | // ToInterfaceArray transforms an array of JSNValues to the array of interfaces 23 | func ToInterfaceArray(ja []JSONValue) []interface{} { 24 | var ret []interface{} 25 | for _, v := range ja { 26 | ret = append(ret, v.(interface{})) 27 | } 28 | return ret 29 | } 30 | 31 | // ConvertToJSONSupportedValue converts any type of Go into a type that is supported by JSON 32 | func ConvertToJSONSupportedValue(t interface{}) JSONValue { 33 | switch v := t.(type) { 34 | // all number types are stored as float64, i.e., IEEE 754 64 bits floating point type. 35 | case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, 36 | *int, *int8, *int16, *int32, *int64, *uint, *uint8, *uint16, *uint32, *uint64: 37 | var f64 float64 38 | switch vv := v.(type) { 39 | case int: 40 | f64 = float64(vv) 41 | case int8: 42 | f64 = float64(vv) 43 | case int16: 44 | f64 = float64(vv) 45 | case int32: 46 | f64 = float64(vv) 47 | case int64: 48 | f64 = float64(vv) 49 | case uint: 50 | f64 = float64(vv) 51 | case uint8: 52 | f64 = float64(vv) 53 | case uint16: 54 | f64 = float64(vv) 55 | case uint32: 56 | f64 = float64(vv) 57 | case uint64: 58 | // if vv > math.MaxFloat64 { 59 | // log.Logger.Warnf("overflow: cannot store an integer more than int64.Max (%d)", math.MaxInt64) 60 | // } 61 | f64 = float64(vv) 62 | case *int: 63 | f64 = float64(*vv) 64 | case *int8: 65 | f64 = float64(*vv) 66 | case *int16: 67 | f64 = float64(*vv) 68 | case *int32: 69 | f64 = float64(*vv) 70 | case *int64: 71 | f64 = float64(*vv) 72 | case *uint: 73 | f64 = float64(*vv) 74 | case *uint8: 75 | f64 = float64(*vv) 76 | case *uint16: 77 | f64 = float64(*vv) 78 | case *uint32: 79 | f64 = float64(*vv) 80 | case *uint64: 81 | // if *vv > math.MaxInt64 { 82 | // log.Logger.Warnf("overflow: cannot store an integer more than int64.Max (%d)", math.MaxInt64) 83 | // } 84 | f64 = float64(*vv) 85 | } 86 | return f64 87 | case float32, float64, *float32, *float64: 88 | var f64 float64 89 | switch vv := v.(type) { 90 | case float32: 91 | f64 = float64(vv) 92 | case float64: 93 | f64 = vv 94 | case *float32: 95 | f64 = float64(*vv) 96 | case *float64: 97 | f64 = *vv 98 | } 99 | return f64 100 | case string: 101 | return v 102 | case *string: 103 | return *v 104 | case bool: 105 | return v 106 | case *bool: 107 | return *v 108 | default: 109 | } 110 | return t 111 | } 112 | -------------------------------------------------------------------------------- /server/schema/clients.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | "strconv" 7 | "time" 8 | 9 | "go.mongodb.org/mongo-driver/bson" 10 | "go.mongodb.org/mongo-driver/mongo" 11 | "go.mongodb.org/mongo-driver/x/bsonx" 12 | ) 13 | 14 | // ClientDoc defines the document for client, stored in MongoDB. 15 | type ClientDoc struct { 16 | CUID string `bson:"_id"` 17 | Alias string `bson:"alias"` 18 | CollectionNum int32 `bson:"colNum"` 19 | Type int8 `bson:"type"` 20 | SyncType int8 `bson:"syncType"` 21 | CreatedAt time.Time `bson:"createdAt"` 22 | UpdatedAt time.Time `bson:"updatedAt"` 23 | } 24 | 25 | // ClientDocFields defines the fields of ClientDoc 26 | var ClientDocFields = struct { 27 | CUID string 28 | Alias string 29 | CollectionNum string 30 | Type string 31 | SyncType string 32 | CreatedAt string 33 | UpdatedAt string 34 | }{ 35 | CUID: "_id", 36 | Alias: "alias", 37 | CollectionNum: "colNum", 38 | Type: "type", 39 | SyncType: "syncType", 40 | CreatedAt: "createdAt", 41 | UpdatedAt: "updatedAt", 42 | } 43 | 44 | func (its *ClientDoc) String() string { 45 | return fmt.Sprintf("(%d)%s:%s", its.CollectionNum, its.Alias, its.CUID) 46 | } 47 | 48 | // ToUpdateBSON returns a bson from a ClientDoc 49 | func (its *ClientDoc) ToUpdateBSON() bson.D { 50 | return bson.D{ 51 | {"$set", bson.D{ 52 | {ClientDocFields.Alias, its.Alias}, 53 | {ClientDocFields.CollectionNum, its.CollectionNum}, 54 | {ClientDocFields.Type, its.Type}, 55 | {ClientDocFields.SyncType, its.SyncType}, 56 | {ClientDocFields.CreatedAt, its.CreatedAt}, 57 | }}, 58 | {"$currentDate", bson.D{ 59 | {ClientDocFields.UpdatedAt, true}, 60 | }}, 61 | } 62 | } 63 | 64 | // GetIndexModel returns the index models of ClientDoc 65 | func (its *ClientDoc) GetIndexModel() []mongo.IndexModel { 66 | return []mongo.IndexModel{ 67 | { 68 | Keys: bsonx.Doc{ 69 | {Key: ClientDocFields.CollectionNum, Value: bsonx.Int32(1)}, 70 | }, 71 | }, 72 | { 73 | Keys: bsonx.Doc{ 74 | {Key: ClientDocFields.Type, Value: bsonx.Int32(1)}, 75 | }, 76 | }, 77 | } 78 | } 79 | 80 | // ClientModelToBson returns a ClientDoc from a model.Client 81 | func ClientModelToBson(model *model.Client, collectionNum int32) *ClientDoc { 82 | return &ClientDoc{ 83 | CUID: model.CUID, 84 | Alias: model.Alias, 85 | CollectionNum: collectionNum, 86 | Type: int8(model.Type), 87 | SyncType: int8(model.SyncType), 88 | } 89 | } 90 | 91 | // GetModel returns model.Client 92 | func (its *ClientDoc) GetModel() *model.Client { 93 | return &model.Client{ 94 | CUID: its.CUID, 95 | Alias: its.Alias, 96 | Collection: strconv.Itoa(int(its.CollectionNum)), 97 | Type: model.ClientType(its.Type), 98 | SyncType: model.SyncType(its.SyncType), 99 | } 100 | } 101 | 102 | // ToString returns ClientDoc string 103 | func (its *ClientDoc) ToString() string { 104 | return fmt.Sprintf("%s(%s)", its.Alias, its.CUID) 105 | } 106 | 107 | // GetType returns model.ClientType 108 | func (its *ClientDoc) GetType() model.ClientType { 109 | return model.ClientType(its.Type) 110 | } 111 | -------------------------------------------------------------------------------- /server/service/service_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | gocontext "context" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | "github.com/orda-io/orda/client/pkg/orda" 9 | "github.com/orda-io/orda/server/service" 10 | "github.com/orda-io/orda/server/testonly" 11 | "github.com/orda-io/orda/server/wrapper" 12 | integration "github.com/orda-io/orda/test" 13 | "github.com/stretchr/testify/require" 14 | "strings" 15 | "testing" 16 | ) 17 | 18 | func TestOrdaService(t *testing.T) { 19 | 20 | _, ctx, _ := integration.InitTestDBCollection(t, testonly.TestDBName) 21 | managers, err := integration.NewTestManagers(ctx, testonly.TestDBName) 22 | svc := service.NewOrdaService(managers) 23 | require.NoError(t, err) 24 | collectionName := t.Name() 25 | 26 | conf := &orda.ClientConfig{ 27 | CollectionName: collectionName, 28 | SyncType: model.SyncType_MANUALLY, 29 | } 30 | 31 | t.Run("Can avoid to create admin client", func(t *testing.T) { 32 | cm := &model.Client{ 33 | CUID: "!@#$OrdaPatchAPI", 34 | Alias: "", 35 | Collection: "test", 36 | Type: 0, 37 | SyncType: 0, 38 | } 39 | req := model.NewClientMessage(cm) 40 | res, err := svc.ProcessClient(ctx, req) 41 | require.NotNil(t, err) 42 | require.True(t, strings.Contains(err.Error(), errors.ServerNoPermission.New(nil, "").Error())) 43 | require.Nil(t, res) 44 | }) 45 | 46 | t.Run("Can do ", func(t *testing.T) { 47 | 48 | client1 := orda.NewClient(conf, t.Name()+"1") 49 | counter1 := client1.SubscribeOrCreateCounter(t.Name(), nil) 50 | 51 | wrapper1 := wrapper.NewDatatypeWrapper(counter1) 52 | _, _ = counter1.Increase() 53 | testonly.RegisterClient(t, svc, wrapper1.GetClientModel()) 54 | 55 | res, err := svc.ProcessPushPull(gocontext.TODO(), wrapper1.CreatePushPullMessage()) 56 | require.NoError(t, err) 57 | ppp1 := res.GetPushPullPacks()[0] 58 | require.True(t, ppp1.CheckPoint.Compare(model.NewSetCheckPoint(2, 2))) 59 | require.Equal(t, len(ppp1.Operations), 0) 60 | require.Equal(t, ppp1.GetOption(), uint32(model.PushPullBitCreate)) 61 | log.Logger.Infof("%v", ppp1.ToString(false)) 62 | wrapper1.ApplyPushPullPack(ppp1) 63 | 64 | client2 := orda.NewClient(conf, t.Name()+"2") 65 | counter2 := client2.SubscribeCounter(t.Name(), nil) 66 | wrapper2 := wrapper.NewDatatypeWrapper(counter2) 67 | testonly.RegisterClient(t, svc, wrapper2.GetClientModel()) 68 | 69 | res, err = svc.ProcessPushPull(gocontext.TODO(), wrapper2.CreatePushPullMessage()) 70 | require.NoError(t, err) 71 | ppp2 := res.GetPushPullPacks()[0] 72 | log.Logger.Infof("%v", ppp2.ToString(false)) 73 | require.True(t, ppp2.CheckPoint.Compare(model.NewSetCheckPoint(2, 0))) 74 | require.Equal(t, len(ppp2.Operations), 2) 75 | require.Equal(t, ppp2.GetOption(), uint32(model.PushPullBitSubscribe)) 76 | wrapper2.ApplyPushPullPack(ppp2) 77 | 78 | _, _ = counter2.Increase() 79 | _, _ = counter2.Increase() 80 | 81 | res, err = svc.ProcessPushPull(gocontext.TODO(), wrapper2.CreatePushPullMessage()) 82 | require.NoError(t, err) 83 | ppp3 := res.GetPushPullPacks()[0] 84 | log.Logger.Infof("%v", ppp3.ToString(false)) 85 | require.True(t, ppp3.CheckPoint.Compare(model.NewSetCheckPoint(4, 2))) 86 | require.Equal(t, len(ppp3.Operations), 0) 87 | require.Equal(t, ppp3.GetOption(), uint32(model.PushPullBitNormal)) 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/orda-io/orda 2 | 3 | replace ( 4 | github.com/orda-io/orda/client => ./client 5 | github.com/orda-io/orda/server => ./server 6 | ) 7 | 8 | require ( 9 | github.com/orda-io/orda/client v0.0.0-20220818033301-4a9396b77850 10 | github.com/orda-io/orda/server v0.0.0-20220801082945-cf9794afb5e4 11 | github.com/stretchr/testify v1.8.0 12 | google.golang.org/protobuf v1.28.1 13 | ) 14 | 15 | require ( 16 | github.com/KyleBanks/depth v1.2.1 // indirect 17 | github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect 18 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 19 | github.com/davecgh/go-spew v1.1.1 // indirect 20 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 21 | github.com/eclipse/paho.mqtt.golang v1.4.1 // indirect 22 | github.com/fatih/color v1.13.0 // indirect 23 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 24 | github.com/go-openapi/jsonreference v0.20.0 // indirect 25 | github.com/go-openapi/spec v0.20.7 // indirect 26 | github.com/go-openapi/swag v0.22.3 // indirect 27 | github.com/go-redis/redis/v8 v8.11.5 // indirect 28 | github.com/go-redsync/redsync/v4 v4.5.1 // indirect 29 | github.com/golang/protobuf v1.5.2 // indirect 30 | github.com/golang/snappy v0.0.4 // indirect 31 | github.com/gorilla/websocket v1.5.0 // indirect 32 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect 33 | github.com/hashicorp/errwrap v1.1.0 // indirect 34 | github.com/hashicorp/go-multierror v1.1.1 // indirect 35 | github.com/josharian/intern v1.0.0 // indirect 36 | github.com/klauspost/compress v1.15.9 // indirect 37 | github.com/logrusorgru/aurora v2.0.3+incompatible // indirect 38 | github.com/mailru/easyjson v0.7.7 // indirect 39 | github.com/matoous/go-nanoid/v2 v2.0.0 // indirect 40 | github.com/mattn/go-colorable v0.1.13 // indirect 41 | github.com/mattn/go-isatty v0.0.16 // indirect 42 | github.com/mitchellh/mapstructure v1.5.0 // indirect 43 | github.com/montanaflynn/stats v0.6.6 // indirect 44 | github.com/pkg/errors v0.9.1 // indirect 45 | github.com/pmezard/go-difflib v1.0.0 // indirect 46 | github.com/sirupsen/logrus v1.9.0 // indirect 47 | github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect 48 | github.com/swaggo/http-swagger v1.3.3 // indirect 49 | github.com/swaggo/swag v1.8.5 // indirect 50 | github.com/tidwall/gjson v1.14.3 // indirect 51 | github.com/tidwall/match v1.1.1 // indirect 52 | github.com/tidwall/pretty v1.2.0 // indirect 53 | github.com/viney-shih/go-lock v1.1.2 // indirect 54 | github.com/wI2L/jsondiff v0.2.0 // indirect 55 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 56 | github.com/xdg-go/scram v1.1.1 // indirect 57 | github.com/xdg-go/stringprep v1.0.3 // indirect 58 | github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect 59 | github.com/ztrue/tracerr v0.3.0 // indirect 60 | go.mongodb.org/mongo-driver v1.10.1 // indirect 61 | golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d // indirect 62 | golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect 63 | golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect 64 | golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 // indirect 65 | golang.org/x/text v0.3.7 // indirect 66 | golang.org/x/tools v0.1.12 // indirect 67 | google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect 68 | google.golang.org/grpc v1.49.0 // indirect 69 | gopkg.in/yaml.v3 v3.0.1 // indirect 70 | ) 71 | 72 | go 1.18 73 | -------------------------------------------------------------------------------- /server/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/orda-io/orda/server 2 | 3 | go 1.18 4 | 5 | replace github.com/orda-io/orda/client => ../client 6 | 7 | replace github.com/orda-io/orda => ../ 8 | 9 | require ( 10 | github.com/eclipse/paho.mqtt.golang v1.4.1 11 | github.com/go-redis/redis/v8 v8.11.5 12 | github.com/go-redsync/redsync/v4 v4.5.1 13 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 14 | github.com/orda-io/orda v0.0.0-00010101000000-000000000000 15 | github.com/orda-io/orda/client v0.0.0-20220818033301-4a9396b77850 16 | github.com/stretchr/testify v1.8.0 17 | github.com/swaggo/http-swagger v1.3.3 18 | github.com/swaggo/swag v1.8.5 19 | github.com/urfave/cli/v2 v2.11.2 20 | github.com/viney-shih/go-lock v1.1.2 21 | go.mongodb.org/mongo-driver v1.10.1 22 | google.golang.org/grpc v1.49.0 23 | ) 24 | 25 | require ( 26 | github.com/KyleBanks/depth v1.2.1 // indirect 27 | github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect 28 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 29 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 30 | github.com/davecgh/go-spew v1.1.1 // indirect 31 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 32 | github.com/fatih/color v1.13.0 // indirect 33 | github.com/go-openapi/jsonpointer v0.19.5 // indirect 34 | github.com/go-openapi/jsonreference v0.20.0 // indirect 35 | github.com/go-openapi/spec v0.20.7 // indirect 36 | github.com/go-openapi/swag v0.22.3 // indirect 37 | github.com/golang/protobuf v1.5.2 // indirect 38 | github.com/golang/snappy v0.0.4 // indirect 39 | github.com/gorilla/websocket v1.5.0 // indirect 40 | github.com/hashicorp/errwrap v1.1.0 // indirect 41 | github.com/hashicorp/go-multierror v1.1.1 // indirect 42 | github.com/josharian/intern v1.0.0 // indirect 43 | github.com/klauspost/compress v1.15.9 // indirect 44 | github.com/logrusorgru/aurora v2.0.3+incompatible // indirect 45 | github.com/mailru/easyjson v0.7.7 // indirect 46 | github.com/matoous/go-nanoid/v2 v2.0.0 // indirect 47 | github.com/mattn/go-colorable v0.1.13 // indirect 48 | github.com/mattn/go-isatty v0.0.16 // indirect 49 | github.com/mitchellh/mapstructure v1.5.0 // indirect 50 | github.com/montanaflynn/stats v0.6.6 // indirect 51 | github.com/pkg/errors v0.9.1 // indirect 52 | github.com/pmezard/go-difflib v1.0.0 // indirect 53 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 54 | github.com/sirupsen/logrus v1.9.0 // indirect 55 | github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect 56 | github.com/tidwall/gjson v1.14.3 // indirect 57 | github.com/tidwall/match v1.1.1 // indirect 58 | github.com/tidwall/pretty v1.2.0 // indirect 59 | github.com/wI2L/jsondiff v0.2.0 // indirect 60 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 61 | github.com/xdg-go/scram v1.1.1 // indirect 62 | github.com/xdg-go/stringprep v1.0.3 // indirect 63 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect 64 | github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect 65 | github.com/ztrue/tracerr v0.3.0 // indirect 66 | golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d // indirect 67 | golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b // indirect 68 | golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect 69 | golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64 // indirect 70 | golang.org/x/text v0.3.7 // indirect 71 | golang.org/x/tools v0.1.12 // indirect 72 | google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect 73 | google.golang.org/protobuf v1.28.1 // indirect 74 | gopkg.in/yaml.v3 v3.0.1 // indirect 75 | ) 76 | -------------------------------------------------------------------------------- /server/mongodb/collection_operations.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/iface" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | "github.com/orda-io/orda/server/schema" 9 | 10 | "go.mongodb.org/mongo-driver/bson" 11 | "go.mongodb.org/mongo-driver/mongo/options" 12 | 13 | "github.com/orda-io/orda/server/constants" 14 | ) 15 | 16 | // InsertOperations inserts operations into MongoDB 17 | func (its *MongoCollections) InsertOperations( 18 | ctx iface.OrdaContext, 19 | operations []interface{}, 20 | ) errors.OrdaError { 21 | if operations == nil { 22 | return nil 23 | } 24 | result, err := its.operations.InsertMany(ctx, operations) 25 | if err != nil { 26 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 27 | } 28 | if len(result.InsertedIDs) != len(operations) { 29 | msg := fmt.Sprintf("the inserted operations (%d) are less than all the intended ones (%d)", 30 | len(result.InsertedIDs), len(operations)) 31 | return errors.ServerDBQuery.New(ctx.L(), msg) 32 | } 33 | return nil 34 | } 35 | 36 | // DeleteOperation deletes operations for the specified sseq 37 | func (its *MongoCollections) DeleteOperation( 38 | ctx iface.OrdaContext, 39 | duid string, 40 | sseq uint32, 41 | ) (int64, errors.OrdaError) { 42 | f := schema.GetFilter(). 43 | AddFilterEQ(schema.OperationDocFields.DUID, duid). 44 | AddFilterEQ(schema.OperationDocFields.Sseq, sseq) 45 | result, err := its.operations.DeleteOne(ctx, f) 46 | if err != nil { 47 | return 0, errors.ServerDBQuery.New(ctx.L(), err.Error()) 48 | } 49 | return result.DeletedCount, nil 50 | } 51 | 52 | // GetOperations gets operations of the specified range. For each operation, a given handler is called. 53 | func (its *MongoCollections) GetOperations( 54 | ctx iface.OrdaContext, 55 | duid string, 56 | from, to uint64, 57 | ) (model.OpList, []uint64, errors.OrdaError) { 58 | f := schema.GetFilter(). 59 | AddFilterEQ(schema.OperationDocFields.DUID, duid). 60 | AddFilterGTE(schema.OperationDocFields.Sseq, from) 61 | if to != constants.InfinitySseq { 62 | f.AddFilterLTE(schema.OperationDocFields.Sseq, to) 63 | } 64 | opt := options.Find() 65 | 66 | opt.SetSort(bson.D{{ 67 | Key: schema.OperationDocFields.Sseq, 68 | Value: 1, 69 | }}) 70 | cursor, err := its.operations.Find(ctx, f, opt) 71 | if err != nil { 72 | return nil, nil, errors.ServerDBQuery.New(ctx.L(), err.Error()) 73 | } 74 | var opList []*model.Operation 75 | var sseqList []uint64 76 | for cursor.Next(ctx) { 77 | var opDoc schema.OperationDoc 78 | if err := cursor.Decode(&opDoc); err != nil { 79 | return nil, nil, errors.ServerDBDecode.New(ctx.L(), err.Error()) 80 | } 81 | opList = append(opList, opDoc.GetOperation()) 82 | sseqList = append(sseqList, opDoc.Sseq) 83 | } 84 | return opList, sseqList, nil 85 | } 86 | 87 | // PurgeOperations purges operations for the specified datatype. 88 | func (its *MongoCollections) PurgeOperations(ctx iface.OrdaContext, collectionNum int32, duid string) errors.OrdaError { 89 | f := schema.GetFilter(). 90 | AddFilterEQ(schema.OperationDocFields.CollectionNum, collectionNum). 91 | AddFilterEQ(schema.OperationDocFields.DUID, duid) 92 | result, err := its.operations.DeleteMany(ctx, f) 93 | if err != nil { 94 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 95 | } 96 | if result.DeletedCount > 0 { 97 | ctx.L().Infof("deleted %d operations of %s(%d)", result.DeletedCount, duid, collectionNum) 98 | return nil 99 | } 100 | ctx.L().Warnf("deleted no operations") 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /server/mongodb/collection_collections.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/server/schema" 7 | "time" 8 | 9 | "go.mongodb.org/mongo-driver/mongo" 10 | ) 11 | 12 | // GetCollection gets a collectionDoc with the specified name. 13 | func (its *MongoCollections) GetCollection(ctx iface.OrdaContext, name string) (*schema.CollectionDoc, errors.OrdaError) { 14 | sr := its.collections.FindOne(ctx, schema.FilterByID(name)) 15 | if err := sr.Err(); err != nil { 16 | if err == mongo.ErrNoDocuments { 17 | return nil, nil 18 | } 19 | return nil, errors.ServerDBQuery.New(ctx.L(), err.Error()) 20 | } 21 | var collection schema.CollectionDoc 22 | if err := sr.Decode(&collection); err != nil { 23 | return nil, errors.ServerDBDecode.New(ctx.L(), err.Error()) 24 | } 25 | return &collection, nil 26 | } 27 | 28 | // DeleteCollection deletes collections with the specified name. 29 | func (its *MongoCollections) DeleteCollection(ctx iface.OrdaContext, name string) errors.OrdaError { 30 | result, err := its.collections.DeleteOne(ctx, schema.FilterByID(name)) 31 | if err != nil { 32 | return errors.ServerDBQuery.New(ctx.L(), err.Error()) 33 | } 34 | ctx.L().Infof("Collection '%s' is successfully removed:%d", name, result.DeletedCount) 35 | return nil 36 | } 37 | 38 | // InsertCollection inserts a document for the specified collection. 39 | func (its *MongoCollections) InsertCollection( 40 | ctx iface.OrdaContext, 41 | name string, 42 | ) (collection *schema.CollectionDoc, err errors.OrdaError) { 43 | if err := its.doTransaction(ctx, func() errors.OrdaError { 44 | num, err := its.GetNextCollectionNum(ctx) 45 | if err != nil { 46 | return err 47 | } 48 | collection = &schema.CollectionDoc{ 49 | Name: name, 50 | Num: num, 51 | CreatedAt: time.Now(), 52 | } 53 | _, err2 := its.collections.InsertOne(ctx, collection) 54 | if err2 != nil { 55 | return errors.ServerDBQuery.New(ctx.L(), err2.Error()) 56 | } 57 | ctx.L().Infof("insert collection: %+v", collection) 58 | return nil 59 | }); err != nil { 60 | return nil, err 61 | } 62 | return collection, nil 63 | } 64 | 65 | // PurgeAllDocumentsOfCollection purges all data for the specified collection. 66 | func (its *MongoCollections) PurgeAllDocumentsOfCollection(ctx iface.OrdaContext, name string) errors.OrdaError { 67 | if err := its.doTransaction(ctx, func() errors.OrdaError { 68 | collectionDoc, err := its.GetCollection(ctx, name) 69 | if err != nil { 70 | return err 71 | } 72 | if collectionDoc == nil { 73 | return nil 74 | } 75 | ctx.L().Infof("purge collection#%d '%s'", collectionDoc.Num, name) 76 | return its.purgeAllDocumentsOfCollectionNum(ctx, collectionDoc.Num) 77 | }); err != nil { 78 | return err 79 | } 80 | return nil 81 | } 82 | 83 | // PurgeAllDocumentsOfCollection purges all data for the specified collection. 84 | func (its *MongoCollections) purgeAllDocumentsOfCollectionNum(ctx iface.OrdaContext, collectionNum int32) errors.OrdaError { 85 | if err := its.purgeAllCollectionDatatypes(ctx, collectionNum); err != nil { 86 | return err 87 | } 88 | if err := its.purgeAllCollectionClients(ctx, collectionNum); err != nil { 89 | return err 90 | } 91 | filter := schema.GetFilter().AddFilterEQ(schema.CollectionDocFields.Name, collectionNum) 92 | 93 | result, err2 := its.collections.DeleteOne(ctx, filter) 94 | if err2 != nil { 95 | return errors.ServerDBQuery.New(ctx.L(), err2.Error()) 96 | } 97 | ctx.L().Infof("delete %d collection#%d", result.DeletedCount, collectionNum) 98 | return nil 99 | 100 | } 101 | -------------------------------------------------------------------------------- /server/mongodb/mongo_collection_test.go: -------------------------------------------------------------------------------- 1 | package mongodb_test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | "github.com/orda-io/orda/client/pkg/operations" 9 | "github.com/orda-io/orda/client/pkg/types" 10 | "github.com/orda-io/orda/server/schema" 11 | "github.com/orda-io/orda/server/testonly" 12 | 13 | "github.com/orda-io/orda/server/constants" 14 | integration "github.com/orda-io/orda/test" 15 | "github.com/stretchr/testify/require" 16 | "go.mongodb.org/mongo-driver/bson" 17 | 18 | "sync" 19 | "testing" 20 | ) 21 | 22 | func TestMongo(t *testing.T) { 23 | mongo, ctx, collectionNum := integration.InitTestDBCollection(t, testonly.TestDBName) 24 | 25 | t.Run("Can make collections simultaneously", func(t *testing.T) { 26 | madeCollections := make(map[int32]*schema.CollectionDoc) 27 | 28 | for i := 0; i < 10; i++ { 29 | require.NoError(t, mongo.DeleteCollection(ctx, fmt.Sprintf("hello_%d", i))) 30 | } 31 | 32 | wg := sync.WaitGroup{} 33 | wg.Add(10) 34 | for i := 0; i < 10; i++ { 35 | go func(idx int) { 36 | collection, err := mongo.InsertCollection(ctx, fmt.Sprintf("hello_%d", idx)) 37 | require.NoError(t, err) 38 | madeCollections[collection.Num] = collection 39 | ctx.L().Infof("made collection %d", collection.Num) 40 | wg.Done() 41 | }(i) 42 | } 43 | 44 | wg.Wait() 45 | require.Equal(t, 10, len(madeCollections)) 46 | }) 47 | 48 | t.Run("Can manipulate datatypeDoc", func(t *testing.T) { 49 | d := schema.NewDatatypeDoc("test_duid2", "test_key", collectionNum, "test_datatype") 50 | 51 | d.AddNewClient("aaaa", int8(model.ClientType_EPHEMERAL), true) 52 | 53 | require.NoError(t, mongo.UpdateDatatype(ctx, d)) 54 | 55 | datatypeDoc1, err := mongo.GetDatatype(ctx, d.DUID) 56 | require.NoError(t, err) 57 | 58 | log.Logger.Infof("%v", datatypeDoc1) 59 | datatypeDoc2, err := mongo.GetDatatype(ctx, "not exist") 60 | require.NoError(t, err) 61 | require.Nil(t, datatypeDoc2) 62 | 63 | datatypeDoc3, err := mongo.GetDatatypeByKey(ctx, d.CollectionNum, d.Key) 64 | require.NoError(t, err) 65 | 66 | log.Logger.Infof("%+v", datatypeDoc3) 67 | }) 68 | 69 | t.Run("Can manipulate operationDoc", func(t *testing.T) { 70 | snap, err := json.Marshal(&testSnapshot{Value: 1}) 71 | require.NoError(t, err) 72 | op := operations.NewSnapshotOperation(model.TypeOfDatatype_DOCUMENT, snap) 73 | 74 | op.ID = model.NewOperationIDWithCUID(types.NewUID()) 75 | modelOp := op.ToModelOperation() 76 | 77 | var oplist []interface{} 78 | opDoc := schema.NewOperationDoc(modelOp, "test_duid", 1, collectionNum) 79 | log.Logger.Infof("%+v", opDoc.GetOperation()) 80 | log.Logger.Infof("%+v", modelOp) 81 | oplist = append(oplist, opDoc) 82 | 83 | _, err = mongo.DeleteOperation(ctx, opDoc.DUID, 1) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | 88 | if err := mongo.InsertOperations(ctx, oplist); err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | opList, _, err := mongo.GetOperations(ctx, opDoc.DUID, 1, constants.InfinitySseq) 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | require.Equal(t, 1, len(opList)) 97 | 98 | }) 99 | 100 | t.Run("Can change json to bson", func(t *testing.T) { 101 | j := &struct { 102 | Key string 103 | Array []string 104 | }{ 105 | Key: "world", 106 | Array: []string{"x", "y"}, 107 | } 108 | data1, err := bson.Marshal(j) 109 | require.NoError(t, err) 110 | log.Logger.Infof("%v", data1) 111 | }) 112 | } 113 | 114 | type testSnapshot struct { 115 | Value int32 `json:"value"` 116 | } 117 | 118 | func (its *testSnapshot) ToJSON() interface{} { 119 | return its 120 | } 121 | -------------------------------------------------------------------------------- /client/pkg/internal/managers/notify.go: -------------------------------------------------------------------------------- 1 | package managers 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/orda-io/orda/client/pkg/context" 6 | "github.com/orda-io/orda/client/pkg/errors" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | 9 | mqtt "github.com/eclipse/paho.mqtt.golang" 10 | ) 11 | 12 | // NotifyManager manages notifications from Orda Server 13 | type NotifyManager struct { 14 | client mqtt.Client 15 | ctx *context.ClientContext 16 | channel chan *notificationMsg 17 | receiver notificationReceiver 18 | } 19 | 20 | type notificationReceiver interface { 21 | ReceiveNotification(topic string, notification model.Notification) 22 | } 23 | 24 | type pubSubNotificationType uint8 25 | 26 | type notificationMsg struct { 27 | typeOf pubSubNotificationType 28 | topic string 29 | msg interface{} 30 | } 31 | 32 | const ( 33 | notificationError pubSubNotificationType = iota 34 | notificationQuit 35 | notificationPushPull 36 | ) 37 | 38 | // NewNotifyManager creates an instance of NotifyManager 39 | func NewNotifyManager(ctx *context.ClientContext, pubSubAddr string, cm *model.Client) *NotifyManager { 40 | pubSubOpts := mqtt.NewClientOptions(). 41 | AddBroker(pubSubAddr). 42 | SetClientID(cm.GetCUID()). 43 | SetUsername(cm.Alias) 44 | client := mqtt.NewClient(pubSubOpts) 45 | channel := make(chan *notificationMsg) 46 | return &NotifyManager{ 47 | ctx: ctx, 48 | client: client, 49 | channel: channel, 50 | } 51 | } 52 | 53 | // SubscribeNotification subscribes notification for a topic. 54 | func (its *NotifyManager) SubscribeNotification(topic string) errors.OrdaError { 55 | token := its.client.Subscribe(topic, 0, its.notificationSubscribeFunc) 56 | if token.Wait() && token.Error() != nil { 57 | return errors.ClientConnect.New(its.ctx.L(), "notification ", token.Error()) 58 | } 59 | return nil 60 | } 61 | 62 | func (its *NotifyManager) notificationSubscribeFunc(client mqtt.Client, msg mqtt.Message) { 63 | notification := model.Notification{} 64 | if err := json.Unmarshal(msg.Payload(), ¬ification); err != nil { 65 | its.channel <- ¬ificationMsg{ 66 | typeOf: notificationError, 67 | msg: err, 68 | } 69 | return 70 | } 71 | 72 | notificationPushPull := ¬ificationMsg{ 73 | typeOf: notificationPushPull, 74 | topic: msg.Topic(), 75 | msg: notification, 76 | } 77 | its.channel <- notificationPushPull 78 | } 79 | 80 | // Connect make a connection with Orda notification server. 81 | func (its *NotifyManager) Connect() errors.OrdaError { 82 | if token := its.client.Connect(); token.Wait() && token.Error() != nil { 83 | return errors.ClientConnect.New(its.ctx.L(), "notification server") 84 | } 85 | its.ctx.L().Infof("connect to notification server") 86 | go its.notificationLoop() 87 | return nil 88 | } 89 | 90 | // Close closes a connection with Orda notification server. 91 | func (its *NotifyManager) Close() { 92 | its.channel <- ¬ificationMsg{ 93 | typeOf: notificationQuit, 94 | } 95 | its.client.Disconnect(0) 96 | } 97 | 98 | // SetReceiver sets receiver which is going to receive notifications, i.e., DatatypeManager 99 | func (its *NotifyManager) SetReceiver(receiver notificationReceiver) { 100 | its.receiver = receiver 101 | } 102 | 103 | func (its *NotifyManager) notificationLoop() { 104 | for { 105 | note := <-its.channel 106 | switch note.typeOf { 107 | case notificationError: 108 | err := note.msg.(error) 109 | its.ctx.L().Errorf("receive a notification error: %s", err.Error()) 110 | case notificationQuit: 111 | its.ctx.L().Infof("quit notification loop") 112 | return 113 | case notificationPushPull: 114 | notification := note.msg.(model.Notification) 115 | its.receiver.ReceiveNotification(note.topic, notification) 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /client/pkg/operations/meta.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/errors" 5 | "github.com/orda-io/orda/client/pkg/model" 6 | ) 7 | 8 | // ////////////////// TransactionOperation //////////////////// 9 | 10 | // NewTransactionOperation creates a TransactionOperation. 11 | func NewTransactionOperation(tag string) *TransactionOperation { 12 | return &TransactionOperation{ 13 | baseOperation: newBaseOperation( 14 | model.TypeOfOperation_TRANSACTION, 15 | nil, 16 | &TransactionBody{ 17 | Tag: tag, 18 | }, 19 | ), 20 | } 21 | } 22 | 23 | // TransactionBody is the body of TransactionOperation 24 | type TransactionBody struct { 25 | Tag string 26 | NumOfOps int32 27 | } 28 | 29 | // TransactionOperation is used to begin a transaction. 30 | type TransactionOperation struct { 31 | baseOperation 32 | } 33 | 34 | // GetBody returns the body 35 | func (its *TransactionOperation) GetBody() *TransactionBody { 36 | return its.Body.(*TransactionBody) 37 | } 38 | 39 | // SetNumOfOps sets the number operations in the transaction. 40 | func (its *TransactionOperation) SetNumOfOps(numOfOps int) { 41 | its.GetBody().NumOfOps = int32(numOfOps) 42 | } 43 | 44 | // GetNumOfOps returns the number operations in the transaction. 45 | func (its *TransactionOperation) GetNumOfOps() int32 { 46 | return its.GetBody().NumOfOps 47 | } 48 | 49 | // ////////////////// ErrorOperation //////////////////// 50 | 51 | // NewErrorOperation creates an ErrorOperation. 52 | func NewErrorOperation(err errors.OrdaError) *ErrorOperation { 53 | return &ErrorOperation{ 54 | baseOperation: newBaseOperation( 55 | model.TypeOfOperation_ERROR, 56 | model.NewOperationID(), 57 | &errorBody{ 58 | Code: err.GetCode(), 59 | Msg: err.Error(), 60 | }, 61 | ), 62 | } 63 | } 64 | 65 | // NewErrorOperationWithCodeAndMsg creates a new ErrorOperation with code and message 66 | func NewErrorOperationWithCodeAndMsg(code errors.ErrorCode, msg string) *ErrorOperation { 67 | return &ErrorOperation{ 68 | baseOperation: newBaseOperation( 69 | model.TypeOfOperation_ERROR, 70 | model.NewOperationID(), 71 | &errorBody{ 72 | Code: code, 73 | Msg: msg, 74 | }, 75 | ), 76 | } 77 | } 78 | 79 | type errorBody struct { 80 | Code errors.ErrorCode 81 | Msg string 82 | } 83 | 84 | // ErrorOperation is used to deliver an error. 85 | type ErrorOperation struct { 86 | baseOperation 87 | } 88 | 89 | func (its *ErrorOperation) getBody() *errorBody { 90 | return its.Body.(*errorBody) 91 | } 92 | 93 | // GetPushPullError returns PushPullError from ErrorOperation 94 | func (its *ErrorOperation) GetPushPullError() *errors.PushPullError { 95 | return &errors.PushPullError{ 96 | Code: its.getBody().Code, 97 | Msg: its.getBody().Msg, 98 | } 99 | } 100 | 101 | // GetCode returns an error code 102 | func (its *ErrorOperation) GetCode() errors.ErrorCode { 103 | return its.Body.(*errorBody).Code 104 | } 105 | 106 | // GetMessage returns an error message 107 | func (its *ErrorOperation) GetMessage() string { 108 | return its.Body.(*errorBody).Msg 109 | } 110 | 111 | // ////////////////// SnapshotOperation //////////////////// 112 | 113 | // NewSnapshotOperation creates a SnapshotOperation 114 | func NewSnapshotOperation(typeOf model.TypeOfDatatype, snapshot []byte) *SnapshotOperation { 115 | var typeOfOp = model.TypeOfOperation(typeOf*10 + 10) 116 | 117 | return &SnapshotOperation{ 118 | baseOperation: newBaseOperation(typeOfOp, model.NewOperationID(), snapshot), 119 | } 120 | } 121 | 122 | // SnapshotOperation is used to deliver the snapshot of a datatype. 123 | type SnapshotOperation struct { 124 | baseOperation 125 | } 126 | 127 | // GetBody returns the body 128 | func (its *SnapshotOperation) GetBody() []byte { 129 | return its.Body.([]byte) 130 | } 131 | -------------------------------------------------------------------------------- /client/pkg/internal/managers/sync.go: -------------------------------------------------------------------------------- 1 | package managers 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/context" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/model" 7 | "google.golang.org/grpc" 8 | ) 9 | 10 | // SyncManager is a manager exchanging request and response with Orda server. 11 | type SyncManager struct { 12 | seq uint32 13 | ctx *context.ClientContext 14 | conn *grpc.ClientConn 15 | client *model.Client 16 | serverAddr string 17 | serviceClient model.OrdaServiceClient 18 | notifyManager *NotifyManager 19 | } 20 | 21 | // NewSyncManager creates an instance of SyncManager. 22 | func NewSyncManager( 23 | ctx *context.ClientContext, 24 | client *model.Client, 25 | serverAddr string, 26 | notificationAddr string, 27 | ) *SyncManager { 28 | var notifyManager *NotifyManager 29 | switch client.SyncType { 30 | case model.SyncType_LOCAL_ONLY, model.SyncType_MANUALLY: 31 | notifyManager = nil 32 | case model.SyncType_REALTIME: 33 | notifyManager = NewNotifyManager(ctx, notificationAddr, client) 34 | } 35 | return &SyncManager{ 36 | seq: 0, 37 | ctx: ctx, 38 | serverAddr: serverAddr, 39 | client: client, 40 | notifyManager: notifyManager, 41 | } 42 | } 43 | 44 | func (its *SyncManager) nextSeq() uint32 { 45 | currentSeq := its.seq 46 | its.seq++ 47 | return currentSeq 48 | } 49 | 50 | // Connect makes connections with Orda GRPC and notification servers. 51 | func (its *SyncManager) Connect() errors.OrdaError { 52 | conn, err := grpc.Dial(its.serverAddr, grpc.WithInsecure()) 53 | if err != nil { 54 | return errors.ClientConnect.New(its.ctx.L(), err.Error()) 55 | } 56 | its.conn = conn 57 | its.serviceClient = model.NewOrdaServiceClient(its.conn) 58 | its.ctx.L().Info("connect to grpc server") 59 | if its.notifyManager != nil { 60 | if err := its.notifyManager.Connect(); err != nil { 61 | return err 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | // Close closes connections with Orda GRPC and notification servers. 68 | func (its *SyncManager) Close() errors.OrdaError { 69 | if its.notifyManager != nil { 70 | its.notifyManager.Close() 71 | } 72 | if err := its.conn.Close(); err != nil { 73 | return errors.ClientClose.New(its.ctx.L(), err.Error()) 74 | } 75 | return nil 76 | } 77 | 78 | // Sync exchanges PUSHPULL_REQUEST and PUSHPULL_RESPONSE 79 | func (its *SyncManager) Sync(pppList ...*model.PushPullPack) (*model.PushPullMessage, errors.OrdaError) { 80 | request := model.NewPushPullMessage(its.nextSeq(), its.client, pppList...) 81 | its.ctx.L().Infof("REQ[PUPU] %s", request.ToString(false)) 82 | response, err := its.serviceClient.ProcessPushPull(its.ctx, request) 83 | if err != nil { 84 | return nil, errors.ClientSync.New(its.ctx.L(), err.Error()) 85 | } 86 | its.ctx.L().Infof("RES[PUPU] %v", response.ToString(false)) 87 | return response, nil 88 | } 89 | 90 | // ExchangeClientRequestResponse exchanges CLIENT_REQUEST and CLIENT_RESPONSE 91 | func (its *SyncManager) ExchangeClientRequestResponse() errors.OrdaError { 92 | request := model.NewClientMessage(its.client) 93 | 94 | response, err := its.serviceClient.ProcessClient(its.ctx, request) 95 | if err != nil { 96 | return errors.ClientSync.New(its.ctx.L(), err.Error()) 97 | } 98 | its.ctx.L().Infof("RES[CLIE] response: %s", response.ToString()) 99 | return nil 100 | } 101 | 102 | func (its *SyncManager) subscribeNotification(topic string) errors.OrdaError { 103 | if its.notifyManager != nil { 104 | return its.notifyManager.SubscribeNotification(topic) 105 | } 106 | return nil 107 | } 108 | 109 | func (its *SyncManager) setNotificationReceiver(receiver notificationReceiver) { 110 | if its.notifyManager != nil { 111 | its.notifyManager.SetReceiver(receiver) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /client/pkg/operations/converter.go: -------------------------------------------------------------------------------- 1 | package operations 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | ) 9 | 10 | // ModelToOperation changes a model.Operation to an operations.Operation 11 | func ModelToOperation(op *model.Operation) iface.Operation { 12 | switch op.OpType { 13 | case model.TypeOfOperation_COUNTER_SNAPSHOT, 14 | model.TypeOfOperation_MAP_SNAPSHOT, 15 | model.TypeOfOperation_LIST_SNAPSHOT, 16 | model.TypeOfOperation_DOC_SNAPSHOT: 17 | return &SnapshotOperation{ 18 | baseOperation: newBaseOperation(op.OpType, op.ID, op.Body), 19 | } 20 | case model.TypeOfOperation_ERROR: 21 | return &ErrorOperation{ 22 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &errorBody{})), 23 | } 24 | case model.TypeOfOperation_TRANSACTION: 25 | return &TransactionOperation{ 26 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &TransactionBody{})), 27 | } 28 | case model.TypeOfOperation_COUNTER_INCREASE: 29 | return &IncreaseOperation{ 30 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &increaseBody{})), 31 | } 32 | case model.TypeOfOperation_MAP_PUT: 33 | return &PutOperation{ 34 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &PutBody{})), 35 | } 36 | case model.TypeOfOperation_MAP_REMOVE: 37 | return &RemoveOperation{ 38 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &RemoveBody{})), 39 | } 40 | case model.TypeOfOperation_LIST_INSERT: 41 | return &InsertOperation{ 42 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &InsertBody{})), 43 | } 44 | case model.TypeOfOperation_LIST_DELETE: 45 | return &DeleteOperation{ 46 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &DeleteBody{})), 47 | } 48 | case model.TypeOfOperation_LIST_UPDATE: 49 | return &UpdateOperation{ 50 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &UpdateBody{})), 51 | } 52 | case model.TypeOfOperation_DOC_OBJ_PUT: 53 | return &DocPutInObjOperation{ 54 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &DocPutInObjBody{})), 55 | } 56 | case model.TypeOfOperation_DOC_OBJ_RMV: 57 | return &DocRemoveInObjOperation{ 58 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &DocRemoveInObjectBody{})), 59 | } 60 | case model.TypeOfOperation_DOC_ARR_INS: 61 | return &DocInsertToArrayOperation{ 62 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &DocInsertToArrayBody{})), 63 | } 64 | case model.TypeOfOperation_DOC_ARR_DEL: 65 | return &DocDeleteInArrayOperation{ 66 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &DocDeleteInArrayBody{})), 67 | } 68 | case model.TypeOfOperation_DOC_ARR_UPD: 69 | return &DocUpdateInArrayOperation{ 70 | baseOperation: newBaseOperation(op.OpType, op.ID, unmarshalBody(op.Body, &DocUpdateInArrayBody{})), 71 | } 72 | } 73 | panic("unsupported type of operation") 74 | } 75 | 76 | func unmarshalBody(b []byte, c interface{}) interface{} { 77 | switch c.(type) { 78 | case string: 79 | return string(b) 80 | case []byte: 81 | return b 82 | } 83 | if err := json.Unmarshal(b, c); err != nil { 84 | log.Logger.Errorf("%v", string(b)) 85 | panic(err) // TODO: this should ne handled 86 | } 87 | return c 88 | } 89 | 90 | func marshalBody(c interface{}) []byte { 91 | switch cast := c.(type) { 92 | case string: 93 | return []byte(cast) 94 | case []byte: 95 | return cast 96 | } 97 | j, err := json.Marshal(c) 98 | if err != nil { 99 | panic(err) // TODO: this should ne handled 100 | } 101 | return j 102 | } 103 | -------------------------------------------------------------------------------- /server/snapshot/manager.go: -------------------------------------------------------------------------------- 1 | package snapshot 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/iface" 7 | "github.com/orda-io/orda/client/pkg/orda" 8 | 9 | "github.com/orda-io/orda/server/constants" 10 | "github.com/orda-io/orda/server/managers" 11 | "github.com/orda-io/orda/server/schema" 12 | ) 13 | 14 | // Manager is a struct that updates snapshot of a datatype in Orda server 15 | type Manager struct { 16 | ctx iface.OrdaContext 17 | managers *managers.Managers 18 | datatypeDoc *schema.DatatypeDoc 19 | collectionDoc *schema.CollectionDoc 20 | } 21 | 22 | // NewManager returns an instance of Snapshot Manager 23 | func NewManager( 24 | ctx iface.OrdaContext, 25 | managers *managers.Managers, 26 | datatypeDoc *schema.DatatypeDoc, 27 | collectionDoc *schema.CollectionDoc, 28 | ) *Manager { 29 | return &Manager{ 30 | ctx: ctx, 31 | managers: managers, 32 | datatypeDoc: datatypeDoc, 33 | collectionDoc: collectionDoc, 34 | } 35 | } 36 | 37 | // GetLatestDatatype returns the datatype from database 38 | func (its *Manager) GetLatestDatatype() (iface.Datatype, uint64, errors.OrdaError) { 39 | var lastSseq uint64 = 0 40 | client := orda.NewClient(orda.NewLocalClientConfig(its.collectionDoc.Name), "orda-server") 41 | datatype := client.CreateDatatype(its.datatypeDoc.Key, its.datatypeDoc.GetType(), nil).(iface.Datatype) 42 | datatype.SetLogger(its.ctx.L()) 43 | if its.datatypeDoc.DUID == "" { 44 | return datatype, lastSseq, nil 45 | } 46 | 47 | datatype.SetDUID(its.datatypeDoc.DUID) 48 | 49 | snapshotDoc, err := its.managers.Mongo.GetLatestSnapshot(its.ctx, its.datatypeDoc.CollectionNum, its.datatypeDoc.DUID) 50 | if err != nil { 51 | return nil, 0, err 52 | } 53 | if snapshotDoc != nil { 54 | lastSseq = snapshotDoc.Sseq 55 | if err = datatype.SetMetaAndSnapshot([]byte(snapshotDoc.Meta), snapshotDoc.Snapshot); err != nil { 56 | return nil, 0, err 57 | } 58 | datatype.ResetWired() 59 | } 60 | opList, sseqList, err := its.managers.Mongo.GetOperations(its.ctx, its.datatypeDoc.DUID, lastSseq+1, constants.InfinitySseq) 61 | if err != nil { 62 | return nil, 0, err 63 | } 64 | 65 | if len(sseqList) <= 0 { 66 | return datatype, lastSseq, nil 67 | } 68 | 69 | its.ctx.L().Infof("apply %d operations: %+v", len(opList), opList.ToString(false)) 70 | if _, err = datatype.ReceiveRemoteModelOperations(opList, false); err != nil { 71 | // TODO: should fix corruption 72 | return nil, 0, err 73 | } 74 | lastSseq = sseqList[len(sseqList)-1] 75 | return datatype, lastSseq, nil 76 | } 77 | 78 | func (its *Manager) getLockKey() string { 79 | return fmt.Sprintf("US:%d:%s", its.collectionDoc.Num, its.datatypeDoc.Key) 80 | } 81 | 82 | // UpdateSnapshot updates snapshot for specified datatype 83 | func (its *Manager) UpdateSnapshot() errors.OrdaError { 84 | lock := its.managers.GetLock(its.ctx, its.getLockKey()) 85 | if !lock.TryLock() { 86 | return errors.ServerUpdateSnapshot.New(its.ctx.L(), "try lock failure") 87 | } 88 | defer lock.Unlock() 89 | its.ctx.L().Infof("BEGIN UPD_SNAP: '%v'", its.datatypeDoc.Key) 90 | datatype, lastSseq, err := its.GetLatestDatatype() 91 | if err != nil { 92 | return err 93 | } 94 | 95 | meta, snap, err := datatype.GetMetaAndSnapshot() 96 | if err != nil { 97 | return err 98 | } 99 | 100 | if err := its.managers.Mongo.InsertSnapshot(its.ctx, its.collectionDoc.Num, its.datatypeDoc.DUID, lastSseq, meta, snap); err != nil { 101 | return err 102 | } 103 | 104 | data := datatype.ToJSON() 105 | 106 | if err := its.managers.Mongo.InsertRealSnapshot(its.ctx, its.collectionDoc.Name, its.datatypeDoc.Key, data, lastSseq); err != nil { 107 | return err 108 | } 109 | its.ctx.L().Infof("FINISH UPD_SNAP: '%v': %d", its.datatypeDoc.Key, lastSseq) 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /test/lock_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "sync" 9 | "testing" 10 | "time" 11 | 12 | "github.com/orda-io/orda/server/redis" 13 | "github.com/orda-io/orda/server/utils" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | var wg = new(sync.WaitGroup) 18 | 19 | func (its *IntegrationTestSuite) TestLock() { 20 | 21 | its.Run("Can lock with redis", func() { 22 | ts1 := time.Now() 23 | wg.Add(2) 24 | go its.tryRedisLock(its.T(), "cli1", true) // 3 seconds 25 | go its.tryRedisLock(its.T(), "cli2", true) // 3 seconds 26 | 27 | wg.Wait() 28 | ts2 := time.Now().Sub(ts1) // more than 6 seconds 29 | its.ctx.L().Infof("%v", ts2) 30 | require.Equal(its.T(), int(ts2.Seconds()), 6) 31 | 32 | wg.Add(2) 33 | its.tryRedisLock(its.T(), "cli3", false) // 3 seconds 34 | its.tryRedisLock(its.T(), "cli4", false) // 5 seconds 35 | wg.Wait() 36 | ts3 := time.Now().Sub(ts1) 37 | require.Equal(its.T(), int(ts3.Seconds()), 14) 38 | }) 39 | 40 | its.Run("Can lock locally", func() { 41 | ts1 := time.Now() 42 | wg.Add(2) 43 | go its.tryLocalLock(its.T(), "cli1", true) 44 | go its.tryLocalLock(its.T(), "cli2", true) 45 | 46 | wg.Wait() 47 | ts2 := time.Now().Sub(ts1) 48 | its.ctx.L().Infof("%v", ts2) 49 | require.Equal(its.T(), int(ts2.Seconds()), 6) 50 | 51 | wg.Add(2) 52 | its.tryLocalLock(its.T(), "cli3", false) // 3 seconds 53 | its.tryLocalLock(its.T(), "cli4", false) // 5 seconds 54 | wg.Wait() 55 | ts3 := time.Now().Sub(ts1) 56 | require.Equal(its.T(), int(ts3.Seconds()), 14) 57 | }) 58 | 59 | its.Run("Can patch documents concurrently", func() { 60 | 61 | jsonFormat := "{ \"json\": \"{\\\"hello\\\": %d}\"}" 62 | wg.Add(2) 63 | json1 := fmt.Sprintf(jsonFormat, 1) 64 | json2 := fmt.Sprintf(jsonFormat, 2) 65 | its.ctx.L().Infof("%s", json1) 66 | go func() { 67 | its.patch(its.T(), json1) 68 | wg.Done() 69 | }() 70 | 71 | go func() { 72 | its.patch(its.T(), json2) 73 | wg.Done() 74 | }() 75 | wg.Wait() 76 | }) 77 | } 78 | 79 | func (its *IntegrationTestSuite) patch(t *testing.T, json string) { 80 | reqBody := bytes.NewBufferString(json) 81 | addr := fmt.Sprintf("http://localhost:%d/api/v1/collections/%s/documents/%s", its.conf.RestfulPort, its.collectionName, its.getTestName()) 82 | its.ctx.L().Infof("%v", addr) 83 | resp, err := http.Post(addr, "application/json", reqBody) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | defer resp.Body.Close() 89 | 90 | // Response 체크. 91 | respBody, err := ioutil.ReadAll(resp.Body) 92 | if err == nil { 93 | str := string(respBody) 94 | println(str) 95 | } 96 | time.Sleep(1 * time.Second) 97 | } 98 | 99 | func (its *IntegrationTestSuite) tryLocalLock(t *testing.T, cliName string, unlock bool) { 100 | 101 | defer func() { 102 | wg.Done() 103 | }() 104 | lock := utils.GetLocalLock(its.ctx, its.getTestName()) 105 | if lock.TryLock() { 106 | its.ctx.L().Infof("locked by %v", cliName) 107 | if unlock { 108 | defer func() { 109 | its.ctx.L().Infof("released by %v", cliName) 110 | lock.Unlock() 111 | }() 112 | } 113 | time.Sleep(3 * time.Second) 114 | } 115 | 116 | } 117 | 118 | func (its *IntegrationTestSuite) tryRedisLock(t *testing.T, cliName string, unlock bool) { 119 | cli1, err1 := redis.New(its.ctx, its.conf.Redis) 120 | require.NoError(t, err1) 121 | defer func() { 122 | require.NoError(its.T(), cli1.Close()) 123 | wg.Done() 124 | }() 125 | 126 | lock := cli1.GetLock(its.ctx, its.getTestName()) 127 | 128 | its.ctx.L().Infof("try to lock at %v", cliName) 129 | if lock.TryLock() { 130 | if unlock { 131 | defer func() { 132 | lock.Unlock() 133 | its.ctx.L().Infof("released by %v", cliName) 134 | }() 135 | } 136 | 137 | its.ctx.L().Infof("locked by %v", cliName) 138 | time.Sleep(3 * time.Second) 139 | } 140 | 141 | } 142 | -------------------------------------------------------------------------------- /test/test_helper.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | gocontext "context" 5 | "github.com/orda-io/orda/client/pkg/context" 6 | "github.com/orda-io/orda/client/pkg/errors" 7 | "github.com/orda-io/orda/client/pkg/iface" 8 | "github.com/orda-io/orda/client/pkg/model" 9 | "github.com/orda-io/orda/client/pkg/orda" 10 | "github.com/orda-io/orda/server/constants" 11 | "github.com/orda-io/orda/server/managers" 12 | redis "github.com/orda-io/orda/server/redis" 13 | "github.com/stretchr/testify/require" 14 | "path/filepath" 15 | "runtime" 16 | "strings" 17 | "sync" 18 | "testing" 19 | "time" 20 | 21 | "github.com/orda-io/orda/server/mongodb" 22 | ) 23 | 24 | var mongoDB = make(map[string]*mongodb.RepositoryMongo) 25 | 26 | // GetFunctionName returns the function name which calls this function. 27 | func GetFunctionName() string { 28 | pc, _, _, _ := runtime.Caller(1) 29 | fn := runtime.FuncForPC(pc).Name() 30 | return fn[strings.LastIndex(fn, ".")+1:] 31 | } 32 | 33 | // GetFileName returns the file name which calls this function. 34 | func GetFileName() string { 35 | _, file, _, _ := runtime.Caller(2) 36 | file = strings.Replace(file, ".", "_", -1) 37 | return filepath.Base(file) 38 | } 39 | 40 | // GetMongo returns an instance of RepositoryMongo for testing. 41 | func GetMongo(ctx iface.OrdaContext, dbName string) (*mongodb.RepositoryMongo, errors.OrdaError) { 42 | if m, ok := mongoDB[dbName]; ok { 43 | return m, nil 44 | } 45 | conf := NewTestMongoDBConfig(dbName) 46 | mongo, err := mongodb.New(ctx, conf) 47 | if err != nil { 48 | return nil, err 49 | } 50 | mongoDB[dbName] = mongo 51 | return mongo, nil 52 | } 53 | 54 | // NewTestOrdaClientConfig generates an OrdaClientConfig for testing. 55 | func NewTestOrdaClientConfig(collectionName string, syncType model.SyncType) *orda.ClientConfig { 56 | return &orda.ClientConfig{ 57 | ServerAddr: "localhost:59062", 58 | CollectionName: collectionName, 59 | NotificationAddr: "tcp://localhost:18181", 60 | SyncType: syncType, 61 | } 62 | } 63 | 64 | // NewTestOrdaServerConfig generates an OrdaServerConfig for testing. 65 | func NewTestOrdaServerConfig(dbName string) *managers.OrdaServerConfig { 66 | return &managers.OrdaServerConfig{ 67 | RPCServerPort: 59062, 68 | RestfulPort: 59862, 69 | SwaggerJSON: "../resources/orda.grpc.swagger.json", 70 | Notification: "tcp://localhost:18181", 71 | Mongo: NewTestMongoDBConfig(dbName), 72 | Redis: &redis.Config{ 73 | Addrs: []string{"127.0.0.1:16379"}, 74 | }, 75 | } 76 | } 77 | 78 | // WaitTimeout waits for timeout of the WaitGroup during the specified duration 79 | func WaitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { 80 | c := make(chan struct{}) 81 | go func() { 82 | defer close(c) 83 | wg.Wait() 84 | }() 85 | select { 86 | case <-c: 87 | return true 88 | case <-time.After(timeout): 89 | return false 90 | } 91 | } 92 | 93 | // NewTestMongoDBConfig creates a new MongoDBConfig for Test 94 | func NewTestMongoDBConfig(dbName string) *mongodb.Config { 95 | return &mongodb.Config{ 96 | Host: "localhost:27017", 97 | OrdaDB: dbName, 98 | User: "root", 99 | Password: "orda-test", 100 | } 101 | } 102 | 103 | // NewTestManagers creates a new Managers for Test 104 | func NewTestManagers(ctx iface.OrdaContext, dbName string) (*managers.Managers, errors.OrdaError) { 105 | conf := NewTestOrdaServerConfig(dbName) 106 | return managers.New(ctx, conf) 107 | } 108 | 109 | // InitTestDBCollection initializes db collection for testing 110 | func InitTestDBCollection(t *testing.T, dbName string) (*mongodb.RepositoryMongo, iface.OrdaContext, int32) { 111 | ctx := context.NewOrdaContext(gocontext.TODO(), constants.TagTest). 112 | UpdateCollectionTags(t.Name(), 0) 113 | mongo, err := GetMongo(ctx, dbName) 114 | require.NoError(t, err) 115 | 116 | err = mongo.PurgeCollection(ctx, t.Name()) 117 | require.NoError(t, err) 118 | collectionNum, err := mongodb.MakeCollection(ctx, mongo, t.Name()) 119 | require.NoError(t, err) 120 | ctx.UpdateCollectionTags(t.Name(), collectionNum) 121 | ctx.L().Infof("Init Test DB Collection %v(%d) in %v", t.Name(), collectionNum, dbName) 122 | return mongo, ctx, collectionNum 123 | } 124 | -------------------------------------------------------------------------------- /client/pkg/orda/counter_test.go: -------------------------------------------------------------------------------- 1 | package orda 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/iface" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | "github.com/orda-io/orda/client/pkg/testonly" 9 | "github.com/stretchr/testify/require" 10 | "testing" 11 | ) 12 | 13 | func TestCounterTransactions(t *testing.T) { 14 | t.Run("Can transaction for Counter", func(t *testing.T) { 15 | tw := testonly.NewTestWire(true) 16 | 17 | counter1, _ := newCounter(testonly.NewBase("key1", model.TypeOfDatatype_COUNTER), tw, nil) 18 | 19 | require.NoError(t, counter1.Transaction("transaction1", func(counter CounterInTx) error { 20 | _, _ = counter.IncreaseBy(2) 21 | require.Equal(t, int32(2), counter.Get()) 22 | _, _ = counter.IncreaseBy(4) 23 | require.Equal(t, int32(6), counter.Get()) 24 | return nil 25 | })) 26 | 27 | require.Equal(t, int32(6), counter1.Get()) 28 | 29 | require.Error(t, counter1.Transaction("transaction2", func(intCounter CounterInTx) error { 30 | _, _ = intCounter.IncreaseBy(3) 31 | require.Equal(t, int32(9), intCounter.Get()) 32 | _, _ = intCounter.IncreaseBy(5) 33 | require.Equal(t, int32(14), intCounter.Get()) 34 | return fmt.Errorf("err") 35 | })) 36 | require.Equal(t, int32(6), counter1.Get()) 37 | 38 | sOp, err := counter1.(iface.Datatype).CreateSnapshotOperation() 39 | require.NoError(t, err) 40 | log.Logger.Infof("%v", sOp) 41 | log.Logger.Infof("%v", sOp.ToModelOperation()) 42 | 43 | counter2, _ := newCounter(testonly.NewBase("key1", model.TypeOfDatatype_COUNTER), tw, nil) 44 | counter2.(iface.Datatype).ExecuteRemote(sOp) 45 | log.Logger.Infof("%v", counter1.ToJSON()) 46 | }) 47 | 48 | t.Run("Can sync Counter operations with Test wire", func(t *testing.T) { 49 | tw := testonly.NewTestWire(true) 50 | counter1, _ := newCounter(testonly.NewBase("key1", model.TypeOfDatatype_COUNTER), tw, nil) 51 | counter2, _ := newCounter(testonly.NewBase("key2", model.TypeOfDatatype_COUNTER), tw, nil) 52 | 53 | tw.SetDatatypes(counter1.(*counter).WiredDatatype, counter2.(*counter).WiredDatatype) 54 | 55 | i, oErr := counter1.Increase() 56 | require.NoError(t, oErr) 57 | 58 | require.Equal(t, i, int32(1)) 59 | require.Equal(t, counter1.Get(), counter2.Get()) 60 | 61 | err := counter1.Transaction("transaction1", func(intCounter CounterInTx) error { 62 | _, _ = intCounter.IncreaseBy(-1) 63 | require.Equal(t, int32(0), intCounter.Get()) 64 | _, _ = intCounter.IncreaseBy(-2) 65 | _, _ = intCounter.IncreaseBy(-3) 66 | require.Equal(t, int32(-5), intCounter.Get()) 67 | return nil 68 | }) 69 | require.NoError(t, err) 70 | 71 | log.Logger.Infof("%#v vs. %#v", counter1.Get(), counter2.Get()) 72 | require.Equal(t, counter1.Get(), counter2.Get()) 73 | 74 | err = counter1.Transaction("transaction2", func(intCounter CounterInTx) error { 75 | _, _ = intCounter.IncreaseBy(1) 76 | _, _ = intCounter.IncreaseBy(2) 77 | require.Equal(t, int32(-2), intCounter.Get()) 78 | _, _ = intCounter.IncreaseBy(3) 79 | _, _ = intCounter.IncreaseBy(4) 80 | return fmt.Errorf("fail to do the transaction") 81 | }) 82 | require.Error(t, err) 83 | 84 | log.Logger.Infof("%#v vs. %#v", counter1.Get(), counter2.Get()) 85 | require.Equal(t, counter1.Get(), counter2.Get()) 86 | }) 87 | 88 | t.Run("Can set and get counterSnapshot", func(t *testing.T) { 89 | counter1, _ := newCounter(testonly.NewBase("key1", model.TypeOfDatatype_COUNTER), nil, nil) 90 | _, _ = counter1.Increase() 91 | _, _ = counter1.IncreaseBy(1234) 92 | clone, _ := newCounter(testonly.NewBase("key2", model.TypeOfDatatype_COUNTER), nil, nil) 93 | meta1, snap1, err := counter1.(iface.Datatype).GetMetaAndSnapshot() 94 | require.NoError(t, err) 95 | err = clone.(iface.Datatype).SetMetaAndSnapshot(meta1, snap1) 96 | require.NoError(t, err) 97 | meta2, snap2, err := clone.(iface.Datatype).GetMetaAndSnapshot() 98 | require.NoError(t, err) 99 | 100 | log.Logger.Infof("%v", string(snap1)) 101 | log.Logger.Infof("%v", string(snap2)) 102 | require.Equal(t, snap1, snap2) 103 | require.Equal(t, meta1, meta2) 104 | 105 | require.Equal(t, counter1.Get(), clone.Get()) 106 | }) 107 | } 108 | -------------------------------------------------------------------------------- /test/realtime_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "fmt" 5 | "github.com/orda-io/orda/client/pkg/errors" 6 | "github.com/orda-io/orda/client/pkg/log" 7 | "github.com/orda-io/orda/client/pkg/model" 8 | "github.com/orda-io/orda/client/pkg/orda" 9 | "sync" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func (its *IntegrationTestSuite) TestNotification() { 16 | key := GetFunctionName() 17 | 18 | its.Run("Can notify remote change", func() { 19 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_REALTIME) 20 | client1 := orda.NewClient(config, "client1") 21 | require.NoError(its.T(), client1.Connect()) 22 | defer func() { 23 | _ = client1.Close() 24 | }() 25 | 26 | client2 := orda.NewClient(config, "client2") 27 | require.NoError(its.T(), client2.Connect()) 28 | defer func() { 29 | _ = client2.Close() 30 | }() 31 | 32 | intCounter1 := client1.CreateCounter(key, nil) 33 | _, _ = intCounter1.Increase() 34 | require.NoError(its.T(), client1.Sync()) 35 | 36 | fmt.Printf("Subscribed by client2\n") 37 | opCount := 0 38 | wg1 := sync.WaitGroup{} 39 | wg1.Add(2) 40 | wg2 := sync.WaitGroup{} 41 | wg2.Add(1) 42 | intCounter2 := client2.SubscribeCounter(key, orda.NewHandlers( 43 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 44 | intCounter := dt.(orda.Counter) 45 | log.Logger.Infof("STATE: %s -> %s %d", old, new, intCounter.Get()) 46 | require.Equal(its.T(), int32(1), intCounter.Get()) 47 | wg1.Done() // one time 48 | }, 49 | func(dt orda.Datatype, opList []interface{}) { 50 | log.Logger.Infof("opList.size: %v", len(opList)) 51 | for _, op := range opList { 52 | opCount++ 53 | log.Logger.Infof("%d) OPERATION %+v", opCount, op) 54 | 55 | } 56 | 57 | if opCount == 2 { 58 | wg1.Done() // two times 59 | } else if opCount == 3 { 60 | wg2.Done() 61 | } 62 | }, 63 | func(dt orda.Datatype, err ...errors.OrdaError) { 64 | require.NoError(its.T(), err[0]) 65 | })) 66 | 67 | wg1.Wait() 68 | 69 | _, _ = intCounter1.IncreaseBy(10) 70 | wg2.Wait() 71 | require.Equal(its.T(), intCounter1.Get(), intCounter2.Get()) 72 | }) 73 | 74 | its.Run("Can test realtime delivery", func() { 75 | key := key + "-rt" 76 | config := NewTestOrdaClientConfig(its.collectionName, model.SyncType_REALTIME) 77 | client1 := orda.NewClient(config, "realtime_client1") 78 | require.NoError(its.T(), client1.Connect()) 79 | defer func() { 80 | _ = client1.Close() 81 | }() 82 | 83 | client2 := orda.NewClient(config, "realtime_client2") 84 | require.NoError(its.T(), client2.Connect()) 85 | defer func() { 86 | _ = client2.Close() 87 | }() 88 | wg1 := new(sync.WaitGroup) 89 | wg1.Add(1) 90 | wg3 := new(sync.WaitGroup) 91 | wg3.Add(1) 92 | counter1 := client1.CreateCounter(key, orda.NewHandlers( 93 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 94 | if new == model.StateOfDatatype_SUBSCRIBED { 95 | wg1.Done() 96 | return 97 | } 98 | require.Fail(its.T(), "fail state") 99 | }, func(dt orda.Datatype, opList []interface{}) { 100 | for _, op := range opList { 101 | log.Logger.Infof("%v", op) 102 | wg3.Done() 103 | } 104 | }, nil)) 105 | _, _ = counter1.Increase() 106 | 107 | require.True(its.T(), WaitTimeout(wg1, time.Second*5)) 108 | 109 | wg2 := new(sync.WaitGroup) 110 | wg2.Add(3) 111 | log.Logger.Infof("SUBSCRIBED by client2") 112 | counter2 := client2.SubscribeCounter(key, orda.NewHandlers( 113 | func(dt orda.Datatype, old model.StateOfDatatype, new model.StateOfDatatype) { 114 | log.Logger.Infof("subscribe:%s -> %s", old, new) 115 | if new == model.StateOfDatatype_SUBSCRIBED { 116 | wg2.Done() 117 | return 118 | } 119 | require.Fail(its.T(), "fail state") 120 | }, 121 | func(dt orda.Datatype, opList []interface{}) { 122 | for _, op := range opList { 123 | log.Logger.Infof("%v", op) 124 | wg2.Done() 125 | } 126 | }, nil)) 127 | require.True(its.T(), WaitTimeout(wg2, time.Second*5)) 128 | require.Equal(its.T(), int32(1), counter2.Get()) 129 | 130 | _, _ = counter2.IncreaseBy(10) 131 | require.True(its.T(), WaitTimeout(wg3, time.Second*5)) 132 | require.Equal(its.T(), counter1.Get(), counter2.Get()) 133 | 134 | }) 135 | } 136 | -------------------------------------------------------------------------------- /client/pkg/errors/errorcode.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "github.com/orda-io/orda/client/pkg/log" 5 | ) 6 | 7 | // ErrorCode is a type for error code of OrdaError 8 | type ErrorCode uint32 9 | 10 | // New creates an error related to the datatype 11 | func (its ErrorCode) New(l *log.OrdaLog, args ...interface{}) OrdaError { 12 | code := uint32(its) / 100 13 | switch code { 14 | case 1: 15 | return newSingleOrdaError(l, its, "ClientError", clientErrFormats[its], args...) 16 | case 2: 17 | return newSingleOrdaError(l, its, "DatatypeError", datatypeErrFormats[its], args...) 18 | case 3: 19 | return newSingleOrdaError(l, its, "PushPullError", pushPullErrFormats[its], args...) 20 | case 4: 21 | return newSingleOrdaError(l, its, "ServerError", serverErrFormats[its], args...) 22 | 23 | } 24 | panic("Unsupported error") 25 | } 26 | 27 | const ( 28 | baseBasicCode ErrorCode = 0 29 | baseClientCode ErrorCode = 100 30 | baseDatatypeCode ErrorCode = 200 31 | basePushPullCode ErrorCode = 300 32 | baseServerCode ErrorCode = 400 33 | ) 34 | 35 | const ( 36 | // MultipleErrors is an error code that includes many OrdaErrors 37 | MultipleErrors = baseBasicCode + iota 38 | ) 39 | 40 | // ClientXXXX defines the error related to client 41 | const ( 42 | ClientConnect = baseClientCode + iota 43 | ClientClose 44 | ClientSync 45 | ) 46 | 47 | var clientErrFormats = map[ErrorCode]string{ 48 | ClientConnect: "fail to connect: %v", 49 | ClientClose: "fail to close: %v", 50 | ClientSync: "fail to sync: %v", 51 | } 52 | 53 | // DatatypeXXX defines an error related to Datatype 54 | const ( 55 | DatatypeCreate = baseDatatypeCode + iota 56 | DatatypeSubscribe 57 | DatatypeTransaction 58 | DatatypeSnapshot 59 | DatatypeIllegalParameters 60 | DatatypeIllegalOperation 61 | DatatypeInvalidParent 62 | DatatypeNoOp 63 | DatatypeMarshal 64 | DatatypeNoTarget 65 | DatatypeInvalidPatch 66 | ) 67 | 68 | var datatypeErrFormats = map[ErrorCode]string{ 69 | DatatypeCreate: "fail to create datatype: %s", 70 | DatatypeSubscribe: "fail to subscribe datatype: %s", 71 | DatatypeTransaction: "fail to proceed transaction: %v", 72 | DatatypeSnapshot: "fail to make a snapshot: %s", 73 | DatatypeIllegalParameters: "fail to execute the operations due to illegal parameters: %v", 74 | DatatypeIllegalOperation: "fail to execute the illegal operation for %v: %v", 75 | DatatypeInvalidParent: "fail to modify due to the invalid parent: %v", 76 | DatatypeNoOp: "fail to issue operation: %v", 77 | DatatypeMarshal: "fail to (un)marshal: %v", 78 | DatatypeNoTarget: "fail to find target: %v", 79 | DatatypeInvalidPatch: "fail to patch: %v", 80 | } 81 | 82 | // ServerXXX denotes the errors when Server is running. 83 | const ( 84 | ServerDBQuery = baseServerCode + iota 85 | ServerDBInit 86 | ServerDBDecode 87 | ServerNoResource 88 | ServerNoPermission 89 | ServerInit 90 | ServerNotify 91 | ServerBadRequest 92 | ServerDBClose 93 | ServerUpdateSnapshot 94 | ServerInternal 95 | ) 96 | 97 | var serverErrFormats = map[ErrorCode]string{ 98 | ServerDBQuery: "fail to succeed DB query: %v", 99 | ServerDBDecode: "fail to decode in DB: %v", 100 | ServerNoResource: "find no resource: %v", 101 | ServerNoPermission: "have no permission: %v", 102 | ServerInit: "fail to initialize server: %v", 103 | ServerNotify: "fail to notify push-pull: %v", 104 | ServerBadRequest: "fail to process due to bad request: %v", 105 | ServerDBClose: "fail to close mongoDB: %v", 106 | ServerUpdateSnapshot: "fail to update snapshot: %v", 107 | ServerInternal: "internal server error: %v", 108 | } 109 | 110 | // PushPullXXX denotes the errors during PushPull 111 | const ( 112 | PushPullAbortionOfServer = basePushPullCode + iota 113 | PushPullAbortionOfClient 114 | PushPullDuplicateKey 115 | PushPullMissingOps 116 | PushPullNoDatatypeToSubscribe 117 | ) 118 | 119 | var pushPullErrFormats = map[ErrorCode]string{ 120 | PushPullAbortionOfServer: "aborted push-pull due to server: %v", 121 | PushPullAbortionOfClient: "aborted push-pull due to client: %v", 122 | PushPullDuplicateKey: "duplicate datatype key: %v", 123 | PushPullMissingOps: "aborted push due to missing operations: %v", 124 | PushPullNoDatatypeToSubscribe: "no datatype to subscribe: %v", 125 | } 126 | --------------------------------------------------------------------------------