├── .gitignore ├── kv ├── kvutil │ ├── util_test.go │ └── util.go ├── tikv │ ├── Makefile │ ├── iter.go │ ├── docker │ │ ├── docker-compose.yaml │ │ └── pd.toml │ ├── tx.go │ ├── tikv.go │ ├── locker.go │ └── tikv_test.go ├── registry │ └── registry.go ├── badger │ ├── iter.go │ ├── tx.go │ ├── badger.go │ ├── locker.go │ └── badger_test.go └── kv.go ├── testutil ├── testdata │ ├── scripts.js │ ├── task.yaml │ ├── account.yaml │ └── user.yaml └── testutil.go ├── benchmarks ├── benchmark_get_test.go ├── benchmark_set_test.go ├── benchmark_create_test.go └── benchmark_query_test.go ├── metakeys.go ├── opts.go ├── metadata_test.go ├── .golangci.yaml ├── errors ├── errors_test.go └── errors.go ├── cdc.yaml ├── .github └── workflows │ ├── golangci-lint.yml │ └── coverage.yaml ├── indexing_test.go ├── javascript_test.go ├── util ├── util.go └── util_test.go ├── dag_test.go ├── indexing.go ├── metadata.go ├── dag.go ├── optimizer.go ├── schema_test.go ├── model_test.go ├── authorization_test.go ├── go.mod ├── example_test.go ├── optimizer_test.go ├── db_util.go ├── authorization.go ├── util.go ├── util_test.go ├── api.go ├── tx_test.go ├── LICENSE ├── tx.go └── javascript.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | tmp 3 | kv/tikv/docker/data 4 | examples/tasks/tmp 5 | *tmp -------------------------------------------------------------------------------- /kv/kvutil/util_test.go: -------------------------------------------------------------------------------- 1 | package kvutil_test 2 | 3 | import ( 4 | "bytes" 5 | "github.com/autom8ter/myjson/kv/kvutil" 6 | "github.com/stretchr/testify/assert" 7 | "testing" 8 | ) 9 | 10 | func TestKVUtil(t *testing.T) { 11 | const input = "hello" 12 | next := kvutil.NextPrefix([]byte(input)) 13 | assert.Equal(t, 1, bytes.Compare(next, []byte(input))) 14 | } 15 | -------------------------------------------------------------------------------- /kv/tikv/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: up 2 | up: ## start local containers 3 | docker-compose -f docker/docker-compose.yaml pull 4 | docker-compose -f docker/docker-compose.yaml up -d 5 | 6 | .PHONY: down 7 | down: ## shuts down local docker containers 8 | docker-compose -f docker/docker-compose.yaml down --remove-orphans 9 | @rm -rf docker/data 10 | 11 | .PHONY: test 12 | test: 13 | go test -race . --tags=tikv -------------------------------------------------------------------------------- /testutil/testdata/scripts.js: -------------------------------------------------------------------------------- 1 | 2 | function setDocTimestamp(doc) { 3 | doc.Set('timestamp', new Date().toISOString()) 4 | } 5 | 6 | function isSuperUser(meta) { 7 | return contains(meta.Get('roles'), 'super_user') 8 | } 9 | 10 | function accountQueryAuth(query, meta) { 11 | return query.where?.length > 0 && query.where[0].field == '_id' && query.where[0].op == 'eq' && contains(meta.Get('groups'), query.where[0].value) 12 | } -------------------------------------------------------------------------------- /kv/kvutil/util.go: -------------------------------------------------------------------------------- 1 | package kvutil 2 | 3 | // NextPrefix returns a prefix that is lexicographically larger than the input prefix 4 | func NextPrefix(prefix []byte) []byte { 5 | buf := make([]byte, len(prefix)) 6 | copy(buf, prefix) 7 | var i int 8 | for i = len(prefix) - 1; i >= 0; i-- { 9 | buf[i]++ 10 | if buf[i] != 0 { 11 | break 12 | } 13 | } 14 | if i == -1 { 15 | buf = make([]byte, 0) 16 | } 17 | return buf 18 | } 19 | -------------------------------------------------------------------------------- /testutil/testdata/task.yaml: -------------------------------------------------------------------------------- 1 | type: object 2 | x-collection: task 3 | required: 4 | - _id 5 | - user 6 | - content 7 | properties: 8 | _id: 9 | type: string 10 | description: The user's id. 11 | x-primary: true 12 | user: 13 | type: string 14 | description: The id of the user who owns the task 15 | x-foreign: 16 | collection: user 17 | field: _id 18 | cascade: true 19 | content: 20 | type: string 21 | description: The content of the task -------------------------------------------------------------------------------- /benchmarks/benchmark_get_test.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson" 8 | "github.com/autom8ter/myjson/testutil" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // BenchmarkGet-12 5100 221980 ns/op 174096 B/op 2198 allocs/op 13 | func BenchmarkGet(b *testing.B) { 14 | b.ReportAllocs() 15 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 16 | b.ResetTimer() 17 | for i := 0; i < b.N; i++ { 18 | _, err := db.Get(ctx, "account", "1") 19 | assert.NoError(b, err) 20 | } 21 | })) 22 | } 23 | -------------------------------------------------------------------------------- /metakeys.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type internalMetaKey string 8 | 9 | const ( 10 | internalKey internalMetaKey = "_internal" 11 | isIndexingKey internalMetaKey = "_is_indexing" 12 | ) 13 | 14 | func isInternal(ctx context.Context) bool { 15 | return ctx.Value(internalKey) == true 16 | } 17 | 18 | func isIndexing(ctx context.Context) bool { 19 | return ctx.Value(isIndexingKey) == true 20 | } 21 | 22 | // SetIsInternal sets a context value to indicate that the request is internal (it should only be used to bypass things like authorization, validation, etc) 23 | func SetIsInternal(ctx context.Context) context.Context { 24 | return context.WithValue(ctx, internalKey, true) 25 | } 26 | -------------------------------------------------------------------------------- /kv/registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "github.com/autom8ter/myjson/errors" 5 | "github.com/autom8ter/myjson/kv" 6 | ) 7 | 8 | // KVDBOpener opens a key value database 9 | type KVDBOpener func(params map[string]interface{}) (kv.DB, error) 10 | 11 | var registeredOpeners = map[string]KVDBOpener{} 12 | 13 | // Register registers a KVDBOpener opener by name 14 | func Register(name string, opener KVDBOpener) { 15 | registeredOpeners[name] = opener 16 | } 17 | 18 | // Open opens a registered key value database 19 | func Open(name string, params map[string]interface{}) (kv.DB, error) { 20 | opener, ok := registeredOpeners[name] 21 | if !ok { 22 | return nil, errors.New(errors.NotFound, "%s is not registered", name) 23 | } 24 | return opener(params) 25 | } 26 | -------------------------------------------------------------------------------- /opts.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // DBOpt is an option for configuring a collection 9 | type DBOpt func(d *defaultDB) 10 | 11 | // WithOptimizer overrides the default query optimizer provider 12 | func WithOptimizer(o Optimizer) DBOpt { 13 | return func(d *defaultDB) { 14 | d.optimizer = o 15 | } 16 | } 17 | 18 | // WithJavascriptOverrides adds global variables or methods to the embedded javascript vm(s) 19 | func WithJavascriptOverrides(overrides map[string]any) DBOpt { 20 | return func(d *defaultDB) { 21 | d.jsOverrides = overrides 22 | } 23 | } 24 | 25 | // WithGlobalJavasciptFunctions adds global javascript functions to the embedded javascript vm(s) 26 | func WithGlobalJavascriptFunctions(scripts []string) DBOpt { 27 | return func(d *defaultDB) { 28 | d.globalScripts = fmt.Sprintln(strings.Join(scripts, "\n")) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /kv/badger/iter.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/autom8ter/myjson/kv" 7 | "github.com/dgraph-io/badger/v3" 8 | ) 9 | 10 | type badgerIterator struct { 11 | opts kv.IterOpts 12 | iter *badger.Iterator 13 | } 14 | 15 | func (b *badgerIterator) Seek(key []byte) { 16 | b.iter.Seek(key) 17 | } 18 | 19 | func (b *badgerIterator) Close() { 20 | b.iter.Close() 21 | } 22 | 23 | func (b *badgerIterator) Valid() bool { 24 | if b.opts.Prefix != nil && !b.iter.ValidForPrefix(b.opts.Prefix) { 25 | return false 26 | } 27 | if b.opts.UpperBound != nil && bytes.Compare(b.Key(), b.opts.UpperBound) == 1 { 28 | return false 29 | } 30 | return b.iter.Valid() 31 | } 32 | 33 | func (b *badgerIterator) Key() []byte { 34 | return b.iter.Item().Key() 35 | } 36 | 37 | func (b *badgerIterator) Value() ([]byte, error) { 38 | return b.iter.Item().ValueCopy(nil) 39 | } 40 | 41 | func (b *badgerIterator) Next() error { 42 | b.iter.Next() 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /kv/tikv/iter.go: -------------------------------------------------------------------------------- 1 | package tikv 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/autom8ter/myjson/kv" 7 | ) 8 | 9 | type unionStoreIterator interface { 10 | Valid() bool 11 | Key() []byte 12 | Value() []byte 13 | Next() error 14 | Close() 15 | } 16 | 17 | type tikvIterator struct { 18 | opts kv.IterOpts 19 | iter unionStoreIterator 20 | } 21 | 22 | func (b *tikvIterator) Seek(key []byte) { 23 | //b.iter.Seek(key) 24 | // TODO: how to seek? 25 | } 26 | 27 | func (b *tikvIterator) Close() { 28 | b.iter.Close() 29 | } 30 | 31 | func (b *tikvIterator) Valid() bool { 32 | if b.opts.Prefix != nil && !bytes.HasPrefix(b.Key(), b.opts.Prefix) { 33 | return false 34 | } 35 | if b.opts.UpperBound != nil && bytes.Compare(b.Key(), b.opts.UpperBound) == 1 { 36 | return false 37 | } 38 | return b.iter.Valid() 39 | } 40 | 41 | func (b *tikvIterator) Key() []byte { 42 | return b.iter.Key() 43 | } 44 | 45 | func (b *tikvIterator) Value() ([]byte, error) { 46 | return b.iter.Value(), nil 47 | } 48 | 49 | func (b *tikvIterator) Next() error { 50 | return b.iter.Next() 51 | } 52 | -------------------------------------------------------------------------------- /metadata_test.go: -------------------------------------------------------------------------------- 1 | package myjson_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestMetadata(t *testing.T) { 12 | t.Run("basic", func(t *testing.T) { 13 | md := myjson.ExtractMetadata(context.Background()) 14 | assert.NotNil(t, md) 15 | assert.Equal(t, "default", md.GetString(myjson.MetadataKeyNamespace)) 16 | }) 17 | 18 | t.Run("set context", func(t *testing.T) { 19 | ctx := context.Background() 20 | ctx = myjson.SetMetadataGroups(ctx, []string{"group1", "group2"}) 21 | ctx = myjson.SetMetadataRoles(ctx, []string{"role1", "role2"}) 22 | ctx = myjson.SetMetadataNamespace(ctx, "acme") 23 | ctx = myjson.SetMetadataUserID(ctx, "123") 24 | md := myjson.ExtractMetadata(ctx) 25 | assert.NotNil(t, md) 26 | assert.Equal(t, "acme", md.GetString(myjson.MetadataKeyNamespace)) 27 | assert.Equal(t, []any{"role1", "role2"}, md.GetArray(myjson.MetadataKeyRoles)) 28 | assert.Equal(t, []any{"group1", "group2"}, md.GetArray(myjson.MetadataKeyGroups)) 29 | assert.Equal(t, "123", md.GetString(myjson.MetadataKeyUserID)) 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | # Options for analysis running. 2 | run: 3 | # The default concurrency value is the number of available CPU. 4 | concurrency: 4 5 | # Timeout for analysis, e.g. 30s, 5m. 6 | # Default: 1m 7 | timeout: 5m 8 | # exit code when at least one issue was found, default is 1 9 | issues-exit-code: 1 10 | # Include test files or not. 11 | # Default: true 12 | tests: false 13 | allow-parallel-runners: false 14 | go: '1.18' 15 | linters: 16 | disable-all: true 17 | enable: 18 | - bodyclose 19 | - deadcode 20 | - depguard 21 | - dupl 22 | - errcheck 23 | - exportloopref 24 | # - exhaustive 25 | # - funlen 26 | # - goconst 27 | # - gocritic 28 | # - gocyclo 29 | - gofmt 30 | - goimports 31 | # - gomnd 32 | # - goprintffuncname 33 | # - gosec 34 | - gosimple 35 | - govet 36 | - ineffassign 37 | - revive 38 | # - lll 39 | # - misspell 40 | - nakedret 41 | # - noctx 42 | - nolintlint 43 | # - rowserrcheck 44 | - staticcheck 45 | # - structcheck 46 | # - stylecheck 47 | - typecheck 48 | - unconvert 49 | # - unparam 50 | - unused 51 | - varcheck 52 | - nilerr 53 | # - whitespace 54 | -------------------------------------------------------------------------------- /benchmarks/benchmark_set_test.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson" 8 | "github.com/autom8ter/myjson/kv" 9 | "github.com/autom8ter/myjson/testutil" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | // BenchmarkSet-12 1347 925214 ns/op 500704 B/op 5961 allocs/op 14 | func BenchmarkSet(b *testing.B) { 15 | doc := testutil.NewUserDoc() 16 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 17 | b.ReportAllocs() 18 | b.ResetTimer() 19 | for i := 0; i < b.N; i++ { 20 | assert.Nil(b, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 21 | return tx.Set(ctx, "user", doc) 22 | })) 23 | } 24 | })) 25 | } 26 | 27 | // BenchmarkSet100000-12 1 59297905703 ns/op 29676942072 B/op 345757621 allocs/op 28 | func BenchmarkSet100000(b *testing.B) { 29 | doc := testutil.NewUserDoc() 30 | b.ReportAllocs() 31 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 32 | b.ResetTimer() 33 | for i := 0; i < b.N; i++ { 34 | assert.Nil(b, db.Tx(ctx, kv.TxOpts{IsReadOnly: false, IsBatch: true}, func(ctx context.Context, tx myjson.Tx) error { 35 | for v := 0; v < 100000; v++ { 36 | if err := tx.Set(ctx, "user", doc); err != nil { 37 | return err 38 | } 39 | } 40 | return nil 41 | })) 42 | } 43 | })) 44 | } 45 | -------------------------------------------------------------------------------- /benchmarks/benchmark_create_test.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson" 8 | "github.com/autom8ter/myjson/kv" 9 | "github.com/autom8ter/myjson/testutil" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | // BenchmarkCreate-12 954 1237137 ns/op 601310 B/op 7459 allocs/op 14 | func BenchmarkCreate(b *testing.B) { 15 | doc := testutil.NewUserDoc() 16 | b.ReportAllocs() 17 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 18 | b.ResetTimer() 19 | for i := 0; i < b.N; i++ { 20 | assert.Nil(b, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 21 | _, err := tx.Create(ctx, "user", doc) 22 | return err 23 | })) 24 | } 25 | })) 26 | } 27 | 28 | // BenchmarkCreate1000-12 1 2662527299 ns/op 403135712 B/op 4319974 allocs/op 29 | func BenchmarkCreate1000(b *testing.B) { 30 | doc := testutil.NewUserDoc() 31 | b.ReportAllocs() 32 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 33 | b.ResetTimer() 34 | for i := 0; i < b.N; i++ { 35 | assert.Nil(b, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 36 | for v := 0; v < 1000; v++ { 37 | if _, err := tx.Create(ctx, "user", doc); err != nil { 38 | return err 39 | } 40 | } 41 | return nil 42 | })) 43 | } 44 | })) 45 | } 46 | -------------------------------------------------------------------------------- /benchmarks/benchmark_query_test.go: -------------------------------------------------------------------------------- 1 | package benchmarks 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson" 8 | "github.com/autom8ter/myjson/testutil" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // BenchmarkQuery-12 4530 290003 ns/op 183447 B/op 2365 allocs/op 13 | func BenchmarkQuery(b *testing.B) { 14 | b.ReportAllocs() 15 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 16 | assert.NoError(b, testutil.SeedUsers(ctx, db, 10, 3)) 17 | b.ResetTimer() 18 | for i := 0; i < b.N; i++ { 19 | _, err := db.Query(ctx, "account", myjson.Query{ 20 | Where: []myjson.Where{ 21 | { 22 | Field: "_id", 23 | Op: myjson.WhereOpEq, 24 | Value: "1", 25 | }, 26 | }, 27 | }) 28 | assert.NoError(b, err) 29 | } 30 | })) 31 | } 32 | 33 | // BenchmarkQuery2-12 2974 392871 ns/op 218961 B/op 2691 allocs/op 34 | func BenchmarkQuery2(b *testing.B) { 35 | b.ReportAllocs() 36 | assert.Nil(b, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 37 | assert.NoError(b, testutil.SeedUsers(ctx, db, 10, 3)) 38 | b.ResetTimer() 39 | for i := 0; i < b.N; i++ { 40 | _, err := db.Query(ctx, "user", myjson.Query{ 41 | Where: []myjson.Where{ 42 | { 43 | Field: "age", 44 | Op: myjson.WhereOpGt, 45 | Value: 50, 46 | }, 47 | }, 48 | }) 49 | assert.NoError(b, err) 50 | } 51 | })) 52 | } 53 | -------------------------------------------------------------------------------- /errors/errors_test.go: -------------------------------------------------------------------------------- 1 | package errors_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson/errors" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestErrors(t *testing.T) { 12 | t.Run("wrap nil error", func(t *testing.T) { 13 | var err error 14 | err = errors.Wrap(err, errors.NotFound, "") 15 | assert.NoError(t, err) 16 | }) 17 | t.Run("wrap error", func(t *testing.T) { 18 | var err = fmt.Errorf("not found") 19 | err = errors.Wrap(err, errors.NotFound, "") 20 | assert.Equal(t, errors.NotFound, errors.Extract(err).Code) 21 | }) 22 | t.Run("new error", func(t *testing.T) { 23 | err := errors.New(errors.NotFound, "not found") 24 | assert.Equal(t, errors.NotFound, errors.Extract(err).Code) 25 | }) 26 | t.Run("new error then wrap", func(t *testing.T) { 27 | err := errors.New(0, "not found") 28 | err = errors.Wrap(err, errors.NotFound, "") 29 | assert.Equal(t, errors.NotFound, errors.Extract(err).Code) 30 | }) 31 | t.Run("new error then wrap then remove", func(t *testing.T) { 32 | err := errors.New(0, "not found") 33 | err = errors.Wrap(err, errors.NotFound, "") 34 | e := errors.Extract(err).RemoveError() 35 | assert.Empty(t, e.Err) 36 | }) 37 | t.Run("error json string", func(t *testing.T) { 38 | err := errors.New(0, "not found") 39 | err = errors.Wrap(err, errors.NotFound, "") 40 | e := errors.Extract(err) 41 | assert.JSONEq(t, `{ "code":404, "err": "not found"}`, e.Error()) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /cdc.yaml: -------------------------------------------------------------------------------- 1 | type: object 2 | x-collection: system_cdc 3 | x-immutable: true 4 | x-read-only: true 5 | required: 6 | - _id 7 | - collection 8 | - action 9 | properties: 10 | _id: 11 | type: string 12 | description: The cdc entry id. 13 | x-primary: true 14 | collection: 15 | type: string 16 | description: The collection the document belongs to 17 | x-index: 18 | collection_document_idx: 19 | enabled: true 20 | additional_fields: 21 | - documentID 22 | action: 23 | type: string 24 | description: The action taken upon the document 25 | enum: 26 | - create 27 | - update 28 | - delete 29 | - set 30 | documentID: 31 | type: string 32 | description: The id of the document being changed 33 | x-index: 34 | document_id_idx: 35 | enabled: true 36 | diff: 37 | type: array 38 | description: An array of changes made to a document 39 | items: 40 | type: object 41 | properties: 42 | op: 43 | type: string 44 | enum: 45 | - replace 46 | - add 47 | - remove 48 | path: 49 | type: string 50 | value: { } 51 | timestamp: 52 | type: integer 53 | description: The unix nanosecond timestamp when the change was commited 54 | x-index: 55 | timestamp_idx: 56 | enabled: true 57 | metadata: 58 | type: object 59 | description: The context metadata when the change was commited 60 | -------------------------------------------------------------------------------- /kv/tikv/docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | redis: 5 | container_name: redis 6 | image: redis:5-alpine 7 | ports: 8 | - "6379:6379" 9 | restart: always 10 | pd0: 11 | hostname: pd0 12 | container_name: pd0 13 | image: pingcap/pd:latest 14 | ports: 15 | - "2379:2379" 16 | - "2380:2380" 17 | volumes: 18 | - ./pd.toml:/pd.toml:ro 19 | - ./data:/data 20 | - ./logs:/logs 21 | command: 22 | - --name=pd0 23 | - --client-urls=http://0.0.0.0:2379 24 | - --peer-urls=http://0.0.0.0:2380 25 | - --advertise-client-urls=http://pd0:2379 26 | - --advertise-peer-urls=http://0.0.0.0:2380 27 | # - --initial-cluster=pd0=http://0.0.0.0:2380 28 | - --data-dir=/data/pd0 29 | - --config=/pd.toml 30 | - --force-new-cluster 31 | restart: on-failure 32 | tikv0: 33 | hostname: tikv0 34 | container_name: tikv0 35 | ports: 36 | - "20160:20160" 37 | image: pingcap/tikv:latest 38 | volumes: 39 | - ./tikv.toml:/tikv.toml:ro 40 | - ./data:/data 41 | - ./logs:/logs 42 | command: 43 | - --addr=0.0.0.0:20160 44 | - --advertise-addr=tikv0:20160 45 | - --data-dir=/data/tikv0 46 | - --pd=pd0:2379 47 | - --config=/tikv.toml 48 | depends_on: 49 | - "pd0" 50 | restart: on-failure 51 | 52 | # $ docker run --rm -ti --network eventsourcing alpine sh -c "apk add curl; curl http://tikv0:20160/metrics" 53 | 54 | # docker exec -it tikv0 sh -c "apk add curl; curl http://pd0:2379/metrics" -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | pull_request: 4 | branches: 5 | - master 6 | - dev 7 | permissions: 8 | contents: read 9 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 10 | # pull-requests: read 11 | jobs: 12 | golangci: 13 | name: lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/setup-go@v3 17 | with: 18 | go-version: 1.18.1 19 | - uses: actions/checkout@v3 20 | - name: golangci-lint 21 | uses: golangci/golangci-lint-action@v3 22 | with: 23 | # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 24 | version: v1.50.1 25 | 26 | # Optional: working directory, useful for monorepos 27 | # working-directory: somedir 28 | 29 | # Optional: golangci-lint command line arguments. 30 | # args: --issues-exit-code=0 31 | 32 | # Optional: show only new issues if it's a pull request. The default value is `false`. 33 | # only-new-issues: true 34 | 35 | # Optional: if set to true then the all caching functionality will be complete disabled, 36 | # takes precedence over all other caching options. 37 | # skip-cache: true 38 | 39 | # Optional: if set to true then the action don't cache or restore ~/go/pkg. 40 | # skip-pkg-cache: true 41 | 42 | # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. 43 | # skip-build-cache: true -------------------------------------------------------------------------------- /indexing_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestIndexing(t *testing.T) { 12 | t.Run("seekPrefix", func(t *testing.T) { 13 | { 14 | pfx := seekPrefix(context.Background(), "user", Index{ 15 | Name: "primary_idx", 16 | Fields: []string{"_id"}, 17 | Unique: true, 18 | Primary: true, 19 | ForeignKey: nil, 20 | }, map[string]any{ 21 | "_id": "123", 22 | }) 23 | 24 | assert.Equal(t, 25 | "ZGVmYXVsdABpbmRleAB1c2VyAHByaW1hcnlfaWR4AF9pZAAxMjM=", 26 | base64.StdEncoding.EncodeToString(pfx.Path())) 27 | } 28 | { 29 | pfx := seekPrefix(context.Background(), "user", Index{ 30 | Name: "primary_idx", 31 | Fields: []string{"_id"}, 32 | Unique: true, 33 | Primary: true, 34 | ForeignKey: nil, 35 | }, map[string]any{ 36 | "_id": "123", 37 | }) 38 | assert.Equal(t, 1, len(pfx.Fields())) 39 | assert.Equal(t, 40 | "ZGVmYXVsdABpbmRleAB1c2VyAHByaW1hcnlfaWR4AF9pZAAxMjMAMTIz", 41 | base64.StdEncoding.EncodeToString(pfx.Seek("123").Path())) 42 | } 43 | { 44 | pfx := seekPrefix(context.Background(), "user", Index{ 45 | Name: "testing", 46 | Fields: []string{"account_id", "contact.email"}, 47 | Unique: false, 48 | Primary: false, 49 | ForeignKey: nil, 50 | }, map[string]any{ 51 | "account_id": "123", 52 | "contact.email": "autom8ter@gmail.com", 53 | }) 54 | assert.Equal(t, 2, len(pfx.Fields())) 55 | assert.Empty(t, pfx.SeekValue()) 56 | assert.Equal(t, 57 | "ZGVmYXVsdABpbmRleAB1c2VyAHRlc3RpbmcAYWNjb3VudF9pZAAxMjMAY29udGFjdC5lbWFpbABhdXRvbTh0ZXJAZ21haWwuY29t", 58 | base64.StdEncoding.EncodeToString(pfx.Path())) 59 | } 60 | 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /javascript_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | _ "github.com/autom8ter/myjson/kv/badger" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestJavascript(t *testing.T) { 12 | t.Run("bool expression", func(t *testing.T) { 13 | ctx := context.Background() 14 | db, err := Open(ctx, "badger", nil) 15 | assert.NoError(t, err) 16 | defer db.Close(ctx) 17 | vm, err := getJavascriptVM(ctx, db, map[string]any{}) 18 | assert.NoError(t, err) 19 | assert.NotNil(t, vm) 20 | doc := NewDocument() 21 | assert.NoError(t, doc.Set("age", 10)) 22 | assert.NoError(t, vm.Set("doc", doc)) 23 | v, err := vm.RunString(`doc.Get("age") > 5`) 24 | assert.NoError(t, err) 25 | res := v.Export().(bool) 26 | 27 | assert.True(t, res, v.String()) 28 | }) 29 | t.Run("fetch", func(t *testing.T) { 30 | ctx := context.Background() 31 | db, err := Open(ctx, "badger", nil) 32 | assert.NoError(t, err) 33 | defer db.Close(ctx) 34 | vm, err := getJavascriptVM(ctx, db, map[string]any{}) 35 | assert.NoError(t, err) 36 | assert.NotNil(t, vm) 37 | var fetch = ` 38 | fetch({ 39 | method: "GET", 40 | url: "https://google.com", 41 | query: { 42 | q: "hello world" 43 | } 44 | }) 45 | ` 46 | val, err := vm.RunString(fetch) 47 | assert.NoError(t, err) 48 | assert.NotNil(t, val) 49 | }) 50 | t.Run("fetch error", func(t *testing.T) { 51 | ctx := context.Background() 52 | db, err := Open(ctx, "badger", nil) 53 | assert.NoError(t, err) 54 | defer db.Close(ctx) 55 | vm, err := getJavascriptVM(ctx, db, map[string]any{}) 56 | assert.NoError(t, err) 57 | assert.NotNil(t, vm) 58 | var fetch = ` 59 | fetch({ 60 | method: "GET", 61 | url: "google.com", 62 | query: { 63 | q: "hello world" 64 | } 65 | }) 66 | ` 67 | _, err = vm.RunString(fetch) 68 | assert.Error(t, err) 69 | }) 70 | } 71 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yaml: -------------------------------------------------------------------------------- 1 | name: Generate code coverage badge 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | - dev 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | name: Update coverage badge 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v2 16 | with: 17 | persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token. 18 | fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. 19 | 20 | - name: Setup go 21 | uses: actions/setup-go@v2 22 | with: 23 | go-version: '1.18.1' 24 | 25 | - uses: actions/cache@v2 26 | with: 27 | path: ~/go/pkg/mod 28 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 29 | restore-keys: | 30 | ${{ runner.os }}-go- 31 | 32 | - name: Run Test 33 | run: | 34 | go test -v ./... -covermode=count -coverprofile=coverage.out 35 | go tool cover -func=coverage.out -o=coverage.out 36 | 37 | - name: Go Coverage Badge # Pass the `coverage.out` output to this action 38 | uses: tj-actions/coverage-badge-go@v2 39 | with: 40 | filename: coverage.out 41 | 42 | - name: Verify Changed files 43 | uses: tj-actions/verify-changed-files@v12 44 | id: verify-changed-files 45 | with: 46 | files: README.md 47 | 48 | - name: Commit changes 49 | if: steps.verify-changed-files.outputs.files_changed == 'true' 50 | run: | 51 | git config --local user.email "action@github.com" 52 | git config --local user.name "GitHub Action" 53 | git add README.md 54 | git commit -m "chore: Updated coverage badge." 55 | 56 | - name: Push changes 57 | if: steps.verify-changed-files.outputs.files_changed == 'true' 58 | uses: ad-m/github-push-action@master 59 | with: 60 | github_token: ${{ github.token }} 61 | branch: ${{ github.head_ref }} -------------------------------------------------------------------------------- /testutil/testdata/account.yaml: -------------------------------------------------------------------------------- 1 | type: object 2 | # collection name 3 | x-collection: account 4 | required: 5 | - _id 6 | - name 7 | - created_at 8 | - status 9 | properties: 10 | _id: 11 | type: string 12 | description: The account's id. 13 | x-primary: true 14 | name: 15 | type: string 16 | description: The accounts's name. 17 | x-immutable: true 18 | created_at: 19 | type: integer 20 | description: The time at which the account was created. (UTC) 21 | x-immutable: true 22 | x-compute: 23 | expr: now().UnixMilli() 24 | write: true 25 | read: false 26 | status: 27 | type: string 28 | description: The account's status. 29 | default: inactive 30 | x-index: 31 | status_idx: { } 32 | x-authorization: 33 | rules: 34 | ## allow super users to do anything 35 | - effect: allow 36 | ## match on any action 37 | action: 38 | - "*" 39 | ## context metadata must have is_super_user set to true 40 | match: | 41 | isSuperUser(meta) 42 | 43 | ## dont allow read-only users to create/update/delete/set accounts 44 | - effect: deny 45 | ## match on document mutations 46 | action: 47 | - create 48 | - update 49 | - delete 50 | - set 51 | ## context metadata must have is_read_only set to true 52 | match: | 53 | contains(meta.Get('roles'), 'read_only') 54 | 55 | ## only allow users to update their own account 56 | - effect: allow 57 | ## match on document mutations 58 | action: 59 | - create 60 | - update 61 | - delete 62 | - set 63 | ## the account's _id must match the user's account_id 64 | match: | 65 | doc.Get('_id') == meta.Get('account_id') 66 | 67 | ## only allow users to query their own account 68 | - effect: allow 69 | ## match on document queries (includes ForEach and other Query based methods) 70 | action: 71 | - query 72 | ## user must have a group matching the account's _id 73 | match: | 74 | accountQueryAuth(query, meta) 75 | -------------------------------------------------------------------------------- /testutil/testdata/user.yaml: -------------------------------------------------------------------------------- 1 | type: object 2 | # x-collection specifies the name of the collection the object will be stored in 3 | x-collection: user 4 | # required specifies the required attributes 5 | required: 6 | - _id 7 | - name 8 | - age 9 | - contact 10 | - gender 11 | - account_id 12 | properties: 13 | _id: 14 | type: string 15 | description: The user's id. 16 | # x-primary indicates that the property is the primary key for the object - only one primary key may be specified 17 | x-primary: true 18 | name: 19 | type: string 20 | description: The user's name. 21 | contact: 22 | type: object 23 | properties: 24 | email: 25 | type: string 26 | description: The user's email. 27 | x-unique: true 28 | age: 29 | description: Age in years which must be equal to or greater than zero. 30 | type: integer 31 | minimum: 0 32 | account_id: 33 | type: string 34 | # x-foreign indicates that the property is a foreign key - foreign keys are automatically indexed 35 | x-foreign: 36 | collection: account 37 | field: _id 38 | cascade: true 39 | # x-index specifies a secondary index which can have 1-many fields 40 | x-index: 41 | account_email_idx: 42 | additional_fields: 43 | - contact.email 44 | language: 45 | type: string 46 | description: The user's first language. 47 | x-index: 48 | language_idx: { } 49 | gender: 50 | type: string 51 | description: The user's gender. 52 | enum: 53 | - male 54 | - female 55 | timestamp: 56 | type: string 57 | annotations: 58 | type: object 59 | 60 | # x-triggers are javascript functions that execute based on certain events 61 | x-triggers: 62 | # name of the trigger 63 | setTimestamp: 64 | # order determines the order in which the functions are executed - lower ordered triggers are executed first 65 | order: 1 66 | # events configures the trigger to execute on certain events 67 | events: 68 | - onCreate 69 | - onUpdate 70 | - onSet 71 | # script is the javascript to execute 72 | script: | 73 | setDocTimestamp(doc) 74 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | ) 8 | 9 | // Code is a code associated with an error 10 | type Code int 11 | 12 | const ( 13 | Internal Code = http.StatusInternalServerError 14 | NotFound Code = http.StatusNotFound 15 | Forbidden Code = http.StatusForbidden 16 | Validation Code = http.StatusBadRequest 17 | Unauthorized Code = http.StatusUnauthorized 18 | ) 19 | 20 | // Error is a custom error 21 | type Error struct { 22 | Code Code `json:"code"` 23 | Messages []string `json:"messages,omitempty"` 24 | Err string `json:"err,omitempty"` 25 | } 26 | 27 | // Error returns the Error as a json string 28 | func (e *Error) Error() string { 29 | bits, _ := json.Marshal(e) 30 | return string(bits) 31 | } 32 | 33 | // RemoveError removes the error from the Error and leaves it's messages and code 34 | func (e *Error) RemoveError() *Error { 35 | return &Error{ 36 | Code: e.Code, 37 | Messages: e.Messages, 38 | } 39 | } 40 | 41 | // Extract extracts the custom Error from the given error. If the error is nil, nil will be returned 42 | func Extract(err error) *Error { 43 | if err == nil { 44 | return nil 45 | } 46 | e, ok := err.(*Error) 47 | if !ok { 48 | return &Error{ 49 | Code: 0, 50 | Messages: nil, 51 | Err: err.Error(), 52 | } 53 | } 54 | return e 55 | } 56 | 57 | // New creates a new error and returns it 58 | func New(code Code, msg string, args ...any) error { 59 | e := &Error{ 60 | Code: code, 61 | Err: fmt.Sprintf(msg, args...), 62 | } 63 | return e 64 | } 65 | 66 | // Wrap Wraps the given error and returns a new one. If the error is nil, it will return nil 67 | func Wrap(err error, code Code, msg string, args ...any) error { 68 | if err == nil { 69 | return nil 70 | } 71 | e, ok := err.(*Error) 72 | if ok { 73 | if msg != "" { 74 | e.Messages = append(e.Messages, fmt.Sprintf(msg, args...)) 75 | } 76 | if e.Err == "" { 77 | e.Err = err.Error() 78 | } 79 | if code > 0 { 80 | e.Code = code 81 | } 82 | return e 83 | } 84 | e = &Error{ 85 | Code: code, 86 | Err: err.Error(), 87 | } 88 | if msg != "" { 89 | e.Messages = append(e.Messages, fmt.Sprintf(msg, args...)) 90 | } 91 | return e 92 | } 93 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/autom8ter/myjson/errors" 9 | "github.com/ghodss/yaml" 10 | "github.com/go-playground/validator/v10" 11 | "github.com/mitchellh/mapstructure" 12 | "github.com/spf13/cast" 13 | ) 14 | 15 | var validate = validator.New() 16 | 17 | func ValidateStruct(val any) error { 18 | return errors.Wrap(validate.Struct(val), errors.Validation, "") 19 | } 20 | 21 | // Decode decodes the input into the output based on json tags 22 | func Decode(input any, output any) error { 23 | config := &mapstructure.DecoderConfig{ 24 | WeaklyTypedInput: true, 25 | Result: output, 26 | TagName: "json", 27 | IgnoreUntaggedFields: true, 28 | } 29 | decoder, err := mapstructure.NewDecoder(config) 30 | if err != nil { 31 | return err 32 | } 33 | return decoder.Decode(input) 34 | } 35 | 36 | // JSONString returns a json string of the input 37 | func JSONString(input any) string { 38 | bits, _ := json.Marshal(input) 39 | return string(bits) 40 | } 41 | 42 | // PrettyJSONString returns a pretty json string of the input 43 | func PrettyJSONString(input any) string { 44 | bits, _ := json.MarshalIndent(input, "", " ") 45 | return string(bits) 46 | } 47 | 48 | func EncodeIndexValue(value any) []byte { 49 | if value == nil { 50 | return []byte("") 51 | } 52 | switch value := value.(type) { 53 | case bool: 54 | return EncodeIndexValue(cast.ToString(value)) 55 | case string: 56 | return []byte(value) 57 | case int, int64, int32, float64, float32, uint64, uint32, uint16: 58 | buf := make([]byte, 8) 59 | binary.BigEndian.PutUint64(buf, cast.ToUint64(value)) 60 | return buf 61 | case time.Time: 62 | return EncodeIndexValue(value.UnixNano()) 63 | case time.Duration: 64 | return EncodeIndexValue(int(value)) 65 | default: 66 | return EncodeIndexValue(JSONString(value)) 67 | } 68 | } 69 | 70 | func YAMLToJSON(yamlContent []byte) ([]byte, error) { 71 | if isJSON(string(yamlContent)) { 72 | return yamlContent, nil 73 | } 74 | return yaml.YAMLToJSON(yamlContent) 75 | } 76 | 77 | func JSONToYAML(jsonContent []byte) ([]byte, error) { 78 | return yaml.JSONToYAML(jsonContent) 79 | } 80 | 81 | func isJSON(str string) bool { 82 | var js json.RawMessage 83 | return json.Unmarshal([]byte(str), &js) == nil 84 | } 85 | 86 | func RemoveElement[T any](index int, results []T) []T { 87 | return append(results[:index], results[index+1:]...) 88 | } 89 | -------------------------------------------------------------------------------- /kv/tikv/docker/pd.toml: -------------------------------------------------------------------------------- 1 | # PD Configuration. 2 | 3 | name = "pd" 4 | data-dir = "default.pd" 5 | # 6 | #client-urls = "http://127.0.0.1:2379" 7 | ## if not set, use ${client-urls} 8 | #advertise-client-urls = "" 9 | # 10 | #peer-urls = "http://127.0.0.1:2380" 11 | ## if not set, use ${peer-urls} 12 | #advertise-peer-urls = "" 13 | # 14 | #initial-cluster = "pd=http://127.0.0.1:2380" 15 | initial-cluster-state = "new" 16 | 17 | lease = 3 18 | tso-save-interval = "3s" 19 | 20 | [security] 21 | # Path of file that contains list of trusted SSL CAs. if set, following four settings shouldn't be empty 22 | cacert-path = "" 23 | # Path of file that contains X509 certificate in PEM format. 24 | cert-path = "" 25 | # Path of file that contains X509 key in PEM format. 26 | key-path = "" 27 | 28 | [log] 29 | level = "info" 30 | 31 | # log format, one of json, text, console 32 | #format = "text" 33 | 34 | # disable automatic timestamps in output 35 | #disable-timestamp = false 36 | 37 | # file logging 38 | [log.file] 39 | #filename = "" 40 | # max log file size in MB 41 | #max-size = 300 42 | # max log file keep days 43 | #max-days = 28 44 | # maximum number of old log files to retain 45 | #max-backups = 7 46 | # rotate log by day 47 | #log-rotate = true 48 | 49 | [metric] 50 | # prometheus client push interval, set "0s" to disable prometheus. 51 | #interval = "15s" 52 | ## prometheus pushgateway address, leaves it empty will disable prometheus. 53 | #address = "pushgateway:9091" 54 | 55 | [schedule] 56 | max-merge-region-size = 0 57 | split-merge-interval = "1h" 58 | max-snapshot-count = 3 59 | max-pending-peer-count = 16 60 | max-store-down-time = "30m" 61 | leader-schedule-limit = 4 62 | region-schedule-limit = 4 63 | replica-schedule-limit = 8 64 | merge-schedule-limit = 8 65 | tolerant-size-ratio = 5.0 66 | 67 | # customized schedulers, the format is as below 68 | # if empty, it will use balance-leader, balance-region, hot-region as default 69 | # [[schedule.schedulers]] 70 | # type = "evict-leader" 71 | # args = ["1"] 72 | 73 | [replication] 74 | # The number of replicas for each region. 75 | max-replicas = 3 76 | # The label keys specified the location of a store. 77 | # The placement priorities is implied by the order of label keys. 78 | # For example, ["zone", "rack"] means that we should place replicas to 79 | # different zones first, then to different racks if we don't have enough zones. 80 | location-labels = [] 81 | 82 | [label-property] 83 | # Do not assign region leaders to stores that have these tags. 84 | # [[label-property.reject-leader]] 85 | # key = "zone" 86 | # value = "cn1 -------------------------------------------------------------------------------- /dag_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | 7 | "github.com/autom8ter/dagger" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestDag(t *testing.T) { 12 | t.Run("dag - topological sort", func(t *testing.T) { 13 | dag := &collectionDag{ 14 | dagger: dagger.NewGraph(), 15 | mu: sync.RWMutex{}, 16 | } 17 | u, _ := newCollectionSchema([]byte(userSchema)) 18 | a, _ := newCollectionSchema([]byte(accountSchema)) 19 | tsk, _ := newCollectionSchema([]byte(taskSchema)) 20 | if err := dag.SetSchemas([]CollectionSchema{u, a, tsk}); err != nil { 21 | t.Fatal(err) 22 | } 23 | sorted, err := dag.TopologicalSort() 24 | assert.NoError(t, err) 25 | assert.Equal(t, "task", sorted[0].Collection()) 26 | assert.Equal(t, "user", sorted[1].Collection()) 27 | assert.Equal(t, "account", sorted[2].Collection()) 28 | }) 29 | t.Run("dag - topological sort reverse", func(t *testing.T) { 30 | dag := &collectionDag{ 31 | dagger: dagger.NewGraph(), 32 | mu: sync.RWMutex{}, 33 | } 34 | u, _ := newCollectionSchema([]byte(userSchema)) 35 | a, _ := newCollectionSchema([]byte(accountSchema)) 36 | tsk, _ := newCollectionSchema([]byte(taskSchema)) 37 | if err := dag.SetSchemas([]CollectionSchema{u, a, tsk}); err != nil { 38 | t.Fatal(err) 39 | } 40 | sorted, err := dag.ReverseTopologicalSort() 41 | assert.NoError(t, err) 42 | assert.Equal(t, "account", sorted[0].Collection()) 43 | assert.Equal(t, "user", sorted[1].Collection()) 44 | assert.Equal(t, "task", sorted[2].Collection()) 45 | }) 46 | t.Run("dag - check edges", func(t *testing.T) { 47 | dag := &collectionDag{ 48 | dagger: dagger.NewGraph(), 49 | mu: sync.RWMutex{}, 50 | } 51 | u, _ := newCollectionSchema([]byte(userSchema)) 52 | a, _ := newCollectionSchema([]byte(accountSchema)) 53 | tsk, _ := newCollectionSchema([]byte(taskSchema)) 54 | if err := dag.SetSchemas([]CollectionSchema{u, a, tsk}); err != nil { 55 | t.Fatal(err) 56 | } 57 | { 58 | count := 0 59 | dag.dagger.RangeEdgesTo("foreignkey", dagger.Path{ 60 | XID: "task", 61 | XType: "collection", 62 | }, func(e dagger.Edge) bool { 63 | count++ 64 | return true 65 | }) 66 | assert.Equal(t, 0, count) 67 | } 68 | { 69 | count := 0 70 | dag.dagger.RangeEdgesTo("foreignkey", dagger.Path{ 71 | XID: "user", 72 | XType: "collection", 73 | }, func(e dagger.Edge) bool { 74 | count++ 75 | return true 76 | }) 77 | assert.Equal(t, 1, count) 78 | } 79 | { 80 | count := 0 81 | dag.dagger.RangeEdgesTo("foreignkey", dagger.Path{ 82 | XID: "account", 83 | XType: "collection", 84 | }, func(e dagger.Edge) bool { 85 | count++ 86 | return true 87 | }) 88 | assert.Equal(t, 1, count) 89 | } 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /indexing.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | 7 | "github.com/autom8ter/myjson/util" 8 | "github.com/nqd/flat" 9 | "github.com/spf13/cast" 10 | ) 11 | 12 | var nullByte = []byte("\x00") 13 | 14 | // indexFieldValue is a key value pair 15 | type indexFieldValue struct { 16 | Field string `json:"field"` 17 | Value any `json:"value"` 18 | } 19 | 20 | func seekPrefix(ctx context.Context, collection string, i Index, fields map[string]any) indexPathPrefix { 21 | fields, _ = flat.Flatten(fields, nil) 22 | var prefix = indexPathPrefix{ 23 | prefix: [][]byte{ 24 | []byte(cast.ToString(GetMetadataValue(ctx, MetadataKeyNamespace))), 25 | []byte("index"), 26 | []byte(collection), 27 | []byte(i.Name), 28 | }, 29 | } 30 | if i.Fields == nil { 31 | return prefix 32 | } 33 | for _, k := range i.Fields { 34 | if v, ok := fields[k]; ok { 35 | prefix = prefix.Append(k, v) 36 | } 37 | } 38 | return prefix 39 | } 40 | 41 | type indexPathPrefix struct { 42 | prefix [][]byte 43 | seekValue any 44 | fields [][]byte 45 | fieldMap []indexFieldValue 46 | } 47 | 48 | func (i indexPathPrefix) Append(field string, value any) indexPathPrefix { 49 | fields := append(i.fields, []byte(field), util.EncodeIndexValue(value)) 50 | fieldMap := append(i.fieldMap, indexFieldValue{ 51 | Field: field, 52 | Value: value, 53 | }) 54 | return indexPathPrefix{ 55 | prefix: i.prefix, 56 | fields: fields, 57 | fieldMap: fieldMap, 58 | } 59 | } 60 | 61 | func (i indexPathPrefix) Seek(value any) indexPathPrefix { 62 | return indexPathPrefix{ 63 | prefix: i.prefix, 64 | seekValue: value, 65 | fields: i.fields, 66 | fieldMap: i.fieldMap, 67 | } 68 | } 69 | 70 | func (i indexPathPrefix) Path() []byte { 71 | var path = append(i.prefix, i.fields...) 72 | if i.seekValue != nil { 73 | path = append(path, util.EncodeIndexValue(i.seekValue)) 74 | } 75 | return bytes.Join(path, nullByte) 76 | } 77 | 78 | func (i indexPathPrefix) SeekValue() any { 79 | return i.seekValue 80 | } 81 | 82 | func (i indexPathPrefix) Fields() []indexFieldValue { 83 | return i.fieldMap 84 | } 85 | 86 | func indexPrefix(ctx context.Context, collection, index string) []byte { 87 | path := [][]byte{ 88 | []byte(cast.ToString(GetMetadataValue(ctx, MetadataKeyNamespace))), 89 | []byte("index"), 90 | []byte(collection), 91 | []byte(index), 92 | } 93 | return bytes.Join(path, nullByte) 94 | } 95 | 96 | func collectionPrefix(ctx context.Context, collection string) []byte { 97 | path := [][]byte{ 98 | []byte(cast.ToString(GetMetadataValue(ctx, MetadataKeyNamespace))), 99 | []byte("index"), 100 | []byte(collection), 101 | } 102 | return bytes.Join(path, nullByte) 103 | } 104 | -------------------------------------------------------------------------------- /util/util_test.go: -------------------------------------------------------------------------------- 1 | package util_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "testing" 7 | 8 | "github.com/autom8ter/myjson" 9 | "github.com/autom8ter/myjson/testutil" 10 | "github.com/autom8ter/myjson/util" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestUtil(t *testing.T) { 15 | t.Run("yaml / json conversions", func(t *testing.T) { 16 | doc := testutil.NewUserDoc() 17 | yml, err := util.JSONToYAML([]byte(doc.String())) 18 | assert.NoError(t, err) 19 | jsonData, err := util.YAMLToJSON(yml) 20 | assert.NoError(t, err) 21 | doc2, err := myjson.NewDocumentFromBytes(jsonData) 22 | assert.NoError(t, err) 23 | assert.Equal(t, doc.String(), doc2.String()) 24 | }) 25 | t.Run("json string", func(t *testing.T) { 26 | doc := testutil.NewUserDoc() 27 | bits, _ := json.Marshal(doc) 28 | assert.Equal(t, string(bits), util.JSONString(doc)) 29 | }) 30 | t.Run("decode", func(t *testing.T) { 31 | doc := testutil.NewUserDoc() 32 | data := map[string]any{} 33 | assert.Nil(t, util.Decode(doc.Value(), &data)) 34 | doc2, err := myjson.NewDocumentFrom(data) 35 | assert.NoError(t, err) 36 | assert.Equal(t, doc.String(), doc2.String()) 37 | }) 38 | 39 | t.Run("validate", func(t *testing.T) { 40 | type usr struct { 41 | Name string `validate:"required"` 42 | } 43 | var u = usr{} 44 | assert.NotNil(t, util.ValidateStruct(&u)) 45 | u.Name = "a name" 46 | assert.Nil(t, util.ValidateStruct(&u)) 47 | }) 48 | t.Run("encode value (float)", func(t *testing.T) { 49 | val1 := util.EncodeIndexValue(1.0) 50 | val2 := util.EncodeIndexValue(2.0) 51 | compare := bytes.Compare(val1, val2) 52 | assert.Equal(t, -1, compare) 53 | }) 54 | t.Run("encode value (string)", func(t *testing.T) { 55 | val1 := util.EncodeIndexValue("hello") 56 | val2 := util.EncodeIndexValue("hellz") 57 | compare := bytes.Compare(val1, val2) 58 | assert.Equal(t, -1, compare) 59 | }) 60 | t.Run("encode value (string)", func(t *testing.T) { 61 | val1 := util.EncodeIndexValue(false) 62 | val2 := util.EncodeIndexValue(true) 63 | compare := bytes.Compare(val1, val2) 64 | assert.Equal(t, -1, compare) 65 | }) 66 | t.Run("encode value (json)", func(t *testing.T) { 67 | val1 := util.EncodeIndexValue(map[string]any{ 68 | "message": "hello", 69 | }) 70 | val2 := util.EncodeIndexValue(map[string]any{ 71 | "message": "hellz", 72 | }) 73 | compare := bytes.Compare(val1, val2) 74 | assert.Equal(t, -1, compare) 75 | }) 76 | t.Run("encode value (empty)", func(t *testing.T) { 77 | val1 := util.EncodeIndexValue(nil) 78 | val2 := util.EncodeIndexValue(nil) 79 | compare := bytes.Compare(val1, val2) 80 | assert.Equal(t, 0, compare) 81 | }) 82 | t.Run("remove element", func(t *testing.T) { 83 | var index = []int{1, 2, 3, 4, 5} 84 | index = util.RemoveElement(1, index) 85 | assert.Equal(t, 4, len(index)) 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /metadata.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import "context" 4 | 5 | type ctxKey int 6 | 7 | const ( 8 | metadataKey ctxKey = 0 9 | ) 10 | 11 | var ( 12 | // MetadataKeyNamespace is the key for the database namespace - it will return as "default" if not set 13 | MetadataKeyNamespace = "namespace" 14 | // MetadataKeyUserID is the key for the user id for use in x-authorizers (optional) 15 | MetadataKeyUserID = "userId" 16 | // MetadataKeyRoles is the key for the user roles([]string) for use in x-authorizers (optional) 17 | MetadataKeyRoles = "roles" 18 | // MetadataKeyGroups is the key for the user groups([]string) for use in x-authorizers (optional) 19 | MetadataKeyGroups = "groups" 20 | ) 21 | 22 | // GetMetadataValue gets a metadata value from the context if it exists 23 | func GetMetadataValue(ctx context.Context, key string) any { 24 | m, ok := ctx.Value(metadataKey).(*Document) 25 | if ok { 26 | val := m.Get(key) 27 | if val == nil && key == MetadataKeyNamespace { 28 | return "default" 29 | } 30 | return val 31 | } 32 | if key == MetadataKeyNamespace { 33 | return "default" 34 | } 35 | return nil 36 | } 37 | 38 | // SetMetadataValues sets metadata key value pairs in the context 39 | func SetMetadataValues(ctx context.Context, data map[string]any) context.Context { 40 | m := ExtractMetadata(ctx) 41 | _ = m.SetAll(data) 42 | return context.WithValue(ctx, metadataKey, m) 43 | } 44 | 45 | // SetMetadataNamespace sets the metadata namespace 46 | func SetMetadataNamespace(ctx context.Context, namespace string) context.Context { 47 | m := ExtractMetadata(ctx) 48 | _ = m.Set(MetadataKeyNamespace, namespace) 49 | return context.WithValue(ctx, metadataKey, m) 50 | } 51 | 52 | // SetMetadataUserID sets the metadata userID for targeting in the collections x-authorizers 53 | func SetMetadataUserID(ctx context.Context, userID string) context.Context { 54 | m := ExtractMetadata(ctx) 55 | _ = m.Set(MetadataKeyUserID, userID) 56 | return context.WithValue(ctx, metadataKey, m) 57 | } 58 | 59 | // SetMetadataRoles sets the metadata user roles for targeting in the collections x-authorizers 60 | func SetMetadataRoles(ctx context.Context, roles []string) context.Context { 61 | m := ExtractMetadata(ctx) 62 | _ = m.Set(MetadataKeyRoles, roles) 63 | return context.WithValue(ctx, metadataKey, m) 64 | } 65 | 66 | // SetMetadataGroups sets the metadata user groups for targeting in the collections x-authorizers 67 | func SetMetadataGroups(ctx context.Context, groups []string) context.Context { 68 | m := ExtractMetadata(ctx) 69 | _ = m.Set(MetadataKeyGroups, groups) 70 | return context.WithValue(ctx, metadataKey, m) 71 | } 72 | 73 | // ExtractMetadata extracts metadata from the context and returns it 74 | func ExtractMetadata(ctx context.Context) *Document { 75 | m, ok := ctx.Value(metadataKey).(*Document) 76 | if ok { 77 | return m 78 | } 79 | m = NewDocument() 80 | 81 | _ = m.Set(MetadataKeyNamespace, "default") 82 | return m 83 | } 84 | -------------------------------------------------------------------------------- /dag.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/autom8ter/dagger" 7 | "github.com/autom8ter/myjson/errors" 8 | ) 9 | 10 | type collectionDag struct { 11 | dagger *dagger.Graph 12 | mu sync.RWMutex 13 | } 14 | 15 | func newCollectionDag() *collectionDag { 16 | return &collectionDag{dagger: dagger.NewGraph()} 17 | } 18 | 19 | func (c *collectionDag) SetSchemas(schemas []CollectionSchema) error { 20 | c.mu.Lock() 21 | defer c.mu.Unlock() 22 | var newDag = newCollectionDag() 23 | for _, schema := range schemas { 24 | nodePath := dagger.Path{ 25 | XID: schema.Collection(), 26 | XType: "collection", 27 | } 28 | newDag.dagger.SetNode(nodePath, map[string]interface{}{ 29 | "schema": schema, 30 | }) 31 | } 32 | for _, schema := range schemas { 33 | for _, f := range schema.Properties() { 34 | if f.ForeignKey != nil { 35 | fkeypath := dagger.Path{ 36 | XID: f.ForeignKey.Collection, 37 | XType: "collection", 38 | } 39 | if !newDag.dagger.HasNode(fkeypath) { 40 | return errors.New(errors.Validation, "foreign key collection not found: %s", f.ForeignKey.Collection) 41 | } 42 | 43 | if _, err := newDag.dagger.SetEdge(dagger.Path{ 44 | XID: schema.Collection(), 45 | XType: "collection", 46 | }, fkeypath, dagger.Node{ 47 | Path: dagger.Path{ 48 | XID: f.Name, 49 | XType: "foreignkey", 50 | }, 51 | Attributes: map[string]interface{}{}, 52 | }, 53 | ); err != nil { 54 | panic(err) 55 | } 56 | } 57 | } 58 | } 59 | _, err := newDag.TopologicalSort() 60 | if err != nil { 61 | return err 62 | } 63 | _, err = newDag.ReverseTopologicalSort() 64 | if err != nil { 65 | return err 66 | } 67 | c.dagger = newDag.dagger 68 | return nil 69 | } 70 | 71 | func (c *collectionDag) TopologicalSort() ([]CollectionSchema, error) { 72 | c.mu.RLock() 73 | defer c.mu.RUnlock() 74 | var schemas []CollectionSchema 75 | var err error 76 | c.dagger.TopologicalSort("collection", "foreignkey", func(node dagger.Node) bool { 77 | var collection, ok = c.dagger.GetNode(node.Path) 78 | if !ok { 79 | err = errors.New(errors.Validation, "schema not found: %s", node.Path.XID) 80 | return false 81 | } 82 | schemas = append(schemas, collection.Attributes["schema"].(CollectionSchema)) 83 | return true 84 | }) 85 | return schemas, err 86 | } 87 | 88 | func (c *collectionDag) ReverseTopologicalSort() ([]CollectionSchema, error) { 89 | c.mu.RLock() 90 | defer c.mu.RUnlock() 91 | var schemas []CollectionSchema 92 | var err error 93 | c.dagger.ReverseTopologicalSort("collection", "foreignkey", func(node dagger.Node) bool { 94 | var collection, ok = c.dagger.GetNode(node.Path) 95 | if !ok { 96 | err = errors.New(errors.Validation, "schema not found: %s", node.Path.XID) 97 | return false 98 | } 99 | schemas = append(schemas, collection.Attributes["schema"].(CollectionSchema)) 100 | return true 101 | }) 102 | return schemas, err 103 | } 104 | -------------------------------------------------------------------------------- /kv/badger/tx.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/autom8ter/machine/v4" 8 | "github.com/autom8ter/myjson/kv" 9 | "github.com/dgraph-io/badger/v3" 10 | ) 11 | 12 | type badgerTx struct { 13 | mu sync.Mutex 14 | opts kv.TxOpts 15 | batch *badger.WriteBatch 16 | txn *badger.Txn 17 | db *badgerKV 18 | machine machine.Machine 19 | entries []kv.CDC 20 | } 21 | 22 | func (b *badgerTx) NewIterator(kopts kv.IterOpts) (kv.Iterator, error) { 23 | if b.txn == nil { 24 | b.txn = b.db.db.NewTransaction(!b.opts.IsReadOnly) 25 | } 26 | opts := badger.DefaultIteratorOptions 27 | opts.PrefetchValues = true 28 | opts.PrefetchSize = 10 29 | opts.Prefix = kopts.Prefix 30 | opts.Reverse = kopts.Reverse 31 | if kopts.Seek == nil && kopts.UpperBound != nil && kopts.Reverse { 32 | kopts.Seek = kopts.UpperBound 33 | } 34 | iter := b.txn.NewIterator(opts) 35 | if kopts.Seek == nil { 36 | iter.Rewind() 37 | } 38 | iter.Seek(kopts.Seek) 39 | return &badgerIterator{iter: iter, opts: kopts}, nil 40 | } 41 | 42 | func (b *badgerTx) Get(ctx context.Context, key []byte) ([]byte, error) { 43 | if b.txn == nil { 44 | b.txn = b.db.db.NewTransaction(!b.opts.IsReadOnly) 45 | } 46 | i, err := b.txn.Get(key) 47 | if err != nil { 48 | if err == badger.ErrKeyNotFound { 49 | return nil, nil 50 | } 51 | return nil, err 52 | } 53 | val, err := i.ValueCopy(nil) 54 | return val, err 55 | } 56 | 57 | func (b *badgerTx) Set(ctx context.Context, key, value []byte) error { 58 | b.mu.Lock() 59 | defer b.mu.Unlock() 60 | var e = &badger.Entry{ 61 | Key: key, 62 | Value: value, 63 | } 64 | if b.batch != nil { 65 | if err := b.batch.SetEntry(e); err != nil { 66 | return err 67 | } 68 | } else if b.txn != nil { 69 | if err := b.txn.SetEntry(e); err != nil { 70 | return err 71 | } 72 | } 73 | b.entries = append(b.entries, kv.CDC{ 74 | Operation: kv.SETOP, 75 | Key: key, 76 | Value: value, 77 | }) 78 | return nil 79 | } 80 | 81 | func (b *badgerTx) Delete(ctx context.Context, key []byte) error { 82 | b.mu.Lock() 83 | defer b.mu.Unlock() 84 | b.entries = append(b.entries, kv.CDC{ 85 | Operation: kv.DELOP, 86 | Key: key, 87 | }) 88 | if b.batch != nil { 89 | return b.batch.Delete(key) 90 | } 91 | return b.txn.Delete(key) 92 | } 93 | 94 | func (b *badgerTx) Rollback(ctx context.Context) error { 95 | b.mu.Lock() 96 | defer b.mu.Unlock() 97 | if b.batch != nil { 98 | b.batch.Cancel() 99 | } 100 | if b.txn != nil { 101 | b.txn.Discard() 102 | } 103 | b.entries = []kv.CDC{} 104 | return nil 105 | } 106 | 107 | func (b *badgerTx) Commit(ctx context.Context) error { 108 | b.mu.Lock() 109 | defer b.mu.Unlock() 110 | if b.batch != nil { 111 | if err := b.batch.Flush(); err != nil { 112 | return err 113 | } 114 | } else if b.txn != nil { 115 | if err := b.txn.Commit(); err != nil { 116 | return err 117 | } 118 | } 119 | for _, e := range b.entries { 120 | b.machine.Publish(ctx, machine.Message{ 121 | Channel: string(e.Key), 122 | Body: e, 123 | }) 124 | } 125 | return nil 126 | } 127 | 128 | func (b *badgerTx) Close(ctx context.Context) { 129 | b.mu.Lock() 130 | defer b.mu.Unlock() 131 | if b.txn != nil { 132 | b.txn.Discard() 133 | } 134 | if b.batch != nil { 135 | b.batch.Cancel() 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /optimizer.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "github.com/autom8ter/myjson/errors" 5 | "github.com/samber/lo" 6 | ) 7 | 8 | type defaultOptimizer struct{} 9 | 10 | func defaultExplain(c CollectionSchema) Explain { 11 | return Explain{ 12 | Collection: c.Collection(), 13 | Index: c.PrimaryIndex(), 14 | MatchedFields: []string{}, 15 | MatchedValues: map[string]any{}, 16 | SeekFields: []string{}, 17 | SeekValues: map[string]any{}, 18 | Reverse: false, 19 | } 20 | } 21 | 22 | func (o defaultOptimizer) Optimize(c CollectionSchema, where []Where) (Explain, error) { 23 | if len(c.PrimaryIndex().Fields) == 0 { 24 | return Explain{}, errors.New(errors.Internal, "zero configured indexes") 25 | } 26 | indexes := c.Indexing() 27 | if len(indexes) == 0 { 28 | return Explain{}, errors.New(errors.Internal, "zero configured indexes") 29 | } 30 | if len(where) == 0 { 31 | return defaultExplain(c), nil 32 | } 33 | if c.PrimaryIndex().Fields[0] == where[0].Field && where[0].Op == WhereOpEq { 34 | return Explain{ 35 | Index: c.PrimaryIndex(), 36 | MatchedFields: []string{c.PrimaryKey()}, 37 | MatchedValues: getMatchedFieldValues([]string{c.PrimaryKey()}, where), 38 | }, nil 39 | } 40 | var ( 41 | opt = &Explain{ 42 | Collection: c.Collection(), 43 | } 44 | ) 45 | for _, index := range indexes { 46 | if len(index.Fields) == 0 { 47 | continue 48 | } 49 | var ( 50 | matchedFields []string 51 | seekFields []string 52 | reverse bool 53 | ) 54 | for i, field := range index.Fields { 55 | if len(where) > i { 56 | if field == where[i].Field && where[i].Op == WhereOpEq { 57 | matchedFields = append(matchedFields, field) 58 | } else if field == where[i].Field && len(index.Fields)-1 == i { 59 | switch { 60 | case where[i].Op == WhereOpGt: 61 | seekFields = append(seekFields, field) 62 | case where[i].Op == WhereOpGte: 63 | seekFields = append(seekFields, field) 64 | case where[i].Op == WhereOpLt: 65 | seekFields = append(seekFields, field) 66 | reverse = true 67 | case where[i].Op == WhereOpLte: 68 | seekFields = append(seekFields, field) 69 | reverse = true 70 | } 71 | } 72 | } 73 | } 74 | matchedFields = lo.Uniq(matchedFields) 75 | if len(matchedFields)+len(seekFields) >= len(opt.MatchedFields)+len(opt.SeekFields) { 76 | opt.Index = index 77 | opt.MatchedFields = matchedFields 78 | opt.Reverse = reverse 79 | opt.SeekFields = seekFields 80 | } 81 | } 82 | if len(opt.MatchedFields)+len(opt.SeekFields) > 0 { 83 | opt.MatchedValues = getMatchedFieldValues(opt.MatchedFields, where) 84 | opt.SeekValues = getMatchedFieldValues(opt.SeekFields, where) 85 | return *opt, nil 86 | } 87 | if c.RequireQueryIndex() { 88 | return Explain{}, errors.New(errors.Forbidden, "index is required for query in collection: %s", c.Collection()) 89 | } 90 | return defaultExplain(c), nil 91 | } 92 | 93 | func getMatchedFieldValues(fields []string, where []Where) map[string]any { 94 | if len(fields) == 0 { 95 | return map[string]any{} 96 | } 97 | var whereValues = map[string]any{} 98 | for _, f := range fields { 99 | for _, w := range where { 100 | if w.Field != f { 101 | continue 102 | } 103 | whereValues[w.Field] = w.Value 104 | } 105 | } 106 | return whereValues 107 | } 108 | -------------------------------------------------------------------------------- /kv/badger/badger.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "time" 7 | 8 | "github.com/autom8ter/machine/v4" 9 | "github.com/autom8ter/myjson/errors" 10 | "github.com/autom8ter/myjson/kv" 11 | "github.com/autom8ter/myjson/kv/registry" 12 | "github.com/dgraph-io/badger/v3" 13 | "github.com/segmentio/ksuid" 14 | "github.com/spf13/cast" 15 | ) 16 | 17 | func init() { 18 | registry.Register("badger", func(params map[string]interface{}) (kv.DB, error) { 19 | return open(cast.ToString(params["storage_path"])) 20 | }) 21 | } 22 | 23 | type badgerKV struct { 24 | db *badger.DB 25 | machine machine.Machine 26 | } 27 | 28 | func open(storagePath string) (kv.DB, error) { 29 | opts := badger.DefaultOptions(storagePath) 30 | if storagePath == "" { 31 | opts.InMemory = true 32 | opts.Dir = "" 33 | opts.ValueDir = "" 34 | } 35 | opts = opts.WithLoggingLevel(badger.ERROR) 36 | db, err := badger.Open(opts) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return &badgerKV{ 41 | db: db, 42 | machine: machine.New(), 43 | }, nil 44 | } 45 | 46 | func (b *badgerKV) Tx(opts kv.TxOpts, fn func(kv.Tx) error) error { 47 | tx, err := b.NewTx(opts) 48 | if err != nil { 49 | return err 50 | } 51 | err = fn(tx) 52 | if err != nil { 53 | //nolint:errcheck 54 | tx.Rollback(context.Background()) 55 | return err 56 | } 57 | if err := tx.Commit(context.Background()); err != nil { 58 | return err 59 | } 60 | return nil 61 | } 62 | 63 | func (b *badgerKV) NewTx(opts kv.TxOpts) (kv.Tx, error) { 64 | if opts.IsBatch { 65 | return &badgerTx{ 66 | opts: opts, 67 | batch: b.db.NewWriteBatch(), 68 | db: b, 69 | machine: b.machine, 70 | }, nil 71 | } 72 | return &badgerTx{ 73 | opts: opts, 74 | txn: b.db.NewTransaction(!opts.IsReadOnly), 75 | db: b, 76 | machine: b.machine, 77 | }, nil 78 | } 79 | 80 | func (b *badgerKV) Close(ctx context.Context) error { 81 | b.machine.Close() 82 | if err := b.db.Sync(); err != nil { 83 | return err 84 | } 85 | return b.db.Close() 86 | } 87 | 88 | func (b *badgerKV) DropPrefix(ctx context.Context, prefix ...[]byte) error { 89 | return b.db.DropPrefix(prefix...) 90 | } 91 | 92 | func (b *badgerKV) NewLocker(key []byte, leaseInterval time.Duration) (kv.Locker, error) { 93 | return &badgerLock{ 94 | id: ksuid.New().String(), 95 | key: key, 96 | db: b, 97 | leaseInterval: leaseInterval, 98 | unlock: make(chan struct{}), 99 | hasUnlocked: make(chan struct{}), 100 | }, nil 101 | } 102 | 103 | func (b *badgerKV) ChangeStream(ctx context.Context, prefix []byte, fn kv.ChangeStreamHandler) error { 104 | return b.machine.Subscribe(ctx, "*", func(ctx context.Context, msg machine.Message) (bool, error) { 105 | cdc, ok := msg.Body.(kv.CDC) 106 | if !ok { 107 | return false, errors.New(errors.Internal, "invalid cdc") 108 | } 109 | if bytes.HasPrefix(cdc.Key, prefix) { 110 | return fn(cdc) 111 | } 112 | return true, nil 113 | }) 114 | } 115 | 116 | // 117 | //func (b *badgerKV) easyGet(ctx context.Context, key []byte) ([]byte, error) { 118 | // var ( 119 | // val []byte 120 | // err error 121 | // ) 122 | // err = b.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 123 | // val, err = tx.Get(ctx, key) 124 | // return err 125 | // }) 126 | // return val, err 127 | //} 128 | -------------------------------------------------------------------------------- /kv/tikv/tx.go: -------------------------------------------------------------------------------- 1 | package tikv 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/autom8ter/myjson/kv" 9 | "github.com/autom8ter/myjson/kv/kvutil" 10 | tikvErr "github.com/tikv/client-go/v2/error" 11 | "github.com/tikv/client-go/v2/txnkv/transaction" 12 | ) 13 | 14 | type tikvTx struct { 15 | txn *transaction.KVTxn 16 | opts kv.TxOpts 17 | db *tikvKV 18 | entries []kv.CDC 19 | } 20 | 21 | func (t *tikvTx) NewIterator(kopts kv.IterOpts) (kv.Iterator, error) { 22 | if kopts.Reverse { 23 | if kopts.Seek == nil { 24 | iter, err := t.txn.IterReverse(kvutil.NextPrefix(kopts.UpperBound)) 25 | if err != nil { 26 | return nil, err 27 | } 28 | // iter.Seek(kopts.Seek) // TODO: how to seek? 29 | return &tikvIterator{iter: iter, opts: kopts}, nil 30 | } 31 | iter, err := t.txn.IterReverse(kvutil.NextPrefix(kopts.Seek)) 32 | if err != nil { 33 | return nil, err 34 | } 35 | return &tikvIterator{iter: iter, opts: kopts}, nil 36 | } 37 | iter, err := t.txn.Iter(kopts.Prefix, kvutil.NextPrefix(kopts.UpperBound)) 38 | if err != nil { 39 | return nil, err 40 | } 41 | // iter.Seek(kopts.Seek) // TODO: how to seek? 42 | return &tikvIterator{iter: iter, opts: kopts}, nil 43 | } 44 | 45 | func (t *tikvTx) Get(ctx context.Context, key []byte) ([]byte, error) { 46 | { 47 | val, _ := t.db.cache.Get(ctx, string(key)).Result() 48 | if val != "" { 49 | return []byte(val), nil 50 | } 51 | } 52 | 53 | val, err := t.txn.Get(ctx, key) 54 | if err != nil { 55 | if tikvErr.IsErrNotFound(err) { 56 | return nil, nil 57 | } 58 | return nil, err 59 | } 60 | return val, err 61 | } 62 | 63 | func (t *tikvTx) Set(ctx context.Context, key, value []byte) error { 64 | if t.opts.IsReadOnly { 65 | return fmt.Errorf("writes forbidden in read-only transaction") 66 | } 67 | if err := t.txn.Set(key, value); err != nil { 68 | return err 69 | } 70 | t.entries = append(t.entries, kv.CDC{ 71 | Operation: kv.SETOP, 72 | Key: key, 73 | Value: value, 74 | }) 75 | return nil 76 | } 77 | 78 | func (t *tikvTx) Delete(ctx context.Context, key []byte) error { 79 | if t.opts.IsReadOnly { 80 | return fmt.Errorf("writes forbidden in read-only transaction") 81 | } 82 | if err := t.txn.Delete(key); err != nil { 83 | return err 84 | } 85 | t.db.cache.Del(ctx, string(key)) 86 | t.entries = append(t.entries, kv.CDC{ 87 | Operation: kv.DELOP, 88 | Key: key, 89 | }) 90 | return nil 91 | } 92 | 93 | func (t *tikvTx) Rollback(ctx context.Context) error { 94 | t.entries = []kv.CDC{} 95 | return t.txn.Rollback() 96 | } 97 | 98 | func (t *tikvTx) Commit(ctx context.Context) error { 99 | if err := t.txn.Commit(ctx); err != nil { 100 | return err 101 | } 102 | var toSet = map[string][]byte{} 103 | for _, e := range t.entries { 104 | bits, err := json.Marshal(e) 105 | if err != nil { 106 | return err 107 | } 108 | t.db.cache.Publish(ctx, string(e.Key), bits) 109 | switch e.Operation { 110 | case kv.DELOP: 111 | toSet[string(e.Key)] = []byte("") 112 | case kv.SETOP: 113 | toSet[string(e.Key)] = e.Value 114 | } 115 | } 116 | if err := t.db.cache.MSet(ctx, toSet).Err(); err != nil { 117 | return err 118 | } 119 | t.entries = []kv.CDC{} 120 | return nil 121 | } 122 | 123 | func (t *tikvTx) Close(ctx context.Context) { 124 | t.entries = []kv.CDC{} 125 | } 126 | -------------------------------------------------------------------------------- /schema_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | // import embed package 6 | _ "embed" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestSchema(t *testing.T) { 13 | t.Run("basic", func(t *testing.T) { 14 | schema, err := newCollectionSchema([]byte(userSchema)) 15 | assert.NoError(t, err) 16 | assert.NotNil(t, schema.Indexing()) 17 | assert.NotEmpty(t, schema.Indexing()["_id.primaryidx"]) 18 | assert.NotEmpty(t, schema.Indexing()["account_id.foreignidx"]) 19 | assert.NotEmpty(t, schema.Indexing()["contact.email.uniqueidx"]) 20 | assert.NotEmpty(t, schema.Indexing()["account_email_idx"]) 21 | assert.NotEmpty(t, schema.Indexing()["language_idx"]) 22 | }) 23 | t.Run("json schema validation", func(t *testing.T) { 24 | schema, err := newCollectionSchema([]byte(userSchema)) 25 | assert.NoError(t, err) 26 | assert.NoError(t, schema.ValidateDocument(context.Background(), newUserDoc())) 27 | assert.Error(t, schema.ValidateDocument(context.Background(), NewDocument())) 28 | }) 29 | t.Run("primary key", func(t *testing.T) { 30 | schema, err := newCollectionSchema([]byte(taskSchema)) 31 | assert.NoError(t, err) 32 | assert.Equal(t, "_id", schema.PrimaryKey()) 33 | }) 34 | t.Run("collection", func(t *testing.T) { 35 | schema, err := newCollectionSchema([]byte(taskSchema)) 36 | assert.NoError(t, err) 37 | assert.Equal(t, "task", schema.Collection()) 38 | }) 39 | t.Run("indexing not nil", func(t *testing.T) { 40 | schema, err := newCollectionSchema([]byte(userSchema)) 41 | assert.NoError(t, err) 42 | assert.NotNil(t, schema.Indexing()) 43 | }) 44 | t.Run("authz not nil", func(t *testing.T) { 45 | schema, err := newCollectionSchema([]byte(accountSchema)) 46 | assert.NoError(t, err) 47 | assert.NotNil(t, schema.Authz()) 48 | }) 49 | t.Run("MarshalJSON/UnmarshalJSON", func(t *testing.T) { 50 | schema, err := newCollectionSchema([]byte(userSchema)) 51 | assert.NoError(t, err) 52 | bits, _ := schema.MarshalJSON() 53 | assert.NoError(t, schema.UnmarshalJSON(bits)) 54 | }) 55 | t.Run("MarshalYAML/UnmarshalYAML", func(t *testing.T) { 56 | schema, err := newCollectionSchema([]byte(userSchema)) 57 | assert.NoError(t, err) 58 | before, _ := schema.MarshalJSON() 59 | bits, _ := schema.MarshalYAML() 60 | assert.NoError(t, schema.UnmarshalYAML(bits)) 61 | after, _ := schema.MarshalJSON() 62 | assert.JSONEq(t, string(before), string(after)) 63 | }) 64 | t.Run("properties", func(t *testing.T) { 65 | schema, err := newCollectionSchema([]byte(userSchema)) 66 | assert.NoError(t, err) 67 | assert.Equal(t, "age", schema.Properties()["age"].Name) 68 | assert.Equal(t, "name", schema.Properties()["name"].Name) 69 | assert.Equal(t, "account_id", schema.Properties()["account_id"].Name) 70 | }) 71 | t.Run("properties foreign", func(t *testing.T) { 72 | schema, err := newCollectionSchema([]byte(userSchema)) 73 | assert.NoError(t, err) 74 | assert.Equal(t, "account", schema.Properties()["account_id"].ForeignKey.Collection) 75 | assert.Equal(t, true, schema.Properties()["account_id"].ForeignKey.Cascade) 76 | assert.Equal(t, true, schema.PropertyPaths()["account_id"].ForeignKey.Cascade) 77 | }) 78 | t.Run("properties nested", func(t *testing.T) { 79 | schema, err := newCollectionSchema([]byte(userSchema)) 80 | assert.NoError(t, err) 81 | assert.Equal(t, "email", schema.Properties()["contact"].Properties["email"].Name) 82 | assert.Equal(t, "contact.email", schema.Properties()["contact"].Properties["email"].Path) 83 | assert.Equal(t, "contact.email", schema.PropertyPaths()["contact.email"].Path) 84 | }) 85 | 86 | } 87 | -------------------------------------------------------------------------------- /model_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestQuery(t *testing.T) { 11 | t.Run("validate empty query", func(t *testing.T) { 12 | q := Query{ 13 | Select: nil, 14 | Where: nil, 15 | } 16 | assert.NotNil(t, q.Validate(context.Background())) 17 | }) 18 | t.Run("validate no select", func(t *testing.T) { 19 | a := Query{} 20 | assert.NotNil(t, a.Validate(context.Background())) 21 | }) 22 | t.Run("validate bad group by", func(t *testing.T) { 23 | a := Query{ 24 | GroupBy: []string{"account_id"}, 25 | Select: []Select{ 26 | { 27 | Field: "age", 28 | Aggregate: AggregateFunctionSum, 29 | As: "age_sum", 30 | }, 31 | }, 32 | OrderBy: []OrderBy{ 33 | { 34 | Field: "account_id", 35 | Direction: OrderByDirectionAsc, 36 | }, 37 | }, 38 | } 39 | assert.NotNil(t, a.Validate(context.Background())) 40 | }) 41 | t.Run("validate bad order by", func(t *testing.T) { 42 | a := Query{ 43 | Select: []Select{ 44 | { 45 | Field: "*", 46 | }, 47 | }, 48 | OrderBy: []OrderBy{ 49 | { 50 | Field: "account_id", 51 | Direction: "dsc", 52 | }, 53 | }, 54 | } 55 | assert.NotNil(t, a.Validate(context.Background())) 56 | }) 57 | t.Run("validate bad where op", func(t *testing.T) { 58 | a := Query{ 59 | Select: []Select{ 60 | { 61 | Field: "*", 62 | }, 63 | }, 64 | Where: []Where{ 65 | { 66 | Field: "account_id", 67 | Op: "==", 68 | Value: 9, 69 | }, 70 | }, 71 | } 72 | assert.NotNil(t, a.Validate(context.Background())) 73 | }) 74 | t.Run("validate bad where field", func(t *testing.T) { 75 | a := Query{ 76 | Select: []Select{ 77 | { 78 | Field: "*", 79 | }, 80 | }, 81 | Where: []Where{ 82 | { 83 | Field: "", 84 | Op: WhereOpEq, 85 | Value: 9, 86 | }, 87 | }, 88 | } 89 | assert.NotNil(t, a.Validate(context.Background())) 90 | }) 91 | t.Run("validate bad where value", func(t *testing.T) { 92 | a := Query{ 93 | Select: []Select{ 94 | { 95 | Field: "*", 96 | }, 97 | }, 98 | Where: []Where{ 99 | { 100 | Field: "name", 101 | Op: WhereOpEq, 102 | }, 103 | }, 104 | } 105 | assert.NotNil(t, a.Validate(context.Background())) 106 | }) 107 | t.Run("validate bad limit", func(t *testing.T) { 108 | a := Query{ 109 | Select: []Select{ 110 | { 111 | Field: "*", 112 | }, 113 | }, 114 | Limit: -1, 115 | } 116 | assert.NotNil(t, a.Validate(context.Background())) 117 | }) 118 | t.Run("validate bad page", func(t *testing.T) { 119 | a := Query{ 120 | Select: []Select{ 121 | { 122 | Field: "*", 123 | }, 124 | }, 125 | Page: -1, 126 | } 127 | assert.NotNil(t, a.Validate(context.Background())) 128 | }) 129 | t.Run("validate good query", func(t *testing.T) { 130 | a := Query{ 131 | Select: []Select{ 132 | { 133 | Field: "test", 134 | Aggregate: AggregateFunctionMax, 135 | As: "max_test", 136 | }, 137 | }, 138 | } 139 | assert.Nil(t, a.Validate(context.Background())) 140 | }) 141 | } 142 | 143 | func TestMetadataContext(t *testing.T) { 144 | ctx := context.Background() 145 | assert.Equal(t, "default", GetMetadataValue(ctx, MetadataKeyNamespace)) 146 | c := ExtractMetadata(ctx) 147 | assert.NotNil(t, c) 148 | assert.Equal(t, "default", c.GetString(MetadataKeyNamespace)) 149 | c.Set("testing", true) 150 | assert.NotNil(t, c) 151 | 152 | c.Set(MetadataKeyNamespace, "acme.com") 153 | assert.Equal(t, "acme.com", c.GetString(MetadataKeyNamespace)) 154 | } 155 | -------------------------------------------------------------------------------- /kv/tikv/tikv.go: -------------------------------------------------------------------------------- 1 | package tikv 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/autom8ter/myjson/kv" 11 | "github.com/autom8ter/myjson/kv/registry" 12 | "github.com/go-redis/redis/v9" 13 | "github.com/segmentio/ksuid" 14 | "github.com/spf13/cast" 15 | "github.com/tikv/client-go/v2/txnkv" 16 | ) 17 | 18 | func init() { 19 | registry.Register("tikv", func(params map[string]interface{}) (kv.DB, error) { 20 | if params["pd_addr"] == nil { 21 | return nil, fmt.Errorf("'pd_addr' is a required paramater") 22 | } 23 | if params["redis_addr"] == nil { 24 | return nil, fmt.Errorf("'redis_addr' is a required paramater") 25 | } 26 | return open(params) 27 | }) 28 | } 29 | 30 | type tikvKV struct { 31 | db *txnkv.Client 32 | cache *redis.Client 33 | } 34 | 35 | func open(params map[string]interface{}) (kv.DB, error) { 36 | pdAddr := cast.ToStringSlice(params["pd_addr"]) 37 | if len(pdAddr) == 0 { 38 | return nil, fmt.Errorf("empty pd address") 39 | } 40 | client, err := txnkv.NewClient(pdAddr) 41 | if err != nil { 42 | return nil, err 43 | } 44 | cache := redis.NewClient(&redis.Options{ 45 | Addr: cast.ToString(params["redis_addr"]), 46 | Username: cast.ToString(params["redis_user"]), 47 | Password: cast.ToString(params["redis_password"]), 48 | }) 49 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 50 | defer cancel() 51 | if err := cache.Ping(ctx); err != nil && err.Err() != nil { 52 | return nil, fmt.Errorf("failed to ping redis instance(%s): %s", cast.ToString(params["redis_addr"]), err.Err()) 53 | } 54 | return &tikvKV{ 55 | db: client, 56 | cache: cache, 57 | }, nil 58 | } 59 | 60 | func (b *tikvKV) Tx(opts kv.TxOpts, fn func(kv.Tx) error) error { 61 | tx, err := b.NewTx(opts) 62 | if err != nil { 63 | return err 64 | } 65 | err = fn(tx) 66 | if err != nil { 67 | if rollbackErr := tx.Rollback(context.Background()); rollbackErr != nil { 68 | return fmt.Errorf("%s - failed to rollback transaction: %s", err.Error(), rollbackErr.Error()) 69 | } 70 | return err 71 | } 72 | if err := tx.Commit(context.Background()); err != nil { 73 | return err 74 | } 75 | return nil 76 | } 77 | 78 | func (b *tikvKV) NewTx(opts kv.TxOpts) (kv.Tx, error) { 79 | tx, err := b.db.Begin() 80 | if err != nil { 81 | return nil, err 82 | } 83 | if !tx.Valid() { 84 | return nil, fmt.Errorf("invalid transaction") 85 | } 86 | return &tikvTx{txn: tx, db: b, opts: opts}, nil 87 | } 88 | 89 | func (b *tikvKV) Close(ctx context.Context) error { 90 | return b.db.Close() 91 | } 92 | 93 | func (b *tikvKV) DropPrefix(ctx context.Context, prefix ...[]byte) error { 94 | for _, p := range prefix { 95 | if _, err := b.db.DeleteRange(ctx, p, nil, 1); err != nil { 96 | return err 97 | } 98 | } 99 | return nil 100 | } 101 | 102 | func (b *tikvKV) NewLocker(key []byte, leaseInterval time.Duration) (kv.Locker, error) { 103 | return &tikvLock{ 104 | id: ksuid.New().String(), 105 | key: key, 106 | db: b, 107 | leaseInterval: leaseInterval, 108 | unlock: make(chan struct{}), 109 | hasUnlocked: make(chan struct{}), 110 | }, nil 111 | } 112 | 113 | func (b *tikvKV) ChangeStream(ctx context.Context, prefix []byte, fn kv.ChangeStreamHandler) error { 114 | ch := b.cache.PSubscribe(ctx, "*").Channel() 115 | for { 116 | select { 117 | case <-ctx.Done(): 118 | return nil 119 | case msg := <-ch: 120 | var cdc kv.CDC 121 | //nolint:errcheck 122 | json.Unmarshal([]byte(msg.Payload), &cdc) 123 | if bytes.HasPrefix(cdc.Key, prefix) { 124 | contn, err := fn(cdc) 125 | if err != nil { 126 | return err 127 | } 128 | if !contn { 129 | return nil 130 | } 131 | } 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /authorization_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/autom8ter/myjson/kv" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var globalScript = ` 12 | function isSuperUser(meta) { 13 | return contains(meta.Get('roles'), 'super_user') 14 | } 15 | function accountQueryAuth(query, meta) { 16 | return query.where?.length > 0 && query.where[0].field == '_id' && query.where[0].op == 'eq' && contains(meta.Get('groups'), query.where[0].value) 17 | } 18 | ` 19 | 20 | func TestAuthorization(t *testing.T) { 21 | t.Run("set as super user (allow)", func(t *testing.T) { 22 | ctx := SetMetadataRoles(context.Background(), []string{"super_user"}) 23 | db, err := Open(ctx, "badger", map[string]any{}, WithGlobalJavascriptFunctions([]string{globalScript})) 24 | assert.NoError(t, err) 25 | assert.NoError(t, db.Configure(ctx, "", []string{accountSchema, userSchema, taskSchema})) 26 | assert.NoError(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 27 | _, err := tx.Create(ctx, "account", db.NewDoc().Set(map[string]any{ 28 | "name": "acme", 29 | }).Doc()) 30 | return err 31 | })) 32 | }) 33 | t.Run("set as readonly user (deny)", func(t *testing.T) { 34 | ctx := SetMetadataRoles(context.Background(), []string{"read_only"}) 35 | db, err := Open(ctx, "badger", map[string]any{}, WithGlobalJavascriptFunctions([]string{globalScript})) 36 | assert.NoError(t, err) 37 | assert.NoError(t, db.Configure(ctx, "", []string{accountSchema, userSchema, taskSchema})) 38 | assert.Error(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 39 | _, err := tx.Create(ctx, "account", db.NewDoc().Set(map[string]any{ 40 | "name": "acme", 41 | }).Doc()) 42 | return err 43 | })) 44 | }) 45 | t.Run("set as no role user (deny)", func(t *testing.T) { 46 | ctx := context.Background() 47 | db, err := Open(ctx, "badger", map[string]any{}, WithGlobalJavascriptFunctions([]string{globalScript})) 48 | assert.NoError(t, err) 49 | assert.NoError(t, db.Configure(ctx, "", []string{accountSchema, userSchema, taskSchema})) 50 | assert.Error(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 51 | _, err := tx.Create(ctx, "account", db.NewDoc().Set(map[string]any{ 52 | "name": "acme", 53 | }).Doc()) 54 | return err 55 | })) 56 | }) 57 | t.Run("read other account as readonly user (deny)", func(t *testing.T) { 58 | ctx := SetMetadataRoles(context.Background(), []string{"read_only"}) 59 | db, err := Open(ctx, "badger", map[string]any{}, WithGlobalJavascriptFunctions([]string{globalScript})) 60 | assert.NoError(t, err) 61 | assert.NoError(t, db.Configure(ctx, "", []string{accountSchema, userSchema, taskSchema})) 62 | assert.Error(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 63 | _, err := tx.Get(ctx, "account", "1") 64 | return err 65 | })) 66 | }) 67 | t.Run("read account as readonly user with proper group (allow)", func(t *testing.T) { 68 | ctx := context.Background() 69 | db, err := Open(ctx, "badger", map[string]any{}, WithGlobalJavascriptFunctions([]string{globalScript})) 70 | assert.NoError(t, err) 71 | ctx = SetMetadataRoles(context.Background(), []string{"super_user"}) 72 | assert.NoError(t, db.Configure(ctx, "", []string{accountSchema, userSchema, taskSchema})) 73 | assert.NoError(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 74 | assert.NoError(t, tx.Set(ctx, "account", db.NewDoc().Set(map[string]any{ 75 | "_id": "1", 76 | "name": "acme", 77 | }).Doc())) 78 | _, err := tx.Get(ctx, "account", "1") 79 | return err 80 | })) 81 | ctx = SetMetadataRoles(context.Background(), []string{"read_only"}) 82 | ctx = SetMetadataGroups(context.Background(), []string{"1"}) 83 | assert.NoError(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 84 | _, err := tx.Get(ctx, "account", "1") 85 | return err 86 | })) 87 | }) 88 | } 89 | -------------------------------------------------------------------------------- /kv/tikv/locker.go: -------------------------------------------------------------------------------- 1 | package tikv 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/autom8ter/myjson/kv" 9 | tikvErr "github.com/tikv/client-go/v2/error" 10 | ) 11 | 12 | type tikvLock struct { 13 | id string 14 | key []byte 15 | db *tikvKV 16 | leaseInterval time.Duration 17 | start time.Time 18 | hasUnlocked chan struct{} 19 | unlock chan struct{} 20 | } 21 | 22 | type lockMeta struct { 23 | ID string `json:"id"` 24 | Start time.Time `json:"start"` 25 | LastUpdate time.Time `json:"lastUpdate"` 26 | Key []byte `json:"key"` 27 | } 28 | 29 | func (b *tikvLock) IsLocked(ctx context.Context) (bool, error) { 30 | isLocked := true 31 | err := b.db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 32 | val, err := tx.Get(ctx, b.key) 33 | if err != nil { 34 | if !tikvErr.IsErrNotFound(err) { 35 | return err 36 | } 37 | isLocked = false 38 | return nil 39 | } 40 | var current lockMeta 41 | //nolint:errcheck 42 | json.Unmarshal(val, ¤t) 43 | if time.Since(current.LastUpdate) > 4*b.leaseInterval && current.ID != b.id { 44 | isLocked = false 45 | return nil 46 | } 47 | return nil 48 | }) 49 | return isLocked, err 50 | } 51 | 52 | func (b *tikvLock) TryLock(ctx context.Context) (bool, error) { 53 | b.start = time.Now() 54 | gotLock := false 55 | err := b.db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 56 | val, err := tx.Get(ctx, b.key) 57 | if err != nil { 58 | if !tikvErr.IsErrNotFound(err) { 59 | return err 60 | } 61 | if err := b.setLock(ctx, tx); err != nil { 62 | return err 63 | } 64 | gotLock = true 65 | return nil 66 | } 67 | var current lockMeta 68 | //nolint:errcheck 69 | json.Unmarshal(val, ¤t) 70 | if time.Since(current.LastUpdate) > 4*b.leaseInterval && current.ID != b.id { 71 | if err := b.setLock(ctx, tx); err != nil { 72 | return err 73 | } 74 | gotLock = true 75 | return nil 76 | } 77 | return nil 78 | }) 79 | if err == nil && gotLock { 80 | //nolint:errcheck 81 | go b.keepalive(ctx) 82 | } 83 | return gotLock, err 84 | } 85 | 86 | func (b *tikvLock) Unlock() { 87 | b.unlock <- struct{}{} 88 | <-b.hasUnlocked 89 | } 90 | 91 | func (b *tikvLock) setLock(ctx context.Context, tx kv.Tx) error { 92 | meta := &lockMeta{ 93 | ID: b.id, 94 | Start: b.start, 95 | LastUpdate: time.Now(), 96 | Key: b.key, 97 | } 98 | bytes, _ := json.Marshal(meta) 99 | if err := tx.Set( 100 | ctx, 101 | b.key, 102 | bytes, 103 | ); err != nil { 104 | return err 105 | } 106 | return nil 107 | } 108 | 109 | func (b *tikvLock) delLock(ctx context.Context, tx kv.Tx) error { 110 | return tx.Delete(ctx, b.key) 111 | } 112 | 113 | func (b *tikvLock) getLock(ctx context.Context, tx kv.Tx) (*lockMeta, error) { 114 | val, err := tx.Get(ctx, b.key) 115 | if err != nil { 116 | return nil, err 117 | } 118 | var m lockMeta 119 | if err := json.Unmarshal(val, &m); err != nil { 120 | return nil, err 121 | } 122 | return &m, nil 123 | } 124 | 125 | func (b *tikvLock) keepalive(ctx context.Context) error { 126 | ctx, cancel := context.WithCancel(ctx) 127 | defer cancel() 128 | ticker := time.NewTicker(b.leaseInterval) 129 | defer ticker.Stop() 130 | for { 131 | select { 132 | case <-ticker.C: 133 | // update lease 134 | err := b.db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 135 | val, err := b.getLock(ctx, tx) 136 | if err != nil { 137 | return err 138 | } 139 | if val.ID == b.id { 140 | return b.setLock(ctx, tx) 141 | } 142 | return nil 143 | }) 144 | if err != nil { 145 | return err 146 | } 147 | case <-b.unlock: 148 | err := b.db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 149 | val, err := b.getLock(ctx, tx) 150 | if err != nil { 151 | return err 152 | } 153 | if val.ID == b.id { 154 | return b.delLock(ctx, tx) 155 | } 156 | return nil 157 | }) 158 | b.hasUnlocked <- struct{}{} 159 | return err 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /kv/badger/locker.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/autom8ter/myjson/kv" 9 | "github.com/dgraph-io/badger/v3" 10 | ) 11 | 12 | type badgerLock struct { 13 | id string 14 | key []byte 15 | db *badgerKV 16 | leaseInterval time.Duration 17 | start time.Time 18 | hasUnlocked chan struct{} 19 | unlock chan struct{} 20 | } 21 | 22 | type lockMeta struct { 23 | ID string `json:"id"` 24 | Start time.Time `json:"start"` 25 | LastUpdate time.Time `json:"lastUpdate"` 26 | Key []byte `json:"key"` 27 | } 28 | 29 | func (b *badgerLock) IsLocked(ctx context.Context) (bool, error) { 30 | //return false, nil 31 | isLocked := true 32 | err := b.db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 33 | val, err := tx.Get(ctx, b.key) 34 | if err != nil { 35 | if err != badger.ErrKeyNotFound { 36 | return err 37 | } 38 | isLocked = false 39 | return nil 40 | } 41 | var current lockMeta 42 | //nolint:errcheck 43 | json.Unmarshal(val, ¤t) 44 | if time.Since(current.LastUpdate) > 4*b.leaseInterval && current.ID != b.id { 45 | isLocked = false 46 | return nil 47 | } 48 | return nil 49 | }) 50 | return isLocked, err 51 | } 52 | 53 | func (b *badgerLock) TryLock(ctx context.Context) (bool, error) { 54 | b.start = time.Now() 55 | gotLock := false 56 | err := b.db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 57 | val, err := tx.Get(ctx, b.key) 58 | if err != nil { 59 | if err != badger.ErrKeyNotFound { 60 | return err 61 | } 62 | if err := b.setLock(ctx, tx); err != nil { 63 | return err 64 | } 65 | gotLock = true 66 | return nil 67 | } 68 | var current lockMeta 69 | //nolint:errcheck 70 | json.Unmarshal(val, ¤t) 71 | if time.Since(current.LastUpdate) > 4*b.leaseInterval && current.ID != b.id { 72 | if err := b.setLock(ctx, tx); err != nil { 73 | return err 74 | } 75 | gotLock = true 76 | return nil 77 | } 78 | return nil 79 | }) 80 | if err == nil && gotLock { 81 | //nolint:errcheck 82 | go b.keepalive(ctx) 83 | } 84 | return gotLock, err 85 | } 86 | 87 | func (b *badgerLock) Unlock() { 88 | b.unlock <- struct{}{} 89 | <-b.hasUnlocked 90 | } 91 | 92 | func (b *badgerLock) setLock(ctx context.Context, tx kv.Tx) error { 93 | meta := &lockMeta{ 94 | ID: b.id, 95 | Start: b.start, 96 | LastUpdate: time.Now(), 97 | Key: b.key, 98 | } 99 | bytes, _ := json.Marshal(meta) 100 | if err := tx.Set( 101 | ctx, 102 | b.key, 103 | bytes, 104 | ); err != nil { 105 | return err 106 | } 107 | return nil 108 | } 109 | 110 | func (b *badgerLock) delLock(ctx context.Context, tx kv.Tx) error { 111 | return tx.Delete(ctx, b.key) 112 | } 113 | 114 | func (b *badgerLock) getLock(ctx context.Context, tx kv.Tx) (*lockMeta, error) { 115 | val, err := tx.Get(ctx, b.key) 116 | if err != nil { 117 | return nil, err 118 | } 119 | var m lockMeta 120 | if err := json.Unmarshal(val, &m); err != nil { 121 | return nil, err 122 | } 123 | return &m, nil 124 | } 125 | 126 | func (b *badgerLock) keepalive(ctx context.Context) error { 127 | ctx, cancel := context.WithCancel(ctx) 128 | defer cancel() 129 | ticker := time.NewTicker(b.leaseInterval) 130 | defer ticker.Stop() 131 | for { 132 | select { 133 | case <-ticker.C: 134 | // update lease 135 | err := b.db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 136 | val, err := b.getLock(ctx, tx) 137 | if err != nil { 138 | return err 139 | } 140 | if val.ID == b.id { 141 | return b.setLock(ctx, tx) 142 | } 143 | return nil 144 | }) 145 | if err != nil { 146 | return err 147 | } 148 | case <-b.unlock: 149 | err := b.db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 150 | val, err := b.getLock(ctx, tx) 151 | if err != nil { 152 | return err 153 | } 154 | if val.ID == b.id { 155 | return b.delLock(ctx, tx) 156 | } 157 | return nil 158 | }) 159 | b.hasUnlocked <- struct{}{} 160 | return err 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /kv/kv.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // DB is a key value database implementation 9 | type DB interface { 10 | // Tx executes the given function against a database transaction 11 | Tx(opts TxOpts, fn func(Tx) error) error 12 | // NewTx creates a new database transaction. 13 | NewTx(opts TxOpts) (Tx, error) 14 | // NewLocker returns a mutex/locker with the given lease duration 15 | NewLocker(key []byte, leaseInterval time.Duration) (Locker, error) 16 | // DropPrefix drops keys with the given prefix(s) from the database 17 | DropPrefix(ctx context.Context, prefix ...[]byte) error 18 | ChangeStreamer 19 | // Close closes the key value database 20 | Close(ctx context.Context) error 21 | } 22 | 23 | // IterOpts are options when creating an iterator 24 | type IterOpts struct { 25 | // Prefix indicates that keys must match the given prefix 26 | Prefix []byte `json:"prefix"` 27 | // UpperBound indicates that keys must be <= the upper bound 28 | UpperBound []byte `json:"upperBound"` 29 | // Seek seeks to the given bytes before beginning to iterate 30 | Seek []byte `json:"seek"` 31 | // Reverse scans the index in reverse 32 | Reverse bool `json:"reverse"` 33 | } 34 | 35 | // TxOpts are options when creating a database transaction 36 | type TxOpts struct { 37 | IsReadOnly bool `json:"isReadOnly,omitempty"` 38 | IsBatch bool `json:"isBatch,omitempty"` 39 | } 40 | 41 | // Tx is a database transaction interface 42 | type Tx interface { 43 | // Getter gets the specified key in the database(if it exists) 44 | Getter 45 | // Mutator executes mutations against the database 46 | Mutator 47 | // NewIterator creates a new iterator 48 | NewIterator(opts IterOpts) (Iterator, error) 49 | // Commit commits the transaction 50 | Commit(ctx context.Context) error 51 | // Rollback rolls back any changes made by the transaction 52 | Rollback(ctx context.Context) error 53 | // Close closes the transaction 54 | Close(ctx context.Context) 55 | } 56 | 57 | // Getter gets the specified key in the database(if it exists). If the key does not exist, a nil byte slice and no error is returned 58 | type Getter interface { 59 | Get(ctx context.Context, key []byte) ([]byte, error) 60 | } 61 | 62 | // Iterator is a key value database iterator. Keys should be sorted lexicographically. 63 | type Iterator interface { 64 | // Seek seeks to the given key 65 | Seek(key []byte) 66 | // Close closes the iterator 67 | Close() 68 | // Valid returns true if the iterator is still valid 69 | Valid() bool 70 | // Key returns the key at the current cursor position 71 | Key() []byte 72 | // Value returns the value at the current cursor position 73 | Value() ([]byte, error) 74 | // Next iterates to the next item 75 | Next() error 76 | } 77 | 78 | // Item is a key value pair in the database 79 | type Item interface { 80 | // Key is the items key - it is a unique identifier for it's value 81 | Key() []byte 82 | // Value is the bytes that correspond to the item's key 83 | Value() ([]byte, error) 84 | } 85 | 86 | // Setter sets specified key/value in the database. If ttl is empty, the key should never expire 87 | type Setter interface { 88 | Set(ctx context.Context, key, value []byte) error 89 | } 90 | 91 | // Deleter deletes specified keys from the database 92 | type Deleter interface { 93 | Delete(ctx context.Context, key []byte) error 94 | } 95 | 96 | // Mutator executes mutations against the database 97 | type Mutator interface { 98 | // Setter sets specified key/value in the database 99 | Setter 100 | // Deleter deletes specified keys from the database 101 | Deleter 102 | } 103 | 104 | type Locker interface { 105 | TryLock(ctx context.Context) (bool, error) 106 | Unlock() 107 | IsLocked(ctx context.Context) (bool, error) 108 | } 109 | 110 | // TxOp is an transaction operation type 111 | type TxOp string 112 | 113 | const ( 114 | // SETOP sets a key value pair 115 | SETOP TxOp = "SET" 116 | // DELOP deletes a key value pair 117 | DELOP TxOp = "DEL" 118 | ) 119 | 120 | // CDC or change data capture holds an transaction operation type and a key value pair. 121 | // It is used as a container for streaming changes to key value pairs 122 | type CDC struct { 123 | Operation TxOp `json:"operation"` 124 | Key []byte `json:"key"` 125 | Value []byte `json:"value,omitempty"` 126 | } 127 | 128 | type ChangeStreamHandler func(cdc CDC) (bool, error) 129 | 130 | type ChangeStreamer interface { 131 | ChangeStream(ctx context.Context, prefix []byte, fn ChangeStreamHandler) error 132 | } 133 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/autom8ter/myjson 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/autom8ter/machine/v4 v4.0.0-20221003043928-593fc3a020bb 7 | github.com/brianvoe/gofakeit/v6 v6.19.0 8 | github.com/dgraph-io/badger/v3 v3.2103.2 9 | github.com/dop251/goja v0.0.0-20221224150820-cc4634e76e9a 10 | github.com/ghodss/yaml v1.0.0 11 | github.com/go-playground/validator/v10 v10.11.1 12 | github.com/go-redis/redis/v9 v9.0.0-rc.2 13 | github.com/google/uuid v1.3.0 14 | github.com/huandu/xstrings v1.4.0 15 | github.com/mitchellh/mapstructure v1.5.0 16 | github.com/nqd/flat v0.1.1 17 | github.com/samber/lo v1.28.2 18 | github.com/segmentio/ksuid v1.0.4 19 | github.com/spf13/cast v1.5.0 20 | github.com/stretchr/testify v1.8.1 21 | github.com/thoas/go-funk v0.9.1 22 | github.com/tidwall/gjson v1.14.3 23 | github.com/tidwall/sjson v1.2.5 24 | github.com/tikv/client-go/v2 v2.0.3 25 | github.com/xeipuuv/gojsonschema v1.2.0 26 | golang.org/x/sync v0.1.0 27 | ) 28 | 29 | require ( 30 | github.com/Masterminds/goutils v1.1.1 // indirect 31 | github.com/Masterminds/semver/v3 v3.2.0 // indirect 32 | github.com/Masterminds/sprig/v3 v3.2.3 // indirect 33 | github.com/autom8ter/dagger v1.0.1 // indirect 34 | github.com/benbjohnson/clock v1.3.0 // indirect 35 | github.com/beorn7/perks v1.0.1 // indirect 36 | github.com/cespare/xxhash v1.1.0 // indirect 37 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 38 | github.com/coreos/go-semver v0.3.0 // indirect 39 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 40 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect 41 | github.com/davecgh/go-spew v1.1.1 // indirect 42 | github.com/dgraph-io/ristretto v0.1.0 // indirect 43 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect 44 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 45 | github.com/dlclark/regexp2 v1.7.0 // indirect 46 | github.com/dustin/go-humanize v1.0.0 // indirect 47 | github.com/go-playground/locales v0.14.0 // indirect 48 | github.com/go-playground/universal-translator v0.18.0 // indirect 49 | github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect 50 | github.com/gogo/protobuf v1.3.2 // indirect 51 | github.com/golang/glog v1.0.0 // indirect 52 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 53 | github.com/golang/protobuf v1.5.2 // indirect 54 | github.com/golang/snappy v0.0.4 // indirect 55 | github.com/google/btree v1.1.2 // indirect 56 | github.com/google/flatbuffers v1.12.1 // indirect 57 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect 58 | github.com/imdario/mergo v0.3.13 // indirect 59 | github.com/klauspost/compress v1.13.6 // indirect 60 | github.com/leodido/go-urn v1.2.1 // indirect 61 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 62 | github.com/mitchellh/copystructure v1.2.0 // indirect 63 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 64 | github.com/opentracing/opentracing-go v1.2.0 // indirect 65 | github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect 66 | github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect 67 | github.com/pingcap/kvproto v0.0.0-20221227030452-22819f5b377a // indirect 68 | github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect 69 | github.com/pkg/errors v0.9.1 // indirect 70 | github.com/pmezard/go-difflib v1.0.0 // indirect 71 | github.com/prometheus/client_golang v1.14.0 // indirect 72 | github.com/prometheus/client_model v0.3.0 // indirect 73 | github.com/prometheus/common v0.39.0 // indirect 74 | github.com/prometheus/procfs v0.9.0 // indirect 75 | github.com/remyoudompheng/bigfft v0.0.0-20220927061507-ef77025ab5aa // indirect 76 | github.com/segmentio/fasthash v1.0.3 // indirect 77 | github.com/shopspring/decimal v1.3.1 // indirect 78 | github.com/stathat/consistent v1.0.0 // indirect 79 | github.com/tidwall/match v1.1.1 // indirect 80 | github.com/tidwall/pretty v1.2.1 // indirect 81 | github.com/tikv/pd/client v0.0.0-20221230063818-531d9f32dcbc // indirect 82 | github.com/twmb/murmur3 v1.1.6 // indirect 83 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect 84 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect 85 | github.com/zyedidia/generic v1.2.1 // indirect 86 | go.etcd.io/etcd/api/v3 v3.5.6 // indirect 87 | go.etcd.io/etcd/client/pkg/v3 v3.5.6 // indirect 88 | go.etcd.io/etcd/client/v3 v3.5.6 // indirect 89 | go.opencensus.io v0.23.0 // indirect 90 | go.uber.org/atomic v1.10.0 // indirect 91 | go.uber.org/multierr v1.9.0 // indirect 92 | go.uber.org/zap v1.24.0 // indirect 93 | golang.org/x/crypto v0.6.0 // indirect 94 | golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect 95 | golang.org/x/net v0.6.0 // indirect 96 | golang.org/x/sys v0.5.0 // indirect 97 | golang.org/x/text v0.7.0 // indirect 98 | google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect 99 | google.golang.org/grpc v1.51.0 // indirect 100 | google.golang.org/protobuf v1.28.1 // indirect 101 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect 102 | gopkg.in/yaml.v2 v2.4.0 // indirect 103 | gopkg.in/yaml.v3 v3.0.1 // indirect 104 | ) 105 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package myjson_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/autom8ter/myjson" 8 | "github.com/autom8ter/myjson/kv" 9 | ) 10 | 11 | func ExampleOpen() { 12 | ctx, cancel := context.WithCancel(context.Background()) 13 | defer cancel() 14 | db, err := myjson.Open(ctx, "badger", map[string]any{ 15 | // leave empty for in-memory 16 | "storage_path": "", 17 | }) 18 | if err != nil { 19 | panic(err) 20 | } 21 | defer db.Close(ctx) 22 | var accountSchema = ` 23 | type: object 24 | # collection name 25 | x-collection: account 26 | required: 27 | - _id 28 | - name 29 | properties: 30 | _id: 31 | type: string 32 | description: The account's id. 33 | x-primary: true 34 | name: 35 | type: string 36 | description: The accounts's name. 37 | x-authorization: 38 | rules: 39 | ## allow super users to do anything 40 | - effect: allow 41 | ## match on any action 42 | action: 43 | - "*" 44 | ## context metadata must have is_super_user set to true 45 | match: | 46 | contains(meta.Get('roles'), 'super_user') 47 | 48 | ## dont allow read-only users to create/update/delete/set accounts 49 | - effect: deny 50 | ## match on document mutations 51 | action: 52 | - create 53 | - update 54 | - delete 55 | - set 56 | ## context metadata must have is_read_only set to true 57 | match: | 58 | contains(meta.Get('roles'), 'read_only') 59 | 60 | ## only allow users to update their own account 61 | - effect: allow 62 | ## match on document mutations 63 | action: 64 | - create 65 | - update 66 | - delete 67 | - set 68 | ## the account's _id must match the user's account_id 69 | match: | 70 | doc.Get('_id') == meta.Get('account_id') 71 | 72 | ## only allow users to query their own account 73 | - effect: allow 74 | ## match on document queries (includes ForEach and other Query based methods) 75 | action: 76 | - query 77 | ## user must have a group matching the account's _id 78 | match: | 79 | query.where?.length > 0 && query.where[0].field == '_id' && query.where[0].op == 'eq' && contains(meta.Get('groups'), query.where[0].value) 80 | 81 | ` 82 | if err := db.Configure(ctx, "", []string{accountSchema}); err != nil { 83 | panic(err) 84 | } 85 | if err := db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 86 | // create a new account document 87 | account, err := myjson.NewDocumentFrom(map[string]any{ 88 | "name": "acme.com", 89 | }) 90 | if err != nil { 91 | return err 92 | } 93 | // create the account 94 | _, err = tx.Create(ctx, "account", account) 95 | if err != nil { 96 | return err 97 | } 98 | return nil 99 | }); err != nil { 100 | panic(err) 101 | } 102 | } 103 | 104 | func ExampleQ() { 105 | query := myjson.Q(). 106 | Select(myjson.Select{ 107 | Field: "*", 108 | }). 109 | Where(myjson.Where{ 110 | Field: "description", 111 | Op: myjson.WhereOpContains, 112 | Value: "testing", 113 | }).Query() 114 | fmt.Println(query.String()) 115 | // Output: 116 | // {"select":[{"field":"*"}],"where":[{"field":"description","op":"contains","value":"testing"}],"page":0} 117 | } 118 | 119 | func ExampleD() { 120 | doc := myjson.D().Set(map[string]any{ 121 | "name": "John Doe", 122 | "email": "johndoe@gmail.com", 123 | }).Doc() 124 | fmt.Println(doc.String()) 125 | } 126 | 127 | func ExampleSetMetadataGroups() { 128 | ctx := context.Background() 129 | ctx = myjson.SetMetadataGroups(ctx, []string{"group1", "group2"}) 130 | fmt.Println(myjson.ExtractMetadata(ctx).GetArray("groups")) 131 | } 132 | 133 | func ExampleSetMetadataRoles() { 134 | ctx := context.Background() 135 | ctx = myjson.SetMetadataRoles(ctx, []string{"super_user"}) 136 | fmt.Println(myjson.ExtractMetadata(ctx).GetArray("roles")) 137 | } 138 | 139 | func ExampleExtractMetadata() { 140 | ctx := context.Background() 141 | meta := myjson.ExtractMetadata(ctx) 142 | fmt.Println(meta.String()) 143 | // Output: 144 | // {"namespace":"default"} 145 | } 146 | 147 | func ExampleNewDocumentFrom() { 148 | doc, _ := myjson.NewDocumentFrom(map[string]any{ 149 | "name": "autom8ter", 150 | }) 151 | fmt.Println(doc.String()) 152 | // Output: 153 | // {"name":"autom8ter"} 154 | } 155 | 156 | func ExampleDocument_Scan() { 157 | type User struct { 158 | Name string `json:"name"` 159 | } 160 | doc, _ := myjson.NewDocumentFrom(map[string]any{ 161 | "name": "autom8ter", 162 | }) 163 | 164 | var usr User 165 | 166 | doc.Scan(&usr) 167 | fmt.Println(usr.Name) 168 | // Output: 169 | // autom8ter 170 | } 171 | 172 | func ExampleDocument_Set() { 173 | 174 | doc := myjson.NewDocument() 175 | doc.Set("name", "autom8ter") 176 | doc.Set("contact.email", "coleman@autom8ter.com") 177 | 178 | fmt.Println(doc.String()) 179 | // Output: 180 | // {"name":"autom8ter","contact":{"email":"coleman@autom8ter.com"}} 181 | } 182 | 183 | func ExampleDocument_Get() { 184 | 185 | doc := myjson.NewDocument() 186 | doc.Set("name", "autom8ter") 187 | doc.Set("contact.email", "coleman@autom8ter.com") 188 | 189 | fmt.Println(doc.String()) 190 | // Output: 191 | // {"name":"autom8ter","contact":{"email":"coleman@autom8ter.com"}} 192 | } 193 | -------------------------------------------------------------------------------- /optimizer_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/brianvoe/gofakeit/v6" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestOptimizer(t *testing.T) { 12 | o := defaultOptimizer{} 13 | schema, err := newCollectionSchema([]byte(userSchema)) 14 | assert.NoError(t, err) 15 | indexes := schema 16 | t.Run("select secondary index", func(t *testing.T) { 17 | explain, err := o.Optimize(indexes, []Where{ 18 | { 19 | Field: "contact.email", 20 | Op: WhereOpEq, 21 | Value: gofakeit.Email(), 22 | }, 23 | }) 24 | assert.NoError(t, err) 25 | assert.Equal(t, false, explain.Index.Primary) 26 | assert.Equal(t, "contact.email", explain.MatchedFields[0]) 27 | }) 28 | 29 | t.Run("select primary index", func(t *testing.T) { 30 | explain, err := o.Optimize(indexes, []Where{ 31 | { 32 | Field: "_id", 33 | Op: WhereOpEq, 34 | Value: gofakeit.Email(), 35 | }, 36 | }) 37 | assert.NoError(t, err) 38 | assert.Equal(t, true, explain.Index.Primary, explain.MatchedFields) 39 | assert.Equal(t, "_id", explain.MatchedFields[0], explain.MatchedFields) 40 | }) 41 | 42 | t.Run("select secondary index (multi-field)", func(t *testing.T) { 43 | explain, err := o.Optimize(indexes, []Where{ 44 | { 45 | Field: "account_id", 46 | Op: WhereOpEq, 47 | Value: "1", 48 | }, 49 | { 50 | Field: "contact.email", 51 | Op: WhereOpEq, 52 | Value: gofakeit.Email(), 53 | }, 54 | }) 55 | assert.NoError(t, err) 56 | assert.Equal(t, false, explain.Index.Primary) 57 | assert.Equal(t, "account_id", explain.MatchedFields[0]) 58 | assert.Equal(t, "contact.email", explain.MatchedFields[1]) 59 | }) 60 | t.Run("select secondary index 2", func(t *testing.T) { 61 | explain, err := o.Optimize(indexes, []Where{ 62 | { 63 | Field: "contact.email", 64 | Op: WhereOpEq, 65 | Value: gofakeit.Email(), 66 | }, 67 | { 68 | Field: "account_id", 69 | Op: WhereOpEq, 70 | Value: "1", 71 | }, 72 | }) 73 | assert.NoError(t, err) 74 | assert.EqualValues(t, false, explain.Index.Primary) 75 | assert.Equal(t, "contact.email", explain.MatchedFields[0]) 76 | }) 77 | t.Run("select secondary index (multi-field partial match)", func(t *testing.T) { 78 | explain, err := o.Optimize(indexes, []Where{ 79 | { 80 | Field: "account_id", 81 | Op: WhereOpEq, 82 | Value: "1", 83 | }, 84 | }) 85 | assert.NoError(t, err) 86 | assert.Equal(t, false, explain.Index.Primary) 87 | assert.Equal(t, "account_id", explain.MatchedFields[0]) 88 | }) 89 | t.Run("select secondary index (multi-field partial match (!=))", func(t *testing.T) { 90 | explain, err := o.Optimize(indexes, []Where{ 91 | { 92 | Field: "account_id", 93 | Op: "!=", 94 | Value: "1", 95 | }, 96 | }) 97 | assert.NoError(t, err) 98 | assert.Equal(t, true, explain.Index.Primary) 99 | assert.Equal(t, 0, len(explain.MatchedFields)) 100 | }) 101 | t.Run("select secondary index (>)", func(t *testing.T) { 102 | cdc, err := newCollectionSchema([]byte(cdcSchema)) 103 | assert.NoError(t, err) 104 | explain, err := o.Optimize(cdc, []Where{ 105 | { 106 | Field: "timestamp", 107 | Op: WhereOpGt, 108 | Value: time.Now().String(), 109 | }, 110 | }) 111 | assert.NoError(t, err) 112 | assert.Equal(t, false, explain.Index.Primary) 113 | assert.Equal(t, "timestamp", explain.SeekFields[0]) 114 | assert.NotEmpty(t, explain.SeekValues["timestamp"]) 115 | }) 116 | t.Run("select primary index (neq)", func(t *testing.T) { 117 | explain, err := o.Optimize(indexes, []Where{ 118 | { 119 | Field: "_id", 120 | Op: WhereOpNeq, 121 | Value: gofakeit.Email(), 122 | }, 123 | }) 124 | assert.NoError(t, err) 125 | assert.Equal(t, true, explain.Index.Primary) 126 | }) 127 | t.Run("select primary index (hasPrefix)", func(t *testing.T) { 128 | explain, err := o.Optimize(indexes, []Where{ 129 | { 130 | Field: "_id", 131 | Op: WhereOpHasPrefix, 132 | Value: gofakeit.Email(), 133 | }, 134 | }) 135 | assert.NoError(t, err) 136 | assert.Equal(t, true, explain.Index.Primary) 137 | }) 138 | t.Run("select primary index (hasSuffix)", func(t *testing.T) { 139 | explain, err := o.Optimize(indexes, []Where{ 140 | { 141 | Field: "_id", 142 | Op: WhereOpHasSuffix, 143 | Value: gofakeit.Email(), 144 | }, 145 | }) 146 | assert.NoError(t, err) 147 | assert.Equal(t, true, explain.Index.Primary) 148 | }) 149 | t.Run("select primary index (contains)", func(t *testing.T) { 150 | explain, err := o.Optimize(indexes, []Where{ 151 | { 152 | Field: "_id", 153 | Op: WhereOpContains, 154 | Value: gofakeit.Email(), 155 | }, 156 | }) 157 | assert.NoError(t, err) 158 | assert.Equal(t, true, explain.Index.Primary) 159 | }) 160 | t.Run("select primary index (in)", func(t *testing.T) { 161 | explain, err := o.Optimize(indexes, []Where{ 162 | { 163 | Field: "_id", 164 | Op: WhereOpIn, 165 | Value: []string{gofakeit.Email()}, 166 | }, 167 | }) 168 | assert.NoError(t, err) 169 | assert.Equal(t, true, explain.Index.Primary) 170 | }) 171 | t.Run("select primary index (containsAll)", func(t *testing.T) { 172 | explain, err := o.Optimize(indexes, []Where{ 173 | { 174 | Field: "_id", 175 | Op: WhereOpContainsAll, 176 | Value: []string{gofakeit.Email()}, 177 | }, 178 | }) 179 | assert.NoError(t, err) 180 | assert.Equal(t, true, explain.Index.Primary) 181 | }) 182 | } 183 | -------------------------------------------------------------------------------- /testutil/testutil.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/autom8ter/myjson" 11 | "github.com/autom8ter/myjson/kv" 12 | "github.com/brianvoe/gofakeit/v6" 13 | "github.com/stretchr/testify/assert" 14 | 15 | // import embed package 16 | _ "embed" 17 | 18 | // import badger kv provider 19 | _ "github.com/autom8ter/myjson/kv/badger" 20 | ) 21 | 22 | var ( 23 | //go:embed testdata/scripts.js 24 | GlobalScript string 25 | //go:embed testdata/task.yaml 26 | TaskSchema string 27 | //go:embed testdata/user.yaml 28 | UserSchema string 29 | //go:embed testdata/account.yaml 30 | AccountSchema string 31 | AllCollections = []string{AccountSchema, UserSchema, TaskSchema} 32 | ) 33 | 34 | func NewUserDoc() *myjson.Document { 35 | doc, err := myjson.NewDocumentFrom(map[string]interface{}{ 36 | "_id": gofakeit.UUID(), 37 | "name": gofakeit.Name(), 38 | "contact": map[string]interface{}{ 39 | "email": fmt.Sprintf("%v.%s", gofakeit.IntRange(0, 100), gofakeit.Email()), 40 | }, 41 | "account_id": fmt.Sprint(gofakeit.IntRange(0, 100)), 42 | "language": gofakeit.Language(), 43 | "birthday_month": gofakeit.Month(), 44 | "favorite_number": gofakeit.Second(), 45 | "gender": gofakeit.Gender(), 46 | "age": gofakeit.IntRange(0, 100), 47 | "timestamp": gofakeit.DateRange(time.Now().Truncate(7200*time.Hour), time.Now()), 48 | "annotations": gofakeit.Map(), 49 | }) 50 | if err != nil { 51 | panic(err) 52 | } 53 | return doc 54 | } 55 | 56 | func NewTaskDoc(usrID string) *myjson.Document { 57 | doc, err := myjson.NewDocumentFrom(map[string]interface{}{ 58 | "_id": gofakeit.UUID(), 59 | "user": usrID, 60 | "content": gofakeit.LoremIpsumSentence(5), 61 | }) 62 | if err != nil { 63 | panic(err) 64 | } 65 | return doc 66 | } 67 | 68 | type TestFunc func(ctx context.Context, t *testing.T, db myjson.Database) 69 | 70 | type TestConfig struct { 71 | Opts []myjson.DBOpt 72 | Persist bool 73 | Collections []string 74 | Values string 75 | Roles []string 76 | Timeout time.Duration 77 | } 78 | 79 | func Test(t *testing.T, cfg TestConfig, fn TestFunc) func(*testing.T) { 80 | ctx := context.Background() 81 | var ( 82 | db myjson.Database 83 | err error 84 | ) 85 | var closers []func() 86 | if cfg.Persist { 87 | _ = os.MkdirAll("tmp", 0700) 88 | dir, err := os.MkdirTemp("./tmp", "") 89 | assert.NoError(t, err) 90 | closers = append(closers, func() { 91 | os.RemoveAll(dir) 92 | }) 93 | db, err = myjson.Open(ctx, "badger", map[string]any{ 94 | "storage_path": dir, 95 | }, cfg.Opts...) 96 | } else { 97 | db, err = myjson.Open(ctx, "badger", map[string]any{}, cfg.Opts...) 98 | } 99 | assert.NoError(t, err) 100 | closers = append(closers, func() { 101 | db.Close(ctx) 102 | }) 103 | assert.NoError(t, err) 104 | if len(cfg.Collections) > 0 { 105 | assert.NoError(t, db.Configure(ctx, cfg.Values, cfg.Collections)) 106 | } 107 | return func(t *testing.T) { 108 | if cfg.Timeout == 0 { 109 | cfg.Timeout = 5 * time.Minute 110 | } 111 | ctx, cancel := context.WithTimeout(ctx, cfg.Timeout) 112 | defer cancel() 113 | if len(cfg.Roles) > 0 { 114 | ctx = myjson.SetMetadataRoles(ctx, cfg.Roles) 115 | } 116 | fn(ctx, t, db) 117 | for _, closer := range closers { 118 | closer() 119 | } 120 | } 121 | } 122 | 123 | func TestDB(fn func(ctx context.Context, db myjson.Database), opts ...myjson.DBOpt) error { 124 | opts = append(opts, myjson.WithGlobalJavascriptFunctions([]string{GlobalScript})) 125 | _ = os.MkdirAll("tmp", 0700) 126 | dir, err := os.MkdirTemp("./tmp", "") 127 | if err != nil { 128 | return err 129 | } 130 | defer os.RemoveAll(dir) 131 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 132 | defer cancel() 133 | ctx = myjson.SetMetadataRoles(ctx, []string{"super_user"}) 134 | db, err := myjson.Open(ctx, "badger", map[string]any{ 135 | "storage_path": dir, 136 | }, opts...) 137 | if err != nil { 138 | return err 139 | } 140 | if err := db.Configure(ctx, "", AllCollections); err != nil { 141 | return err 142 | } 143 | if err := db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 144 | for i := 0; i <= 100; i++ { 145 | d, _ := myjson.NewDocumentFrom(map[string]any{ 146 | "_id": fmt.Sprint(i), 147 | "name": gofakeit.Company(), 148 | }) 149 | if err := tx.Set(ctx, "account", d); err != nil { 150 | return err 151 | } 152 | } 153 | return nil 154 | }); err != nil { 155 | return err 156 | } 157 | 158 | defer db.Close(ctx) 159 | fn(ctx, db) 160 | return nil 161 | } 162 | 163 | func Seed(ctx context.Context, db myjson.Database, accounts int, usersPerAccount int, tasksPerUser int) error { 164 | if err := db.Tx(ctx, kv.TxOpts{IsBatch: true}, func(ctx context.Context, tx myjson.Tx) error { 165 | for i := 0; i <= accounts; i++ { 166 | d, _ := myjson.NewDocumentFrom(map[string]any{ 167 | "_id": fmt.Sprint(i), 168 | "name": gofakeit.Company(), 169 | }) 170 | if err := tx.Set(ctx, "account", d); err != nil { 171 | return err 172 | } 173 | } 174 | return nil 175 | }); err != nil { 176 | return err 177 | } 178 | if err := SeedUsers(ctx, db, usersPerAccount, tasksPerUser); err != nil { 179 | return err 180 | } 181 | return nil 182 | } 183 | 184 | func SeedUsers(ctx context.Context, db myjson.Database, perAccount int, tasksPerUser int) error { 185 | results, err := db.Query(ctx, "account", myjson.Q().Query()) 186 | if err != nil { 187 | return err 188 | } 189 | if err := db.Tx(ctx, kv.TxOpts{IsBatch: true}, func(ctx context.Context, tx myjson.Tx) error { 190 | for _, a := range results.Documents { 191 | for i := 0; i < perAccount; i++ { 192 | u := NewUserDoc() 193 | if err := u.Set("account_id", a.Get("_id")); err != nil { 194 | return err 195 | } 196 | if err := tx.Set(ctx, "user", u); err != nil { 197 | return err 198 | } 199 | 200 | for i := 0; i < tasksPerUser; i++ { 201 | t := NewTaskDoc(u.GetString("_id")) 202 | _, err := tx.Create(ctx, "task", t) 203 | if err != nil { 204 | return err 205 | } 206 | } 207 | } 208 | } 209 | return nil 210 | }); err != nil { 211 | return err 212 | } 213 | return nil 214 | } 215 | -------------------------------------------------------------------------------- /db_util.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/autom8ter/myjson/errors" 9 | "github.com/autom8ter/myjson/kv" 10 | ) 11 | 12 | func (d *defaultDB) lockCollection(ctx context.Context, collection string) (func(), error) { 13 | lock, err := d.kv.NewLocker([]byte(fmt.Sprintf("cache.internal.locks.%s", collection)), 1*time.Minute) 14 | if err != nil { 15 | return nil, errors.Wrap(err, errors.Internal, "failed to acquire lock on collection %s", collection) 16 | } 17 | gotLock, err := lock.TryLock(ctx) 18 | if err != nil { 19 | return nil, errors.Wrap(err, errors.Internal, "failed to acquire lock on collection %s", collection) 20 | } 21 | if !gotLock { 22 | return nil, errors.New(errors.Forbidden, "collection: %s is locked", collection) 23 | } 24 | return lock.Unlock, nil 25 | } 26 | 27 | // 28 | //func (d *defaultDB) awaitCollectionLock(ctx context.Context, ttl time.Duration, collection string) (func(), error) { 29 | // ctx, cancel := context.WithTimeout(ctx, ttl) 30 | // defer cancel() 31 | // ticker := time.NewTicker(50 * time.Millisecond) 32 | // lock := d.kv.NewLocker([]byte(fmt.Sprintf("cache.internal.locks.%s", collection)), 1*time.Minute) 33 | // for { 34 | // select { 35 | // case <-ctx.Done(): 36 | // return nil, errors.Open(errors.Forbidden, "failed to await lock release on collection: %s", collection) 37 | // case <-ticker.C: 38 | // gotLock, err := lock.TryLock() 39 | // if err != nil { 40 | // return nil, errors.Wrap(err, errors.Internal, "failed to acquire lock on collection %s", collection) 41 | // } 42 | // if !gotLock { 43 | // continue 44 | // } 45 | // return lock.Unlock, nil 46 | // } 47 | // } 48 | //} 49 | 50 | func (d *defaultDB) collectionIsLocked(ctx context.Context, collection string) bool { 51 | lock, _ := d.kv.NewLocker([]byte(fmt.Sprintf("cache.internal.locks.%s", collection)), 1*time.Minute) 52 | if lock == nil { 53 | return true 54 | } 55 | is, _ := lock.IsLocked(ctx) 56 | return is 57 | } 58 | 59 | func (d *defaultDB) addIndex(ctx context.Context, collection string, index Index) error { 60 | if index.Name == "" { 61 | return errors.New(errors.Validation, "%s - empty index name", collection) 62 | } 63 | schema, ctx := d.getSchema(ctx, collection) 64 | if err := d.persistCollectionConfig(ctx, schema); err != nil { 65 | return err 66 | } 67 | ctx = context.WithValue(ctx, isIndexingKey, true) 68 | ctx = context.WithValue(ctx, internalKey, true) 69 | 70 | if !index.Primary { 71 | if err := d.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx Tx) error { 72 | _, err := d.ForEach(ctx, collection, ForEachOpts{}, func(doc *Document) (bool, error) { 73 | if err := tx.Set(ctx, collection, doc); err != nil { 74 | return false, err 75 | } 76 | return true, nil 77 | }) 78 | if err != nil { 79 | return err 80 | } 81 | return nil 82 | }); err != nil { 83 | return errors.Wrap(err, 0, "indexing: failed to add index %s - %s", collection, index.Name) 84 | } 85 | } 86 | return nil 87 | } 88 | 89 | func (d *defaultDB) getSchema(ctx context.Context, collection string) (CollectionSchema, context.Context) { 90 | schema := schemaFromCtx(ctx, collection) 91 | if schema == nil { 92 | c, _ := d.collections.Load(collection) 93 | if c == nil { 94 | return nil, ctx 95 | } 96 | return c.(CollectionSchema), schemaToCtx(ctx, c.(CollectionSchema)) 97 | } 98 | return schema, ctx 99 | } 100 | 101 | func (d *defaultDB) removeIndex(ctx context.Context, collection string, index Index) error { 102 | schema, ctx := d.getSchema(ctx, collection) 103 | if err := d.kv.DropPrefix(ctx, indexPrefix(ctx, schema.Collection(), index.Name)); err != nil { 104 | return errors.Wrap(err, 0, "indexing: failed to remove index %s - %s", collection, index.Name) 105 | } 106 | return nil 107 | } 108 | 109 | func (d *defaultDB) persistCollectionConfig(ctx context.Context, val CollectionSchema) error { 110 | if err := d.kv.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 111 | bits, err := val.MarshalJSON() 112 | if err != nil { 113 | return err 114 | } 115 | err = tx.Set(ctx, collectionConfigKey(ctx, val.Collection()), bits) 116 | if err != nil { 117 | return err 118 | } 119 | return nil 120 | }); err != nil { 121 | return err 122 | } 123 | d.collections.Store(val.Collection(), val) 124 | return nil 125 | } 126 | 127 | func (d *defaultDB) deleteCollectionConfig(ctx context.Context, collection string) error { 128 | if err := d.kv.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 129 | err := tx.Delete(ctx, collectionConfigKey(ctx, collection)) 130 | if err != nil { 131 | return err 132 | } 133 | return nil 134 | }); err != nil { 135 | return err 136 | } 137 | d.collections.Delete(collection) 138 | return nil 139 | } 140 | 141 | func (d *defaultDB) getPersistedCollections(ctx context.Context) ([]CollectionSchema, error) { 142 | var existing []CollectionSchema 143 | if err := d.kv.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 144 | i, err := tx.NewIterator(kv.IterOpts{ 145 | Prefix: collectionConfigPrefix(ctx), 146 | }) 147 | if err != nil { 148 | return err 149 | } 150 | defer i.Close() 151 | for i.Valid() { 152 | bits, err := i.Value() 153 | if err != nil { 154 | return err 155 | } 156 | if len(bits) > 0 { 157 | cfg, err := newCollectionSchema(bits) 158 | if err != nil { 159 | return err 160 | } 161 | existing = append(existing, cfg) 162 | } 163 | if err := i.Next(); err != nil { 164 | return err 165 | } 166 | } 167 | return nil 168 | }); err != nil { 169 | return nil, err 170 | } 171 | return existing, nil 172 | } 173 | 174 | func (d *defaultDB) getPersistedCollection(ctx context.Context, collection string) (CollectionSchema, error) { 175 | var cfg CollectionSchema 176 | if err := d.kv.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 177 | bits, err := tx.Get(ctx, collectionConfigKey(ctx, collection)) 178 | if err != nil { 179 | return err 180 | } 181 | cfg, err = newCollectionSchema(bits) 182 | if err != nil { 183 | return err 184 | } 185 | return nil 186 | }); err != nil { 187 | return cfg, err 188 | } 189 | if cfg == nil { 190 | return nil, errors.New(errors.Validation, "collection not found") 191 | } 192 | return cfg, nil 193 | } 194 | 195 | func (d *defaultDB) getCachedCollections() []CollectionSchema { 196 | var existing []CollectionSchema 197 | d.collections.Range(func(key, value interface{}) bool { 198 | existing = append(existing, value.(CollectionSchema)) 199 | return true 200 | }) 201 | return existing 202 | } 203 | -------------------------------------------------------------------------------- /authorization.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/autom8ter/myjson/errors" 7 | "github.com/samber/lo" 8 | ) 9 | 10 | func (t *transaction) authorizeCommand(ctx context.Context, schema CollectionSchema, command *persistCommand) (bool, error) { 11 | if isInternal(ctx) { 12 | return true, nil 13 | } 14 | if len(schema.Authz().Rules) == 0 { 15 | return true, nil 16 | } 17 | deny := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 18 | if a.Action[0] == "*" && a.Effect == Deny { 19 | return true 20 | } 21 | return lo.Contains(a.Action, command.Action) && a.Effect == Deny 22 | }) 23 | if len(deny) > 0 { 24 | for _, d := range deny { 25 | d.Match = t.db.globalScripts + d.Match 26 | result, err := t.vm.RunString(d.Match) 27 | if err != nil { 28 | return false, errors.Wrap(err, 0, "failed to run authz match script") 29 | } 30 | if result.ToBoolean() { 31 | return false, nil 32 | } 33 | } 34 | } 35 | allow := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 36 | if a.Action[0] == "*" && a.Effect == Allow { 37 | return true 38 | } 39 | return lo.Contains(a.Action, command.Action) && a.Effect == Allow 40 | }) 41 | if len(allow) == 0 { 42 | return true, nil 43 | } 44 | for _, d := range allow { 45 | d.Match = t.db.globalScripts + d.Match 46 | result, err := t.vm.RunString(d.Match) 47 | if err != nil { 48 | return false, errors.Wrap(err, 0, "failed to run authz match script") 49 | } 50 | if result.ToBoolean() { 51 | return true, nil 52 | } 53 | } 54 | return false, nil 55 | } 56 | 57 | func (t *transaction) authorizeQuery(ctx context.Context, schema CollectionSchema, query *Query) (bool, error) { 58 | if isInternal(ctx) || isIndexing(ctx) { 59 | return true, nil 60 | } 61 | if len(schema.Authz().Rules) == 0 { 62 | return true, nil 63 | } 64 | if err := t.vm.Set(string(JavascriptGlobalCtx), ctx); err != nil { 65 | return false, err 66 | } 67 | if err := t.vm.Set(string(JavascriptGlobalSchema), schema); err != nil { 68 | return false, err 69 | } 70 | if err := t.vm.Set(string(JavascriptGlobalQuery), *query); err != nil { 71 | return false, err 72 | } 73 | meta := ExtractMetadata(ctx) 74 | if err := t.vm.Set(string(JavascriptGlobalMeta), meta); err != nil { 75 | return false, err 76 | } 77 | 78 | deny := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 79 | if a.Action[0] == "*" && a.Effect == Deny { 80 | return true 81 | } 82 | return lo.Contains(a.Action, QueryAction) && a.Effect == Deny 83 | }) 84 | if len(deny) > 0 { 85 | for _, d := range deny { 86 | d.Match = t.db.globalScripts + d.Match 87 | result, err := t.vm.RunString(d.Match) 88 | if err != nil { 89 | return false, errors.Wrap(err, 0, "failed to run authz match script") 90 | } 91 | if result.ToBoolean() { 92 | return false, nil 93 | } 94 | } 95 | } 96 | allow := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 97 | if a.Action[0] == "*" && a.Effect == Allow { 98 | return true 99 | } 100 | return lo.Contains(a.Action, QueryAction) && a.Effect == Allow 101 | }) 102 | if len(allow) == 0 { 103 | return true, nil 104 | } 105 | for _, d := range allow { 106 | d.Match = t.db.globalScripts + d.Match 107 | result, err := t.vm.RunString(d.Match) 108 | if err != nil { 109 | return false, errors.Wrap(err, 0, "failed to run authz match script") 110 | } 111 | if result.ToBoolean() { 112 | return true, nil 113 | } 114 | } 115 | return false, nil 116 | } 117 | 118 | func (t *defaultDB) authorizeConfigure(ctx context.Context, schema CollectionSchema) (bool, error) { 119 | if isInternal(ctx) || isIndexing(ctx) { 120 | return true, nil 121 | } 122 | if len(schema.Authz().Rules) == 0 { 123 | return true, nil 124 | } 125 | vm := <-t.vmPool 126 | if err := vm.Set(string(JavascriptGlobalCtx), ctx); err != nil { 127 | return false, err 128 | } 129 | if err := vm.Set(string(JavascriptGlobalSchema), schema); err != nil { 130 | return false, err 131 | } 132 | meta := ExtractMetadata(ctx) 133 | if err := vm.Set(string(JavascriptGlobalMeta), meta); err != nil { 134 | return false, err 135 | } 136 | 137 | deny := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 138 | if a.Action[0] == "*" && a.Effect == Deny { 139 | return true 140 | } 141 | return lo.Contains(a.Action, ConfigureAction) && a.Effect == Deny 142 | }) 143 | if len(deny) > 0 { 144 | for _, d := range deny { 145 | d.Match = t.globalScripts + d.Match 146 | result, err := vm.RunString(d.Match) 147 | if err != nil { 148 | return false, errors.Wrap(err, 0, "failed to run authz match script") 149 | } 150 | if result.ToBoolean() { 151 | return false, nil 152 | } 153 | } 154 | } 155 | allow := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 156 | if a.Action[0] == "*" && a.Effect == Allow { 157 | return true 158 | } 159 | return lo.Contains(a.Action, ConfigureAction) && a.Effect == Allow 160 | }) 161 | if len(allow) == 0 { 162 | return true, nil 163 | } 164 | for _, d := range allow { 165 | d.Match = t.globalScripts + d.Match 166 | result, err := vm.RunString(d.Match) 167 | if err != nil { 168 | return false, errors.Wrap(err, 0, "failed to run authz match script") 169 | } 170 | if result.ToBoolean() { 171 | return true, nil 172 | } 173 | } 174 | return false, nil 175 | } 176 | 177 | func (t *defaultDB) authorizeChangeStream(ctx context.Context, schema CollectionSchema, filter []Where) (bool, error) { 178 | if isInternal(ctx) || isIndexing(ctx) { 179 | return true, nil 180 | } 181 | if len(schema.Authz().Rules) == 0 { 182 | return true, nil 183 | } 184 | vm := <-t.vmPool 185 | if err := vm.Set(string(JavascriptGlobalCtx), ctx); err != nil { 186 | return false, err 187 | } 188 | if err := vm.Set(string(JavascriptGlobalSchema), schema); err != nil { 189 | return false, err 190 | } 191 | if err := vm.Set(string(JavascriptGlobalFilter), filter); err != nil { 192 | return false, err 193 | } 194 | meta := ExtractMetadata(ctx) 195 | if err := vm.Set(string(JavascriptGlobalMeta), meta); err != nil { 196 | return false, err 197 | } 198 | 199 | deny := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 200 | if a.Action[0] == "*" && a.Effect == Deny { 201 | return true 202 | } 203 | return lo.Contains(a.Action, ChangeStreamAction) && a.Effect == Deny 204 | }) 205 | if len(deny) > 0 { 206 | for _, d := range deny { 207 | d.Match = t.globalScripts + d.Match 208 | result, err := vm.RunString(d.Match) 209 | if err != nil { 210 | return false, errors.Wrap(err, 0, "failed to run authz match script") 211 | } 212 | if result.ToBoolean() { 213 | return false, nil 214 | } 215 | } 216 | } 217 | allow := lo.Filter(schema.Authz().Rules, func(a AuthzRule, i int) bool { 218 | if a.Action[0] == "*" && a.Effect == Allow { 219 | return true 220 | } 221 | return lo.Contains(a.Action, ChangeStreamAction) && a.Effect == Allow 222 | }) 223 | if len(allow) == 0 { 224 | return true, nil 225 | } 226 | for _, d := range allow { 227 | d.Match = t.globalScripts + d.Match 228 | result, err := vm.RunString(d.Match) 229 | if err != nil { 230 | return false, errors.Wrap(err, 0, "failed to run authz match script") 231 | } 232 | if result.ToBoolean() { 233 | return true, nil 234 | } 235 | } 236 | return false, nil 237 | } 238 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | "sort" 8 | "strings" 9 | "time" 10 | 11 | "github.com/autom8ter/myjson/errors" 12 | "github.com/autom8ter/myjson/util" 13 | "github.com/samber/lo" 14 | "github.com/spf13/cast" 15 | ) 16 | 17 | type stringKey string 18 | 19 | type indexDiff struct { 20 | toRemove []Index 21 | toAdd []Index 22 | toUpdate []Index 23 | } 24 | 25 | func getIndexDiff(after, before map[string]Index) (indexDiff, error) { 26 | var ( 27 | toRemove []Index 28 | toAdd []Index 29 | toUpdate []Index 30 | ) 31 | for _, index := range after { 32 | if _, ok := before[index.Name]; !ok { 33 | toAdd = append(toAdd, index) 34 | } 35 | } 36 | 37 | for _, current := range before { 38 | if _, ok := after[current.Name]; !ok { 39 | toRemove = append(toRemove, current) 40 | } else { 41 | if !reflect.DeepEqual(current.Fields, current.Fields) { 42 | toUpdate = append(toUpdate, current) 43 | } 44 | } 45 | } 46 | return indexDiff{ 47 | toRemove: toRemove, 48 | toAdd: toAdd, 49 | toUpdate: toUpdate, 50 | }, nil 51 | } 52 | 53 | func defaultAs(function AggregateFunction, field string) string { 54 | if function != "" { 55 | return fmt.Sprintf("%s_%s", function, field) 56 | } 57 | return field 58 | } 59 | 60 | func compareField(field string, i, j *Document) bool { 61 | iFieldVal := i.Get(field) 62 | jFieldVal := j.Get(field) 63 | switch val := i.Get(field).(type) { 64 | case time.Time: 65 | return val.After(cast.ToTime(jFieldVal)) 66 | case bool: 67 | return val && !cast.ToBool(jFieldVal) 68 | case float64: 69 | return val > cast.ToFloat64(jFieldVal) 70 | case string: 71 | return val > cast.ToString(jFieldVal) 72 | default: 73 | return util.JSONString(iFieldVal) > util.JSONString(jFieldVal) 74 | } 75 | } 76 | 77 | func orderByDocs(d Documents, orderBys []OrderBy) Documents { 78 | if len(orderBys) == 0 { 79 | return d 80 | } 81 | orderBy := orderBys[0] 82 | 83 | if orderBy.Direction == OrderByDirectionDesc { 84 | sort.Slice(d, func(i, j int) bool { 85 | index := 1 86 | if d[i].Get(orderBy.Field) != d[j].Get(orderBy.Field) { 87 | return compareField(orderBy.Field, d[i], d[j]) 88 | } 89 | for index < len(orderBys) { 90 | order := orderBys[index] 91 | if order.Direction == OrderByDirectionDesc { 92 | if d[i].Get(order.Field) != d[j].Get(order.Field) { 93 | return compareField(order.Field, d[i], d[j]) 94 | } 95 | } else { 96 | if d[i].Get(order.Field) != d[j].Get(order.Field) { 97 | return !compareField(order.Field, d[i], d[j]) 98 | } 99 | } 100 | index++ 101 | } 102 | return false 103 | }) 104 | } else { 105 | sort.Slice(d, func(i, j int) bool { 106 | index := 1 107 | if d[i].Get(orderBy.Field) != d[j].Get(orderBy.Field) { 108 | return !compareField(orderBy.Field, d[i], d[j]) 109 | } 110 | for index < len(orderBys) { 111 | order := orderBys[index] 112 | if d[i].Get(order.Field) != d[j].Get(order.Field) { 113 | if order.Direction == OrderByDirectionDesc { 114 | if d[i].Get(order.Field) != d[j].Get(order.Field) { 115 | return compareField(order.Field, d[i], d[j]) 116 | } 117 | } else { 118 | if d[i].Get(order.Field) != d[j].Get(order.Field) { 119 | return !compareField(order.Field, d[i], d[j]) 120 | } 121 | } 122 | } 123 | index++ 124 | } 125 | return false 126 | 127 | }) 128 | } 129 | return d 130 | } 131 | 132 | func groupByDocs(documents Documents, fields []string) map[string]Documents { 133 | var grouped = map[string]Documents{} 134 | for _, d := range documents { 135 | var values []string 136 | for _, g := range fields { 137 | values = append(values, cast.ToString(d.Get(g))) 138 | } 139 | group := strings.Join(values, ".") 140 | grouped[group] = append(grouped[group], d) 141 | } 142 | return grouped 143 | } 144 | 145 | func aggregateDocs(d Documents, selects []Select) (*Document, error) { 146 | var ( 147 | aggregated *Document 148 | ) 149 | var aggregates = lo.Filter[Select](selects, func(s Select, i int) bool { 150 | return s.Aggregate != "" 151 | }) 152 | var nonAggregates = lo.Filter[Select](selects, func(s Select, i int) bool { 153 | return s.Aggregate == "" 154 | }) 155 | for _, next := range d { 156 | if aggregated == nil || !aggregated.Valid() { 157 | aggregated = NewDocument() 158 | for _, nagg := range nonAggregates { 159 | if err := applyNonAggregates(nagg, aggregated, next); err != nil { 160 | return nil, err 161 | } 162 | } 163 | } 164 | for _, agg := range aggregates { 165 | if agg.As == "" { 166 | agg.As = defaultAs(agg.Aggregate, agg.Field) 167 | } 168 | if err := applyAggregates(agg, aggregated, next); err != nil { 169 | return nil, err 170 | } 171 | } 172 | } 173 | return aggregated, nil 174 | } 175 | 176 | func applyNonAggregates(selct Select, aggregated, next *Document) error { 177 | value := next.Get(selct.Field) 178 | if selct.As == "" { 179 | if err := aggregated.Set(selct.Field, value); err != nil { 180 | return err 181 | } 182 | } else { 183 | if err := aggregated.Set(selct.As, value); err != nil { 184 | return err 185 | } 186 | } 187 | return nil 188 | } 189 | 190 | func applyAggregates(agg Select, aggregated, next *Document) error { 191 | current := aggregated.GetFloat(agg.As) 192 | switch agg.Aggregate { 193 | case AggregateFunctionCount: 194 | current++ 195 | case AggregateFunctionMax: 196 | if value := next.GetFloat(agg.Field); value > current { 197 | current = value 198 | } 199 | case AggregateFunctionMin: 200 | if value := next.GetFloat(agg.Field); value < current { 201 | current = value 202 | } 203 | case AggregateFunctionSum: 204 | current += next.GetFloat(agg.Field) 205 | default: 206 | return errors.New(errors.Validation, "unsupported aggregate function: %s/%s", agg.Field, agg.Aggregate) 207 | } 208 | if err := aggregated.Set(agg.As, current); err != nil { 209 | return err 210 | } 211 | return nil 212 | } 213 | 214 | func selectDocument(d *Document, fields []Select) error { 215 | if len(fields) == 0 || fields[0].Field == "*" { 216 | return nil 217 | } 218 | patch := map[string]interface{}{} 219 | for _, f := range fields { 220 | if f.As == "" { 221 | if f.Aggregate != "" { 222 | f.As = defaultAs(f.Aggregate, f.Field) 223 | } 224 | } 225 | if f.As == "" { 226 | patch[f.Field] = d.Get(f.Field) 227 | } else { 228 | patch[f.As] = d.Get(f.Field) 229 | } 230 | } 231 | err := d.Overwrite(patch) 232 | if err != nil { 233 | return err 234 | } 235 | return nil 236 | } 237 | 238 | func collectionConfigKey(ctx context.Context, collection string) []byte { 239 | return []byte(fmt.Sprintf("cache.internal.collections.%s", collection)) 240 | } 241 | 242 | func collectionConfigPrefix(ctx context.Context) []byte { 243 | return []byte("cache.internal.collections.") 244 | } 245 | 246 | func schemaToCtx(ctx context.Context, schema CollectionSchema) context.Context { 247 | if schema == nil { 248 | return ctx 249 | } 250 | return context.WithValue(ctx, stringKey(fmt.Sprintf("%s.schema", schema.Collection())), schema) 251 | } 252 | 253 | func schemaFromCtx(ctx context.Context, collection string) CollectionSchema { 254 | c, ok := ctx.Value(stringKey(fmt.Sprintf("%s.schema", collection))).(CollectionSchema) 255 | if !ok { 256 | return nil 257 | } 258 | return c 259 | } 260 | 261 | func isAggregateQuery(q Query) bool { 262 | for _, a := range q.Select { 263 | if a.Aggregate != "" { 264 | return true 265 | } 266 | } 267 | return false 268 | } 269 | -------------------------------------------------------------------------------- /kv/badger/badger_test.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "sync" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | 12 | "github.com/autom8ter/myjson/kv" 13 | "github.com/samber/lo" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func Test(t *testing.T) { 18 | db, err := open("") 19 | assert.NoError(t, err) 20 | data := map[string]string{} 21 | for i := 0; i < 100; i++ { 22 | data[fmt.Sprint(i)] = fmt.Sprint(i) 23 | } 24 | t.Run("batch set", func(t *testing.T) { 25 | assert.Nil(t, db.Tx(kv.TxOpts{IsBatch: true}, func(tx kv.Tx) error { 26 | for k, v := range data { 27 | assert.Nil(t, tx.Set(context.Background(), []byte(k), []byte(v))) 28 | } 29 | return nil 30 | })) 31 | }) 32 | t.Run("set", func(t *testing.T) { 33 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 34 | for k, v := range data { 35 | assert.Nil(t, tx.Set(context.Background(), []byte(k), []byte(v))) 36 | } 37 | return nil 38 | })) 39 | }) 40 | 41 | t.Run("get", func(t *testing.T) { 42 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 43 | for k, v := range data { 44 | data, err := tx.Get(context.Background(), []byte(k)) 45 | assert.NoError(t, err) 46 | assert.EqualValues(t, string(v), string(data)) 47 | } 48 | return nil 49 | })) 50 | }) 51 | t.Run("iterate", func(t *testing.T) { 52 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 53 | iter, err := tx.NewIterator(kv.IterOpts{ 54 | Prefix: nil, 55 | Seek: nil, 56 | Reverse: false, 57 | }) 58 | assert.NoError(t, err) 59 | defer iter.Close() 60 | i := 0 61 | for iter.Valid() { 62 | i++ 63 | val, _ := iter.Value() 64 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 65 | iter.Next() 66 | } 67 | assert.Equal(t, len(data), i) 68 | return nil 69 | })) 70 | }) 71 | t.Run("iterate w/ prefix", func(t *testing.T) { 72 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 73 | iter, err := tx.NewIterator(kv.IterOpts{ 74 | Prefix: []byte("1"), 75 | Seek: nil, 76 | Reverse: false, 77 | }) 78 | assert.NoError(t, err) 79 | defer iter.Close() 80 | i := 0 81 | for iter.Valid() { 82 | i++ 83 | assert.True(t, bytes.HasPrefix(iter.Key(), []byte("1"))) 84 | val, _ := iter.Value() 85 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 86 | iter.Next() 87 | } 88 | assert.Equal(t, 11, i) 89 | return nil 90 | })) 91 | }) 92 | t.Run("iterate w/ upper bound", func(t *testing.T) { 93 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 94 | iter, err := tx.NewIterator(kv.IterOpts{ 95 | Prefix: []byte("1"), 96 | Seek: nil, 97 | Reverse: false, 98 | UpperBound: []byte("10"), 99 | }) 100 | assert.NoError(t, err) 101 | defer iter.Close() 102 | i := 0 103 | for iter.Valid() { 104 | i++ 105 | val, _ := iter.Value() 106 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 107 | iter.Next() 108 | } 109 | assert.Equal(t, 2, i) 110 | return nil 111 | })) 112 | }) 113 | t.Run("iterate in reverse", func(t *testing.T) { 114 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 115 | iter, err := tx.NewIterator(kv.IterOpts{ 116 | Prefix: []byte("1"), 117 | Reverse: true, 118 | UpperBound: []byte("10"), 119 | }) 120 | assert.NoError(t, err) 121 | defer iter.Close() 122 | var found [][]byte 123 | for iter.Valid() { 124 | val, _ := iter.Value() 125 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 126 | found = append(found, iter.Key()) 127 | iter.Next() 128 | } 129 | assert.Equal(t, 2, len(found)) 130 | assert.Equal(t, []byte("10"), found[0]) 131 | return nil 132 | })) 133 | }) 134 | t.Run("delete", func(t *testing.T) { 135 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: false}, func(tx kv.Tx) error { 136 | for k, _ := range data { 137 | assert.Nil(t, tx.Delete(context.Background(), []byte(k))) 138 | } 139 | for k, _ := range data { 140 | bytes, _ := tx.Get(context.Background(), []byte(k)) 141 | assert.Nil(t, bytes) 142 | } 143 | return nil 144 | })) 145 | }) 146 | t.Run("locker", func(t *testing.T) { 147 | lock, err := db.NewLocker([]byte("testing"), 1*time.Second) 148 | assert.NoError(t, err) 149 | { 150 | gotLock, err := lock.TryLock(context.Background()) 151 | assert.NoError(t, err) 152 | assert.True(t, gotLock) 153 | is, err := lock.IsLocked(context.Background()) 154 | assert.NoError(t, err) 155 | assert.True(t, is) 156 | } 157 | { 158 | gotLock, err := lock.TryLock(context.Background()) 159 | assert.NoError(t, err) 160 | assert.False(t, gotLock) 161 | } 162 | { 163 | lock.Unlock() 164 | assert.NoError(t, err) 165 | } 166 | 167 | newLock, err := db.NewLocker([]byte("testing"), 1*time.Second) 168 | assert.NoError(t, err) 169 | gotLock, err := newLock.TryLock(context.Background()) 170 | assert.NoError(t, err) 171 | assert.True(t, gotLock) 172 | 173 | gotLock, err = lock.TryLock(context.Background()) 174 | assert.NoError(t, err) 175 | assert.False(t, gotLock) 176 | }) 177 | t.Run("set", func(t *testing.T) { 178 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 179 | for k, v := range data { 180 | assert.Nil(t, tx.Set(context.Background(), []byte(k), []byte(v))) 181 | } 182 | for k, _ := range data { 183 | _, err := tx.Get(context.Background(), []byte(k)) 184 | assert.NoError(t, err) 185 | } 186 | return nil 187 | })) 188 | }) 189 | t.Run("new tx", func(t *testing.T) { 190 | tx, err := db.NewTx(kv.TxOpts{}) 191 | assert.NoError(t, err) 192 | defer func() { 193 | assert.NoError(t, tx.Commit(context.Background())) 194 | }() 195 | for k, v := range data { 196 | assert.Nil(t, tx.Set(context.Background(), []byte(k), []byte(v))) 197 | } 198 | for k, _ := range data { 199 | _, err := tx.Get(context.Background(), []byte(k)) 200 | assert.NoError(t, err) 201 | } 202 | }) 203 | t.Run("new tx w/ rollback", func(t *testing.T) { 204 | tx, err := db.NewTx(kv.TxOpts{}) 205 | assert.NoError(t, err) 206 | for k, v := range data { 207 | assert.Nil(t, tx.Set(context.Background(), []byte(k), []byte(v))) 208 | } 209 | tx.Rollback(context.Background()) 210 | for k, _ := range data { 211 | val, _ := tx.Get(context.Background(), []byte(k)) 212 | assert.Empty(t, val) 213 | } 214 | }) 215 | t.Run("drop prefix", func(t *testing.T) { 216 | { 217 | tx, err := db.NewTx(kv.TxOpts{}) 218 | assert.NoError(t, err) 219 | for k, v := range data { 220 | assert.Nil(t, tx.Set(context.Background(), []byte(fmt.Sprintf("testing.%s", k)), []byte(v))) 221 | } 222 | assert.NoError(t, tx.Commit(context.Background())) 223 | } 224 | assert.NoError(t, db.DropPrefix(context.Background(), []byte("testing."))) 225 | count := 0 226 | assert.NoError(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 227 | iter, err := tx.NewIterator(kv.IterOpts{Prefix: []byte("testing.")}) 228 | assert.NoError(t, err) 229 | defer iter.Close() 230 | for iter.Valid() { 231 | _, err = iter.Value() 232 | assert.NoError(t, err) 233 | count++ 234 | iter.Next() 235 | } 236 | return nil 237 | })) 238 | assert.Equal(t, 0, count) 239 | }) 240 | 241 | } 242 | 243 | func TestChangeStream(t *testing.T) { 244 | t.Run("change stream set", func(t *testing.T) { 245 | db, err := open("") 246 | assert.NoError(t, err) 247 | data := map[string]string{} 248 | for i := 0; i < 100; i++ { 249 | data[fmt.Sprint(i)] = fmt.Sprint(i) 250 | } 251 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 252 | defer cancel() 253 | wg := sync.WaitGroup{} 254 | wg.Add(1) 255 | count := lo.ToPtr(int64(0)) 256 | go func() { 257 | defer wg.Done() 258 | assert.NoError(t, db.ChangeStream(ctx, []byte("testing."), func(cdc kv.CDC) (bool, error) { 259 | atomic.AddInt64(count, 1) 260 | return true, nil 261 | })) 262 | }() 263 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 264 | for k, v := range data { 265 | assert.Nil(t, tx.Set(context.Background(), []byte(fmt.Sprintf("testing.%s", k)), []byte(v))) 266 | } 267 | return nil 268 | })) 269 | wg.Wait() 270 | assert.Equal(t, int64(len(data)), *count) 271 | }) 272 | } 273 | -------------------------------------------------------------------------------- /kv/tikv/tikv_test.go: -------------------------------------------------------------------------------- 1 | //go:build tikv 2 | // +build tikv 3 | 4 | package tikv 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "fmt" 10 | "sync" 11 | "sync/atomic" 12 | "testing" 13 | "time" 14 | 15 | "github.com/autom8ter/myjson/kv" 16 | "github.com/samber/lo" 17 | "github.com/stretchr/testify/assert" 18 | ) 19 | 20 | func Test(t *testing.T) { 21 | ctx := context.Background() 22 | db, err := open(map[string]interface{}{ 23 | "pd_addr": []string{"http://pd0:2379"}, 24 | "redis_addr": "localhost:6379", 25 | }) 26 | assert.NoError(t, err) 27 | data := map[string]string{} 28 | for i := 0; i < 100; i++ { 29 | data[fmt.Sprint(i)] = fmt.Sprint(i) 30 | } 31 | t.Run("set", func(t *testing.T) { 32 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 33 | for k, v := range data { 34 | assert.Nil(t, tx.Set(ctx, []byte(k), []byte(v))) 35 | g, err := tx.Get(ctx, []byte(k)) 36 | assert.NoError(t, err) 37 | assert.NotNil(t, g) 38 | } 39 | return nil 40 | })) 41 | }) 42 | t.Run("get", func(t *testing.T) { 43 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 44 | for k, v := range data { 45 | d, err := tx.Get(ctx, []byte(k)) 46 | assert.NoError(t, err) 47 | assert.EqualValues(t, string(v), string(d), string(k)) 48 | } 49 | return nil 50 | })) 51 | }) 52 | t.Run("iterate", func(t *testing.T) { 53 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 54 | iter, err := tx.NewIterator(kv.IterOpts{ 55 | UpperBound: []byte("999"), 56 | }) 57 | assert.NoError(t, err) 58 | defer iter.Close() 59 | i := 0 60 | for iter.Valid() { 61 | i++ 62 | val, err := iter.Value() 63 | assert.NoError(t, err) 64 | assert.EqualValues(t, data[string(iter.Key())], string(val)) 65 | assert.NoError(t, iter.Next()) 66 | } 67 | assert.Equal(t, len(data), i) 68 | return nil 69 | })) 70 | }) 71 | t.Run("iterate w/ prefix", func(t *testing.T) { 72 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 73 | iter, err := tx.NewIterator(kv.IterOpts{ 74 | Prefix: []byte("1"), 75 | Seek: nil, 76 | Reverse: false, 77 | UpperBound: []byte("999"), 78 | }) 79 | assert.NoError(t, err) 80 | defer iter.Close() 81 | i := 0 82 | for iter.Valid() { 83 | i++ 84 | assert.True(t, bytes.HasPrefix(iter.Key(), []byte("1")), string(iter.Key())) 85 | val, _ := iter.Value() 86 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 87 | assert.NoError(t, iter.Next()) 88 | } 89 | assert.Equal(t, 11, i) 90 | return nil 91 | })) 92 | }) 93 | t.Run("iterate w/ upper bound", func(t *testing.T) { 94 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 95 | iter, err := tx.NewIterator(kv.IterOpts{ 96 | Prefix: []byte("1"), 97 | Seek: nil, 98 | Reverse: false, 99 | UpperBound: []byte("10"), 100 | }) 101 | assert.NoError(t, err) 102 | defer iter.Close() 103 | i := 0 104 | for iter.Valid() { 105 | i++ 106 | val, _ := iter.Value() 107 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 108 | assert.NoError(t, iter.Next()) 109 | } 110 | assert.Equal(t, 2, i) 111 | return nil 112 | })) 113 | }) 114 | t.Run("iterate in reverse", func(t *testing.T) { 115 | assert.Nil(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 116 | iter, err := tx.NewIterator(kv.IterOpts{ 117 | Prefix: []byte("1"), 118 | //Seek: []byte("100"), 119 | Reverse: true, 120 | UpperBound: []byte("10"), 121 | }) 122 | assert.NoError(t, err) 123 | defer iter.Close() 124 | var found [][]byte 125 | for iter.Valid() { 126 | val, _ := iter.Value() 127 | assert.EqualValues(t, string(val), data[string(iter.Key())]) 128 | found = append(found, iter.Key()) 129 | assert.NoError(t, iter.Next()) 130 | } 131 | assert.Equal(t, 2, len(found)) 132 | assert.Equal(t, []byte("10"), found[0]) 133 | return nil 134 | })) 135 | }) 136 | 137 | t.Run("locker", func(t *testing.T) { 138 | lock, err := db.NewLocker([]byte("testing"), 1*time.Second) 139 | assert.NoError(t, err) 140 | { 141 | gotLock, err := lock.TryLock(ctx) 142 | assert.NoError(t, err) 143 | assert.True(t, gotLock) 144 | is, err := lock.IsLocked(ctx) 145 | assert.NoError(t, err) 146 | assert.True(t, is) 147 | } 148 | { 149 | gotLock, err := lock.TryLock(ctx) 150 | assert.NoError(t, err) 151 | assert.False(t, gotLock) 152 | } 153 | { 154 | lock.Unlock() 155 | assert.NoError(t, err) 156 | } 157 | 158 | newLock, err := db.NewLocker([]byte("testing"), 1*time.Second) 159 | assert.NoError(t, err) 160 | gotLock, err := newLock.TryLock(ctx) 161 | assert.NoError(t, err) 162 | assert.True(t, gotLock) 163 | 164 | gotLock, err = lock.TryLock(ctx) 165 | assert.NoError(t, err) 166 | assert.False(t, gotLock) 167 | }) 168 | t.Run("set", func(t *testing.T) { 169 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 170 | for k, v := range data { 171 | assert.Nil(t, tx.Set(ctx, []byte(k), []byte(v))) 172 | } 173 | for k, _ := range data { 174 | _, err := tx.Get(ctx, []byte(k)) 175 | assert.NoError(t, err) 176 | } 177 | return nil 178 | })) 179 | }) 180 | t.Run("new tx", func(t *testing.T) { 181 | tx, err := db.NewTx(kv.TxOpts{}) 182 | assert.NoError(t, err) 183 | defer func() { 184 | assert.NoError(t, tx.Commit(ctx)) 185 | }() 186 | for k, v := range data { 187 | assert.Nil(t, tx.Set(ctx, []byte(k), []byte(v))) 188 | } 189 | for k, _ := range data { 190 | _, err := tx.Get(ctx, []byte(k)) 191 | assert.NoError(t, err) 192 | } 193 | }) 194 | t.Run("delete", func(t *testing.T) { 195 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 196 | for k, _ := range data { 197 | assert.Nil(t, tx.Delete(ctx, []byte(k))) 198 | } 199 | for k, _ := range data { 200 | bytes, _ := tx.Get(ctx, []byte(k)) 201 | assert.Nil(t, bytes) 202 | } 203 | return nil 204 | })) 205 | }) 206 | t.Run("new tx w/ rollback", func(t *testing.T) { 207 | { 208 | tx, err := db.NewTx(kv.TxOpts{}) 209 | assert.NoError(t, err) 210 | for k, v := range data { 211 | assert.Nil(t, tx.Set(ctx, []byte(k), []byte(v))) 212 | } 213 | tx.Rollback(ctx) 214 | tx.Close(ctx) 215 | } 216 | tx, err := db.NewTx(kv.TxOpts{}) 217 | assert.NoError(t, err) 218 | for k, _ := range data { 219 | val, _ := tx.Get(ctx, []byte(k)) 220 | assert.Empty(t, string(val)) 221 | } 222 | }) 223 | //t.Run("drop prefix", func(t *testing.T) { 224 | // { 225 | // tx, err := db.NewTx(false) 226 | // assert.NoError(t, err) 227 | // for k, v := range data { 228 | // assert.Nil(t, tx.Set(ctx, []byte(fmt.Sprintf("testing.%s", k)), []byte(v))) 229 | // } 230 | // assert.NoError(t, tx.Commit(ctx)) 231 | // } 232 | // assert.NoError(t, db.DropPrefix(ctx, []byte("testing."))) 233 | // count := 0 234 | // assert.NoError(t, db.Tx(kv.TxOpts{IsReadOnly: true}, func(tx kv.Tx) error { 235 | // iter, err := tx.NewIterator(kv.IterOpts{Prefix: []byte("testing.")}) 236 | // assert.NoError(t, err) 237 | // defer iter.Close() 238 | // for iter.Valid() { 239 | // _, err = iter.Value() 240 | // assert.NoError(t, err) 241 | // count++ 242 | // iter.Next() 243 | // } 244 | // return nil 245 | // })) 246 | // assert.Equal(t, 0, count) 247 | //}) 248 | } 249 | 250 | func TestChangeStream(t *testing.T) { 251 | t.Run("change stream set", func(t *testing.T) { 252 | db, err := open(map[string]interface{}{ 253 | "pd_addr": []string{"http://pd0:2379"}, 254 | "redis_addr": "localhost:6379", 255 | }) 256 | assert.NoError(t, err) 257 | data := map[string]string{} 258 | for i := 0; i < 100; i++ { 259 | data[fmt.Sprint(i)] = fmt.Sprint(i) 260 | } 261 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) 262 | defer cancel() 263 | wg := sync.WaitGroup{} 264 | wg.Add(1) 265 | count := lo.ToPtr(int64(0)) 266 | go func() { 267 | defer wg.Done() 268 | assert.NoError(t, db.ChangeStream(ctx, []byte("testing."), func(cdc kv.CDC) (bool, error) { 269 | atomic.AddInt64(count, 1) 270 | return true, nil 271 | })) 272 | }() 273 | assert.Nil(t, db.Tx(kv.TxOpts{}, func(tx kv.Tx) error { 274 | for k, v := range data { 275 | assert.Nil(t, tx.Set(context.Background(), []byte(fmt.Sprintf("testing.%s", k)), []byte(v))) 276 | } 277 | return nil 278 | })) 279 | wg.Wait() 280 | assert.Equal(t, int64(len(data)), *count) 281 | }) 282 | } 283 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/autom8ter/myjson/util" 10 | "github.com/zyedidia/generic/set" 11 | 12 | // import embed package 13 | _ "embed" 14 | 15 | "github.com/brianvoe/gofakeit/v6" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func newUserDoc() *Document { 20 | doc, err := NewDocumentFrom(map[string]interface{}{ 21 | "_id": gofakeit.UUID(), 22 | "name": gofakeit.Name(), 23 | "contact": map[string]interface{}{ 24 | "email": fmt.Sprintf("%v.%s", gofakeit.IntRange(0, 100), gofakeit.Email()), 25 | }, 26 | "account_id": fmt.Sprint(gofakeit.IntRange(0, 100)), 27 | "language": gofakeit.Language(), 28 | "birthday_month": gofakeit.Month(), 29 | "favorite_number": gofakeit.Second(), 30 | "gender": gofakeit.Gender(), 31 | "age": gofakeit.IntRange(0, 100), 32 | "timestamp": gofakeit.DateRange(time.Now().Truncate(7200*time.Hour), time.Now()), 33 | "annotations": gofakeit.Map(), 34 | }) 35 | if err != nil { 36 | panic(err) 37 | } 38 | return doc 39 | } 40 | 41 | var ( 42 | //go:embed testutil/testdata/task.yaml 43 | taskSchema string 44 | //go:embed testutil/testdata/user.yaml 45 | userSchema string 46 | //go:embed testutil/testdata/account.yaml 47 | accountSchema string 48 | allCollections = [][]byte{[]byte(userSchema), []byte(taskSchema), []byte(accountSchema)} 49 | ) 50 | 51 | func TestUtil(t *testing.T) { 52 | type contact struct { 53 | Email string `json:"email"` 54 | Phone string `json:"phone,omitempty"` 55 | } 56 | type user struct { 57 | ID string `json:"id"` 58 | Contact contact `json:"contact"` 59 | Name string `json:"name"` 60 | Age int `json:"age"` 61 | Enabled bool `json:"enabled"` 62 | } 63 | const email = "john.smith@yahoo.com" 64 | usr := user{ 65 | ID: gofakeit.UUID(), 66 | Contact: contact{ 67 | Email: email, 68 | Phone: gofakeit.Phone(), 69 | }, 70 | Name: "john smith", 71 | Age: 50, 72 | } 73 | r, err := NewDocumentFrom(&usr) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | t.Run("compareField", func(t *testing.T) { 78 | d, err := NewDocumentFrom(map[string]any{ 79 | "age": 50, 80 | "name": "coleman", 81 | "isMale": true, 82 | }) 83 | assert.NoError(t, err) 84 | d1, err := NewDocumentFrom(map[string]any{ 85 | "age": 51, 86 | "name": "lacee", 87 | }) 88 | assert.NoError(t, err) 89 | t.Run("compare age", func(t *testing.T) { 90 | assert.False(t, compareField("age", d, d1)) 91 | }) 92 | t.Run("compare age (reverse)", func(t *testing.T) { 93 | assert.True(t, compareField("age", d1, d)) 94 | }) 95 | t.Run("compare name", func(t *testing.T) { 96 | assert.False(t, compareField("name", d, d1)) 97 | }) 98 | t.Run("compare name (reverse)", func(t *testing.T) { 99 | assert.True(t, compareField("name", d1, d)) 100 | }) 101 | t.Run("compare isMale", func(t *testing.T) { 102 | assert.True(t, compareField("isMale", d, d1)) 103 | }) 104 | t.Run("compare name (reverse)", func(t *testing.T) { 105 | assert.False(t, compareField("isMale", d1, d)) 106 | }) 107 | }) 108 | t.Run("decode", func(t *testing.T) { 109 | d, err := NewDocumentFrom(map[string]any{ 110 | "age": 50, 111 | "name": "coleman", 112 | "isMale": true, 113 | }) 114 | assert.NoError(t, err) 115 | d1 := NewDocument() 116 | assert.Nil(t, util.Decode(d, d1)) 117 | assert.Equal(t, d.String(), d1.String()) 118 | }) 119 | t.Run("selectDoc", func(t *testing.T) { 120 | before := r.Get("contact.email") 121 | err := selectDocument(r, []Select{{Field: "contact.email"}}) 122 | assert.NoError(t, err) 123 | after := r.Get("contact.email") 124 | assert.Equal(t, before, after) 125 | assert.Nil(t, r.Get("name")) 126 | }) 127 | t.Run("sum age", func(t *testing.T) { 128 | var expected = float64(0) 129 | var docs Documents 130 | for i := 0; i < 5; i++ { 131 | u := newUserDoc() 132 | expected += u.GetFloat("age") 133 | docs = append(docs, u) 134 | } 135 | reduced, err := aggregateDocs(docs, []Select{ 136 | { 137 | Field: "age", 138 | Aggregate: AggregateFunctionSum, 139 | As: "age_sum", 140 | }, 141 | }) 142 | assert.NoError(t, err) 143 | assert.Equal(t, expected, reduced.GetFloat("age_sum")) 144 | }) 145 | t.Run("documents - orderBy (desc/desc)", func(t *testing.T) { 146 | var docs Documents 147 | for i := 0; i < 100; i++ { 148 | doc := newUserDoc() 149 | assert.Nil(t, doc.Set("account_id", gofakeit.IntRange(1, 5))) 150 | docs = append(docs, doc) 151 | } 152 | docs = orderByDocs(docs, []OrderBy{ 153 | { 154 | Field: "account_id", 155 | Direction: OrderByDirectionDesc, 156 | }, 157 | { 158 | Field: "age", 159 | Direction: OrderByDirectionDesc, 160 | }, 161 | }) 162 | docs.ForEach(func(next *Document, i int) { 163 | if len(docs) > i+1 { 164 | assert.GreaterOrEqual(t, next.GetFloat("account_id"), docs[i+1].GetFloat("account_id"), i) 165 | if next.GetFloat("account_id") == docs[i+1].GetFloat("account_id") { 166 | assert.GreaterOrEqual(t, next.GetFloat("age"), docs[i+1].GetFloat("age"), i) 167 | } 168 | } 169 | }) 170 | }) 171 | t.Run("documents - orderBy (asc/asc)", func(t *testing.T) { 172 | var docs Documents 173 | for i := 0; i < 100; i++ { 174 | doc := newUserDoc() 175 | assert.Nil(t, doc.Set("account_id", gofakeit.IntRange(1, 5))) 176 | docs = append(docs, doc) 177 | } 178 | docs = orderByDocs(docs, []OrderBy{ 179 | { 180 | Field: "account_id", 181 | Direction: OrderByDirectionAsc, 182 | }, 183 | { 184 | Field: "age", 185 | Direction: OrderByDirectionAsc, 186 | }, 187 | }) 188 | docs.ForEach(func(next *Document, i int) { 189 | if len(docs) > i+1 { 190 | assert.LessOrEqual(t, next.GetFloat("account_id"), docs[i+1].GetFloat("account_id"), i) 191 | if next.GetFloat("account_id") == docs[i+1].GetFloat("account_id") { 192 | assert.LessOrEqual(t, next.GetFloat("age"), docs[i+1].GetFloat("age"), i) 193 | } 194 | } 195 | }) 196 | }) 197 | t.Run("documents - orderBy (asc/desc)", func(t *testing.T) { 198 | var docs Documents 199 | for i := 0; i < 100; i++ { 200 | doc := newUserDoc() 201 | assert.Nil(t, doc.Set("account_id", gofakeit.IntRange(1, 5))) 202 | docs = append(docs, doc) 203 | } 204 | docs = orderByDocs(docs, []OrderBy{ 205 | { 206 | Field: "account_id", 207 | Direction: OrderByDirectionAsc, 208 | }, 209 | { 210 | Field: "age", 211 | Direction: OrderByDirectionDesc, 212 | }, 213 | }) 214 | docs.ForEach(func(next *Document, i int) { 215 | if len(docs) > i+1 { 216 | assert.LessOrEqual(t, next.GetFloat("account_id"), docs[i+1].GetFloat("account_id"), i) 217 | if next.GetFloat("account_id") == docs[i+1].GetFloat("account_id") { 218 | assert.GreaterOrEqual(t, next.GetFloat("age"), docs[i+1].GetFloat("age"), i) 219 | } 220 | } 221 | }) 222 | }) 223 | t.Run("documents - orderBy (desc/asc)", func(t *testing.T) { 224 | var docs Documents 225 | for i := 0; i < 100; i++ { 226 | doc := newUserDoc() 227 | assert.Nil(t, doc.Set("account_id", gofakeit.IntRange(1, 5))) 228 | docs = append(docs, doc) 229 | } 230 | docs = orderByDocs(docs, []OrderBy{ 231 | { 232 | Field: "account_id", 233 | Direction: OrderByDirectionAsc, 234 | }, 235 | { 236 | Field: "age", 237 | Direction: OrderByDirectionDesc, 238 | }, 239 | }) 240 | docs.ForEach(func(next *Document, i int) { 241 | if len(docs) > i+1 { 242 | assert.LessOrEqual(t, next.GetFloat("account_id"), docs[i+1].GetFloat("account_id"), i) 243 | if next.GetFloat("account_id") == docs[i+1].GetFloat("account_id") { 244 | assert.GreaterOrEqual(t, next.GetFloat("age"), docs[i+1].GetFloat("age"), i) 245 | } 246 | } 247 | }) 248 | }) 249 | t.Run("schemaToCtx", func(t *testing.T) { 250 | ctx := context.Background() 251 | s, err := newCollectionSchema([]byte(userSchema)) 252 | assert.NoError(t, err) 253 | ctx = schemaToCtx(ctx, s) 254 | s2 := schemaFromCtx(ctx, "user") 255 | assert.Equal(t, util.JSONString(s), util.JSONString(s2)) 256 | }) 257 | t.Run("defaultAs", func(t *testing.T) { 258 | assert.Equal(t, "count_account_id", defaultAs(AggregateFunctionCount, "account_id")) 259 | }) 260 | 261 | } 262 | 263 | func TestSets(t *testing.T) { 264 | var all = set.NewMapset[int]() 265 | for i := 0; i < 100; i++ { 266 | all.Put(i) 267 | } 268 | var evens = set.NewMapset[int]() 269 | for i := 0; i < 100; i++ { 270 | if i%2 == 0 { 271 | evens.Put(i) 272 | } 273 | } 274 | fmt.Println(all.Difference(evens).Keys()) 275 | //assert.True(t, set1.Equal(set2)) 276 | } 277 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/autom8ter/myjson/kv" 9 | ) 10 | 11 | // CollectionSchema is a database collection configuration 12 | type CollectionSchema interface { 13 | // Collection is the collection name 14 | Collection() string 15 | // ValidateDocument validates the input document against the collection's JSON schema 16 | ValidateDocument(ctx context.Context, doc *Document) error 17 | // Indexing returns a copy the schemas indexing 18 | Indexing() map[string]Index 19 | // PrimaryIndex returns the collection's primary index 20 | PrimaryIndex() Index 21 | // PrimaryKey returns the collection's primary key 22 | PrimaryKey() string 23 | // GetPrimaryKey gets the document's primary key 24 | GetPrimaryKey(doc *Document) string 25 | // SetPrimaryKey sets the document's primary key 26 | SetPrimaryKey(doc *Document, id string) error 27 | // RequireQueryIndex returns whether the collection requires that queries are appropriately indexed 28 | RequireQueryIndex() bool 29 | // Properties returns a map of the schema's properties 30 | Properties() map[string]SchemaProperty 31 | // PropertyPaths returns a flattened map of the schema's properties - nested properties will be keyed in dot notation 32 | PropertyPaths() map[string]SchemaProperty 33 | // Triggers returns a map of triggers keyed by name that are assigned to the collection 34 | Triggers() []Trigger 35 | // IsReadOnly returns whether the collection is read only 36 | IsReadOnly() bool 37 | // Authz returns the collection's authz if it exists 38 | Authz() Authz 39 | // Immutable returns whether the collection is immutable 40 | Immutable() bool 41 | // PreventDeletes returns whether the collection prevents deletes 42 | PreventDeletes() bool 43 | // Equals returns whether the given collection schema is equal to the current schema 44 | Equals(schema CollectionSchema) bool 45 | // MarshalYAML returns the collection schema as yaml bytes 46 | MarshalYAML() ([]byte, error) 47 | // UnmarshalYAML refreshes the collection schema with the given json bytes 48 | UnmarshalYAML(bytes []byte) error 49 | json.Marshaler 50 | json.Unmarshaler 51 | } 52 | 53 | // ChangeStreamHandler handles changes to documents which are emitted as a change data capture stream 54 | type ChangeStreamHandler func(ctx context.Context, cdc CDC) (bool, error) 55 | 56 | // CollectionConfiguration is a map of collection names to collection schemas - it declarative represents the database collection configuration 57 | type CollectionConfiguration map[string]string 58 | 59 | // Database is a NoSQL database built on top of key value storage 60 | type Database interface { 61 | // Collections returns a list of collection names that are registered in the database 62 | Collections(ctx context.Context) []string 63 | // GetSchema gets a collection schema by name (if it exists) 64 | GetSchema(ctx context.Context, collection string) CollectionSchema 65 | // HasCollection reports whether a collection exists in the database 66 | HasCollection(ctx context.Context, collection string) bool 67 | // Plan returns a configuration plan for the given values yaml(optional for templating) and collectionSchemas yaml schemas. 68 | // The plan will contain the changes that need to be made to the database in order to match the given configuration. 69 | Plan(ctx context.Context, valuesYaml string, collectionSchemas []string) (*ConfigurationPlan, error) 70 | // ConfigurePlan applies the given configuration plan to the database 71 | // This method is useful for applying a plan that was generated by the Plan method 72 | // If the plan is empty, this method will do nothing 73 | ConfigurePlan(ctx context.Context, plan ConfigurationPlan) error 74 | // Configure sets the database collection configurations. It will create/update/delete the necessary collections and indexes to 75 | // match the given configuration. Each element in the config should be a YAML string representing a CollectionSchema. 76 | Configure(ctx context.Context, valuesYaml string, yamlSchemas []string) error 77 | // Tx executes the given function against a new transaction. 78 | // if the function returns an error, all changes will be rolled back. 79 | // otherwise, the changes will be commited to the database 80 | Tx(ctx context.Context, opts kv.TxOpts, fn TxFunc) error 81 | // NewTx returns a new transaction. a transaction must call Commit method in order to persist changes 82 | NewTx(opts kv.TxOpts) (Txn, error) 83 | // ChangeStream streams changes to documents in the given collection. CDC Persistence must be enabled to use this method. 84 | ChangeStream(ctx context.Context, collection string, filter []Where, fn ChangeStreamHandler) error 85 | // Get gets a single document by id 86 | Get(ctx context.Context, collection, id string) (*Document, error) 87 | // ForEach scans the optimal index for a collection's documents passing its filters. 88 | // results will not be ordered unless an index supporting the order by(s) was found by the optimizer 89 | // Query should be used when order is more important than performance/resource-usage 90 | ForEach(ctx context.Context, collection string, opts ForEachOpts, fn ForEachFunc) (Explain, error) 91 | // Query queries a list of documents 92 | Query(ctx context.Context, collection string, query Query) (Page, error) 93 | // RunScript executes a javascript function within the script 94 | // The following global variables will be injected: 95 | // 'db' - a database instance, 96 | // 'ctx' - the context passed to RunScript, 97 | // and 'params' - the params passed to RunScript 98 | RunScript(ctx context.Context, script string, params map[string]any) (any, error) 99 | // RawKV returns the database key value provider - it should be used with caution and only when standard database functionality is insufficient. 100 | RawKV() kv.DB 101 | // Serve serves the database over the given transport 102 | Serve(ctx context.Context, t Transport) error 103 | // NewDoc creates a new document builder instance 104 | NewDoc() *DocBuilder 105 | // Close closes the database 106 | Close(ctx context.Context) error 107 | } 108 | 109 | // Optimizer selects the best index from a set of indexes based on where clauses 110 | type Optimizer interface { 111 | // Optimize selects the optimal index to use based on the given where clauses 112 | Optimize(c CollectionSchema, where []Where) (Explain, error) 113 | } 114 | 115 | // Txn is a database transaction interface - it holds the methods used while using a transaction + commit,rollback,and close functionality 116 | type Txn interface { 117 | // Commit commits the transaction to the database 118 | Commit(ctx context.Context) error 119 | // Rollback rollsback all changes made to the datbase 120 | Rollback(ctx context.Context) error 121 | // Close closes the transaction - it should be deferred after 122 | Close(ctx context.Context) 123 | Tx 124 | } 125 | 126 | // Tx is a database transaction interface - it holds the primary methods used while using a transaction 127 | type Tx interface { 128 | // Cmd is a generic command that can be used to execute any command against the database 129 | Cmd(ctx context.Context, cmd TxCmd) TxResponse 130 | // Query executes a query against the database 131 | Query(ctx context.Context, collection string, query Query) (Page, error) 132 | // Get returns a document by id 133 | Get(ctx context.Context, collection string, id string) (*Document, error) 134 | // Create creates a new document - if the documents primary key is unset, it will be set as a sortable unique id 135 | Create(ctx context.Context, collection string, document *Document) (string, error) 136 | // Update updates a value in the database 137 | Update(ctx context.Context, collection, id string, document map[string]any) error 138 | // Set sets the specified key/value in the database 139 | Set(ctx context.Context, collection string, document *Document) error 140 | // Delete deletes the specified key from the database 141 | Delete(ctx context.Context, collection string, id string) error 142 | // ForEach scans the optimal index for a collection's documents passing its filters. 143 | // results will not be ordered unless an index supporting the order by(s) was found by the optimizer 144 | // Query should be used when order is more important than performance/resource-usage 145 | ForEach(ctx context.Context, collection string, opts ForEachOpts, fn ForEachFunc) (Explain, error) 146 | // TimeTravel sets the document to the value it was at the given timestamp. 147 | // If the document did not exist at the given timestamp, it will return the first version of the document 148 | TimeTravel(ctx context.Context, collection string, documentID string, timestamp time.Time) (*Document, error) 149 | // Revert reverts the document to the value it was at the given timestamp. 150 | // If the document did not exist at the given timestamp, it will persist the first version of the document 151 | Revert(ctx context.Context, collection string, documentID string, timestamp time.Time) error 152 | // DB returns the transactions underlying database 153 | DB() Database 154 | } 155 | 156 | // Transport serves the database over a network (optional for integration with different transport mechanisms) 157 | type Transport interface { 158 | Serve(ctx context.Context, db Database) error 159 | } 160 | -------------------------------------------------------------------------------- /tx_test.go: -------------------------------------------------------------------------------- 1 | package myjson_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/autom8ter/myjson" 9 | "github.com/autom8ter/myjson/kv" 10 | "github.com/autom8ter/myjson/testutil" 11 | "github.com/brianvoe/gofakeit/v6" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestTx(t *testing.T) { 16 | t.Run("set then get", func(t *testing.T) { 17 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 18 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 19 | doc := testutil.NewUserDoc() 20 | err := tx.Set(ctx, "user", doc) 21 | assert.NoError(t, err) 22 | d, err := tx.Get(ctx, "user", doc.GetString("_id")) 23 | assert.NoError(t, err) 24 | assert.NotNil(t, d) 25 | assert.Equal(t, doc.Get("contact.email"), d.GetString("contact.email")) 26 | return nil 27 | })) 28 | })) 29 | }) 30 | t.Run("create then get", func(t *testing.T) { 31 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 32 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 33 | doc := testutil.NewUserDoc() 34 | id, err := tx.Create(ctx, "user", doc) 35 | assert.NoError(t, err) 36 | d, err := tx.Get(ctx, "user", id) 37 | assert.NoError(t, err) 38 | assert.NotNil(t, d) 39 | assert.Equal(t, doc.Get("contact.email"), d.GetString("contact.email")) 40 | return nil 41 | })) 42 | })) 43 | }) 44 | t.Run("create then update then get", func(t *testing.T) { 45 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 46 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 47 | doc := testutil.NewUserDoc() 48 | id, err := tx.Create(ctx, "user", doc) 49 | assert.NoError(t, err) 50 | err = tx.Update(ctx, "user", id, map[string]any{ 51 | "age": 10, 52 | }) 53 | assert.NoError(t, err) 54 | d, err := tx.Get(ctx, "user", id) 55 | assert.NoError(t, err) 56 | assert.NotNil(t, d) 57 | assert.Equal(t, doc.Get("contact.email"), d.GetString("contact.email")) 58 | return nil 59 | })) 60 | })) 61 | }) 62 | t.Run("create then delete then get", func(t *testing.T) { 63 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 64 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 65 | doc := testutil.NewUserDoc() 66 | id, err := tx.Create(ctx, "user", doc) 67 | assert.NoError(t, err) 68 | err = tx.Delete(ctx, "user", id) 69 | assert.NoError(t, err) 70 | d, err := tx.Get(ctx, "user", id) 71 | assert.NotNil(t, err) 72 | assert.Nil(t, d) 73 | return nil 74 | })) 75 | })) 76 | }) 77 | t.Run("set 10 then forEach", func(t *testing.T) { 78 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 79 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 80 | var usrs = map[string]*myjson.Document{} 81 | for i := 0; i < 10; i++ { 82 | doc := testutil.NewUserDoc() 83 | err := tx.Set(ctx, "user", doc) 84 | assert.NoError(t, err) 85 | usrs[doc.GetString("_id")] = doc 86 | } 87 | var count = 0 88 | _, err := tx.ForEach(ctx, "user", myjson.ForEachOpts{}, func(d *myjson.Document) (bool, error) { 89 | assert.NotEmpty(t, usrs[d.GetString("_id")]) 90 | count++ 91 | return true, nil 92 | }) 93 | assert.NoError(t, err) 94 | assert.Equal(t, 10, count) 95 | return nil 96 | })) 97 | })) 98 | }) 99 | t.Run("set then edit then time travel", func(t *testing.T) { 100 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 101 | document := testutil.NewUserDoc() 102 | now := time.Now() 103 | before := document.Clone() 104 | mid := time.Now() 105 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 106 | assert.NoError(t, tx.Set(ctx, "user", document)) 107 | assert.NoError(t, tx.Update(ctx, "user", document.GetString("_id"), map[string]any{ 108 | "age": 10, 109 | })) 110 | mid = time.Now() 111 | assert.NoError(t, tx.Update(ctx, "user", document.GetString("_id"), map[string]any{ 112 | "age": 9, 113 | })) 114 | assert.NoError(t, tx.Update(ctx, "user", document.GetString("_id"), map[string]any{ 115 | "age": 28, 116 | })) 117 | assert.NoError(t, tx.Update(ctx, "user", document.GetString("_id"), map[string]any{ 118 | "name": gofakeit.Name(), 119 | "contact.email": gofakeit.Email(), 120 | })) 121 | return nil 122 | })) 123 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: true}, func(ctx context.Context, tx myjson.Tx) error { 124 | result, err := tx.TimeTravel(ctx, "user", document.GetString("_id"), now) 125 | assert.NoError(t, err) 126 | assert.NotNil(t, result) 127 | assert.Equal(t, before.Get("age"), result.Get("age")) 128 | assert.Equal(t, before.Get("name"), result.Get("name")) 129 | assert.Equal(t, before.Get("language"), result.Get("language")) 130 | assert.Equal(t, before.Get("contact.email"), result.Get("contact.email")) 131 | return nil 132 | })) 133 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: true}, func(ctx context.Context, tx myjson.Tx) error { 134 | result, err := tx.TimeTravel(ctx, "user", document.GetString("_id"), mid) 135 | assert.NoError(t, err) 136 | assert.NotNil(t, result) 137 | assert.Equal(t, float64(10), result.Get("age")) 138 | return nil 139 | })) 140 | })) 141 | }) 142 | 143 | t.Run("DB() not nil", func(t *testing.T) { 144 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 145 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 146 | assert.NotNil(t, tx.DB()) 147 | return nil 148 | })) 149 | })) 150 | }) 151 | t.Run("cmd - no cmds", func(t *testing.T) { 152 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 153 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 154 | result := tx.Cmd(ctx, myjson.TxCmd{ 155 | Create: nil, 156 | Get: nil, 157 | Set: nil, 158 | Update: nil, 159 | Delete: nil, 160 | Query: nil, 161 | }) 162 | assert.Error(t, result.Error) 163 | return nil 164 | })) 165 | })) 166 | }) 167 | t.Run("cmd - set then get", func(t *testing.T) { 168 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 169 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 170 | result := tx.Cmd(ctx, myjson.TxCmd{ 171 | Set: &myjson.SetCmd{Collection: "user", Document: testutil.NewUserDoc()}, 172 | }) 173 | assert.Nil(t, result.Error) 174 | result2 := tx.Cmd(ctx, myjson.TxCmd{ 175 | Get: &myjson.GetCmd{Collection: "user", ID: result.Set.GetString("_id")}, 176 | }) 177 | assert.Nil(t, result2.Error) 178 | assert.JSONEq(t, result.Set.String(), result2.Get.String()) 179 | return nil 180 | })) 181 | })) 182 | }) 183 | t.Run("cmd - set then update then get", func(t *testing.T) { 184 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 185 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 186 | result := tx.Cmd(ctx, myjson.TxCmd{ 187 | Set: &myjson.SetCmd{Collection: "user", Document: testutil.NewUserDoc()}, 188 | }) 189 | assert.Nil(t, result.Error) 190 | id := result.Set.GetString("_id") 191 | result2 := tx.Cmd(ctx, myjson.TxCmd{ 192 | Update: &myjson.UpdateCmd{ 193 | Collection: "user", 194 | ID: id, 195 | Update: map[string]any{ 196 | "age": 20, 197 | }, 198 | }, 199 | }) 200 | assert.Nil(t, result2.Error) 201 | result3 := tx.Cmd(ctx, myjson.TxCmd{ 202 | Get: &myjson.GetCmd{Collection: "user", ID: id}, 203 | }) 204 | assert.Nil(t, result3.Error) 205 | assert.JSONEq(t, result2.Update.String(), result3.Get.String()) 206 | return nil 207 | })) 208 | })) 209 | }) 210 | t.Run("cmd - set then delete", func(t *testing.T) { 211 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 212 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 213 | result := tx.Cmd(ctx, myjson.TxCmd{ 214 | Set: &myjson.SetCmd{Collection: "user", Document: testutil.NewUserDoc()}, 215 | }) 216 | assert.Nil(t, result.Error) 217 | id := result.Set.GetString("_id") 218 | result = tx.Cmd(ctx, myjson.TxCmd{ 219 | Delete: &myjson.DeleteCmd{ 220 | Collection: "user", 221 | ID: id, 222 | }, 223 | }) 224 | assert.Nil(t, result.Error) 225 | return nil 226 | })) 227 | })) 228 | }) 229 | t.Run("cmd - query accounts", func(t *testing.T) { 230 | assert.Nil(t, testutil.TestDB(func(ctx context.Context, db myjson.Database) { 231 | assert.Nil(t, db.Tx(ctx, kv.TxOpts{IsReadOnly: false}, func(ctx context.Context, tx myjson.Tx) error { 232 | result := tx.Cmd(ctx, myjson.TxCmd{ 233 | Query: &myjson.QueryCmd{Collection: "account", Query: myjson.Query{}}, 234 | }) 235 | assert.Nil(t, result.Error) 236 | assert.NotEqual(t, 0, len(result.Query.Documents)) 237 | return nil 238 | })) 239 | })) 240 | }) 241 | } 242 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /tx.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/autom8ter/myjson/errors" 8 | "github.com/autom8ter/myjson/kv" 9 | "github.com/autom8ter/myjson/util" 10 | "github.com/dop251/goja" 11 | "github.com/samber/lo" 12 | "github.com/segmentio/ksuid" 13 | ) 14 | 15 | // TxFunc is a function executed against a transaction - if the function returns an error, all changes will be rolled back. 16 | // Otherwise, the changes will be commited to the database 17 | type TxFunc func(ctx context.Context, tx Tx) error 18 | 19 | // ForEachFunc returns false to stop scanning and an error if one occurred 20 | type ForEachFunc func(d *Document) (bool, error) 21 | 22 | type transaction struct { 23 | db *defaultDB 24 | tx kv.Tx 25 | isBatch bool 26 | cdc []CDC 27 | vm *goja.Runtime 28 | docs map[string]struct{} 29 | } 30 | 31 | func (t *transaction) Commit(ctx context.Context) error { 32 | if err := t.tx.Commit(ctx); err != nil { 33 | return err 34 | } 35 | t.cdc = []CDC{} 36 | return nil 37 | } 38 | 39 | func (t *transaction) Rollback(ctx context.Context) error { 40 | if err := t.tx.Rollback(ctx); err != nil { 41 | return err 42 | } 43 | t.cdc = []CDC{} 44 | return nil 45 | } 46 | 47 | func (t *transaction) Update(ctx context.Context, collection string, id string, update map[string]any) error { 48 | schema, ctx := t.db.getSchema(ctx, collection) 49 | if schema == nil { 50 | return errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 51 | } 52 | doc, err := NewDocumentFrom(update) 53 | if err != nil { 54 | return errors.Wrap(err, 0, "tx: failed to update") 55 | } 56 | if err := schema.SetPrimaryKey(doc, id); err != nil { 57 | return errors.Wrap(err, 0, "tx: failed to set primary key") 58 | } 59 | if err := t.persistCommand(ctx, &persistCommand{ 60 | Collection: collection, 61 | Action: UpdateAction, 62 | Document: doc, 63 | Timestamp: time.Now().UnixNano(), 64 | Metadata: ExtractMetadata(ctx), 65 | }); err != nil { 66 | return errors.Wrap(err, 0, "tx: failed to commit update") 67 | } 68 | return nil 69 | } 70 | 71 | func (t *transaction) Create(ctx context.Context, collection string, document *Document) (string, error) { 72 | c, ctx := t.db.getSchema(ctx, collection) 73 | if c == nil { 74 | return "", errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 75 | } 76 | var id = c.GetPrimaryKey(document) 77 | if id == "" { 78 | id = ksuid.New().String() 79 | err := c.SetPrimaryKey(document, id) 80 | if err != nil { 81 | return "", err 82 | } 83 | } 84 | if err := t.persistCommand(ctx, &persistCommand{ 85 | Collection: collection, 86 | Action: CreateAction, 87 | Document: document, 88 | Timestamp: time.Now().UnixNano(), 89 | Metadata: ExtractMetadata(ctx), 90 | }); err != nil { 91 | return "", errors.Wrap(err, 0, "tx: failed to commit create") 92 | } 93 | return id, nil 94 | } 95 | 96 | func (t *transaction) Set(ctx context.Context, collection string, document *Document) error { 97 | schema, ctx := t.db.getSchema(ctx, collection) 98 | if schema == nil { 99 | return errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 100 | } 101 | if err := t.persistCommand(ctx, &persistCommand{ 102 | Collection: collection, 103 | Action: SetAction, 104 | Document: document, 105 | Timestamp: time.Now().UnixNano(), 106 | Metadata: ExtractMetadata(ctx), 107 | }); err != nil { 108 | return errors.Wrap(err, 0, "tx: failed to commit set") 109 | } 110 | return nil 111 | } 112 | 113 | func (t *transaction) Delete(ctx context.Context, collection string, id string) error { 114 | schema, ctx := t.db.getSchema(ctx, collection) 115 | if schema == nil { 116 | return errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 117 | } 118 | d, _ := NewDocumentFrom(map[string]any{ 119 | t.db.GetSchema(ctx, collection).PrimaryKey(): id, 120 | }) 121 | if err := t.persistCommand(ctx, &persistCommand{ 122 | Collection: collection, 123 | Action: DeleteAction, 124 | Document: d, 125 | Timestamp: time.Now().UnixNano(), 126 | Metadata: ExtractMetadata(ctx), 127 | }); err != nil { 128 | return errors.Wrap(err, 0, "tx: failed to commit delete") 129 | } 130 | return nil 131 | } 132 | 133 | func (t *transaction) Query(ctx context.Context, collection string, query Query) (Page, error) { 134 | if len(query.Select) == 0 { 135 | query.Select = append(query.Select, Select{Field: "*"}) 136 | } 137 | if err := query.Validate(ctx); err != nil { 138 | return Page{}, err 139 | } 140 | schema, ctx := t.db.getSchema(ctx, collection) 141 | if schema == nil { 142 | return Page{}, errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 143 | } 144 | allow, err := t.authorizeQuery(ctx, schema, &query) 145 | if err != nil { 146 | return Page{}, err 147 | } 148 | if !allow { 149 | return Page{}, errors.New(errors.Forbidden, "not authorized: %s/%s", collection, QueryAction) 150 | } 151 | if isAggregateQuery(query) { 152 | return t.aggregate(ctx, collection, query) 153 | } 154 | ctx, cancel := context.WithCancel(ctx) 155 | defer cancel() 156 | now := time.Now() 157 | 158 | var results Documents 159 | fullScan := true 160 | match, err := t.queryScan(ctx, collection, query.Where, query.Join, func(d *Document) (bool, error) { 161 | results = append(results, d) 162 | if query.Page == 0 && len(query.OrderBy) == 0 && query.Limit > 0 && len(results) >= query.Limit { 163 | fullScan = false 164 | return false, nil 165 | } 166 | return true, nil 167 | }) 168 | if err != nil { 169 | return Page{}, err 170 | } 171 | results = orderByDocs(results, query.OrderBy) 172 | 173 | if fullScan && query.Limit > 0 && query.Page > 0 { 174 | results = lo.Slice(results, query.Limit*query.Page, (query.Limit*query.Page)+query.Limit) 175 | } 176 | if query.Limit > 0 && len(results) > query.Limit { 177 | results = results[:query.Limit] 178 | } 179 | 180 | if len(query.Select) > 0 && query.Select[0].Field != "*" { 181 | for _, result := range results { 182 | err := selectDocument(result, query.Select) 183 | if err != nil { 184 | return Page{}, err 185 | } 186 | } 187 | } 188 | 189 | return Page{ 190 | Documents: results, 191 | NextPage: query.Page + 1, 192 | Count: len(results), 193 | Stats: PageStats{ 194 | ExecutionTime: time.Since(now), 195 | Explain: &match, 196 | }, 197 | }, nil 198 | } 199 | 200 | func (t *transaction) Get(ctx context.Context, collection string, id string) (*Document, error) { 201 | c, ctx := t.db.getSchema(ctx, collection) 202 | if c == nil { 203 | return nil, errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 204 | } 205 | results, err := t.Query(ctx, collection, Query{Where: []Where{{Field: c.PrimaryKey(), Op: WhereOpEq, Value: id}}, Limit: 1}) 206 | if err != nil { 207 | return nil, errors.Wrap(err, errors.NotFound, "%s not found", id) 208 | } 209 | if results.Count == 0 { 210 | return nil, errors.New(errors.NotFound, "%s not found", id) 211 | } 212 | return results.Documents[0], nil 213 | } 214 | 215 | func (t *transaction) Cmd(ctx context.Context, cmd TxCmd) TxResponse { 216 | switch { 217 | case cmd.Commit != nil: 218 | if err := t.Commit(ctx); err != nil { 219 | return TxResponse{Commit: &struct{}{}, Error: errors.Extract(err)} 220 | } 221 | return TxResponse{Commit: &struct{}{}} 222 | case cmd.Rollback != nil: 223 | if err := t.Rollback(ctx); err != nil { 224 | return TxResponse{Rollback: &struct{}{}, Error: errors.Extract(err)} 225 | } 226 | return TxResponse{Rollback: &struct{}{}} 227 | case cmd.Query != nil: 228 | results, err := t.Query(ctx, cmd.Query.Collection, cmd.Query.Query) 229 | if err != nil { 230 | return TxResponse{Error: errors.Extract(err)} 231 | } 232 | return TxResponse{ 233 | Query: &results, 234 | } 235 | case cmd.Create != nil: 236 | _, err := t.Create(ctx, cmd.Create.Collection, cmd.Create.Document) 237 | if err != nil { 238 | return TxResponse{Error: errors.Extract(err)} 239 | } 240 | return TxResponse{ 241 | Create: cmd.Create.Document, 242 | } 243 | case cmd.Set != nil: 244 | err := t.Set(ctx, cmd.Set.Collection, cmd.Set.Document) 245 | if err != nil { 246 | return TxResponse{Error: errors.Extract(err)} 247 | } 248 | return TxResponse{ 249 | Set: cmd.Set.Document, 250 | } 251 | case cmd.Delete != nil: 252 | err := t.Delete(ctx, cmd.Delete.Collection, cmd.Delete.ID) 253 | if err != nil { 254 | return TxResponse{Error: errors.Extract(err)} 255 | } 256 | return TxResponse{ 257 | Delete: &struct{}{}, 258 | } 259 | case cmd.Get != nil: 260 | doc, err := t.Get(ctx, cmd.Get.Collection, cmd.Get.ID) 261 | if err != nil { 262 | return TxResponse{Error: errors.Extract(err)} 263 | } 264 | return TxResponse{ 265 | Get: doc, 266 | } 267 | case cmd.Update != nil: 268 | err := t.Update(ctx, cmd.Update.Collection, cmd.Update.ID, cmd.Update.Update) 269 | if err != nil { 270 | return TxResponse{Error: errors.Extract(err)} 271 | } 272 | doc, err := t.Get(ctx, cmd.Update.Collection, cmd.Update.ID) 273 | if err != nil { 274 | return TxResponse{Error: errors.Extract(err)} 275 | } 276 | return TxResponse{ 277 | Update: doc, 278 | } 279 | case cmd.TimeTravel != nil: 280 | doc, err := t.TimeTravel(ctx, cmd.TimeTravel.Collection, cmd.TimeTravel.ID, cmd.TimeTravel.Timestamp) 281 | if err != nil { 282 | return TxResponse{Error: errors.Extract(err)} 283 | } 284 | return TxResponse{ 285 | TimeTravel: doc, 286 | } 287 | case cmd.Revert != nil: 288 | if err := t.Revert(ctx, cmd.Revert.Collection, cmd.Revert.ID, cmd.Revert.Timestamp); err != nil { 289 | return TxResponse{Error: errors.Extract(err)} 290 | } 291 | return TxResponse{ 292 | Revert: &struct{}{}, 293 | } 294 | } 295 | return TxResponse{Error: errors.Extract(errors.New(errors.Validation, "tx: unsupported command"))} 296 | } 297 | 298 | // aggregate performs aggregations against the collection 299 | func (t *transaction) aggregate(ctx context.Context, collection string, query Query) (Page, error) { 300 | c, ctx := t.db.getSchema(ctx, collection) 301 | if c == nil { 302 | return Page{}, errors.New(errors.Validation, "tx: unsupported collection: %s", collection) 303 | } 304 | ctx, cancel := context.WithCancel(ctx) 305 | defer cancel() 306 | now := time.Now() 307 | var results Documents 308 | match, err := t.queryScan(ctx, collection, query.Where, query.Join, func(d *Document) (bool, error) { 309 | results = append(results, d) 310 | return true, nil 311 | }) 312 | if err != nil { 313 | return Page{}, err 314 | } 315 | var reduced Documents 316 | for _, values := range groupByDocs(results, query.GroupBy) { 317 | value, err := aggregateDocs(values, query.Select) 318 | if err != nil { 319 | return Page{}, err 320 | } 321 | reduced = append(reduced, value) 322 | } 323 | reduced, err = docsHaving(query.Having, reduced) 324 | if err != nil { 325 | return Page{}, errors.Wrap(err, errors.Internal, "") 326 | } 327 | reduced = orderByDocs(reduced, query.OrderBy) 328 | if query.Limit > 0 && query.Page > 0 { 329 | reduced = lo.Slice(reduced, query.Limit*query.Page, (query.Limit*query.Page)+query.Limit) 330 | } 331 | if query.Limit > 0 && len(reduced) > query.Limit { 332 | reduced = reduced[:query.Limit] 333 | } 334 | return Page{ 335 | Documents: reduced, 336 | NextPage: query.Page + 1, 337 | Count: len(reduced), 338 | Stats: PageStats{ 339 | ExecutionTime: time.Since(now), 340 | Explain: &match, 341 | }, 342 | }, nil 343 | } 344 | 345 | func docsHaving(where []Where, results Documents) (Documents, error) { 346 | if len(where) > 0 { 347 | for i, document := range results { 348 | pass, err := document.Where(where) 349 | if err != nil { 350 | return nil, err 351 | } 352 | if pass { 353 | results = util.RemoveElement(i, results) 354 | } 355 | } 356 | } 357 | return results, nil 358 | } 359 | 360 | func (t *transaction) ForEach(ctx context.Context, collection string, opts ForEachOpts, fn ForEachFunc) (Explain, error) { 361 | pass, err := t.authorizeQuery(ctx, t.db.GetSchema(ctx, collection), &Query{ 362 | Where: opts.Where, 363 | Join: opts.Join, 364 | }) 365 | if err != nil { 366 | return Explain{}, err 367 | } 368 | if !pass { 369 | return Explain{}, errors.New(errors.Forbidden, "not authorized: %s", QueryAction) 370 | } 371 | return t.queryScan(ctx, collection, opts.Where, opts.Join, fn) 372 | } 373 | 374 | func (t *transaction) Close(ctx context.Context) { 375 | t.tx.Close(ctx) 376 | t.cdc = []CDC{} 377 | } 378 | 379 | func (t *transaction) CDC() []CDC { 380 | return t.cdc 381 | } 382 | 383 | func (t *transaction) DB() Database { 384 | return t.db 385 | } 386 | -------------------------------------------------------------------------------- /javascript.go: -------------------------------------------------------------------------------- 1 | package myjson 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "crypto/sha1" 7 | "crypto/sha256" 8 | "crypto/sha512" 9 | "encoding/base64" 10 | "encoding/hex" 11 | "encoding/json" 12 | "fmt" 13 | "io/ioutil" 14 | "net/http" 15 | "reflect" 16 | "strconv" 17 | "strings" 18 | "time" 19 | 20 | "github.com/dop251/goja" 21 | "github.com/google/uuid" 22 | "github.com/huandu/xstrings" 23 | "github.com/segmentio/ksuid" 24 | "github.com/spf13/cast" 25 | "github.com/thoas/go-funk" 26 | ) 27 | 28 | // JavascriptGlobal is a global variable injected into a javascript function (triggers/authorizers/etc) 29 | type JavascriptGlobal string 30 | 31 | const ( 32 | // JavascriptGlobalDB is the global variable for the database instance - it is injected into all javascript functions 33 | // All methods, fields are available within the script 34 | JavascriptGlobalDB JavascriptGlobal = "db" 35 | // JavascriptGlobalCtx is the context of the request when the function is called - it is injected into all javascript functions 36 | // All methods, fields are available within the script 37 | JavascriptGlobalCtx JavascriptGlobal = "ctx" 38 | // JavascriptGlobalMeta is the context metadta of the request when the function is called - it is injected into all javascript functions 39 | // All methods, fields are available within the script 40 | JavascriptGlobalMeta JavascriptGlobal = "meta" 41 | // JavascriptGlobalTx is the transaction instance - it is injected into all javascript functions except ChangeStream Authorizers 42 | // All methods, fields are available within the script 43 | JavascriptGlobalTx JavascriptGlobal = "tx" 44 | // JavascriptGlobalSchema is the collection schema instance - it is injected into all javascript functions 45 | // All methods, fields are available within the script 46 | JavascriptGlobalSchema JavascriptGlobal = "schema" 47 | // JavascriptGlobalQuery is the query instance - it is injected into only Query-based based javascript functions (including foreach) 48 | // All methods, fields are available within the script 49 | JavascriptGlobalQuery JavascriptGlobal = "query" 50 | // JavascriptGlobalFilter is an array of where clauses - it is injected into ChangeStream based javascript functions 51 | // All methods, fields are available within the script 52 | JavascriptGlobalFilter JavascriptGlobal = "filter" 53 | // JavascriptGlobalDoc is a myjson document - it is injected into document based javascript functions 54 | // All methods, fields are available within the script 55 | JavascriptGlobalDoc JavascriptGlobal = "doc" 56 | ) 57 | 58 | func getJavascriptVM(ctx context.Context, db Database, overrides map[string]any) (*goja.Runtime, error) { 59 | vm := goja.New() 60 | vm.SetFieldNameMapper(goja.TagFieldNameMapper("json", false)) 61 | if err := vm.Set(string(JavascriptGlobalDB), db); err != nil { 62 | return nil, err 63 | } 64 | if err := vm.Set(string(JavascriptGlobalCtx), ctx); err != nil { 65 | return nil, err 66 | } 67 | if err := vm.Set(string(JavascriptGlobalMeta), ExtractMetadata(ctx)); err != nil { 68 | return nil, err 69 | } 70 | for k, v := range JavascriptBuiltIns { 71 | if err := vm.Set(k, v); err != nil { 72 | return nil, err 73 | } 74 | } 75 | for k, v := range overrides { 76 | if err := vm.Set(k, v); err != nil { 77 | return nil, err 78 | } 79 | } 80 | return vm, nil 81 | } 82 | 83 | var JavascriptBuiltIns = map[string]any{ 84 | // newDocumentFrom is a helper function to create a new document from a map 85 | "newDocumentFrom": NewDocumentFrom, 86 | // newDocument is a helper function to create an empty document 87 | "newDocument": NewDocument, 88 | // ksuid is a helper function to create a new ksuid id (time sortable id) 89 | "ksuid": func() string { 90 | return ksuid.New().String() 91 | }, 92 | // uuid is a helper function to create a new uuid 93 | "uuid": func() string { 94 | return uuid.New().String() 95 | }, 96 | // now is a helper function to get the current time object 97 | "now": time.Now, 98 | // chunk is a helper function to chunk an array of values 99 | "chunk": funk.Chunk, 100 | // map is a helper function to map an array of values 101 | "map": funk.Map, 102 | // filter is a helper function to filter an array of values 103 | "filter": funk.Filter, 104 | // find is a helper function to find an element in an array of values 105 | "find": funk.Find, 106 | // contains is a helper function to check if an array of values contains a value or if a string contains a substring 107 | "contains": func(v any, e any) bool { 108 | if v == nil { 109 | return false 110 | } 111 | switch v := v.(type) { 112 | case string: 113 | return strings.Contains(v, cast.ToString(e)) 114 | default: 115 | return funk.Contains(v, e) 116 | } 117 | }, 118 | // equals is a helper function to check if two objects are equal 119 | "equal": funk.Equal, 120 | // equals is a helper function to check if two objects are equal 121 | "deepEqual": reflect.DeepEqual, 122 | // keys is a helper function to get the keys of a object 123 | "keys": funk.Keys, 124 | // values is a helper function to get the values of an object 125 | "values": funk.Values, 126 | // flatten is a helper function to flatten an array of values 127 | "flatten": funk.Flatten, 128 | // uniq is a helper function to get the unique values of an array 129 | "uniq": funk.Uniq, 130 | // drop is a helper function to drop the first n elements of an array 131 | "drop": funk.Drop, 132 | // last is a helper function to get the last element of an array 133 | "last": funk.Last, 134 | // empty is a helper function to check if a value is empty 135 | "empty": funk.IsEmpty, 136 | // notEmpty is a helper function to check if a value is not empty 137 | "notEmpty": funk.NotEmpty, 138 | // difference is a helper function to get the difference between two arrays 139 | "difference": funk.Difference, 140 | // isZero is a helper function to check if a value is zero 141 | "isZero": funk.IsZero, 142 | // sum is a helper function to sum an array of values 143 | "sum": funk.Sum, 144 | // set is a helper function to set a value in an object 145 | "set": funk.Set, 146 | // getOr is a helper function to get a value from an object or return a default value 147 | "getOr": funk.GetOrElse, 148 | // prune is a helper function to prune an object of empty values 149 | "prune": funk.Prune, 150 | // len is a helper function to get the length of an array 151 | "len": func(v any) int { return len(cast.ToSlice(v)) }, 152 | // toSlice is a helper function to cast a value to a slice 153 | "toSlice": cast.ToSlice, 154 | // toMap is a helper function to cast a value to a map 155 | "toMap": cast.ToStringMap, 156 | // toStr is a helper function to cast a value to a string 157 | "toStr": cast.ToString, 158 | // toInt is a helper function to cast a value to an int 159 | "toInt": cast.ToInt, 160 | // toFloat is a helper function to cast a value to a float 161 | "toFloat": cast.ToFloat64, 162 | // toBool is a helper function to cast a value to a bool 163 | "toBool": cast.ToBool, 164 | // toTime is a helper function to cast a value to a date 165 | "toTime": cast.ToTime, 166 | // toDuration is a helper function to cast a value to a duration 167 | "toDuration": cast.ToDuration, 168 | // asDoc is a helper function to cast a value to a document 169 | "asDoc": func(v any) *Document { 170 | d, _ := NewDocumentFrom(v) 171 | return d 172 | }, 173 | // indexOf is a helper function to get the index of a value in an array 174 | "indexOf": funk.IndexOf, 175 | // join is a helper function to join an array of values 176 | "join": strings.Join, 177 | // split is a helper function to split a string 178 | "split": strings.Split, 179 | // replace is a helper function to replace a string 180 | "replace": strings.ReplaceAll, 181 | // lower is a helper function to lower a string 182 | "lower": strings.ToLower, 183 | // upper is a helper function to upper a string 184 | "upper": strings.ToUpper, 185 | // trim is a helper function to trim a string 186 | "trim": strings.TrimSpace, 187 | // trimLeft is a helper function to trim a string 188 | "trimLeft": strings.TrimLeft, 189 | // trimRight is a helper function to trim a string 190 | "trimRight": strings.TrimRight, 191 | // trimPrefix is a helper function to trim a string 192 | "trimPrefix": strings.TrimPrefix, 193 | // trimSuffix is a helper function to trim a string 194 | "trimSuffix": strings.TrimSuffix, 195 | // startsWith is a helper function to check if a string starts with a substring 196 | "startsWith": strings.HasPrefix, 197 | // endsWith is a helper function to check if a string ends with a substring 198 | "endsWith": strings.HasSuffix, 199 | // camelCase is a helper function to convert a string to camelCase 200 | "camelCase": xstrings.ToCamelCase, 201 | // snakeCase is a helper function to convert a string to snake_case 202 | "snakeCase": xstrings.ToSnakeCase, 203 | // kebabCase is a helper function to convert a string to kebab-case 204 | "kebabCase": xstrings.ToKebabCase, 205 | // quote is a helper function to quote a string 206 | "quote": strconv.Quote, 207 | // unquote is a helper function to unquote a string 208 | "unquote": strconv.Unquote, 209 | // parseTime is a helper function to parse a time 210 | "parseTime": time.Parse, 211 | // since is a helper function to get the duration since a time 212 | "since": time.Since, 213 | // until is a helper function to get the duration until a time 214 | "until": time.Until, 215 | // after is a helper function to get the duration after a time 216 | "after": time.After, 217 | // unixMicro is a helper function to get the time from a unix micro timestamp 218 | "unixMicro": time.UnixMicro, 219 | // unixMilli is a helper function to get the time from a unix milli timestamp 220 | "unixMilli": time.UnixMilli, 221 | // unix is a helper function to get the time from a unix timestamp 222 | "unix": time.Unix, 223 | // date is a helper function to get the date from a timestamp 224 | "date": time.Date, 225 | // sha1 is a helper function to get the sha1 hash of a string 226 | "sha1": func(v any) string { 227 | h := sha1.New() 228 | h.Write([]byte(cast.ToString(v))) 229 | return hex.EncodeToString(h.Sum(nil)) 230 | }, 231 | // sha256 is a helper function to get the sha256 hash of a string 232 | "sha256": func(v any) string { 233 | h := sha256.New() 234 | h.Write([]byte(cast.ToString(v))) 235 | return hex.EncodeToString(h.Sum(nil)) 236 | }, 237 | // sha512 is a helper function to get the sha512 hash of a string 238 | "sha512": func(v any) string { 239 | h := sha512.New() 240 | h.Write([]byte(cast.ToString(v))) 241 | return hex.EncodeToString(h.Sum(nil)) 242 | }, 243 | // md5 is a helper function to get the md5 hash of a string 244 | "md5": func(v any) string { 245 | h := md5.New() 246 | h.Write([]byte(cast.ToString(v))) 247 | return hex.EncodeToString(h.Sum(nil)) 248 | }, 249 | // base64Encode is a helper function to encode a string to base64 250 | "base64Encode": func(v any) string { 251 | return base64.StdEncoding.EncodeToString([]byte(cast.ToString(v))) 252 | }, 253 | // base64Decode is a helper function to decode a string from base64 254 | "base64Decode": func(v any) (string, error) { 255 | b, err := base64.StdEncoding.DecodeString(cast.ToString(v)) 256 | return string(b), err 257 | }, 258 | // jsonEncode is a helper function to encode a value to json 259 | "jsonEncode": func(v any) (string, error) { 260 | b, err := json.Marshal(v) 261 | return string(b), err 262 | }, 263 | // jsonDecode is a helper function to decode a value from json 264 | "jsonDecode": func(v string) (any, error) { 265 | var out any 266 | err := json.Unmarshal([]byte(v), &out) 267 | return out, err 268 | }, 269 | // fetch is a helper function to fetch a url 270 | "fetch": func(request map[string]any) (map[string]any, error) { 271 | method := cast.ToString(request["method"]) 272 | if method == "" { 273 | return nil, fmt.Errorf("missing 'method'") 274 | } 275 | url := cast.ToString(request["url"]) 276 | if url == "" { 277 | return nil, fmt.Errorf("missing 'url'") 278 | } 279 | req, err := http.NewRequest(method, url, nil) 280 | if err != nil { 281 | return nil, err 282 | } 283 | if headers, ok := request["headers"]; ok { 284 | for k, v := range cast.ToStringMap(headers) { 285 | req.Header.Set(k, cast.ToString(v)) 286 | } 287 | } 288 | if queryParams, ok := request["query"]; ok { 289 | q := req.URL.Query() 290 | for k, v := range cast.ToStringMap(queryParams) { 291 | q.Set(k, cast.ToString(v)) 292 | } 293 | req.URL.RawQuery = q.Encode() 294 | } 295 | if body, ok := request["body"]; ok { 296 | req.Body = ioutil.NopCloser(strings.NewReader(cast.ToString(body))) 297 | } 298 | 299 | resp, err := http.DefaultClient.Do(req) 300 | if err != nil { 301 | return nil, err 302 | } 303 | defer resp.Body.Close() 304 | b, err := ioutil.ReadAll(resp.Body) 305 | if err != nil { 306 | return nil, err 307 | } 308 | return map[string]any{ 309 | "status": resp.StatusCode, 310 | "headers": resp.Header, 311 | "body": string(b), 312 | }, nil 313 | }, 314 | } 315 | --------------------------------------------------------------------------------