5 |
6 | ----
7 |
8 | There are four tables in Sloop to store data:
9 |
10 | 1. Watch table
11 | 1. Resources summary table
12 | 1. Event count table
13 | 1. Watch activity table
14 |
15 | ----
16 |
17 | Details:
18 |
19 | 1. Watch table:
20 | It has the raw kube watch data. It is the source of truth for the whole data.
21 |
22 | 1. Resource Summary: It stores the resources information including name, creation date, deployment details and last update time.
23 |
24 | 1. Event table: It stores the event details that took place for a resource. The information includes event type, message and time stamp.
25 |
26 | 1. Watch Activity table: It stores any watch activity received. It has the information that was there a change from the last known state or not.
27 |
28 |
29 | ## Data Distribution
30 |
31 | The data distribution in terms of size among the tables is shown below. As expected, watch table occupies the most space as it contains the raw data. Rest of the tables are derived from it.
32 | 
33 |
--------------------------------------------------------------------------------
/pkg/sloop/store/typed/eventcounttable.go:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2019, salesforce.com, inc.
3 | * All rights reserved.
4 | * SPDX-License-Identifier: BSD-3-Clause
5 | * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
6 | */
7 |
8 | package typed
9 |
10 | import (
11 | "fmt"
12 | "time"
13 |
14 | badger "github.com/dgraph-io/badger/v2"
15 | "github.com/salesforce/sloop/pkg/sloop/common"
16 | "github.com/salesforce/sloop/pkg/sloop/store/untyped"
17 | "github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
18 | )
19 |
20 | type EventCountKey struct {
21 | PartitionId string
22 | Kind string
23 | Namespace string
24 | Name string
25 | Uid string
26 | }
27 |
28 | func NewEventCountKey(timestamp time.Time, kind string, namespace string, name string, uid string) *EventCountKey {
29 | partitionId := untyped.GetPartitionId(timestamp)
30 | return &EventCountKey{PartitionId: partitionId, Kind: kind, Namespace: namespace, Name: name, Uid: uid}
31 | }
32 |
33 | func NewEventCountKeyComparator(kind string, namespace string, name string, uid string) *EventCountKey {
34 | return &EventCountKey{Kind: kind, Namespace: namespace, Name: name, Uid: uid}
35 | }
36 |
37 | func (*EventCountKey) TableName() string {
38 | return "eventcount"
39 | }
40 |
41 | func (k *EventCountKey) Parse(key string) error {
42 | err, parts := common.ParseKey(key)
43 | if err != nil {
44 | return err
45 | }
46 |
47 | if parts[1] != k.TableName() {
48 | return fmt.Errorf("Second part of key (%v) should be %v", key, k.TableName())
49 | }
50 | k.PartitionId = parts[2]
51 | k.Kind = parts[3]
52 | k.Namespace = parts[4]
53 | k.Name = parts[5]
54 | k.Uid = parts[6]
55 | return nil
56 | }
57 |
58 | // todo: need to make sure it can work as keyPrefix when some fields are empty
59 | func (k *EventCountKey) String() string {
60 | if k.Uid == "" {
61 | return fmt.Sprintf("/%v/%v/%v/%v/%v", k.TableName(), k.PartitionId, k.Kind, k.Namespace, k.Name)
62 | } else {
63 | return fmt.Sprintf("/%v/%v/%v/%v/%v/%v", k.TableName(), k.PartitionId, k.Kind, k.Namespace, k.Name, k.Uid)
64 | }
65 | }
66 |
67 | func (*EventCountKey) ValidateKey(key string) error {
68 | newKey := EventCountKey{}
69 | return newKey.Parse(key)
70 | }
71 |
72 | func (t *ResourceEventCountsTable) GetOrDefault(txn badgerwrap.Txn, key string) (*ResourceEventCounts, error) {
73 | rec, err := t.Get(txn, key)
74 | if err != nil {
75 | if err != badger.ErrKeyNotFound {
76 | return nil, err
77 | } else {
78 | return &ResourceEventCounts{MapMinToEvents: make(map[int64]*EventCounts)}, nil
79 | }
80 | }
81 | return rec, nil
82 | }
83 |
84 | func (k *EventCountKey) SetPartitionId(newPartitionId string) {
85 | k.PartitionId = newPartitionId
86 | }
87 |
--------------------------------------------------------------------------------
/pkg/sloop/store/typed/eventcounttablegen_test.go:
--------------------------------------------------------------------------------
1 | // This file was automatically generated by genny.
2 | // Any changes will be lost if this file is regenerated.
3 | // see https://github.com/cheekybits/genny
4 |
5 | /*
6 | * Copyright (c) 2019, salesforce.com, inc.
7 | * All rights reserved.
8 | * SPDX-License-Identifier: BSD-3-Clause
9 | * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
10 | */
11 |
12 | package typed
13 |
14 | import (
15 | "fmt"
16 | "reflect"
17 | "testing"
18 | "time"
19 |
20 | "github.com/dgraph-io/badger/v2"
21 | "github.com/salesforce/sloop/pkg/sloop/store/untyped"
22 | "github.com/salesforce/sloop/pkg/sloop/store/untyped/badgerwrap"
23 | "github.com/stretchr/testify/assert"
24 | )
25 |
26 | func helper_ResourceEventCounts_ShouldSkip() bool {
27 | // Tests will not work on the fake types in the template, but we want to run tests on real objects
28 | if "typed.Value"+"Type" == fmt.Sprint(reflect.TypeOf(ResourceEventCounts{})) {
29 | fmt.Printf("Skipping unit test")
30 | return true
31 | }
32 | return false
33 | }
34 |
35 | func Test_ResourceEventCountsTable_SetWorks(t *testing.T) {
36 | if helper_ResourceEventCounts_ShouldSkip() {
37 | return
38 | }
39 |
40 | untyped.TestHookSetPartitionDuration(time.Hour * 24)
41 | db, err := (&badgerwrap.MockFactory{}).Open(badger.DefaultOptions(""))
42 | assert.Nil(t, err)
43 | err = db.Update(func(txn badgerwrap.Txn) error {
44 | k := (&EventCountKey{}).GetTestKey()
45 | vt := OpenResourceEventCountsTable()
46 | err2 := vt.Set(txn, k, (&EventCountKey{}).GetTestValue())
47 | assert.Nil(t, err2)
48 | return nil
49 | })
50 | assert.Nil(t, err)
51 | }
52 |
53 | func helper_update_ResourceEventCountsTable(t *testing.T, keys []string, val *ResourceEventCounts) (badgerwrap.DB, *ResourceEventCountsTable) {
54 | b, err := (&badgerwrap.MockFactory{}).Open(badger.DefaultOptions(""))
55 | assert.Nil(t, err)
56 | wt := OpenResourceEventCountsTable()
57 | err = b.Update(func(txn badgerwrap.Txn) error {
58 | var txerr error
59 | for _, key := range keys {
60 | txerr = wt.Set(txn, key, val)
61 | if txerr != nil {
62 | return txerr
63 | }
64 | }
65 | // Add some keys outside the range
66 | txerr = txn.Set([]byte("/a/123/"), []byte{})
67 | if txerr != nil {
68 | return txerr
69 | }
70 | txerr = txn.Set([]byte("/zzz/123/"), []byte{})
71 | if txerr != nil {
72 | return txerr
73 | }
74 | return nil
75 | })
76 | assert.Nil(t, err)
77 | return b, wt
78 | }
79 |
80 | func Test_ResourceEventCountsTable_GetUniquePartitionList_Success(t *testing.T) {
81 | if helper_ResourceEventCounts_ShouldSkip() {
82 | return
83 | }
84 |
85 | db, wt := helper_update_ResourceEventCountsTable(t, (&EventCountKey{}).SetTestKeys(), (&EventCountKey{}).SetTestValue())
86 | var partList []string
87 | var err1 error
88 | err := db.View(func(txn badgerwrap.Txn) error {
89 | partList, err1 = wt.GetUniquePartitionList(txn)
90 | return nil
91 | })
92 | assert.Nil(t, err)
93 | assert.Nil(t, err1)
94 | assert.Len(t, partList, 3)
95 | assert.Contains(t, partList, someMinPartition)
96 | assert.Contains(t, partList, someMiddlePartition)
97 | assert.Contains(t, partList, someMaxPartition)
98 | }
99 |
100 | func Test_ResourceEventCountsTable_GetUniquePartitionList_EmptyPartition(t *testing.T) {
101 | if helper_ResourceEventCounts_ShouldSkip() {
102 | return
103 | }
104 |
105 | db, wt := helper_update_ResourceEventCountsTable(t, []string{}, &ResourceEventCounts{})
106 | var partList []string
107 | var err1 error
108 | err := db.View(func(txn badgerwrap.Txn) error {
109 | partList, err1 = wt.GetUniquePartitionList(txn)
110 | return err1
111 | })
112 | assert.Nil(t, err)
113 | assert.Len(t, partList, 0)
114 | }
115 |
--------------------------------------------------------------------------------
/pkg/sloop/store/typed/protobufhelper.go:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2019, salesforce.com, inc.
3 | * All rights reserved.
4 | * SPDX-License-Identifier: BSD-3-Clause
5 | * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
6 | */
7 |
8 | package typed
9 |
10 | import (
11 | "time"
12 |
13 | "github.com/golang/protobuf/ptypes"
14 | "github.com/golang/protobuf/ptypes/timestamp"
15 | "github.com/pkg/errors"
16 | )
17 |
18 | func StringToProtobufTimestamp(ts string) (*timestamp.Timestamp, error) {
19 | t, err := time.Parse(time.RFC3339, ts)
20 | if err != nil {
21 | return nil, errors.Wrap(err, "could not parse timestamp")
22 | }
23 |
24 | tspb, err := ptypes.TimestampProto(t)
25 | if err != nil {
26 | return nil, errors.Wrap(err, "could not transform to proto timestamp")
27 | }
28 |
29 | return tspb, nil
30 | }
31 |
--------------------------------------------------------------------------------
/pkg/sloop/store/typed/protobufhelper_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2019, salesforce.com, inc.
3 | * All rights reserved.
4 | * SPDX-License-Identifier: BSD-3-Clause
5 | * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
6 | */
7 |
8 | package typed
9 |
10 | import (
11 | "testing"
12 |
13 | "github.com/golang/protobuf/ptypes/timestamp"
14 | "github.com/stretchr/testify/assert"
15 | )
16 |
17 | func Test_StringToProtobufTimestamp_Success(t *testing.T) {
18 | expectedResult := ×tamp.Timestamp{
19 | Seconds: 1562962332,
20 | Nanos: 0,
21 | }
22 | ts, err := StringToProtobufTimestamp("2019-07-12T20:12:12Z")
23 | assert.Nil(t, err)
24 | assert.Equal(t, expectedResult, ts)
25 | }
26 |
27 | func Test_StringToProtobufTimestamp_FailureCannotParse(t *testing.T) {
28 | ts, err := StringToProtobufTimestamp("2019-070:12:12Z")
29 | assert.NotNil(t, err)
30 | assert.Contains(t, err.Error(), "could not parse timestamp")
31 | assert.Nil(t, ts)
32 | }
33 |
34 | func Test_StringToProtobufTimestamp_FailureCannotTransformToPB(t *testing.T) {
35 | ts, err := StringToProtobufTimestamp("0000-07-12T20:12:12Z")
36 | assert.NotNil(t, err)
37 | assert.Contains(t, err.Error(), "could not transform to proto timestamp")
38 | assert.Nil(t, ts)
39 | }
40 |
--------------------------------------------------------------------------------
/pkg/sloop/store/typed/resourcesummarytable.go:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2019, salesforce.com, inc.
3 | * All rights reserved.
4 | * SPDX-License-Identifier: BSD-3-Clause
5 | * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
6 | */
7 |
8 | package typed
9 |
10 | import (
11 | "fmt"
12 | "time"
13 |
14 | "github.com/salesforce/sloop/pkg/sloop/common"
15 | "github.com/salesforce/sloop/pkg/sloop/store/untyped"
16 | )
17 |
18 | // Key is /
22 | {{.}}
23 | |
| 22 | 30 | |
| Total keys | {{.TotalKeys}} |
| Total Sloop Keys | {{.TotalSloopKeys}} |
| Total Estimated Size | {{.TotalEstimatedSize}} |
| Deleted Keys | {{.DeletedKeys}} |
| Total Internal Keys | {{.TotalInternalKeys}} |
| Total Internal Keys Size | {{.TotalInternalKeysSize}} |
| Total Internal Head Keys | {{.TotalHeadKeys}} |
| Total Internal Move Keys | {{.TotalMoveKeys}} |
| Total Internal Discard Keys | {{.TotalDiscardKeys}} |
| Table | Partition ID | Number of Keys | Estimated Size | Minimum Size | Maximum Size | Average Size |
| {{$key.TableName}} | {{$key.PartitionID}} | {{$value.TotalKeys}} | {{$value.TotalSize}} | {{$value.MinimumSize}} | {{$value.MaximumSize}} | {{$value.AverageSize}} |
| 23 | 65 | |
| Total Keys Matched | {{.KeysMatched}} |
| Total Size of Matched Keys | {{.TotalSize}} |
| Total Keys Searched | {{.TotalKeys}} |
| Level | 24 |KeyCount | 25 |Left | 26 |Right | 27 |ID | 28 |Size | 29 |
|---|---|---|---|---|---|
| {{.Level}} | 33 |{{.KeyCount}} | 34 |{{.LeftKey}} | 35 |{{.RightKey}} | 36 |{{.ID}} | 37 |{{.Size}} | 38 |
| Key | {{.Key}} |
| Payload | {{.Payload}} |
| {{.ExtraName}} | {{.ExtraValue}} |