├── .DS_Store ├── .gitignore ├── LICENSE ├── README.md ├── configs └── config.yml ├── docs ├── List 数据模型.png ├── List 数据模型设计.md ├── List 锁模型设计.md ├── kv 存储引擎选型.md └── lua 脚本支持.md ├── go.mod ├── go.sum ├── internal ├── collection │ └── list │ │ ├── list.go │ │ └── util.go ├── config │ └── config.go ├── model │ └── list.pb.go ├── protobuf │ └── list.proto ├── service │ ├── list.go │ ├── locker.go │ └── lua.go ├── store │ └── store.go └── util │ ├── base64.go │ ├── id.go │ └── log.go ├── scripts └── build_protobuf.sh └── test ├── list_test.go ├── lua_test.go └── store_test.go /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yemingfeng/sdb/2988c9f264795d25f9db0d31128938809d60613c/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | **db -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 2-Clause License 2 | 3 | Copyright (c) 2021, yemingfeng 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## [SDB](https://github.com/yemingfeng/sdb) ：纯 Go 开发、数据结构丰富、持久化、简单易用的 NoSQL 数据库 2 | ------ 3 | 4 | ### 为什么需要 SDB？ 5 | 6 | 试想以下业务场景： 7 | 8 | - 计数服务：对内容的点赞量、播放量进行统计 9 | - 推荐服务：每个用户有一个包含内容和权重推荐列表 10 | - 评论服务：查看内容的评论列表 11 | 12 | 在传统的做法中，我们会通过 MySQL + Redis 的方式实现。其中 MySQL 提供数据的持久化能力，Redis 提供高性能的读写能力。在这样的架构下会面临以下问题： 13 | 14 | - 同时部署 MySQL + Redis，机器成本高 15 | - MySQL + Redis 数据不同步带来的一致性问题 16 | - 随着业务上涨，MySQL 面临海量数据的读写压力 17 | 18 | 回过头来看上面的需求，我们真正需要的其实是有持久化能力的 Redis。 业内也有解决方案，如： 19 | 20 | - [pika](https://github.com/OpenAtomFoundation/pika) 21 | - [kvrocks](https://github.com/apache/incubator-kvrocks) 22 | - [tendis](https://cloud.tencent.com/document/product/1363/50791) 23 | - [memorydb](https://aws.amazon.com/cn/memorydb/) 24 | 25 | SDB 也是应对上面问题提供的解决方案。 26 | 27 | ### RoadMap 28 | 29 | - 数据结构 30 | - [x] List 31 | - [ ] Linked 32 | - [ ] String 33 | - [ ] Set 34 | - [ ] SortedSet 35 | - [ ] BitMap 36 | - [ ] BloomFilter 37 | - [ ] GeoHash 38 | - [ ] Grpc Server 39 | - [ ] Prometheus 监控 40 | - [ ] kv 存储引擎接入 41 | - [ ] badger 42 | - [ ] 集群 43 | - [ ] 主从 44 | - [ ] 分布式 45 | 46 | ### sdb 背后的设计 47 | 48 | - [kv 存储引擎选型](https://github.com/yemingfeng/sdb/blob/master/docs/kv%20%E5%AD%98%E5%82%A8%E5%BC%95%E6%93%8E%E9%80%89%E5%9E%8B.md) 49 | - [List 数据模型设计](https://github.com/yemingfeng/sdb/blob/master/docs/List%20%E6%95%B0%E6%8D%AE%E6%A8%A1%E5%9E%8B%E8%AE%BE%E8%AE%A1.md) 50 | - [List 锁模型设计](https://github.com/yemingfeng/sdb/blob/master/docs/List%20%E9%94%81%E6%A8%A1%E5%9E%8B%E8%AE%BE%E8%AE%A1.md) 51 | - [lua 脚本支持](https://github.com/yemingfeng/sdb/blob/master/docs/lua%20%E8%84%9A%E6%9C%AC%E6%94%AF%E6%8C%81.md) 52 | 53 | ### 友链 54 | 55 | - [rdb](https://github.com/MoSunDay/rdb) -------------------------------------------------------------------------------- /configs/config.yml: -------------------------------------------------------------------------------- 1 | debug: true 2 | 3 | # store config 4 | store: 5 | path: ./master/db 6 | delete_old: false 7 | 8 | # collection config 9 | collection: 10 | locker_count: 128 -------------------------------------------------------------------------------- /docs/List 数据模型.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yemingfeng/sdb/2988c9f264795d25f9db0d31128938809d60613c/docs/List 数据模型.png -------------------------------------------------------------------------------- /docs/List 数据模型设计.md: -------------------------------------------------------------------------------- 1 | ## SDB 背后的思考 ———— List 数据模型设计 2 | ------ 3 | 4 | [我们借助 LSM / B+ 树的实现，已经有了可靠的 kv 存储引擎了](https://github.com/yemingfeng/sdb/blob/master/docs/kv%20%E5%AD%98%E5%82%A8%E5%BC%95%E6%93%8E%E9%80%89%E5%9E%8B.md) 。回顾下我们具备的能力，是一个巨大的、持久化的 sortedMap。提供的接口是： 5 | 6 | - get(storeKey) 7 | - set(storeKey, storeValue) 8 | - delete(storeKey) 9 | - scan(minKey, maxKey) 10 | 11 | 那么，我们如何根据上述接口，实现 List 功能呢？我先抛出 SDB 实现后的结论： 12 | 13 | | 接口 | 时间复杂度 | 说明 14 | | ---- | ---- | ---- | 15 | | LPush(userKey, values) | O(1) | 从左边增加元素 16 | | RPush(userKey, values) | O(1) | 从右边增加元素 17 | | LPop(userKey) | O(1) | 从左边弹出元素 18 | | RPop(userKey) | O(1) | 从右边弹出元素 19 | | Count(userKey) | O(1) | 获取元素个数 20 | | List(userKey) | O(n) | 返回某个 listService 所有元素 21 | | Range(userKey, offset, limit) | O(1) | 遍历 List，若 offset < 0，代表反向迭代 22 | | Get(userKey, index) | O(1) | 返回 List 某下标的元素 23 | | Contain(userKey, values) | O(1) | 判断元素是否存在 24 | | Delete(userKey) | O(1) | 删除某个 List 25 | | Rem(userKey, values) | O(n) | 删除 List 中某些元素 26 | | Ttl(userKey, ttl) | O(1) | 设置 ttl 27 | | LInsert(userKey, insertValue, index) | O(n) | 向 List 指定位置左边插入元素 28 | | RInsert(userKey, insertValue, index) | O(n) | 向 List 指定位置右边插入元素 29 | 30 | ### 存储模型图 31 | 32 | 33 | 34 | | storeKey |生成规则 | storeValue | 作用 35 | | ---- | ---- | ---- | ---- | 36 | | metaKey | am:{userKey}:{version} | {head, tail, count, deleted, ttl} | 存储 List 的原始信息，version 在同一个 list 内是递增的 37 | | deletedKey | ad:{userKey}:{version} | - | 若某 List 的 version 被标记为 deleted，则会写入 deletedKey，仅用于快速回收 list 某版本的数据，后面会详细说明 38 | | ttlKey | at:{userKey}:{ttl}:{version} | - | 若某 List 有 ttl，则会写入 ttlKey，仅用于快速回收 List 某版本的数据，后面会详细说明 39 | | seqKey | as:{userKey}:{version}:{seq}:{value} | - | List 每个元素对应一个 seqKey，用于遍历该 List 40 | | valueKey | av:{userKey}:{version}:{value}:{seq} | - | List 每个元素对应一个 valueKey，用于判断某 value 是否在 list 中 41 | 42 | ### meta 设计 43 | 44 | | 字段 | 生成逻辑 | 作用 45 | | ---- | ---- | ---- | 46 | | head | 程序保障 | 指向 List 第一个元素，每向 List 左边增加一个元素会让 head - 1 47 | | tail | 程序保障 | 指向 List 最后一个元素，每向 List 右边增加一个元素会让 tail - 1 48 | | ttl | 用户设置 | 默认值为 0，若 ttl <= 0，则认为该 List 未启用 ttl 功能 49 | | deleted | 用户设置 | 默认值为 false，代表未被删除 50 | | count | 程序保障 | List 的元素个数 51 | 52 | 假设有一个 listA，其在存储引擎中的数据可能是： 53 | 54 | | userKey | version | metaKey | meta 信息 55 | | ---- | ---- | ---- | ---- | 56 | | listA | 0 | `am:listA:0` | `{version=0, count=10, head=0, tail=10, deleted=true, ttl=0}` 57 | | listA | 1 | `am:listA:1` | `{version=1, count=3, head=-2, tail=0, deleted=true, ttl=0}` 58 | | listA | 2 | `am:listA:2` | `{version=2, count=15, head=3, tail=18, deleted=false, ttl=0}` 59 | 60 | 可以看出，每个 List 的 meta 是包含多版本的。 61 | 62 | SDB 只会认为最高版本的 meta 在 ttl 内并且未 deleted 才是有效的。若最高版本的 meta 无效，向列表增加元素时，SDB 会创建一个新的 meta。 63 | 64 | 非最高版本的数据用户是不能操作的，只会在后面的数据回收任务中回收，后面会详细说明。 65 | 66 | ### ttl 设计 67 | 68 | 为了支持 List 过期功能。SDB 在 List meta 中增加了 ttl 字段。 假设 listA 在版本 3 中设置了 ttl=10，则在存储引擎中 listA 的 meta 数据可能是： 69 | 70 | | userKey | version | metaKey | meta 信息（省略无用信息） 71 | | ---- | ---- | ---- | ---- | 72 | | listA | 0 | `am:listA:0` | `{ttl=0}` 73 | | listA | 1 | `am:listA:1` | `{ttl=0}` 74 | | listA | 2 | `am:listA:2` | `{ttl=0}` 75 | | listA | 3 | `am:listA:3` | `{ttl=10}` 76 | 77 | 在用户操作 SDB 时，只会判断 version=3 的 meta 信息，发现无效后，再向 List 加入元素时，SDB 会创建一个新版本的 meta 信息。 78 | 79 | 假设对 listA 的操作流程如下： 80 | 81 | - 向 listA 增加元素 a 82 | - 设置 listA ttl=1（会快速过期） 83 | - 向 listA 增加元素 b 84 | - 设置 listA ttl=1（会快速过期） 85 | - 向 listA 增加元素 c 86 | 87 | 则 listA 的 meta 信息的数据可能是： 88 | 89 | | userKey | version | metaKey | meta 信息（省略无用信息） 90 | | ---- | ---- | ---- | ---- | 91 | | listA | 0 | `am:listA:0` | `{ttl=1}` 92 | | listA | 1 | `am:listA:1` | `{ttl=1}` 93 | | listA | 2 | `am:listA:2` | `{ttl=0}` 94 | 95 | 其元素的数据可能是： 96 | 97 | | userKey | version | 元素 | seqKey | valueKey 98 | | ---- | ---- | ---- | ---- | ---- | 99 | | listA | 0 | a | `as:listA:0:0:a` | `av:listA:0:a:0` 100 | | listA | 1 | b | `as:listA:1:0:b` | `av:listA:1:b:0` 101 | | listA | 2 | c | `as:listA:2:0:c` | `av:listA:2:c:0` 102 | 103 | 我们可以很快发现 listA 记录中 version=0 和 version=1 相关数据都可以回收。 104 | 105 | 为了支持快速回收无效数据，SDB 为设置了 ttl 的版本增加了 ttlKey： 106 | 107 | | userKey | version | ttlKey 108 | | ---- | ---- | ---- | 109 | | listA | 0 | `at:listA:0` 110 | | listA | 1 | `at:listA:1` 111 | 112 | 有了 ttlKey 后，SDB 就可以根据它快速扫描出已经 ttl 的 List，用于回收相关数据。 113 | 114 | ### deleted 设计 115 | 116 | 同 ttl 设计，不赘述。 117 | 118 | ### seqKey 和 valueKey 设计 119 | 120 | List 中每一个元素都包含 seqKey 和 valueKey，作用已经在上面表述了。 这里补充的是： 121 | 122 | - 向 List 左边增加元素时，seq = head - 1 123 | - 向 List 右边增加元素时，seq = tail + 1 124 | 125 | 假设有一个 listA，它的最高版本是 3，其元素列表如：[a, b, c]，其每个元素在存储引擎中可能是这样的： 126 | 127 | | 元素 | seqKey | valueKey 128 | | ---- | ---- | ---- | 129 | | a | `as:listA:3:-1:a` | `av:listA:3:a:-1` 130 | | b | `as:listA:3:0:b` | `av:listA:3:b:0` 131 | | c | `as:listA:3:1:c` | `av:listA:3:c:1` 132 | 133 | ### 如何在指定位置插入元素？ 134 | 135 | 假设有一个 listA，元素为[a, b, c]，其存储数据为： 136 | 137 | | 元素 | seqKey | valueKey 138 | | ---- | ---- | ---- | 139 | | a | `as:listA:0:-1:a` | `av:listA:0:a:-1` 140 | | b | `as:listA:0:0:b` | `av:listA:0:b:0` 141 | | c | `as:listA:0:1:c` | `av:listA:0:c:1` 142 | 143 | 现在要在 b 后面增加一个元素 d，则存储数据变为： 144 | 145 | | 元素 | seqKey | valueKey 146 | | ---- | ---- | ---- | 147 | | a | `as:listA:0:-1:a` | `av:listA:0:a:-1` 148 | | b | `as:listA:0:0:b` | `av:listA:0:b:0` 149 | | d | `as:listA:0:1:d` | `av:listA:0:d:1` 150 | | c | `as:listA:0:2:c` | `av:listA:0:c:2` 151 | 152 | 意味着所有的元素都会挪动，时间复杂度较高。Rem 接口同理。 -------------------------------------------------------------------------------- /docs/List 锁模型设计.md: -------------------------------------------------------------------------------- 1 | ## SDB 背后的思考 ———— List 数据结构锁模型设计 2 | ------ 3 | 4 | [从上一篇中，我们已经设计好了 List 数据结构在 kv 存储中的数据模型。](https://github.com/yemingfeng/sdb/blob/master/docs/List%20%E6%95%B0%E6%8D%AE%E6%A8%A1%E5%9E%8B%E8%AE%BE%E8%AE%A1.md) 5 | 6 | 这一篇，我们介绍下 List 数据结构中的锁模型设计。 7 | 8 | Q：用户同时进行写入操作请求：LPush、LPush、Delete、Rem 等，如何加锁？ 9 | 10 | A：由于这类请求会操作 metaKey、ttlKey 和 deletedKey。为了保证一致性，SDB 会按照用户写入的 userKey 进行加锁。 11 | 12 | SDB 内部维护了多把锁，每个 userKey hash 后会取到对应的锁，然后对该锁进行加锁操作。伪代码如下： 13 | 14 | ```golang 15 | var lockers []*sync.RWMutex 16 | 17 | // 写操作加锁 18 | func wlock(userKey []byte) { 19 | getLocker(userKey).Lock() 20 | } 21 | 22 | // 读操作加锁 23 | func wUnLock(userKey []byte) { 24 | getLocker(userKey).Unlock() 25 | } 26 | 27 | // 根据 userKey 获取锁 28 | func getLocker(userKey []byte) *sync.RWMutex { 29 | checksum := crc16.Checksum(userKey, crc16.IBMTable) 30 | return lockers[int(checksum)%len(lockers)] 31 | } 32 | ``` 33 | 34 | Q：为什么不考虑每个 userKey 一把锁，而是多个 userKey 经过 hash 后共用一把锁？ 35 | 36 | A：如果每个 userKey 一把锁，可能会出现锁太多带来的性能损耗。虽然多个 userKey 经过 hash 后会共用一把锁，但每次用户的写入请求应该是**快速返回**的，写锁锁住的时间应该是不会太长的。 37 | 38 | Q：用户的写入操作和 Count、Range 的加锁逻辑是什么？ 39 | 40 | A：针对 Count，只是读取 meta 信息，不需要做额外的加锁处理。 而 Range 操作是遍历 List 的，为了防止在遍历的时候，对该 List 进行了写入操作带来的数据混乱问题。SDB 对该操作加了读锁，伪代码如下： 41 | 42 | ```golang 43 | // 写操作加锁 44 | func rlock(userKey []byte) { 45 | getLocker(userKey).RLock() 46 | } 47 | 48 | // 读操作加锁 49 | func rUnLock(userKey []byte) { 50 | getLocker(userKey).RUnlock() 51 | } 52 | ``` 53 | 54 | Q：deleted 数据回收任务和用户的写入请求加锁逻辑是怎样的？ 55 | 56 | A：由于 SDB 是采用多版本的设计，用户的写入请求只会操作最新版本的 metaKey 等，而 deleted 数据回收任务不会回收最新版本的数据，所以二者不存在冲突的问题，不需要额外加锁。 57 | 58 | Q：ttl 数据回收任务和用户的写入请求加锁逻辑是怎样的？ 59 | 60 | A：由于 SDB 是采用多版本的设计，用户的写入请求只会操作最新版本的 metaKey 等，而 ttl 数据回收任务可能会回收最新版本的数据，这二者存在以下组合： 61 | 62 | - 用户请求 LPush（RPush 同理） + ttl 数据回收任务同时进行 63 | - 假设 listA 最高版本号是 3 即将过期，meta 信息是：`{count=3, version=3, ttl=1, head=0, tail=3, deleted=false}`，包含了元素是：[a, b, c]。LPush 的元素是：[a, d]。 64 | - LPush 获取到最高版本号 3，发现未过期（即将过期）。所以会往版本号 3 写入 [a, d] 数据 + meta `{count=5, version=3, ttl=1, head=0, tail=5, deleted=false}` 信息和元素 [a, d]。 65 | - ttl 数据回收任务只回收了版本号 3 的 metaKey：`{count=3, version=3, ttl=1, head=0, tail=3, deleted=false}` 和数据：[a, b, c]。 66 | - **总结：这种情况不需要加锁，只需要在每次 LPush 是再保证写入一次 ttlKey 即可。等下一次回收任务还会回收该 listA 的版本号 3 的所有数据。** 67 | - 用户请求 Delete + ttl 数据回收任务同时进行 68 | - 假设删除 listA 最高版本号是 3，对 listA 进行删除，同时 ttl 数据回收任务发现 listA 即将过期。 69 | - **总结：这种情况不需要加锁，ttl 数据任务会回收一次 listA 数据，deleted 数据回收也会回收一次 listA 数据。** 70 | - Rem 同理。 -------------------------------------------------------------------------------- /docs/kv 存储引擎选型.md: -------------------------------------------------------------------------------- 1 | ## SDB 背后的思考 ———— kv 存储引擎选型 2 | ------ 3 | 4 | 主流的 kv 存储引擎模型主要分为三大类： 5 | 6 | - LSM 树模型 7 | - 针对写多读少场景，顺序写、读比较慢 8 | - B+ 树模型 9 | - 针对读多写少场景，随机写、读比较快 10 | - SLM 模型 11 | - 结合 LSM 树和 B + 树的有点，写入、读取都很快 12 | - 可参考实现 [lotusdb](https://github.com/flower-corp/lotusdb) 13 | 14 | 不管哪种存储模型，我们可以将 kv 存储引擎想象成是一个巨大的、持久化的 sortedMap。其提供的接口是： 15 | 16 | - get(storeKey) 17 | - set(storeKey, storeValue) 18 | - delete(storeKey) 19 | - scan(minKey, maxKey) 20 | 21 | 由于 SDB 定位是纯 Go 开发的 NoSQL 数据库，只会兼容纯 Go 开发的 kv 存储库。为了减少工作量，选择集成已有、成熟的 kv 存储引擎，找到了以下可靠的存储引擎： 22 | 23 | | 项目 | 介绍 | 存储模型 24 | | ---- | ---- | ---- 25 | | [pebble](https://github.com/cockroachdb/pebble) | [cockroach](https://github.com/cockroachdb/cockroach) 推出并兼容 rocksdb。 | LSM 26 | | [goleveldb](https://github.com/syndtr/goleveldb) | Go 版 LevelDB | LSM 27 | | [badger](https://github.com/dgraph-io/badger) | Go 版 badger | LSM 28 | 29 | 目前 SDB 选择了 pebble 作为 kv 存储引擎。PS: 欢迎补充 Go 版本的 kv 存储。 -------------------------------------------------------------------------------- /docs/lua 脚本支持.md: -------------------------------------------------------------------------------- 1 | ## SDB 背后的思考 ———— lua 脚本支持 2 | ------ 3 | 4 | lua 脚本是酷炫的，Redis、Nginx 等开源项目都可以内嵌 lua 实现业务逻辑。所以 SDB 也打算支撑 lua 脚本。还好找到了优秀的开源项目：[gopher-lua](https://github.com/yuin/gopher-lua) 。 5 | 6 | ### 初始化 7 | 8 | ```go 9 | L := lua.NewState() 10 | defer L.Close() 11 | ``` 12 | 13 | ### 方法注册 14 | 15 | 将 SDB 的方法注册到 lua 脚本中，以 LCount 为例子： 16 | 17 | ```go 18 | L.SetGlobal("LCount", L.NewFunction(func(L *lua.LState) int { 19 | userKey := L.CheckString(1) 20 | luaLogger.Printf("[LCount] userKey: %s", userKey) 21 | 22 | res, err := luaService.list.Count(batch, []byte(userKey)) 23 | if err != nil { 24 | L.RaiseError("%s", err) 25 | } 26 | L.Push(lua.LNumber(res)) 27 | return 1 28 | })) 29 | ``` 30 | 31 | ### 如何保证事务操作？ 32 | 33 | lua 脚本是灵活的，在上面我们可以写很多的业务逻辑，如果我们在 lua 中用了：LLPush、LRPush 如何能保证多个方法是操作是事务的呢？也比较简单，我们只需要创建一个 batch 对象，执行完我们的 lua 脚本后进行 commit。保证执行每一次脚本是事务的。 34 | 35 | ### 加锁逻辑？ 36 | 37 | 这其实是最容易被忽略的一点。对比 Redis 来说，由于是单线程的，所以 lua 脚本是不需要考虑锁的。 38 | 39 | 但是 SDB 不同，SDB 首先是支持多线程的，那么对 userKey 的写操作会进行加锁。 lua 脚本在某种程度上破坏了对单个 userKey 的加锁策略。 40 | 41 | 首先 lua 脚本得让用户传入会操作的 userKey 列表，然后对每个 userKey 进行加锁。只有获取了所有锁，lua 脚本才能开始运行。 42 | 43 | 但这是不够的，可能会出现死锁问题。如当一个 lua 脚本操作 a、b 两个 userKey，另一个 lua 脚本操作 b、d 两个 userKey。假设 d 和 a 的锁是同一把。就会出现死锁的问题。 44 | 45 | 防止死锁的解决方法也很简单，只需要保证先对 lua 脚本的 userKey 按某种顺序依次获取锁既可。伪代码如下： 46 | ```go 47 | hashes := make([]bool, len(locker.lockers)) 48 | for i := 0; i < len(userKeys); i++ { 49 | hashes[locker.hash(userKeys[i])] = true 50 | } 51 | for i := 0; i < len(hashes); i++ { 52 | if hashes[i] { 53 | locker.lockers[i].Lock() 54 | } 55 | } 56 | ``` -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/yemingfeng/sdb 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb 7 | github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6 8 | google.golang.org/protobuf v1.27.1 9 | gopkg.in/yaml.v2 v2.4.0 10 | ) 11 | 12 | require ( 13 | github.com/bwmarrin/snowflake v0.3.0 // indirect 14 | github.com/cockroachdb/errors v1.8.1 // indirect 15 | github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect 16 | github.com/cockroachdb/redact v1.0.8 // indirect 17 | github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect 18 | github.com/gogo/protobuf v1.3.2 // indirect 19 | github.com/golang/snappy v0.0.3 // indirect 20 | github.com/google/go-cmp v0.5.6 // indirect 21 | github.com/kr/pretty v0.1.0 // indirect 22 | github.com/kr/text v0.1.0 // indirect 23 | github.com/pkg/errors v0.9.1 // indirect 24 | github.com/stretchr/testify v1.7.0 // indirect 25 | github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 // indirect 26 | golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect 27 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect 28 | ) 29 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 3 | github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= 4 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 5 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 6 | github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= 7 | github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= 8 | github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= 9 | github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= 10 | github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= 11 | github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= 12 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 13 | github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= 14 | github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= 15 | github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= 16 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 17 | github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= 18 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 19 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 20 | github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= 21 | github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= 22 | github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= 23 | github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= 24 | github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= 25 | github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= 26 | github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= 27 | github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb h1:dqFirML/6RMDwkge7Tqf33qE0ORbF6rRJOLjCmmwTNg= 28 | github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= 29 | github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= 30 | github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= 31 | github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= 32 | github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= 33 | github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= 34 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= 35 | github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= 36 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 37 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 38 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 39 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 40 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 41 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 42 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 43 | github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= 44 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 45 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 46 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 47 | github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= 48 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 49 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 50 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 51 | github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= 52 | github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= 53 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 54 | github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= 55 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 56 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 57 | github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= 58 | github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= 59 | github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= 60 | github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 61 | github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= 62 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= 63 | github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= 64 | github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= 65 | github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= 66 | github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= 67 | github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= 68 | github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 69 | github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= 70 | github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= 71 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 72 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 73 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 74 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 75 | github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= 76 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 77 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 78 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 79 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 80 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 81 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 82 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 83 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 84 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 85 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 86 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 87 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 88 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 89 | github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 90 | github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= 91 | github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 92 | github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 93 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 94 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 95 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 96 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 97 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 98 | github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= 99 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 100 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 101 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 102 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 103 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 104 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 105 | github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6 h1:IIVxLyDUYErC950b8kecjoqDet8P5S4lcVRUOM6rdkU= 106 | github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6/go.mod h1:JslaLRrzGsOKJgFEPBP65Whn+rdwDQSk0I0MCRFe2Zw= 107 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 108 | github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= 109 | github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= 110 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 111 | github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= 112 | github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= 113 | github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= 114 | github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= 115 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 116 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 117 | github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= 118 | github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= 119 | github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= 120 | github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= 121 | github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= 122 | github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= 123 | github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= 124 | github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= 125 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 126 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 127 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 128 | github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 129 | github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 130 | github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 131 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 132 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 133 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 134 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 135 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 136 | github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= 137 | github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= 138 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 139 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 140 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 141 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 142 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 143 | github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= 144 | github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= 145 | github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= 146 | github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= 147 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 148 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 149 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 150 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 151 | github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= 152 | github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= 153 | github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= 154 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 155 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 156 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 157 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 158 | github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= 159 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 160 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 161 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 162 | github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= 163 | github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= 164 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 165 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 166 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 167 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 168 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 169 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 170 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 171 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 172 | github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= 173 | github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 174 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 175 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 176 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 177 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 178 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 179 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 180 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 181 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 182 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 183 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 184 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 185 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 186 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 187 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 188 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 189 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 190 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 191 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 192 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 193 | github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 194 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 195 | github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= 196 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 197 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 198 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= 199 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= 200 | github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= 201 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 202 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= 203 | github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= 204 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= 205 | github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= 206 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 207 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 208 | github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= 209 | github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= 210 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 211 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 212 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 213 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 214 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 215 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 216 | golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 217 | golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= 218 | golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= 219 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 220 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 221 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 222 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 223 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 224 | golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= 225 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 226 | golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 227 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 228 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 229 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 230 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 231 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 232 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 233 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 234 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 235 | golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 236 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 237 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 238 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 239 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 240 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 241 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 242 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 243 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 244 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 245 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 246 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 247 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 248 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 249 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 250 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 251 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 252 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 253 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 254 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 255 | golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 256 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 257 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 258 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 259 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 260 | golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 261 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 262 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 263 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 264 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 265 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 266 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320 h1:0jf+tOCoZ3LyutmCOWpVni1chK4VfFLhRsDK7MhqGRY= 267 | golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 268 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 269 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 270 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 271 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 272 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 273 | golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 274 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 275 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 276 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 277 | golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 278 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 279 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 280 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 281 | golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 282 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 283 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 284 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 285 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 286 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 287 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 288 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 289 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 290 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 291 | google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 292 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 293 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 294 | google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 295 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 296 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 297 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 298 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 299 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 300 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 301 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 302 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 303 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 304 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 305 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 306 | google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= 307 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 308 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 309 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 310 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 311 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 312 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 313 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 314 | gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 315 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 316 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 317 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 318 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 319 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 320 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 321 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 322 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 323 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 324 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 325 | -------------------------------------------------------------------------------- /internal/collection/list/list.go: -------------------------------------------------------------------------------- 1 | package list 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/cockroachdb/pebble" 7 | "github.com/yemingfeng/sdb/internal/model" 8 | "github.com/yemingfeng/sdb/internal/store" 9 | "github.com/yemingfeng/sdb/internal/util" 10 | "google.golang.org/protobuf/proto" 11 | "reflect" 12 | ) 13 | 14 | type List struct { 15 | } 16 | 17 | func NewList() *List { 18 | return &List{} 19 | } 20 | 21 | // LPush 22 | // left push elements, such as if the list contains [a, b, c], 23 | // now want to left push [d, e], 24 | // then the list will update be: [e, d, a, b, c] 25 | func (list *List) LPush(batch *pebble.Batch, userKey []byte, values [][]byte) error { 26 | return list.push(batch, userKey, values, true) 27 | } 28 | 29 | // RPush 30 | // right push elements, such as if the list contains [a, b, c], 31 | // now want to left push [d, e], 32 | // then the list will update be: [a, b, c, d, e] 33 | func (list *List) RPush(batch *pebble.Batch, userKey []byte, values [][]byte) error { 34 | return list.push(batch, userKey, values, false) 35 | } 36 | 37 | func (list *List) push(batch *pebble.Batch, userKey []byte, values [][]byte, left bool) error { 38 | if len(userKey) == 0 { 39 | return errors.New("userKey is emtpy") 40 | } 41 | meta, version, err := list.getMeta(batch, userKey) 42 | if err != nil { 43 | return err 44 | } 45 | if meta == nil { 46 | version = util.NextId() 47 | meta = &model.ListMeta{ 48 | Head: 0, 49 | Tail: 1, 50 | Count: 0, 51 | Deleted: false, 52 | } 53 | } 54 | 55 | for i := 0; i < len(values); i++ { 56 | var seq int64 57 | if left { 58 | seq = meta.Head 59 | meta.Head-- 60 | } else { 61 | seq = meta.Tail 62 | meta.Tail++ 63 | } 64 | value := values[i] 65 | seqKey := generateSeqKey(userKey, version, seq, value) 66 | if err := batch.Set(seqKey, nil, nil); err != nil { 67 | return err 68 | } 69 | valueKey := generateValueKey(userKey, version, value, seq) 70 | if err := batch.Set(valueKey, nil, nil); err != nil { 71 | return err 72 | } 73 | } 74 | 75 | meta.Count = uint64(len(values)) + meta.Count 76 | 77 | metaValue, err := proto.Marshal(meta) 78 | if err != nil { 79 | return err 80 | } 81 | if err := batch.Set(generateMetaKey(userKey, version), metaValue, nil); err != nil { 82 | return err 83 | } 84 | return nil 85 | } 86 | 87 | func (list *List) LPop(batch *pebble.Batch, userKey []byte) ([]byte, error) { 88 | return list.pop(batch, userKey, true) 89 | } 90 | 91 | func (list *List) RPop(batch *pebble.Batch, userKey []byte) ([]byte, error) { 92 | return list.pop(batch, userKey, false) 93 | } 94 | 95 | func (list *List) pop(batch *pebble.Batch, userKey []byte, left bool) ([]byte, error) { 96 | if len(userKey) == 0 { 97 | return nil, errors.New("userKey is emtpy") 98 | } 99 | meta, version, err := list.getMeta(batch, userKey) 100 | if err != nil || meta == nil || meta.Count == 0 { 101 | return nil, err 102 | } 103 | 104 | seq := int64(0) 105 | if left { 106 | seq = meta.Head + 1 107 | meta.Head += 1 108 | } else { 109 | seq = meta.Tail - 1 110 | meta.Tail -= 1 111 | } 112 | 113 | seqKey := list.getSeqKey(batch, userKey, version, seq) 114 | _, _, _, value, err := parseSeqKey(seqKey) 115 | if err != nil { 116 | return nil, err 117 | } 118 | if err := batch.Delete(generateValueKey(userKey, version, value, seq), nil); err != nil { 119 | return nil, err 120 | } 121 | if err := batch.Delete(generateSeqKey(userKey, version, seq, value), nil); err != nil { 122 | return nil, err 123 | } 124 | meta.Count -= 1 125 | 126 | metaValue, err := proto.Marshal(meta) 127 | if err != nil { 128 | return nil, err 129 | } 130 | if err := batch.Set(generateMetaKey(userKey, version), metaValue, nil); err != nil { 131 | return nil, err 132 | } 133 | 134 | return value, nil 135 | } 136 | 137 | // Count 138 | // return element counts, such as if the list contains [a, b, c], will return 3 139 | func (list *List) Count(batch *pebble.Batch, userKey []byte) (uint64, error) { 140 | if len(userKey) == 0 { 141 | return 0, errors.New("userKey is emtpy") 142 | } 143 | meta, _, err := list.getMeta(batch, userKey) 144 | if err != nil || meta == nil { 145 | return 0, err 146 | } 147 | return meta.Count, nil 148 | } 149 | 150 | // Range 151 | // range list, such as if the list contains [a, b, c, d, e] 152 | // if pass offset=0,limit=3, will return [a, b, c] 153 | // if pass offset=1,limit=2, will return [a, b] 154 | // if pass offset=-1,limit=2, will return [e, d] 155 | // if pass offset=-2,limit=5, will return [c, b, a] 156 | func (list *List) Range(batch *pebble.Batch, userKey []byte, offset int64, limit uint64) ([][]byte, error) { 157 | if len(userKey) == 0 { 158 | return nil, errors.New("userKey is emtpy") 159 | } 160 | iter := batch.NewIter(&pebble.IterOptions{}) 161 | defer iter.Close() 162 | 163 | meta, version, err := list.getMeta(batch, userKey) 164 | if err != nil || meta == nil || meta.Count == 0 { 165 | return nil, err 166 | } 167 | 168 | reverse := offset < 0 169 | if !reverse { 170 | lowerBound := generateSeqPrefixKey(userKey, version, meta.Head+offset+1) 171 | upperBound := generateSeqPrefixKey(userKey, version, meta.Head+offset+int64(limit)+1) 172 | iter.SetBounds(lowerBound, upperBound) 173 | iter.First() 174 | } else { 175 | lowerBound := generateSeqPrefixKey(userKey, version, meta.Tail+offset-int64(limit)+1) 176 | upperBound := generateSeqPrefixKey(userKey, version, meta.Tail+offset+1) 177 | iter.SetBounds(lowerBound, upperBound) 178 | iter.Last() 179 | } 180 | 181 | res := make([][]byte, limit) 182 | i := uint64(0) 183 | 184 | for { 185 | if !iter.Valid() { 186 | break 187 | } 188 | seqKey := iter.Key() 189 | _, _, _, value, err := parseSeqKey(seqKey) 190 | 191 | if err != nil { 192 | return nil, err 193 | } 194 | res[i] = value 195 | i++ 196 | if !reverse { 197 | iter.Next() 198 | } else { 199 | iter.Prev() 200 | } 201 | } 202 | return res[:i], nil 203 | } 204 | 205 | // Get 206 | // return get elements, such as if the list contains [a, b, c], if pass index = [1, 2], return [b, c] 207 | func (list *List) Get(batch *pebble.Batch, userKey []byte, indexes []int64) (map[int64][]byte, error) { 208 | if len(userKey) == 0 { 209 | return nil, errors.New("userKey is emtpy") 210 | } 211 | meta, version, err := list.getMeta(batch, userKey) 212 | if err != nil || meta == nil || meta.Count == 0 { 213 | return nil, err 214 | } 215 | 216 | res := make(map[int64][]byte) 217 | for i := 0; i < len(indexes); i++ { 218 | index := indexes[i] 219 | seq := meta.Head + 1 + index 220 | if seq > meta.Tail-1 || seq < meta.Head { 221 | return nil, errors.New(fmt.Sprintf("invalid index: %d", index)) 222 | } 223 | seqKey := list.getSeqKey(batch, userKey, version, seq) 224 | if len(seqKey) > 0 { 225 | _, _, _, value, err := parseSeqKey(seqKey) 226 | if err != nil { 227 | return nil, err 228 | } 229 | res[index] = value 230 | } 231 | } 232 | 233 | return res, nil 234 | } 235 | 236 | // Contain 237 | // return dose elements if not contains, such as if the list contains [a, b, c], if pass index = [a, b, d], return [true, true, false] 238 | func (list *List) Contain(batch *pebble.Batch, userKey []byte, values [][]byte) ([]bool, error) { 239 | if len(userKey) == 0 { 240 | return nil, errors.New("userKey is emtpy") 241 | } 242 | iter := batch.NewIter(&pebble.IterOptions{}) 243 | defer iter.Close() 244 | 245 | meta, version, err := list.getMeta(batch, userKey) 246 | if err != nil || meta == nil || meta.Count == 0 { 247 | return nil, err 248 | } 249 | 250 | res := make([]bool, len(values)) 251 | for i := 0; i < len(values); i++ { 252 | value := values[i] 253 | iterOptions := store.NewPrefixIterOptions(generateValuePrefixKey(userKey, version, value)) 254 | iter.SetBounds(iterOptions.LowerBound, iterOptions.UpperBound) 255 | iter.First() 256 | res[i] = iter.Valid() 257 | } 258 | 259 | return res, nil 260 | } 261 | 262 | // Exist 263 | // return dose elements if not exist 264 | func (list *List) Exist(batch *pebble.Batch, userKey []byte) (bool, error) { 265 | if len(userKey) == 0 { 266 | return false, errors.New("userKey is emtpy") 267 | } 268 | iter := batch.NewIter(&pebble.IterOptions{}) 269 | defer iter.Close() 270 | 271 | meta, _, err := list.getMeta(batch, userKey) 272 | if err != nil || meta == nil { 273 | return false, err 274 | } 275 | return true, nil 276 | } 277 | 278 | // Delete 279 | // delete list 280 | func (list *List) Delete(batch *pebble.Batch, userKey []byte) error { 281 | if len(userKey) == 0 { 282 | return errors.New("userKey is emtpy") 283 | } 284 | meta, version, err := list.getMeta(batch, userKey) 285 | if err != nil || meta == nil { 286 | return err 287 | } 288 | 289 | meta.Deleted = true 290 | 291 | metaValue, err := proto.Marshal(meta) 292 | if err != nil { 293 | return err 294 | } 295 | if err := batch.Set(generateMetaKey(userKey, version), metaValue, nil); err != nil { 296 | return err 297 | } 298 | 299 | return nil 300 | } 301 | 302 | // Rem 303 | // if list contains [a, b, c, d, a, e], then pass 'a' 304 | // the list will be [b, c, d, e] 305 | func (list *List) Rem(batch *pebble.Batch, userKey []byte, value []byte) error { 306 | if len(userKey) == 0 { 307 | return errors.New("userKey is emtpy") 308 | } 309 | meta, version, err := list.getMeta(batch, userKey) 310 | if err != nil || meta == nil || meta.Count == 0 { 311 | return err 312 | } 313 | 314 | iter := batch.NewIter(store.NewPrefixIterOptions(generateValuePrefixKey(userKey, version, value))) 315 | defer iter.Close() 316 | 317 | startSeq := int64(0) 318 | found := false 319 | iter.First() 320 | if iter.Valid() { 321 | valueKey := iter.Key() 322 | _, _, _, seq, err := parseValueKey(valueKey) 323 | if err != nil { 324 | return err 325 | } 326 | startSeq = seq 327 | found = true 328 | } 329 | if !found { 330 | return nil 331 | } 332 | 333 | lowerBound := generateSeqPrefixKey(userKey, version, startSeq) 334 | upperBound := store.Next(generateSeqIteratorKey(userKey, version)) 335 | iter.SetBounds(lowerBound, upperBound) 336 | foundCount := int64(0) 337 | //preSeq := startSeq 338 | // [a, a, a, b, a, a, c, d, a] 339 | // [1, 2, 3, 4, 5, 6, 7, 8, 9] 340 | for iter.First(); iter.Valid(); iter.Next() { 341 | // get current seq 342 | _, _, currentSeq, currentValue, err := parseSeqKey(iter.Key()) 343 | if err != nil { 344 | return err 345 | } 346 | 347 | // delete old seq 348 | if err := batch.Delete(generateSeqKey(userKey, version, currentSeq, currentValue), nil); err != nil { 349 | return err 350 | } 351 | if err := batch.Delete(generateValueKey(userKey, version, currentValue, currentSeq), nil); err != nil { 352 | return err 353 | } 354 | if reflect.DeepEqual(currentValue, value) { 355 | foundCount++ 356 | } else { 357 | if err := batch.Set(generateValueKey(userKey, version, currentValue, currentSeq-foundCount), nil, nil); err != nil { 358 | return err 359 | } 360 | if err := batch.Set(generateSeqKey(userKey, version, currentSeq-foundCount, currentValue), nil, nil); err != nil { 361 | return err 362 | } 363 | } 364 | } 365 | 366 | meta.Count = meta.Count - uint64(foundCount) 367 | meta.Tail = meta.Tail - foundCount 368 | 369 | metaValue, err := proto.Marshal(meta) 370 | if err != nil { 371 | return err 372 | } 373 | if err := batch.Set(generateMetaKey(userKey, version), metaValue, nil); err != nil { 374 | return err 375 | } 376 | 377 | return nil 378 | } 379 | 380 | func (list *List) LInsert(batch *pebble.Batch, userKey []byte, insertValue []byte, index int64) error { 381 | return list.insert(batch, userKey, insertValue, index, true) 382 | } 383 | 384 | func (list *List) RInsert(batch *pebble.Batch, userKey []byte, insertValue []byte, index int64) error { 385 | return list.insert(batch, userKey, insertValue, index, false) 386 | } 387 | 388 | // Insert 389 | // if list contains [a, b, c, d, a, e], then pass insertValue='a', index=2 390 | // if pass left=true, the list will be [a, b, a, c, d, a, e] 391 | // if pass left=false, the list will be [a, b, c, a, d, a, e] 392 | // if list not exist, will create new 393 | func (list *List) insert(batch *pebble.Batch, userKey []byte, insertValue []byte, index int64, left bool) error { 394 | if len(userKey) == 0 { 395 | return errors.New("userKey is emtpy") 396 | } 397 | meta, version, err := list.getMeta(batch, userKey) 398 | if err != nil { 399 | return err 400 | } 401 | if meta == nil { 402 | version = util.NextId() 403 | meta = &model.ListMeta{ 404 | Head: 0, 405 | Tail: 1, 406 | Count: 0, 407 | Deleted: false, 408 | } 409 | } 410 | 411 | // if index=0 & count=0, then just insert 412 | if index == 0 && meta.Count == 0 { 413 | seq := meta.Tail 414 | seqKey := generateSeqKey(userKey, version, seq, insertValue) 415 | if err := batch.Set(seqKey, nil, nil); err != nil { 416 | return err 417 | } 418 | valueKey := generateValueKey(userKey, version, insertValue, seq) 419 | if err := batch.Set(valueKey, nil, nil); err != nil { 420 | return err 421 | } 422 | meta.Tail++ 423 | meta.Count++ 424 | 425 | metaValue, err := proto.Marshal(meta) 426 | if err != nil { 427 | return err 428 | } 429 | if err := batch.Set(generateMetaKey(userKey, version), metaValue, nil); err != nil { 430 | return err 431 | } 432 | return nil 433 | } 434 | 435 | insertSeq := meta.Head + 1 + index 436 | if insertSeq < meta.Head || insertSeq > meta.Tail { 437 | return errors.New(fmt.Sprintf("invalid index: %d", index)) 438 | } 439 | 440 | // move [meta.Tail...insertSeq+1] 441 | iter := batch.NewIter(&pebble.IterOptions{ 442 | LowerBound: generateSeqPrefixKey(userKey, version, insertSeq+1), 443 | UpperBound: generateSeqPrefixKey(userKey, version, meta.Tail), 444 | }) 445 | defer iter.Close() 446 | for iter.Last(); iter.Valid(); iter.Prev() { 447 | currentSeqKey := iter.Key() 448 | _, _, currentSeq, currentValue, err := parseSeqKey(currentSeqKey) 449 | if err != nil { 450 | return err 451 | } 452 | // delete old currentKey & valueKey 453 | if err := batch.Delete(currentSeqKey, nil); err != nil { 454 | return err 455 | } 456 | if err := batch.Delete(generateValueKey(userKey, version, currentValue, currentSeq), nil); err != nil { 457 | return err 458 | } 459 | 460 | // insert to new seq 461 | if err := batch.Set(generateSeqKey(userKey, version, currentSeq+1, currentValue), nil, nil); err != nil { 462 | return err 463 | } 464 | if err := batch.Set(generateValueKey(userKey, version, currentValue, currentSeq+1), nil, nil); err != nil { 465 | return err 466 | } 467 | } 468 | insertSeqKey := list.getSeqKey(batch, userKey, version, insertSeq) 469 | _, _, _, insertSeqValue, err := parseSeqKey(insertSeqKey) 470 | if err != nil { 471 | return err 472 | } 473 | if err := batch.Delete(insertSeqKey, nil); err != nil { 474 | return err 475 | } 476 | if err := batch.Delete(generateValueKey(userKey, version, insertSeqValue, insertSeq), nil); err != nil { 477 | return err 478 | } 479 | if left { 480 | if err := batch.Set(generateSeqKey(userKey, version, insertSeq, insertValue), nil, nil); err != nil { 481 | return err 482 | } 483 | if err := batch.Set(generateValueKey(userKey, version, insertValue, insertSeq), nil, nil); err != nil { 484 | return err 485 | } 486 | if err := batch.Set(generateSeqKey(userKey, version, insertSeq+1, insertSeqValue), nil, nil); err != nil { 487 | return err 488 | } 489 | if err := batch.Set(generateValueKey(userKey, version, insertSeqValue, insertSeq+1), nil, nil); err != nil { 490 | return err 491 | } 492 | } else { 493 | if err := batch.Set(generateSeqKey(userKey, version, insertSeq+1, insertValue), nil, nil); err != nil { 494 | return err 495 | } 496 | if err := batch.Set(generateValueKey(userKey, version, insertValue, insertSeq+1), nil, nil); err != nil { 497 | return err 498 | } 499 | if err := batch.Set(generateSeqKey(userKey, version, insertSeq, insertSeqValue), nil, nil); err != nil { 500 | return err 501 | } 502 | if err := batch.Set(generateValueKey(userKey, version, insertSeqValue, insertSeq), nil, nil); err != nil { 503 | return err 504 | } 505 | } 506 | 507 | meta.Count += 1 508 | meta.Tail += 1 509 | 510 | metaValue, err := proto.Marshal(meta) 511 | if err != nil { 512 | return err 513 | } 514 | return batch.Set(generateMetaKey(userKey, version), metaValue, nil) 515 | } 516 | 517 | // getMeta 518 | // return max version && valid meta 519 | // if max version meta is valid, return nil 520 | func (list *List) getMeta(batch *pebble.Batch, userKey []byte) (*model.ListMeta, uint64, error) { 521 | iter := batch.NewIter(store.NewPrefixIterOptions(generateMetaPrefixKey(userKey))) 522 | defer iter.Close() 523 | 524 | if res := iter.Last(); !res { 525 | return nil, 0, nil 526 | } 527 | 528 | metaKey := iter.Key() 529 | metaValue := iter.Value() 530 | 531 | if len(metaKey) == 0 { 532 | return nil, 0, nil 533 | } 534 | _, version, err := parseMetaKey(metaKey) 535 | if err != nil { 536 | return nil, 0, err 537 | } 538 | meta := &model.ListMeta{} 539 | if err := proto.Unmarshal(metaValue, meta); err != nil { 540 | return nil, 0, err 541 | } 542 | valid := meta.Deleted != true 543 | if valid { 544 | return meta, version, nil 545 | } 546 | 547 | return nil, 0, nil 548 | } 549 | 550 | func (list *List) getSeqKey(batch *pebble.Batch, userKey []byte, version uint64, seq int64) []byte { 551 | iter := batch.NewIter(store.NewPrefixIterOptions(generateSeqPrefixKey(userKey, version, seq))) 552 | defer iter.Close() 553 | 554 | if res := iter.Last(); !res { 555 | return nil 556 | } 557 | 558 | return iter.Key() 559 | } 560 | -------------------------------------------------------------------------------- /internal/collection/list/util.go: -------------------------------------------------------------------------------- 1 | package list 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/yemingfeng/sdb/internal/util" 7 | "math" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | var listLogger = util.GetLogger("list") 13 | 14 | const metaKeyMagic = "am" 15 | const seqKeyMagic = "as" 16 | const valueKeyMagic = "av" 17 | 18 | // generateMetaKey, return am:{userKey}: 19 | func generateMetaKey(userKey []byte, version uint64) []byte { 20 | return []byte(fmt.Sprintf("%s:%s:%064d", metaKeyMagic, util.Base64Encode(userKey), version)) 21 | } 22 | 23 | // generateMetaPrefixKey, return am:{userKey}: 24 | func generateMetaPrefixKey(userKey []byte) []byte { 25 | return []byte(fmt.Sprintf("%s:%s:", metaKeyMagic, util.Base64Encode(userKey))) 26 | } 27 | 28 | // parseMetaKey, return userKey, version, error 29 | func parseMetaKey(metaKey []byte) ([]byte, uint64, error) { 30 | infos := strings.Split(string(metaKey), ":") 31 | if len(infos) != 3 { 32 | listLogger.Printf("can not parse metaKey: %s, length != 3", metaKey) 33 | return nil, 0, errors.New(fmt.Sprintf("can not parse metaKey: %s", metaKey)) 34 | } 35 | if infos[0] != metaKeyMagic { 36 | listLogger.Printf("can not parse metaKey: %s, %s is not %s", metaKey, infos[0], metaKeyMagic) 37 | return nil, 0, errors.New(fmt.Sprintf("can not parse metaKey: %s, %s is not %s", metaKey, infos[0], metaKeyMagic)) 38 | } 39 | userKey, err := util.Base64Decode([]byte(infos[1])) 40 | if err != nil { 41 | listLogger.Printf("can not base64Encode metaKey on userKey: %s, error: %+v", metaKey, err) 42 | return nil, 0, err 43 | } 44 | version, err := strconv.ParseUint(infos[2], 10, 64) 45 | if err != nil { 46 | listLogger.Printf("can not parseUint metaKey on seq: %s, error: %+v", metaKey, err) 47 | return nil, 0, err 48 | } 49 | return userKey, version, nil 50 | } 51 | 52 | // generateSeqKey, return as:{userKey}:{version}:{seq}:{value} 53 | func generateSeqKey(userKey []byte, version uint64, seq int64, value []byte) []byte { 54 | return []byte(fmt.Sprintf("%s:%s:%064d:%064d:%s", seqKeyMagic, util.Base64Encode(userKey), version, seq+math.MaxInt32, util.Base64Encode(value))) 55 | } 56 | 57 | // generateSeqPrefixKey, return as:{userKey}:{version}:{seq}: 58 | func generateSeqPrefixKey(userKey []byte, version uint64, seq int64) []byte { 59 | return []byte(fmt.Sprintf("%s:%s:%064d:%064d:", seqKeyMagic, util.Base64Encode(userKey), version, seq+math.MaxInt32)) 60 | } 61 | 62 | // generateSeqIteratorKey, return as:{userKey}:{version}: 63 | func generateSeqIteratorKey(userKey []byte, version uint64) []byte { 64 | return []byte(fmt.Sprintf("%s:%s:%064d:", seqKeyMagic, util.Base64Encode(userKey), version)) 65 | } 66 | 67 | // parseSeqKey, return userKey, version, seq, value, error 68 | func parseSeqKey(seqKey []byte) ([]byte, uint64, int64, []byte, error) { 69 | infos := strings.Split(string(seqKey), ":") 70 | if len(infos) != 5 { 71 | listLogger.Printf("can not parse seqKey: %s, length != 5", seqKey) 72 | return nil, 0, 0, nil, errors.New(fmt.Sprintf("can not parse seqKey: %s, length != 5", seqKey)) 73 | } 74 | if infos[0] != seqKeyMagic { 75 | listLogger.Printf("can not parse seqKey: %s, %s is not %s", seqKey, infos[0], seqKeyMagic) 76 | return nil, 0, 0, nil, errors.New(fmt.Sprintf("can not parse seqKey: %s, %s is not %s", seqKey, infos[0], seqKeyMagic)) 77 | } 78 | userKey, err := util.Base64Decode([]byte(infos[1])) 79 | if err != nil { 80 | listLogger.Printf("can not base64Encode seqKey on userKey: %s, error: %+v", seqKey, err) 81 | return nil, 0, 0, nil, err 82 | } 83 | version, err := strconv.ParseUint(infos[2], 10, 64) 84 | if err != nil { 85 | listLogger.Printf("can not parseUint seqKey on version: %s, error: %+v", seqKey, err) 86 | return nil, 0, 0, nil, err 87 | } 88 | seq, err := strconv.ParseInt(infos[3], 10, 64) 89 | if err != nil { 90 | listLogger.Printf("can not parseInt seqKey on seq: %s, error: %+v", seqKey, err) 91 | return nil, 0, 0, nil, err 92 | } 93 | value, err := util.Base64Decode([]byte(infos[4])) 94 | if err != nil { 95 | listLogger.Printf("can not base64Encode seqKey on value: %s, error: %+v", seqKey, err) 96 | return nil, 0, 0, nil, err 97 | } 98 | return userKey, version, seq - math.MaxInt32, value, nil 99 | } 100 | 101 | // generateValueKey, return av:{userKey}:{version}:{value}:{seq} 102 | func generateValueKey(userKey []byte, version uint64, value []byte, seq int64) []byte { 103 | return []byte(fmt.Sprintf("%s:%s:%064d:%s:%064d", valueKeyMagic, util.Base64Encode(userKey), version, util.Base64Encode(value), seq+math.MaxInt32)) 104 | } 105 | 106 | // generateValuePrefixKey, return av:{userKey}:{version}:{value}: 107 | func generateValuePrefixKey(userKey []byte, version uint64, value []byte) []byte { 108 | return []byte(fmt.Sprintf("%s:%s:%064d:%s:", valueKeyMagic, util.Base64Encode(userKey), version, util.Base64Encode(value))) 109 | } 110 | 111 | // parseValueKey, return userKey, version, value, seq, error 112 | func parseValueKey(valueKey []byte) ([]byte, uint64, []byte, int64, error) { 113 | infos := strings.Split(string(valueKey), ":") 114 | if len(infos) != 5 { 115 | listLogger.Printf("can not parse valueKey: %s, length != 5", valueKey) 116 | return nil, 0, nil, 0, errors.New(fmt.Sprintf("can not parse valueKey: %s", valueKey)) 117 | } 118 | if infos[0] != valueKeyMagic { 119 | listLogger.Printf("can not parse valueKey: %s, %s is not %s", valueKey, infos[0], valueKeyMagic) 120 | return nil, 0, nil, 0, errors.New(fmt.Sprintf("can not parse valueKey: %s, %s is not %s", valueKey, infos[0], valueKeyMagic)) 121 | } 122 | userKey, err := util.Base64Decode([]byte(infos[1])) 123 | if err != nil { 124 | listLogger.Printf("can not base64Encode valueKey on userKey: %s, error: %+v", valueKey, err) 125 | return nil, 0, nil, 0, err 126 | } 127 | version, err := strconv.ParseUint(infos[2], 10, 64) 128 | if err != nil { 129 | listLogger.Printf("can not parseUint valueKey on seq: %s, error: %+v", valueKey, err) 130 | return nil, 0, nil, 0, err 131 | } 132 | value, err := util.Base64Decode([]byte(infos[3])) 133 | if err != nil { 134 | listLogger.Printf("can not base64Encode valueKey on value: %s, error: %+v", valueKey, err) 135 | return nil, 0, nil, 0, err 136 | } 137 | seq, err := strconv.ParseInt(infos[4], 10, 64) 138 | if err != nil { 139 | listLogger.Printf("can not parseInt valueKey on seq: %s, error: %+v", valueKey, err) 140 | return nil, 0, nil, 0, err 141 | } 142 | return userKey, version, value, seq - math.MaxInt32, nil 143 | } 144 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type Config struct { 4 | Debug bool `default:"false" yaml:"debug"` 5 | Store Store `yaml:"store"` 6 | Collection Collection `yaml:"collection"` 7 | } 8 | 9 | type Store struct { 10 | Path string `default:"./db/" yaml:"path"` 11 | DeleteOld bool `default:"false" yaml:"delete_old"` 12 | } 13 | 14 | type Collection struct { 15 | LockerCount int `default:"128" yaml:"locker_count"` 16 | } 17 | 18 | func NewDefaultConfig() *Config { 19 | return &Config{ 20 | Debug: true, 21 | Store: Store{ 22 | Path: "./db/", 23 | }, 24 | Collection: Collection{ 25 | LockerCount: 128, 26 | }, 27 | } 28 | } 29 | 30 | func NewTestConfig() *Config { 31 | return &Config{ 32 | Debug: true, 33 | Store: Store{ 34 | Path: "./db/", 35 | DeleteOld: true, 36 | }, 37 | Collection: Collection{ 38 | LockerCount: 128, 39 | }, 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /internal/model/list.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.27.1 4 | // protoc v3.21.4 5 | // source: internal/protobuf/list.proto 6 | 7 | package model 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type ListMeta struct { 24 | state protoimpl.MessageState 25 | sizeCache protoimpl.SizeCache 26 | unknownFields protoimpl.UnknownFields 27 | 28 | Count uint64 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` 29 | Head int64 `protobuf:"varint,2,opt,name=head,proto3" json:"head,omitempty"` 30 | Tail int64 `protobuf:"varint,3,opt,name=tail,proto3" json:"tail,omitempty"` 31 | Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` 32 | } 33 | 34 | func (x *ListMeta) Reset() { 35 | *x = ListMeta{} 36 | if protoimpl.UnsafeEnabled { 37 | mi := &file_internal_protobuf_list_proto_msgTypes[0] 38 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 39 | ms.StoreMessageInfo(mi) 40 | } 41 | } 42 | 43 | func (x *ListMeta) String() string { 44 | return protoimpl.X.MessageStringOf(x) 45 | } 46 | 47 | func (*ListMeta) ProtoMessage() {} 48 | 49 | func (x *ListMeta) ProtoReflect() protoreflect.Message { 50 | mi := &file_internal_protobuf_list_proto_msgTypes[0] 51 | if protoimpl.UnsafeEnabled && x != nil { 52 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 53 | if ms.LoadMessageInfo() == nil { 54 | ms.StoreMessageInfo(mi) 55 | } 56 | return ms 57 | } 58 | return mi.MessageOf(x) 59 | } 60 | 61 | // Deprecated: Use ListMeta.ProtoReflect.Descriptor instead. 62 | func (*ListMeta) Descriptor() ([]byte, []int) { 63 | return file_internal_protobuf_list_proto_rawDescGZIP(), []int{0} 64 | } 65 | 66 | func (x *ListMeta) GetCount() uint64 { 67 | if x != nil { 68 | return x.Count 69 | } 70 | return 0 71 | } 72 | 73 | func (x *ListMeta) GetHead() int64 { 74 | if x != nil { 75 | return x.Head 76 | } 77 | return 0 78 | } 79 | 80 | func (x *ListMeta) GetTail() int64 { 81 | if x != nil { 82 | return x.Tail 83 | } 84 | return 0 85 | } 86 | 87 | func (x *ListMeta) GetDeleted() bool { 88 | if x != nil { 89 | return x.Deleted 90 | } 91 | return false 92 | } 93 | 94 | var File_internal_protobuf_list_proto protoreflect.FileDescriptor 95 | 96 | var file_internal_protobuf_list_proto_rawDesc = []byte{ 97 | 0x0a, 0x1c, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 98 | 0x62, 0x75, 0x66, 0x2f, 0x6c, 0x69, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 99 | 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x62, 100 | 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 101 | 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 102 | 0x12, 0x12, 0x0a, 0x04, 0x68, 0x65, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 103 | 0x68, 0x65, 0x61, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x69, 0x6c, 0x18, 0x03, 0x20, 0x01, 104 | 0x28, 0x03, 0x52, 0x04, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65, 105 | 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 106 | 0x65, 0x64, 0x42, 0x10, 0x5a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x6d, 107 | 0x6f, 0x64, 0x65, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 108 | } 109 | 110 | var ( 111 | file_internal_protobuf_list_proto_rawDescOnce sync.Once 112 | file_internal_protobuf_list_proto_rawDescData = file_internal_protobuf_list_proto_rawDesc 113 | ) 114 | 115 | func file_internal_protobuf_list_proto_rawDescGZIP() []byte { 116 | file_internal_protobuf_list_proto_rawDescOnce.Do(func() { 117 | file_internal_protobuf_list_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_protobuf_list_proto_rawDescData) 118 | }) 119 | return file_internal_protobuf_list_proto_rawDescData 120 | } 121 | 122 | var file_internal_protobuf_list_proto_msgTypes = make([]protoimpl.MessageInfo, 1) 123 | var file_internal_protobuf_list_proto_goTypes = []interface{}{ 124 | (*ListMeta)(nil), // 0: internal.model.ListMeta 125 | } 126 | var file_internal_protobuf_list_proto_depIdxs = []int32{ 127 | 0, // [0:0] is the sub-list for method output_type 128 | 0, // [0:0] is the sub-list for method input_type 129 | 0, // [0:0] is the sub-list for extension type_name 130 | 0, // [0:0] is the sub-list for extension extendee 131 | 0, // [0:0] is the sub-list for field type_name 132 | } 133 | 134 | func init() { file_internal_protobuf_list_proto_init() } 135 | func file_internal_protobuf_list_proto_init() { 136 | if File_internal_protobuf_list_proto != nil { 137 | return 138 | } 139 | if !protoimpl.UnsafeEnabled { 140 | file_internal_protobuf_list_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 141 | switch v := v.(*ListMeta); i { 142 | case 0: 143 | return &v.state 144 | case 1: 145 | return &v.sizeCache 146 | case 2: 147 | return &v.unknownFields 148 | default: 149 | return nil 150 | } 151 | } 152 | } 153 | type x struct{} 154 | out := protoimpl.TypeBuilder{ 155 | File: protoimpl.DescBuilder{ 156 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 157 | RawDescriptor: file_internal_protobuf_list_proto_rawDesc, 158 | NumEnums: 0, 159 | NumMessages: 1, 160 | NumExtensions: 0, 161 | NumServices: 0, 162 | }, 163 | GoTypes: file_internal_protobuf_list_proto_goTypes, 164 | DependencyIndexes: file_internal_protobuf_list_proto_depIdxs, 165 | MessageInfos: file_internal_protobuf_list_proto_msgTypes, 166 | }.Build() 167 | File_internal_protobuf_list_proto = out.File 168 | file_internal_protobuf_list_proto_rawDesc = nil 169 | file_internal_protobuf_list_proto_goTypes = nil 170 | file_internal_protobuf_list_proto_depIdxs = nil 171 | } 172 | -------------------------------------------------------------------------------- /internal/protobuf/list.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package internal.model; 4 | option go_package = "internal/model"; 5 | 6 | message ListMeta { 7 | uint64 count = 1; 8 | int64 head = 2; 9 | int64 tail = 3; 10 | bool deleted = 4; 11 | } -------------------------------------------------------------------------------- /internal/service/list.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/cockroachdb/pebble" 5 | "github.com/yemingfeng/sdb/internal/collection/list" 6 | ) 7 | 8 | type ListService struct { 9 | db *pebble.DB 10 | locker *Locker 11 | list *list.List 12 | } 13 | 14 | func NewListService(db *pebble.DB, lock *Locker, list *list.List) *ListService { 15 | return &ListService{db: db, locker: lock, list: list} 16 | } 17 | 18 | func (listService *ListService) LPush(userKey []byte, values [][]byte, sync bool) error { 19 | listService.locker.wLock(userKey) 20 | defer listService.locker.wUnLock(userKey) 21 | 22 | batch := listService.db.NewIndexedBatch() 23 | defer batch.Close() 24 | 25 | if err := listService.list.LPush(batch, userKey, values); err != nil { 26 | return err 27 | } 28 | return batch.Commit(&pebble.WriteOptions{Sync: sync}) 29 | } 30 | 31 | func (listService *ListService) RPush(userKey []byte, values [][]byte, sync bool) error { 32 | listService.locker.wLock(userKey) 33 | defer listService.locker.wUnLock(userKey) 34 | 35 | batch := listService.db.NewIndexedBatch() 36 | defer batch.Close() 37 | 38 | if err := listService.list.RPush(batch, userKey, values); err != nil { 39 | return err 40 | } 41 | return batch.Commit(&pebble.WriteOptions{Sync: sync}) 42 | } 43 | 44 | func (listService *ListService) LPop(userKey []byte, sync bool) ([]byte, error) { 45 | listService.locker.wLock(userKey) 46 | defer listService.locker.wUnLock(userKey) 47 | 48 | batch := listService.db.NewIndexedBatch() 49 | defer batch.Close() 50 | 51 | if res, err := listService.list.LPop(batch, userKey); err != nil { 52 | return nil, err 53 | } else { 54 | if err := batch.Commit(&pebble.WriteOptions{Sync: sync}); err != nil { 55 | return nil, err 56 | } 57 | return res, nil 58 | } 59 | } 60 | 61 | func (listService *ListService) Rpop(userKey []byte, sync bool) ([]byte, error) { 62 | listService.locker.wLock(userKey) 63 | defer listService.locker.wUnLock(userKey) 64 | 65 | batch := listService.db.NewIndexedBatch() 66 | defer batch.Close() 67 | 68 | if res, err := listService.list.RPop(batch, userKey); err != nil { 69 | return nil, err 70 | } else { 71 | if err := batch.Commit(&pebble.WriteOptions{Sync: sync}); err != nil { 72 | return nil, err 73 | } 74 | return res, nil 75 | } 76 | } 77 | 78 | func (listService *ListService) Count(userKey []byte) (uint64, error) { 79 | listService.locker.rLock(userKey) 80 | defer listService.locker.rUnLock(userKey) 81 | 82 | batch := listService.db.NewIndexedBatch() 83 | defer batch.Close() 84 | 85 | if res, err := listService.list.Count(batch, userKey); err != nil { 86 | return 0, err 87 | } else { 88 | return res, nil 89 | } 90 | } 91 | 92 | func (listService *ListService) Range(userKey []byte, offset int64, limit uint64) ([][]byte, error) { 93 | listService.locker.rLock(userKey) 94 | defer listService.locker.rUnLock(userKey) 95 | 96 | batch := listService.db.NewIndexedBatch() 97 | defer batch.Close() 98 | 99 | if res, err := listService.list.Range(batch, userKey, offset, limit); err != nil { 100 | return nil, err 101 | } else { 102 | return res, nil 103 | } 104 | } 105 | 106 | func (listService *ListService) Get(userKey []byte, indexes []int64) (map[int64][]byte, error) { 107 | listService.locker.rLock(userKey) 108 | defer listService.locker.rUnLock(userKey) 109 | 110 | batch := listService.db.NewIndexedBatch() 111 | defer batch.Close() 112 | 113 | if res, err := listService.list.Get(batch, userKey, indexes); err != nil { 114 | return nil, err 115 | } else { 116 | return res, nil 117 | } 118 | } 119 | 120 | func (listService *ListService) Contain(userKey []byte, values [][]byte) ([]bool, error) { 121 | listService.locker.rLock(userKey) 122 | defer listService.locker.rUnLock(userKey) 123 | 124 | batch := listService.db.NewIndexedBatch() 125 | defer batch.Close() 126 | 127 | if res, err := listService.list.Contain(batch, userKey, values); err != nil { 128 | return nil, err 129 | } else { 130 | return res, nil 131 | } 132 | } 133 | 134 | func (listService *ListService) Exist(userKey []byte) (bool, error) { 135 | listService.locker.rLock(userKey) 136 | defer listService.locker.rUnLock(userKey) 137 | 138 | batch := listService.db.NewIndexedBatch() 139 | defer batch.Close() 140 | 141 | if res, err := listService.list.Exist(batch, userKey); err != nil { 142 | return false, err 143 | } else { 144 | return res, nil 145 | } 146 | } 147 | 148 | func (listService *ListService) Delete(userKey []byte, sync bool) error { 149 | listService.locker.wLock(userKey) 150 | defer listService.locker.wUnLock(userKey) 151 | 152 | batch := listService.db.NewIndexedBatch() 153 | defer batch.Close() 154 | 155 | if err := listService.list.Delete(batch, userKey); err != nil { 156 | return err 157 | } else { 158 | return batch.Commit(&pebble.WriteOptions{Sync: sync}) 159 | } 160 | } 161 | 162 | func (listService *ListService) Rem(userKey []byte, value []byte, sync bool) error { 163 | listService.locker.wLock(userKey) 164 | defer listService.locker.wUnLock(userKey) 165 | 166 | batch := listService.db.NewIndexedBatch() 167 | defer batch.Close() 168 | 169 | if err := listService.list.Rem(batch, userKey, value); err != nil { 170 | return err 171 | } else { 172 | return batch.Commit(&pebble.WriteOptions{Sync: sync}) 173 | } 174 | } 175 | 176 | func (listService *ListService) LInsert(userKey []byte, insertValue []byte, index int64, sync bool) error { 177 | listService.locker.wLock(userKey) 178 | defer listService.locker.wUnLock(userKey) 179 | 180 | batch := listService.db.NewIndexedBatch() 181 | defer batch.Close() 182 | 183 | if err := listService.list.LInsert(batch, userKey, insertValue, index); err != nil { 184 | return err 185 | } else { 186 | return batch.Commit(&pebble.WriteOptions{Sync: sync}) 187 | } 188 | } 189 | 190 | func (listService *ListService) RInsert(userKey []byte, insertValue []byte, index int64, sync bool) error { 191 | listService.locker.wLock(userKey) 192 | defer listService.locker.wUnLock(userKey) 193 | 194 | batch := listService.db.NewIndexedBatch() 195 | defer batch.Close() 196 | 197 | if err := listService.list.RInsert(batch, userKey, insertValue, index); err != nil { 198 | return err 199 | } else { 200 | return batch.Commit(&pebble.WriteOptions{Sync: sync}) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /internal/service/locker.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/yemingfeng/sdb/internal/config" 5 | "hash/fnv" 6 | "sync" 7 | ) 8 | 9 | type Locker struct { 10 | lockers []*sync.RWMutex 11 | } 12 | 13 | func NewLocker(config *config.Config) *Locker { 14 | lockers := make([]*sync.RWMutex, config.Collection.LockerCount) 15 | for i := 0; i < len(lockers); i++ { 16 | lockers[i] = &sync.RWMutex{} 17 | } 18 | return &Locker{ 19 | lockers: lockers, 20 | } 21 | } 22 | 23 | func (locker *Locker) batchLock(userKeys [][]byte) { 24 | hashes := make([]bool, len(locker.lockers)) 25 | for i := 0; i < len(userKeys); i++ { 26 | hashes[locker.hash(userKeys[i])] = true 27 | } 28 | for i := 0; i < len(hashes); i++ { 29 | if hashes[i] { 30 | locker.lockers[i].Lock() 31 | } 32 | } 33 | } 34 | 35 | func (locker *Locker) batchUnLock(userKeys [][]byte) { 36 | hashes := make([]bool, len(locker.lockers)) 37 | for i := 0; i < len(userKeys); i++ { 38 | hashes[locker.hash(userKeys[i])] = true 39 | } 40 | for i := 0; i < len(hashes); i++ { 41 | if hashes[i] { 42 | locker.lockers[i].Unlock() 43 | } 44 | } 45 | } 46 | 47 | func (locker *Locker) hash(userKey []byte) int { 48 | h := fnv.New32a() 49 | _, _ = h.Write(userKey) 50 | return int(h.Sum32()) % len(locker.lockers) 51 | } 52 | 53 | func (locker *Locker) getLocker(userKey []byte) *sync.RWMutex { 54 | return locker.lockers[locker.hash(userKey)] 55 | } 56 | 57 | func (locker *Locker) rLock(userKey []byte) { 58 | locker.getLocker(userKey).RLock() 59 | } 60 | 61 | func (locker *Locker) rUnLock(userKey []byte) { 62 | locker.getLocker(userKey).RUnlock() 63 | } 64 | 65 | func (locker *Locker) wLock(userKey []byte) { 66 | locker.getLocker(userKey).Lock() 67 | } 68 | 69 | func (locker *Locker) wUnLock(userKey []byte) { 70 | locker.getLocker(userKey).Unlock() 71 | } 72 | -------------------------------------------------------------------------------- /internal/service/lua.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "github.com/cockroachdb/pebble" 6 | "github.com/yemingfeng/sdb/internal/collection/list" 7 | "github.com/yemingfeng/sdb/internal/util" 8 | lua "github.com/yuin/gopher-lua" 9 | ) 10 | 11 | var luaLogger = util.GetLogger("lua") 12 | 13 | type LuaService struct { 14 | db *pebble.DB 15 | locker *Locker 16 | list *list.List 17 | } 18 | 19 | func NewLuaService(db *pebble.DB, locker *Locker, list *list.List) *LuaService { 20 | return &LuaService{db: db, locker: locker, list: list} 21 | } 22 | 23 | func (luaService *LuaService) Execute(userKeys [][]byte, script string) ([]string, error) { 24 | L := lua.NewState() 25 | defer L.Close() 26 | 27 | luaService.locker.batchLock(userKeys) 28 | defer luaService.locker.batchUnLock(userKeys) 29 | 30 | batch := luaService.db.NewIndexedBatch() 31 | defer batch.Close() 32 | 33 | found := func(uk string) bool { 34 | for _, userKey := range userKeys { 35 | if uk == string(userKey) { 36 | return true 37 | } 38 | } 39 | return false 40 | } 41 | 42 | L.SetGlobal("LLPush", L.NewFunction(func(L *lua.LState) int { 43 | top := L.GetTop() 44 | userKey := L.CheckString(1) 45 | if !found(userKey) { 46 | L.RaiseError(fmt.Sprintf("can not opt not lock userKey: %s", userKey)) 47 | } 48 | 49 | values := make([][]byte, top-1) 50 | for i := 0; i < len(values); i++ { 51 | values[i] = []byte(L.CheckString(i + 2)) 52 | } 53 | luaLogger.Printf("[LLPush] userKey: %s, values: %s", userKey, values) 54 | 55 | err := luaService.list.LPush(batch, []byte(userKey), values) 56 | if err != nil { 57 | L.RaiseError("%s", err) 58 | } 59 | return 0 60 | })) 61 | L.SetGlobal("LRPush", L.NewFunction(func(L *lua.LState) int { 62 | top := L.GetTop() 63 | userKey := L.CheckString(1) 64 | if !found(userKey) { 65 | L.RaiseError(fmt.Sprintf("can not opt not lock userKey: %s", userKey)) 66 | } 67 | 68 | values := make([][]byte, top-1) 69 | for i := 0; i < len(values); i++ { 70 | values[i] = []byte(L.CheckString(i + 2)) 71 | } 72 | luaLogger.Printf("[LRPush] userKey: %s, values: %s", userKey, values) 73 | 74 | err := luaService.list.RPush(batch, []byte(userKey), values) 75 | if err != nil { 76 | L.RaiseError("%s", err) 77 | } 78 | return 0 79 | })) 80 | L.SetGlobal("LLPop", L.NewFunction(func(L *lua.LState) int { 81 | userKey := L.CheckString(1) 82 | if !found(userKey) { 83 | L.RaiseError(fmt.Sprintf("can not opt not lock userKey: %s", userKey)) 84 | } 85 | 86 | luaLogger.Printf("[LLPop] userKey: %s", userKey) 87 | 88 | res, err := luaService.list.LPop(batch, []byte(userKey)) 89 | if err != nil { 90 | L.RaiseError("%s", err) 91 | } 92 | L.Push(lua.LString(res)) 93 | return 1 94 | })) 95 | L.SetGlobal("LRPop", L.NewFunction(func(L *lua.LState) int { 96 | userKey := L.CheckString(1) 97 | if !found(userKey) { 98 | L.RaiseError(fmt.Sprintf("can not opt not lock userKey: %s", userKey)) 99 | } 100 | luaLogger.Printf("[LRPop] userKey: %s", userKey) 101 | 102 | res, err := luaService.list.RPop(batch, []byte(userKey)) 103 | if err != nil { 104 | L.RaiseError("%s", err) 105 | } 106 | L.Push(lua.LString(res)) 107 | return 1 108 | })) 109 | L.SetGlobal("LCount", L.NewFunction(func(L *lua.LState) int { 110 | userKey := L.CheckString(1) 111 | luaLogger.Printf("[LCount] userKey: %s", userKey) 112 | 113 | res, err := luaService.list.Count(batch, []byte(userKey)) 114 | if err != nil { 115 | L.RaiseError("%s", err) 116 | } 117 | L.Push(lua.LNumber(res)) 118 | return 1 119 | })) 120 | L.SetGlobal("LRange", L.NewFunction(func(L *lua.LState) int { 121 | userKey := L.CheckString(1) 122 | offset := L.CheckInt64(2) 123 | limit := uint64(L.CheckInt64(3)) 124 | luaLogger.Printf("[LRange] userKey: %s, offset: %d, limit: %d", userKey, offset, limit) 125 | 126 | res, err := luaService.list.Range(batch, []byte(userKey), offset, limit) 127 | if err != nil { 128 | L.RaiseError("%s", err) 129 | } 130 | table := L.NewTable() 131 | for i := 0; i < len(res); i++ { 132 | table.Append(lua.LString(res[i])) 133 | } 134 | L.Push(table) 135 | return 1 136 | })) 137 | L.SetGlobal("LGet", L.NewFunction(func(L *lua.LState) int { 138 | userKey := L.CheckString(1) 139 | table := L.CheckTable(2) 140 | indexes := make([]int64, table.Len()) 141 | for i := 0; i < table.Len(); i++ { 142 | indexes[i] = int64(table.RawGetInt(i + 1).(lua.LNumber)) 143 | } 144 | luaLogger.Printf("[LGet] userKey: %s, indexes: %+v", userKey, indexes) 145 | 146 | res, err := luaService.list.Get(batch, []byte(userKey), indexes) 147 | if err != nil { 148 | L.RaiseError("%s", err) 149 | } 150 | table = L.NewTable() 151 | for index, value := range res { 152 | table.RawSetInt(int(index), lua.LString(value)) 153 | } 154 | L.Push(table) 155 | return 1 156 | })) 157 | L.SetGlobal("LContain", L.NewFunction(func(L *lua.LState) int { 158 | userKey := L.CheckString(1) 159 | table := L.CheckTable(2) 160 | values := make([][]byte, table.Len()) 161 | for i := 0; i < table.Len(); i++ { 162 | values[i] = []byte(table.RawGetInt(i + 1).(lua.LString)) 163 | } 164 | luaLogger.Printf("[LContain] userKey: %s, values: %+s", userKey, values) 165 | 166 | res, err := luaService.list.Contain(batch, []byte(userKey), values) 167 | if err != nil { 168 | L.RaiseError("%s", err) 169 | } 170 | table = L.NewTable() 171 | for i, b := range res { 172 | table.RawSetInt(i, lua.LBool(b)) 173 | } 174 | L.Push(table) 175 | return 1 176 | })) 177 | L.SetGlobal("LExist", L.NewFunction(func(L *lua.LState) int { 178 | userKey := L.CheckString(1) 179 | luaLogger.Printf("[LExist] userKey: %s", userKey) 180 | 181 | res, err := luaService.list.Exist(batch, []byte(userKey)) 182 | if err != nil { 183 | L.RaiseError("%s", err) 184 | } 185 | L.Push(lua.LBool(res)) 186 | return 1 187 | })) 188 | L.SetGlobal("LDelete", L.NewFunction(func(L *lua.LState) int { 189 | userKey := L.CheckString(1) 190 | if !found(userKey) { 191 | L.RaiseError(fmt.Sprintf("can not opt not lock userKey: %s", userKey)) 192 | } 193 | luaLogger.Printf("[LDelete] userKey: %s", userKey) 194 | 195 | err := luaService.list.Delete(batch, []byte(userKey)) 196 | if err != nil { 197 | L.RaiseError("%s", err) 198 | } 199 | return 0 200 | })) 201 | L.SetGlobal("LRem", L.NewFunction(func(L *lua.LState) int { 202 | userKey := L.CheckString(1) 203 | if !found(userKey) { 204 | L.RaiseError(fmt.Sprintf("can not opt not lock userKey: %s", userKey)) 205 | } 206 | value := L.CheckString(2) 207 | luaLogger.Printf("[LRem] userKey: %s, value: %s", userKey, value) 208 | 209 | err := luaService.list.Rem(batch, []byte(userKey), []byte(value)) 210 | if err != nil { 211 | L.RaiseError("%s", err) 212 | } 213 | return 0 214 | })) 215 | 216 | if err := L.DoString(script); err != nil { 217 | return nil, err 218 | } 219 | 220 | top := L.GetTop() 221 | res := make([]string, top) 222 | for i := 1; i <= top; i++ { 223 | res[i-1] = L.Get(i).String() 224 | } 225 | 226 | if err := batch.Commit(&pebble.WriteOptions{Sync: true}); err != nil { 227 | return nil, err 228 | } 229 | return res, nil 230 | } 231 | -------------------------------------------------------------------------------- /internal/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "github.com/cockroachdb/pebble" 5 | "github.com/yemingfeng/sdb/internal/config" 6 | "github.com/yemingfeng/sdb/internal/util" 7 | "os" 8 | ) 9 | 10 | var storeLogger = util.GetLogger("store") 11 | 12 | func NewStore(config *config.Config) *pebble.DB { 13 | dbPath := config.Store.Path 14 | if config.Store.DeleteOld { 15 | if err := os.RemoveAll(dbPath); err != nil { 16 | storeLogger.Fatalf("delete old error: %+v", err) 17 | } 18 | } 19 | db, err := pebble.Open(dbPath, &pebble.Options{}) 20 | if err != nil { 21 | storeLogger.Fatalf("failed to open file: %+v", err) 22 | } 23 | storeLogger.Printf("db init %s complete", dbPath) 24 | 25 | return db 26 | } 27 | 28 | func NewPrefixIterOptions(prefix []byte) *pebble.IterOptions { 29 | return &pebble.IterOptions{ 30 | LowerBound: prefix, 31 | UpperBound: Next(prefix), 32 | } 33 | } 34 | 35 | func Next(key []byte) []byte { 36 | upperBound := func(b []byte) []byte { 37 | end := make([]byte, len(b)) 38 | copy(end, b) 39 | for i := len(end) - 1; i >= 0; i-- { 40 | end[i] = end[i] + 1 41 | if end[i] != 0 { 42 | return end[:i+1] 43 | } 44 | } 45 | return nil 46 | } 47 | return upperBound(key) 48 | } 49 | -------------------------------------------------------------------------------- /internal/util/base64.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "encoding/base64" 4 | 5 | func Base64Encode(src []byte) []byte { 6 | dst := make([]byte, base64.URLEncoding.EncodedLen(len(src))) 7 | base64.URLEncoding.Encode(dst, src) 8 | return dst 9 | } 10 | 11 | func Base64Decode(src []byte) ([]byte, error) { 12 | dst := make([]byte, base64.StdEncoding.DecodedLen(len(src))) 13 | n, err := base64.StdEncoding.Decode(dst, src) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return dst[:n], nil 18 | } 19 | -------------------------------------------------------------------------------- /internal/util/id.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "github.com/bwmarrin/snowflake" 5 | ) 6 | 7 | var node *snowflake.Node 8 | var idLogger = GetLogger("id") 9 | 10 | func init() { 11 | newNode, err := snowflake.NewNode(1) 12 | if err != nil { 13 | idLogger.Fatalf("can not new snowflake node, err: %+v", err) 14 | } 15 | node = newNode 16 | } 17 | 18 | func NextId() uint64 { 19 | return uint64(node.Generate()) 20 | } 21 | -------------------------------------------------------------------------------- /internal/util/log.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | func GetLogger(name string) *log.Logger { 9 | return log.New(os.Stdout, name+": ", log.Lshortfile|log.Lmicroseconds|log.Ldate) 10 | } 11 | -------------------------------------------------------------------------------- /scripts/build_protobuf.sh: -------------------------------------------------------------------------------- 1 | rm -rf pkg/protobuf/*.go 2 | protoc --go_out=./ --go-grpc_out=./ ./internal/protobuf/*.proto -------------------------------------------------------------------------------- /test/list_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/yemingfeng/sdb/internal/collection/list" 6 | "github.com/yemingfeng/sdb/internal/config" 7 | "github.com/yemingfeng/sdb/internal/service" 8 | "github.com/yemingfeng/sdb/internal/store" 9 | "testing" 10 | ) 11 | 12 | func TestList(t *testing.T) { 13 | config := config.NewTestConfig() 14 | list := list.NewList() 15 | locker := service.NewLocker(config) 16 | db := store.NewStore(config) 17 | listService := service.NewListService(db, locker, list) 18 | 19 | for i := 0; i < 2; i++ { 20 | userKey := []byte(fmt.Sprintf("userKey:%d", i)) 21 | existRes, err := listService.Exist(userKey) 22 | if err != nil { 23 | t.Fatalf("err: %+v", err) 24 | } 25 | t.Logf("existRes: %+v", existRes) 26 | 27 | err = listService.LPush(userKey, [][]byte{[]byte("v1"), []byte("v2"), []byte("v2"), []byte("v3")}, true) 28 | if err != nil { 29 | t.Fatalf("err: %+v", err) 30 | } 31 | 32 | err = listService.LPush(userKey, [][]byte{[]byte("v-1")}, true) 33 | if err != nil { 34 | t.Fatalf("err: %+v", err) 35 | } 36 | 37 | err = listService.RPush(userKey, [][]byte{[]byte("v4"), []byte("v5")}, true) 38 | if err != nil { 39 | t.Fatalf("err: %+v", err) 40 | } 41 | 42 | count, err := listService.Count(userKey) 43 | if err != nil { 44 | t.Fatalf("err: %+v", err) 45 | } 46 | t.Logf("Count: %d", count) 47 | 48 | rangeRes, err := listService.Range(userKey, 0, 10) 49 | if err != nil { 50 | t.Fatalf("err: %+v", err) 51 | } 52 | t.Logf("rangeRes: %+v", rangeResToPrintableString(rangeRes)) 53 | 54 | rangeRes, err = listService.Range(userKey, 3, 2) 55 | if err != nil { 56 | t.Fatalf("err: %+v", err) 57 | } 58 | t.Logf("rangeRes: %+v", rangeResToPrintableString(rangeRes)) 59 | 60 | rangeRes, err = listService.Range(userKey, -1, 5) 61 | if err != nil { 62 | t.Fatalf("err: %+v", err) 63 | } 64 | t.Logf("rangeRes: %+v", rangeResToPrintableString(rangeRes)) 65 | 66 | getRes, err := listService.Get(userKey, []int64{0, 1, 2, 3, 4, 5}) 67 | if err != nil { 68 | t.Fatalf("err: %+v", err) 69 | } 70 | t.Logf("getRes: %+v", getResToPrintableString(getRes)) 71 | 72 | containRes, err := listService.Contain(userKey, [][]byte{[]byte("v1"), []byte("v2"), []byte("v3"), []byte("10")}) 73 | if err != nil { 74 | t.Fatalf("err: %+v", err) 75 | } 76 | t.Logf("containRes: %+v", containRes) 77 | 78 | existRes, err = listService.Exist(userKey) 79 | if err != nil { 80 | t.Fatalf("err: %+v", err) 81 | } 82 | t.Logf("existRes: %+v", existRes) 83 | 84 | err = listService.Rem(userKey, []byte("v2"), true) 85 | if err != nil { 86 | t.Fatalf("err: %+v", err) 87 | } 88 | 89 | rangeRes, err = listService.Range(userKey, 0, 10) 90 | if err != nil { 91 | t.Fatalf("err: %+v", err) 92 | } 93 | t.Logf("rangeRes: %+v", rangeResToPrintableString(rangeRes)) 94 | 95 | err = listService.Delete(userKey, true) 96 | if err != nil { 97 | t.Fatalf("err: %+v", err) 98 | } 99 | 100 | rangeRes, err = listService.Range(userKey, 0, 10) 101 | if err != nil { 102 | t.Fatalf("err: %+v", err) 103 | } 104 | t.Logf("rangeRes: %+v", rangeResToPrintableString(rangeRes)) 105 | 106 | t.Log("==================") 107 | } 108 | } 109 | 110 | func getResToPrintableString(getRes map[int64][]byte) string { 111 | str := "" 112 | for index, value := range getRes { 113 | str += fmt.Sprintf("%d=%s, ", index, value) 114 | } 115 | return str 116 | } 117 | 118 | func rangeResToPrintableString(rangeRes [][]byte) string { 119 | str := "" 120 | for _, item := range rangeRes { 121 | str += string(item) + ", " 122 | } 123 | return str 124 | } 125 | -------------------------------------------------------------------------------- /test/lua_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/yemingfeng/sdb/internal/collection/list" 5 | "github.com/yemingfeng/sdb/internal/config" 6 | "github.com/yemingfeng/sdb/internal/service" 7 | "github.com/yemingfeng/sdb/internal/store" 8 | "testing" 9 | ) 10 | 11 | func TestLua(t *testing.T) { 12 | config := config.NewTestConfig() 13 | list := list.NewList() 14 | locker := service.NewLocker(config) 15 | db := store.NewStore(config) 16 | luaService := service.NewLuaService(db, locker, list) 17 | 18 | script := ` 19 | res = LExist("h2") 20 | if (res ~= false) 21 | then 22 | LLPush("h2", "v1", "v2", "v3") 23 | return LCount("h2") 24 | else 25 | LRPush("h2", "vv1", "vv2", "vv3") 26 | return true 27 | end 28 | return false 29 | ` 30 | 31 | res, err := luaService.Execute([][]byte{ 32 | []byte("h1"), 33 | []byte("h2"), 34 | }, script) 35 | 36 | t.Logf("res: %+v, err: %+v", res, err) 37 | } 38 | -------------------------------------------------------------------------------- /test/store_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/cockroachdb/pebble" 5 | "github.com/yemingfeng/sdb/internal/config" 6 | "github.com/yemingfeng/sdb/internal/store" 7 | "testing" 8 | ) 9 | 10 | func TestStore(t *testing.T) { 11 | conf := config.NewTestConfig() 12 | s := store.NewStore(conf) 13 | 14 | iter := s.NewIter(&pebble.IterOptions{}) 15 | defer iter.Close() 16 | 17 | lowerBound := []byte("as:aDE=:0000000000000000000000000000000000000000000000000000000000000004:0000000000000000000000000000000000000000000000000000002147483651:") 18 | upperBound := []byte("as:aDE=:0000000000000000000000000000000000000000000000000000000000000004:0000000000000000000000000000000000000000000000000000002147483658:") 19 | iter.SetBounds(lowerBound, upperBound) 20 | 21 | for iter.First(); iter.Valid(); iter.Next() { 22 | key := iter.Key() 23 | t.Logf("key=%s", key) 24 | } 25 | } 26 | --------------------------------------------------------------------------------