├── docs ├── bitalos.png ├── benchmark-qps.png └── benchmark-qps-vertical.png ├── .gitignore ├── internal ├── base │ ├── db_shard.go │ ├── error.go │ ├── iterator.go │ ├── merger.go │ ├── cleaner.go │ └── filenames_test.go ├── cache │ ├── lrucache │ │ ├── cgo_enabled.go │ │ ├── cgo_disabled.go │ │ ├── clockpro_normal.go │ │ ├── clockpro_tracing.go │ │ ├── value.go │ │ ├── entry_invariants.go │ │ ├── value_normal.go │ │ ├── value_invariants.go │ │ ├── refcnt_normal.go │ │ ├── refcnt_tracing.go │ │ ├── entry_normal.go │ │ └── entry.go │ ├── cache.go │ └── lfucache │ │ ├── internal │ │ ├── arenaskl │ │ │ ├── race_test.go │ │ │ ├── arena_test.go │ │ │ ├── skl_test.go │ │ │ └── flush_iterator.go │ │ └── base │ │ │ └── iterator.go │ │ ├── internal.go │ │ ├── merging_iter_test.go │ │ ├── flushable.go │ │ ├── read_state.go │ │ ├── metrics.go │ │ └── merging_iter_heap.go ├── hash │ ├── crc.go │ ├── md5block_decl.go │ ├── md5block_generic.go │ └── fnv.go ├── invariants │ ├── race_on.go │ ├── race_off.go │ ├── finalizer_off.go │ ├── on.go │ ├── off.go │ └── finalizer_on.go ├── rawalloc │ ├── rawalloc_64bit.go │ ├── rawalloc_32bit.go │ ├── rawalloc_gccgo.go │ ├── rawalloc.go │ ├── rawalloc_go1.9.go │ └── rawalloc_test.go ├── vfs │ ├── fadvise_generic.go │ ├── dir_unix.go │ ├── syncing_file_generic.go │ ├── preallocate_linux.go │ ├── errors_unix_test.go │ ├── preallocate_generic.go │ ├── prefetch_generic.go │ ├── errors_unix.go │ ├── dir_windows.go │ ├── file_lock_generic.go │ ├── errors_windows.go │ ├── fadvise_linux.go │ ├── disk_usage_windows.go │ ├── prefetch_linux.go │ ├── syncing_fs.go │ ├── disk_usage_unix.go │ ├── file_lock_windows.go │ ├── fd_test.go │ ├── fd.go │ ├── disk_usage_linux.go │ ├── file_lock_unix.go │ └── testdata │ │ └── vfs ├── manual │ ├── manual_64bit.go │ ├── manual_32bit.go │ ├── manual_nocgo.go │ └── manual.go ├── fastrand │ ├── fastrand.go │ └── fastrand_test.go ├── crc │ └── crc.go ├── errors │ └── errutil.go ├── mmap │ ├── mmap_plan9.go │ └── mmap_unix.go ├── arenaskl │ ├── race_test.go │ ├── arena_test.go │ ├── flush_iterator.go │ └── arena.go ├── list2 │ ├── stack.go │ └── stack_test.go ├── unsafe2 │ └── unsafe.go ├── options │ └── options.go ├── utils │ └── format.go ├── os2 │ └── dir.go ├── compress │ └── compress.go ├── humanize │ └── humanize.go ├── sortedkv │ └── key.go ├── statemachine │ └── db_state_machine.go └── bitask │ └── memtable_task.go ├── looptest └── sh_loop_test.sh ├── sh_unit_test.sh ├── bithash ├── .gitignore ├── internal.go ├── table_test.go ├── error.go ├── bithash_writer.go ├── stats.go ├── checkpoint.go └── block_test.go ├── bitpage ├── .gitignore ├── stats.go ├── read_state.go ├── page_writer.go ├── internal.go ├── flushable.go ├── merging_iter_heap.go ├── checkpoint.go └── super_table_iterator.go ├── bitree ├── .gitignore └── bdb │ ├── .gitignore │ ├── bdb_arm.go │ ├── bdb_amd64.go │ ├── bdb_arm64.go │ ├── bdb_linux.go │ ├── bdbsync_unix.go │ ├── mlock_windows.go │ ├── bdb_openbsd.go │ ├── mlock_unix.go │ ├── unsafe.go │ ├── allocate_test.go │ ├── errors.go │ ├── manydbs_test.go │ ├── bdb_unix.go │ ├── page_test.go │ └── simulation_no_freelist_sync_test.go ├── go.mod ├── read_state.go ├── internal.go ├── bitable └── batch.go ├── README_CN.md ├── flushable.go ├── internal_test.go ├── log_recycler.go ├── merging_iter_heap.go └── metrics.go /docs/bitalos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zuoyebang/bitalosdb/HEAD/docs/bitalos.png -------------------------------------------------------------------------------- /docs/benchmark-qps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zuoyebang/bitalosdb/HEAD/docs/benchmark-qps.png -------------------------------------------------------------------------------- /docs/benchmark-qps-vertical.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zuoyebang/bitalosdb/HEAD/docs/benchmark-qps-vertical.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | .DS_Store 7 | *.test 8 | *.out 9 | .idea 10 | out 11 | gen 12 | .vscode 13 | *.log 14 | *.zip 15 | *.swp 16 | *.swo 17 | data 18 | -------------------------------------------------------------------------------- /internal/base/db_shard.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import "github.com/zuoyebang/bitalosdb/internal/consts" 4 | 5 | func GetBitowerIndex(i int) int { 6 | return i & consts.DefaultBitowerNumMask 7 | } 8 | -------------------------------------------------------------------------------- /looptest/sh_loop_test.sh: -------------------------------------------------------------------------------- 1 | rm -rf testlog 2 | mkdir -p testlog 3 | 4 | go test -v -timeout 200000s -run TestBitalosdbLoop1 &>./testlog/TestBitalosdbLoop1.log & 5 | go test -v -timeout 200000s -run TestBitalosdbLoop2 &>./testlog/TestBitalosdbLoop2.log & 6 | -------------------------------------------------------------------------------- /sh_unit_test.sh: -------------------------------------------------------------------------------- 1 | cd internal/manifest 2 | go test -v -timeout 2000s 3 | cd - 4 | 5 | cd bithash 6 | go test -v -timeout 2000s 7 | cd - 8 | 9 | cd bitpage 10 | go test -v -timeout 2000s 11 | cd - 12 | 13 | cd bitree 14 | go test -v -timeout 2000s 15 | cd - 16 | 17 | go test -v -timeout 3600s -------------------------------------------------------------------------------- /bithash/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | ### Example user template template 13 | ### Example user template 14 | 15 | # IntelliJ project files 16 | .idea 17 | out 18 | gen 19 | .DS_Store 20 | 21 | # VsCode project files 22 | .vscode/* 23 | 24 | *.log 25 | *.zip 26 | 27 | *.swp 28 | *.swo 29 | 30 | go.sock 31 | 32 | test/ 33 | -------------------------------------------------------------------------------- /bitpage/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | ### Example user template template 13 | ### Example user template 14 | 15 | # IntelliJ project files 16 | .idea 17 | out 18 | gen 19 | .DS_Store 20 | 21 | # VsCode project files 22 | .vscode/* 23 | 24 | *.log 25 | *.zip 26 | 27 | *.swp 28 | *.swo 29 | 30 | go.sock 31 | 32 | test/ 33 | -------------------------------------------------------------------------------- /bitree/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | ### Example user template template 13 | ### Example user template 14 | 15 | # IntelliJ project files 16 | .idea 17 | out 18 | gen 19 | .DS_Store 20 | 21 | # VsCode project files 22 | .vscode/* 23 | 24 | *.log 25 | *.zip 26 | 27 | *.swp 28 | *.swo 29 | 30 | go.sock 31 | 32 | test/ 33 | -------------------------------------------------------------------------------- /bitree/bdb/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.exe~ 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | ### Example user template template 13 | ### Example user template 14 | 15 | # IntelliJ project files 16 | .idea 17 | out 18 | gen 19 | .DS_Store 20 | 21 | # VsCode project files 22 | .vscode/* 23 | 24 | *.log 25 | *.zip 26 | 27 | *.swp 28 | *.swo 29 | 30 | go.sock 31 | 32 | test/ 33 | -------------------------------------------------------------------------------- /bitree/bdb/bdb_arm.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | const maxMapSize = 0x7FFFFFFF 18 | const maxAllocSize = 0xFFFFFFF 19 | -------------------------------------------------------------------------------- /bitree/bdb/bdb_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | const maxMapSize = 0xFFFFFFFFFFFF 18 | const maxAllocSize = 0x7FFFFFFF 19 | -------------------------------------------------------------------------------- /bitree/bdb/bdb_arm64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | const maxMapSize = 0xFFFFFFFFFFFF 18 | const maxAllocSize = 0x7FFFFFFF 19 | -------------------------------------------------------------------------------- /internal/cache/lrucache/cgo_enabled.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build cgo 16 | // +build cgo 17 | 18 | package lrucache 19 | 20 | const cgoEnabled = true 21 | -------------------------------------------------------------------------------- /internal/cache/lrucache/cgo_disabled.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !cgo 16 | // +build !cgo 17 | 18 | package lrucache 19 | 20 | const cgoEnabled = false 21 | -------------------------------------------------------------------------------- /internal/hash/crc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package hash 16 | 17 | import "hash/crc32" 18 | 19 | func Crc32(key []byte) uint32 { 20 | return crc32.ChecksumIEEE(key) 21 | } 22 | -------------------------------------------------------------------------------- /internal/cache/lrucache/clockpro_normal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !tracing 16 | // +build !tracing 17 | 18 | package lrucache 19 | 20 | func (lrc *LruCache) trace(_ string, _ int64) {} 21 | -------------------------------------------------------------------------------- /bitree/bdb/bdb_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import ( 18 | "syscall" 19 | ) 20 | 21 | func fdatasync(db *DB) error { 22 | return syscall.Fdatasync(int(db.file.Fd())) 23 | } 24 | -------------------------------------------------------------------------------- /bitree/bdb/bdbsync_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !windows && !plan9 && !linux && !openbsd 16 | 17 | package bdb 18 | 19 | func fdatasync(db *DB) error { 20 | return db.file.Sync() 21 | } 22 | -------------------------------------------------------------------------------- /internal/invariants/race_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build race 16 | // +build race 17 | 18 | package invariants 19 | 20 | // RaceEnabled is true if we were built with the "race" build tag. 21 | const RaceEnabled = true 22 | -------------------------------------------------------------------------------- /internal/invariants/race_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !race 16 | // +build !race 17 | 18 | package invariants 19 | 20 | // RaceEnabled is true if we were built with the "race" build tag. 21 | const RaceEnabled = false 22 | -------------------------------------------------------------------------------- /internal/hash/md5block_decl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build amd64 || 386 || arm || ppc64le || ppc64 || s390x || arm64 16 | 17 | package hash 18 | 19 | const haveAsm = true 20 | 21 | //go:noescape 22 | func block(dig *digest, p []byte) 23 | -------------------------------------------------------------------------------- /internal/invariants/finalizer_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (!invariants && !tracing) || race 16 | // +build !invariants,!tracing race 17 | 18 | package invariants 19 | 20 | func SetFinalizer(obj, finalizer interface{}) { 21 | } 22 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_64bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 16 | 17 | package rawalloc 18 | 19 | const ( 20 | maxArrayLen = 1<<50 - 1 21 | ) 22 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_32bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build 386 || amd64p32 || arm || armbe || mips || mipsle || mips64p32 || mips64p32le || ppc || sparc 16 | 17 | package rawalloc 18 | 19 | const ( 20 | maxArrayLen = 1<<31 - 1 21 | ) 22 | -------------------------------------------------------------------------------- /internal/vfs/fadvise_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !linux 16 | 17 | package vfs 18 | 19 | func fadviseRandom(f uintptr) error { 20 | return nil 21 | } 22 | 23 | func fadviseSequential(f uintptr) error { 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_gccgo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build gccgo 16 | 17 | package rawalloc 18 | 19 | import "unsafe" 20 | 21 | //extern runtime.mallocgc 22 | func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer 23 | -------------------------------------------------------------------------------- /internal/invariants/on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build invariants || race 16 | // +build invariants race 17 | 18 | package invariants 19 | 20 | // Enabled is true if we were built with the "invariants" or "race" build tags. 21 | const Enabled = true 22 | -------------------------------------------------------------------------------- /internal/invariants/off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !invariants && !race 16 | // +build !invariants,!race 17 | 18 | package invariants 19 | 20 | // Enabled is true if we were built with the "invariants" or "race" build tags. 21 | const Enabled = false 22 | -------------------------------------------------------------------------------- /internal/vfs/dir_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package vfs 16 | 17 | import ( 18 | "os" 19 | "syscall" 20 | ) 21 | 22 | func (defaultFS) OpenDir(name string) (File, error) { 23 | f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) 24 | return f, err 25 | } 26 | -------------------------------------------------------------------------------- /internal/hash/md5block_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !amd64 && !386 && !arm && !ppc64le && !ppc64 && !s390x && !arm64 16 | 17 | package md5hash 18 | 19 | const haveAsm = false 20 | 21 | func block(dig *digest, p []byte) { 22 | blockGeneric(dig, p) 23 | } 24 | -------------------------------------------------------------------------------- /bitree/bdb/mlock_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | func mlock(_ *DB, _ int) error { 18 | panic("mlock is supported only on UNIX systems") 19 | } 20 | 21 | func munlock(_ *DB, _ int) error { 22 | panic("munlock is supported only on UNIX systems") 23 | } 24 | -------------------------------------------------------------------------------- /internal/vfs/syncing_file_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !linux || arm 16 | 17 | package vfs 18 | 19 | func (f *syncingFile) init() { 20 | f.syncTo = f.syncToGeneric 21 | } 22 | 23 | func (f *syncingFile) syncToGeneric(_ int64) error { 24 | return f.Sync() 25 | } 26 | -------------------------------------------------------------------------------- /internal/hash/fnv.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package hash 16 | 17 | import "hash/fnv" 18 | 19 | func Fnv32(key []byte) uint32 { 20 | h := fnv.New32() 21 | h.Write(key) 22 | return h.Sum32() 23 | } 24 | 25 | func Fnv64(key []byte) uint64 { 26 | h := fnv.New64() 27 | h.Write(key) 28 | return h.Sum64() 29 | } 30 | -------------------------------------------------------------------------------- /internal/manual/manual_64bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 16 | // +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 17 | 18 | package manual 19 | 20 | const ( 21 | MaxArrayLen = 1<<50 - 1 22 | ) 23 | -------------------------------------------------------------------------------- /internal/manual/manual_32bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build 386 || amd64p32 || arm || armbe || mips || mipsle || mips64p32 || mips64p32le || ppc || sparc 16 | // +build 386 amd64p32 arm armbe mips mipsle mips64p32 mips64p32le ppc sparc 17 | 18 | package manual 19 | 20 | const ( 21 | MaxArrayLen = 1<<31 - 1 22 | ) 23 | -------------------------------------------------------------------------------- /internal/invariants/finalizer_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (invariants && !race) || (tracing && !race) 16 | // +build invariants,!race tracing,!race 17 | 18 | package invariants 19 | 20 | import "runtime" 21 | 22 | func SetFinalizer(obj, finalizer interface{}) { 23 | runtime.SetFinalizer(obj, finalizer) 24 | } 25 | -------------------------------------------------------------------------------- /internal/manual/manual_nocgo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !cgo 16 | // +build !cgo 17 | 18 | package manual 19 | 20 | // Provides versions of New and Free when cgo is not available (e.g. cross 21 | // compilation). 22 | 23 | func New(n int) []byte { 24 | return make([]byte, n) 25 | } 26 | 27 | func Free(b []byte) { 28 | } 29 | -------------------------------------------------------------------------------- /internal/vfs/preallocate_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build linux 16 | 17 | package vfs 18 | 19 | import ( 20 | "syscall" 21 | 22 | "golang.org/x/sys/unix" 23 | ) 24 | 25 | func preallocExtend(fd uintptr, offset, length int64) error { 26 | return syscall.Fallocate(int(fd), unix.FALLOC_FL_KEEP_SIZE, offset, length) 27 | } 28 | -------------------------------------------------------------------------------- /internal/cache/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cache 16 | 17 | type ICache interface { 18 | Set([]byte, []byte, uint32) error 19 | Delete([]byte, uint32) error 20 | ExistAndDelete([]byte, uint32) error 21 | Get([]byte, uint32) ([]byte, func(), bool) 22 | GetKeyHash([]byte) uint32 23 | MetricsInfo() string 24 | Close() 25 | } 26 | -------------------------------------------------------------------------------- /internal/vfs/errors_unix_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build darwin || dragonfly || freebsd || linux || openbsd 16 | 17 | package vfs 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | "golang.org/x/sys/unix" 24 | ) 25 | 26 | func TestIsNoSpaceError(t *testing.T) { 27 | err := unix.ENOSPC 28 | require.True(t, IsNoSpaceError(err)) 29 | } 30 | -------------------------------------------------------------------------------- /internal/base/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package base 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/errors" 18 | 19 | var ErrNotFound = errors.New("bitalosdb: not found") 20 | 21 | var ErrCorruption = errors.New("bitalosdb: corruption") 22 | 23 | func CorruptionErrorf(format string, args ...interface{}) error { 24 | return errors.Wrapf(ErrCorruption, format, args...) 25 | } 26 | -------------------------------------------------------------------------------- /internal/cache/lrucache/clockpro_tracing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build tracing 16 | // +build tracing 17 | 18 | package lrucache 19 | 20 | import ( 21 | "fmt" 22 | "runtime/debug" 23 | ) 24 | 25 | func (c *Cache) trace(msg string, refs int64) { 26 | s := fmt.Sprintf("%s: refs=%d\n%s", msg, refs, debug.Stack()) 27 | c.tr.Lock() 28 | c.tr.msgs = append(c.tr.msgs, s) 29 | c.tr.Unlock() 30 | } 31 | -------------------------------------------------------------------------------- /bitpage/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | type Stats struct { 22 | Size int64 23 | } 24 | 25 | func newStats() *Stats { 26 | return &Stats{ 27 | Size: 0, 28 | } 29 | } 30 | 31 | func (s *Stats) Reset() { 32 | s.Size = 0 33 | } 34 | 35 | func (s *Stats) String() string { 36 | return fmt.Sprintf("size:%v", 37 | s.Size, 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /internal/fastrand/fastrand.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fastrand 16 | 17 | import _ "unsafe" // required by go:linkname 18 | 19 | // Uint32 returns a lock free uint32 value. 20 | // 21 | //go:linkname Uint32 runtime.fastrand 22 | func Uint32() uint32 23 | 24 | // Uint32n returns a lock free uint32 value in the interval [0, n). 25 | // 26 | //go:linkname Uint32n runtime.fastrandn 27 | func Uint32n(n uint32) uint32 28 | -------------------------------------------------------------------------------- /internal/vfs/preallocate_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !linux 16 | 17 | package vfs 18 | 19 | func preallocExtend(fd uintptr, offset, length int64) error { 20 | // It is ok for correctness to no-op file preallocation. WAL recycling is the 21 | // more important mechanism for WAL sync performance and it doesn't rely on 22 | // fallocate or posix_fallocate in order to be effective. 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /internal/vfs/prefetch_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !linux 16 | 17 | package vfs 18 | 19 | // Prefetch signals the OS (on supported platforms) to fetch the next size 20 | // bytes in file (as returned by os.File.Fd()) after offset into cache. Any 21 | // subsequent reads in that range will not issue disk IO. 22 | func Prefetch(file uintptr, offset uint64, size uint64) error { 23 | // No-op. 24 | return nil 25 | } 26 | -------------------------------------------------------------------------------- /internal/crc/crc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package crc 16 | 17 | import "hash/crc32" 18 | 19 | var table = crc32.MakeTable(crc32.Castagnoli) 20 | 21 | type CRC uint32 22 | 23 | func New(b []byte) CRC { 24 | return CRC(0).Update(b) 25 | } 26 | 27 | func (c CRC) Update(b []byte) CRC { 28 | return CRC(crc32.Update(uint32(c), table, b)) 29 | } 30 | 31 | func (c CRC) Value() uint32 { 32 | return uint32(c>>15|c<<17) + 0xa282ead8 33 | } 34 | -------------------------------------------------------------------------------- /internal/errors/errutil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package errors 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | ) 21 | 22 | func New(text string) error { 23 | return errors.New(text) 24 | } 25 | 26 | func Wrapf(err error, format string, args ...interface{}) error { 27 | return fmt.Errorf("%s err:%s", fmt.Sprintf(format, args...), err) 28 | } 29 | 30 | func Errorf(format string, args ...interface{}) error { 31 | return fmt.Errorf(format, args...) 32 | } 33 | -------------------------------------------------------------------------------- /internal/vfs/errors_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build darwin || dragonfly || freebsd || linux || openbsd 16 | 17 | package vfs 18 | 19 | import ( 20 | "errors" 21 | "syscall" 22 | 23 | "golang.org/x/sys/unix" 24 | ) 25 | 26 | var errNotEmpty = syscall.ENOTEMPTY 27 | 28 | // IsNoSpaceError returns true if the given error indicates that the disk is 29 | // out of space. 30 | func IsNoSpaceError(err error) bool { 31 | return errors.Is(err, unix.ENOSPC) 32 | } 33 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc.go: -------------------------------------------------------------------------------- 1 | /// Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rawalloc 16 | 17 | // New returns a new byte slice of the specified length and capacity where the 18 | // backing memory is uninitialized. This differs from make([]byte) which 19 | // guarantees that the backing memory for the slice is initialized to zero. Use 20 | // carefully. 21 | func New(len, cap int) []byte { 22 | ptr := mallocgc(uintptr(cap), nil, false) 23 | return (*[maxArrayLen]byte)(ptr)[:len:cap] 24 | } 25 | -------------------------------------------------------------------------------- /internal/vfs/dir_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build windows 16 | 17 | package vfs 18 | 19 | import ( 20 | "os" 21 | "syscall" 22 | ) 23 | 24 | type windowsDir struct { 25 | File 26 | } 27 | 28 | func (windowsDir) Sync() error { 29 | return nil 30 | } 31 | 32 | func (defaultFS) OpenDir(name string) (File, error) { 33 | f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return windowsDir{f}, nil 38 | } 39 | -------------------------------------------------------------------------------- /internal/vfs/file_lock_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows 16 | 17 | package vfs 18 | 19 | import ( 20 | "io" 21 | "runtime" 22 | 23 | "github.com/zuoyebang/bitalosdb/internal/errors" 24 | ) 25 | 26 | func (defFS) Lock(name string) (io.Closer, error) { 27 | return nil, errors.Errorf("bitalosdb: file locking is not implemented on %s/%s", runtime.GOOS, runtime.GOARCH) 28 | } 29 | -------------------------------------------------------------------------------- /internal/mmap/mmap_plan9.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package mmap 16 | 17 | import "syscall" 18 | 19 | func mmapfd(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { 20 | return nil, syscall.EPLAN9 21 | } 22 | 23 | func (m Mbuf) flush() error { 24 | return syscall.EPLAN9 25 | } 26 | 27 | func (m Mbuf) lock() error { 28 | return syscall.EPLAN9 29 | } 30 | 31 | func (m Mbuf) unlock() error { 32 | return syscall.EPLAN9 33 | } 34 | 35 | func (m Mbuf) unmap() error { 36 | return syscall.EPLAN9 37 | } 38 | -------------------------------------------------------------------------------- /internal/vfs/errors_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build windows 16 | 17 | package vfs 18 | 19 | import ( 20 | "github.com/zuoyebang/bitalosdb/internal/errors" 21 | "golang.org/x/sys/windows" 22 | ) 23 | 24 | var errNotEmpty = windows.ERROR_DIR_NOT_EMPTY 25 | 26 | // IsNoSpaceError returns true if the given error indicates that the disk is 27 | // out of space. 28 | func IsNoSpaceError(err error) bool { 29 | return errors.Is(err, windows.ERROR_DISK_FULL) || 30 | errors.Is(err, windows.ERROR_HANDLE_DISK_FULL) 31 | } 32 | -------------------------------------------------------------------------------- /internal/vfs/fadvise_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build linux 16 | 17 | package vfs 18 | 19 | import "golang.org/x/sys/unix" 20 | 21 | // Calls Fadvise with FADV_RANDOM to disable readahead on a file descriptor. 22 | func fadviseRandom(f uintptr) error { 23 | return unix.Fadvise(int(f), 0, 0, unix.FADV_RANDOM) 24 | } 25 | 26 | // Calls Fadvise with FADV_SEQUENTIAL to enable readahead on a file descriptor. 27 | func fadviseSequential(f uintptr) error { 28 | return unix.Fadvise(int(f), 0, 0, unix.FADV_SEQUENTIAL) 29 | } 30 | -------------------------------------------------------------------------------- /internal/vfs/disk_usage_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build windows 16 | 17 | package vfs 18 | 19 | import "golang.org/x/sys/windows" 20 | 21 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 22 | p, err := windows.UTF16PtrFromString(path) 23 | if err != nil { 24 | return DiskUsage{}, err 25 | } 26 | var freeBytes uint64 27 | du := DiskUsage{} 28 | err = windows.GetDiskFreeSpaceEx(p, &du.AvailBytes, &du.TotalBytes, &freeBytes) 29 | du.UsedBytes = du.TotalBytes - freeBytes 30 | return du, err 31 | } 32 | -------------------------------------------------------------------------------- /internal/arenaskl/race_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build race 16 | // +build race 17 | 18 | package arenaskl 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestNodeArenaEnd(t *testing.T) { 27 | ikey := makeIkey("a") 28 | val := []byte("b") 29 | 30 | for i := uint32(1); i < 256; i++ { 31 | a := newArena(i) 32 | _, err := newNode(a, 1, ikey, val) 33 | if err == nil { 34 | t.Log(i) 35 | break 36 | } 37 | require.Equal(t, ErrArenaFull, err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /internal/vfs/prefetch_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build linux 16 | 17 | package vfs 18 | 19 | import "syscall" 20 | 21 | // Prefetch signals the OS (on supported platforms) to fetch the next size 22 | // bytes in file (as returned by os.File.Fd()) after offset into cache. Any 23 | // subsequent reads in that range will not issue disk IO. 24 | func Prefetch(file uintptr, offset uint64, size uint64) error { 25 | _, _, err := syscall.Syscall(syscall.SYS_READAHEAD, uintptr(file), uintptr(offset), uintptr(size)) 26 | return err 27 | } 28 | -------------------------------------------------------------------------------- /internal/base/iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package base 16 | 17 | import "fmt" 18 | 19 | type InternalIterator interface { 20 | SeekGE(key []byte) (*InternalKey, []byte) 21 | SeekPrefixGE(prefix, key []byte, trySeekUsingNext bool) (*InternalKey, []byte) 22 | SeekLT(key []byte) (*InternalKey, []byte) 23 | First() (*InternalKey, []byte) 24 | Last() (*InternalKey, []byte) 25 | Next() (*InternalKey, []byte) 26 | Prev() (*InternalKey, []byte) 27 | Error() error 28 | Close() error 29 | SetBounds(lower, upper []byte) 30 | fmt.Stringer 31 | } 32 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_go1.9.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build gc && go1.9 16 | 17 | package rawalloc 18 | 19 | import "unsafe" 20 | 21 | // The go:linkname directives provides backdoor access to private functions in 22 | // the runtime. Below we're accessing the mallocgc function. Note that this 23 | // access is necessarily tied to a specific Go release which is why this file 24 | // is protected by a build tag. 25 | 26 | //go:linkname mallocgc runtime.mallocgc 27 | func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer 28 | -------------------------------------------------------------------------------- /bitree/bdb/bdb_openbsd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import ( 18 | "syscall" 19 | "unsafe" 20 | ) 21 | 22 | const ( 23 | msAsync = 1 << iota 24 | msSync 25 | msInvalidate 26 | ) 27 | 28 | func msync(db *DB) error { 29 | _, _, errno := syscall.Syscall(syscall.SYS_MSYNC, uintptr(unsafe.Pointer(db.data)), uintptr(db.datasz), msInvalidate) 30 | if errno != 0 { 31 | return errno 32 | } 33 | return nil 34 | } 35 | 36 | func fdatasync(db *DB) error { 37 | if db.data != nil { 38 | return msync(db) 39 | } 40 | return db.file.Sync() 41 | } 42 | -------------------------------------------------------------------------------- /bithash/internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/base" 18 | 19 | type InternalKey = base.InternalKey 20 | 21 | const ( 22 | InternalKeyKindDelete = base.InternalKeyKindDelete 23 | InternalKeyKindSet = base.InternalKeyKindSet 24 | InternalKeyKindInvalid = base.InternalKeyKindInvalid 25 | ) 26 | 27 | type Compare = base.Compare 28 | type Equal = base.Equal 29 | type Separator = base.Separator 30 | type Successor = base.Successor 31 | type Split = base.Split 32 | type Comparer = base.Comparer 33 | -------------------------------------------------------------------------------- /internal/cache/lfucache/internal/arenaskl/race_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build race 16 | // +build race 17 | 18 | package arenaskl 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestNodeArenaEnd(t *testing.T) { 27 | ikey := makeIkey("a") 28 | val := []byte("b") 29 | 30 | for i := uint32(1); i < 256; i++ { 31 | a := arena2.newArena(i) 32 | _, err := newNode(a, 1, ikey, val) 33 | if err == nil { 34 | t.Log(i) 35 | break 36 | } 37 | require.Equal(t, ErrArenaFull, err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /internal/cache/lrucache/value.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lrucache 16 | 17 | type Value struct { 18 | buf []byte 19 | ref refcnt 20 | } 21 | 22 | func (v *Value) Buf() []byte { 23 | if v == nil { 24 | return nil 25 | } 26 | return v.buf 27 | } 28 | 29 | func (v *Value) Truncate(n int) { 30 | v.buf = v.buf[:n] 31 | } 32 | 33 | func (v *Value) refs() int32 { 34 | return v.ref.refs() 35 | } 36 | 37 | func (v *Value) acquire() { 38 | v.ref.acquire() 39 | } 40 | 41 | func (v *Value) release() { 42 | if v.ref.release() { 43 | v.free() 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bitpage/read_state.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import "sync/atomic" 18 | 19 | type readState struct { 20 | refcnt atomic.Int32 21 | stMutable *superTable 22 | stQueue flushableList 23 | arrtable *flushableEntry 24 | } 25 | 26 | func (s *readState) ref() { 27 | s.refcnt.Add(1) 28 | } 29 | 30 | func (s *readState) unref() { 31 | if s.refcnt.Add(-1) != 0 { 32 | return 33 | } 34 | 35 | for i := range s.stQueue { 36 | s.stQueue[i].readerUnref() 37 | } 38 | 39 | if s.arrtable != nil { 40 | s.arrtable.readerUnref() 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /internal/cache/lfucache/internal/base/iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package base 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | type InternalIterator interface { 22 | SeekGE(key []byte) (*InternalKey, []byte) 23 | SeekPrefixGE(prefix, key []byte, trySeekUsingNext bool) (*InternalKey, []byte) 24 | SeekLT(key []byte) (*InternalKey, []byte) 25 | First() (*InternalKey, []byte) 26 | Last() (*InternalKey, []byte) 27 | Next() (*InternalKey, []byte) 28 | Prev() (*InternalKey, []byte) 29 | Error() error 30 | Close() error 31 | SetBounds(lower, upper []byte) 32 | 33 | fmt.Stringer 34 | } 35 | -------------------------------------------------------------------------------- /internal/vfs/syncing_fs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package vfs 16 | 17 | // syncingFS wraps a vfs.FS with one that wraps newly created files with 18 | // vfs.NewSyncingFile. 19 | type syncingFS struct { 20 | FS 21 | 22 | syncOpts SyncingFileOptions 23 | } 24 | 25 | func WithSyncingFS(fs FS, opts SyncingFileOptions) FS { 26 | return syncingFS{ 27 | FS: fs, 28 | syncOpts: opts, 29 | } 30 | } 31 | 32 | func (fs syncingFS) Create(name string) (File, error) { 33 | f, err := fs.FS.Create(name) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return NewSyncingFile(f, fs.syncOpts), nil 38 | } 39 | -------------------------------------------------------------------------------- /internal/cache/lfucache/internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lfucache 16 | 17 | import ( 18 | base3 "github.com/zuoyebang/bitalosdb/internal/cache/lfucache/internal/base" 19 | "github.com/zuoyebang/bitalosdb/internal/options" 20 | ) 21 | 22 | const ( 23 | internalKeyKindDelete = base3.InternalKeyKindDelete 24 | internalKeyKindSet = base3.InternalKeyKindSet 25 | internalKeyKindInvalid = base3.InternalKeyKindInvalid 26 | 27 | internalKeySeqNumMax = base3.InternalKeySeqNumMax 28 | ) 29 | 30 | type internalKeyKind = base3.InternalKeyKind 31 | type internalKey = base3.InternalKey 32 | type internalIterator = base3.InternalIterator 33 | type iterOptions = options.IterOptions 34 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package rawalloc 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | var sizes = []int{16, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024} 23 | 24 | func BenchmarkRawalloc(b *testing.B) { 25 | for _, size := range sizes { 26 | b.Run(fmt.Sprintf("rawalloc-%d", size), func(b *testing.B) { 27 | for i := 0; i < b.N; i++ { 28 | _ = New(size, size) 29 | } 30 | }) 31 | } 32 | } 33 | 34 | func BenchmarkMake(b *testing.B) { 35 | for _, size := range sizes { 36 | b.Run(fmt.Sprintf("make-%d", size), func(b *testing.B) { 37 | for i := 0; i < b.N; i++ { 38 | _ = make([]byte, size) 39 | } 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /internal/vfs/disk_usage_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build darwin || openbsd || dragonfly || freebsd 16 | 17 | package vfs 18 | 19 | import "golang.org/x/sys/unix" 20 | 21 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 22 | stat := unix.Statfs_t{} 23 | if err := unix.Statfs(path, &stat); err != nil { 24 | return DiskUsage{}, err 25 | } 26 | 27 | freeBytes := uint64(stat.Bsize) * uint64(stat.Bfree) 28 | availBytes := uint64(stat.Bsize) * uint64(stat.Bavail) 29 | totalBytes := uint64(stat.Bsize) * uint64(stat.Blocks) 30 | return DiskUsage{ 31 | AvailBytes: availBytes, 32 | TotalBytes: totalBytes, 33 | UsedBytes: totalBytes - freeBytes, 34 | }, nil 35 | } 36 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/zuoyebang/bitalosdb 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/RoaringBitmap/roaring v1.9.0 7 | github.com/cockroachdb/redact v1.1.5 8 | github.com/golang/snappy v0.0.4 9 | github.com/stretchr/testify v1.9.0 10 | github.com/zuoyebang/bitalostable v1.0.1 11 | golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 12 | golang.org/x/sys v0.18.0 13 | ) 14 | 15 | require ( 16 | github.com/DataDog/zstd v1.4.5 // indirect 17 | github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect 18 | github.com/bits-and-blooms/bitset v1.12.0 // indirect 19 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 20 | github.com/cockroachdb/errors v1.11.1 // indirect 21 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect 22 | github.com/davecgh/go-spew v1.1.1 // indirect 23 | github.com/getsentry/sentry-go v0.18.0 // indirect 24 | github.com/gogo/protobuf v1.3.2 // indirect 25 | github.com/klauspost/compress v1.15.11 // indirect 26 | github.com/kr/pretty v0.3.1 // indirect 27 | github.com/kr/text v0.2.0 // indirect 28 | github.com/mschoch/smat v0.2.0 // indirect 29 | github.com/pkg/errors v0.9.1 // indirect 30 | github.com/pmezard/go-difflib v1.0.0 // indirect 31 | github.com/rogpeppe/go-internal v1.9.0 // indirect 32 | golang.org/x/text v0.7.0 // indirect 33 | gopkg.in/yaml.v3 v3.0.1 // indirect 34 | ) 35 | -------------------------------------------------------------------------------- /bitpage/page_writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | type PageWriter struct { 18 | p *page 19 | Sentinel []byte 20 | } 21 | 22 | func (w *PageWriter) Set(key internalKey, value []byte) error { 23 | return w.p.set(key, value) 24 | } 25 | 26 | func (w *PageWriter) SetMultiValue(key internalKey, values ...[]byte) error { 27 | return w.p.setMulti(key, values...) 28 | } 29 | 30 | func (w *PageWriter) UpdateMetaTimestamp() { 31 | w.p.bp.meta.updatePagemetaTimestamp(w.p.pn) 32 | } 33 | 34 | func (w *PageWriter) FlushFinish() error { 35 | return w.p.memFlushFinish() 36 | } 37 | 38 | func (w *PageWriter) MaybePageFlush(size uint64) bool { 39 | return w.p.maybeScheduleFlush(size, false) 40 | } 41 | -------------------------------------------------------------------------------- /bitree/bdb/mlock_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !windows 16 | 17 | package bdb 18 | 19 | import "golang.org/x/sys/unix" 20 | 21 | func mlock(db *DB, fileSize int) error { 22 | sizeToLock := fileSize 23 | if sizeToLock > db.datasz { 24 | sizeToLock = db.datasz 25 | } 26 | if err := unix.Mlock(db.dataref[:sizeToLock]); err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | 32 | func munlock(db *DB, fileSize int) error { 33 | if db.dataref == nil { 34 | return nil 35 | } 36 | 37 | sizeToUnlock := fileSize 38 | if sizeToUnlock > db.datasz { 39 | sizeToUnlock = db.datasz 40 | } 41 | 42 | if err := unix.Munlock(db.dataref[:sizeToUnlock]); err != nil { 43 | return err 44 | } 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /internal/cache/lrucache/entry_invariants.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (invariants && !race) || (tracing && !race) 16 | // +build invariants,!race tracing,!race 17 | 18 | package lrucache 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | 24 | "github.com/zuoyebang/bitalosdb/internal/invariants" 25 | ) 26 | 27 | const entriesGoAllocated = true 28 | 29 | func entryAllocNew() *entry { 30 | e := &entry{} 31 | invariants.SetFinalizer(e, func(obj interface{}) { 32 | e := obj.(*entry) 33 | if v := e.ref.refs(); v != 0 { 34 | fmt.Fprintf(os.Stderr, "%p: cache entry has non-zero reference count: %d\n%s", 35 | e, v, e.ref.traces()) 36 | os.Exit(1) 37 | } 38 | }) 39 | return e 40 | } 41 | 42 | func entryAllocFree(e *entry) { 43 | } 44 | -------------------------------------------------------------------------------- /bitree/bdb/unsafe.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import ( 18 | "reflect" 19 | "unsafe" 20 | ) 21 | 22 | func unsafeAdd(base unsafe.Pointer, offset uintptr) unsafe.Pointer { 23 | return unsafe.Pointer(uintptr(base) + offset) 24 | } 25 | 26 | func unsafeIndex(base unsafe.Pointer, offset uintptr, elemsz uintptr, n int) unsafe.Pointer { 27 | return unsafe.Pointer(uintptr(base) + offset + uintptr(n)*elemsz) 28 | } 29 | 30 | func unsafeByteSlice(base unsafe.Pointer, offset uintptr, i, j int) []byte { 31 | return (*[maxAllocSize]byte)(unsafeAdd(base, offset))[i:j:j] 32 | } 33 | 34 | func unsafeSlice(slice, data unsafe.Pointer, len int) { 35 | s := (*reflect.SliceHeader)(slice) 36 | s.Data = uintptr(data) 37 | s.Cap = len 38 | s.Len = len 39 | } 40 | -------------------------------------------------------------------------------- /bitpage/internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import ( 18 | "github.com/zuoyebang/bitalosdb/internal/base" 19 | "github.com/zuoyebang/bitalosdb/internal/options" 20 | ) 21 | 22 | const ( 23 | internalKeyKindDelete = base.InternalKeyKindDelete 24 | internalKeyKindSet = base.InternalKeyKindSet 25 | internalKeyKindSetBithash = base.InternalKeyKindSetBithash 26 | internalKeyKindPrefixDelete = base.InternalKeyKindPrefixDelete 27 | internalKeyKindInvalid = base.InternalKeyKindInvalid 28 | ) 29 | 30 | type internalKeyKind = base.InternalKeyKind 31 | type internalKey = base.InternalKey 32 | type internalIterator = base.InternalIterator 33 | type iterOptions = options.IterOptions 34 | 35 | var iterCompactOpts = iterOptions{DisableCache: true} 36 | -------------------------------------------------------------------------------- /bitree/bdb/allocate_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestTx_allocatePageStats(t *testing.T) { 23 | f := newTestFreelist() 24 | ids := []pgid{2, 3} 25 | f.readIDs(ids) 26 | 27 | tx := &Tx{ 28 | db: &DB{ 29 | freelist: f, 30 | pageSize: defaultPageSize, 31 | }, 32 | meta: &meta{}, 33 | pages: make(map[pgid]*page), 34 | } 35 | 36 | prePageTotal := tx.Stats().PageCount 37 | allocateCnt := f.free_count() 38 | fmt.Println(allocateCnt) 39 | 40 | if _, _, err := tx.allocate(allocateCnt); err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | if tx.Stats().PageCount != prePageTotal+allocateCnt { 45 | t.Errorf("Allocated %d but got %d page in stats", allocateCnt, tx.Stats().PageCount) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/manual/manual.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package manual 16 | 17 | // #include 18 | import "C" 19 | import "unsafe" 20 | 21 | // The go:linkname directives provides backdoor access to private functions in 22 | // the runtime. Below we're accessing the throw function. 23 | 24 | //go:linkname throw runtime.throw 25 | func throw(s string) 26 | 27 | func New(n int) []byte { 28 | if n == 0 { 29 | return make([]byte, 0) 30 | } 31 | 32 | ptr := C.calloc(C.size_t(n), 1) 33 | if ptr == nil { 34 | throw("out of memory") 35 | } 36 | 37 | return (*[MaxArrayLen]byte)(unsafe.Pointer(ptr))[:n:n] 38 | } 39 | 40 | func Free(b []byte) { 41 | if cap(b) != 0 { 42 | if len(b) == 0 { 43 | b = b[:cap(b)] 44 | } 45 | ptr := unsafe.Pointer(&b[0]) 46 | C.free(ptr) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /internal/arenaskl/arena_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arenaskl 16 | 17 | import ( 18 | "math" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func newArena(n uint32) *Arena { 25 | return NewArena(make([]byte, n)) 26 | } 27 | 28 | func TestArenaSizeOverflow(t *testing.T) { 29 | a := newArena(math.MaxUint32) 30 | 31 | offset, _, err := a.alloc(math.MaxUint16, 0, 0) 32 | require.Nil(t, err) 33 | require.Equal(t, uint32(1), offset) 34 | require.Equal(t, uint32(math.MaxUint16)+1, a.Size()) 35 | 36 | _, _, err = a.alloc(math.MaxUint32, 0, 0) 37 | require.Equal(t, ErrArenaFull, err) 38 | require.Equal(t, uint32(math.MaxUint32), a.Size()) 39 | 40 | _, _, err = a.alloc(math.MaxUint16, 0, 0) 41 | require.Equal(t, ErrArenaFull, err) 42 | require.Equal(t, uint32(math.MaxUint32), a.Size()) 43 | } 44 | -------------------------------------------------------------------------------- /internal/list2/stack.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package list2 16 | 17 | import "container/list" 18 | 19 | type Stack struct { 20 | list *list.List 21 | } 22 | 23 | func NewStack() *Stack { 24 | l := list.New() 25 | return &Stack{l} 26 | } 27 | 28 | func (stack *Stack) Push(value interface{}) { 29 | stack.list.PushBack(value) 30 | } 31 | 32 | func (stack *Stack) Pop() interface{} { 33 | e := stack.list.Back() 34 | if e != nil { 35 | stack.list.Remove(e) 36 | return e.Value 37 | } 38 | return nil 39 | } 40 | 41 | func (stack *Stack) Peak() interface{} { 42 | e := stack.list.Back() 43 | if e != nil { 44 | return e.Value 45 | } 46 | 47 | return nil 48 | } 49 | 50 | func (stack *Stack) Len() int { 51 | return stack.list.Len() 52 | } 53 | 54 | func (stack *Stack) Empty() bool { 55 | return stack.list.Len() == 0 56 | } 57 | -------------------------------------------------------------------------------- /internal/cache/lfucache/internal/arenaskl/arena_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arenaskl 16 | 17 | import ( 18 | "math" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/require" 22 | ) 23 | 24 | func newArena(n uint32) *Arena { 25 | return NewArena(make([]byte, n)) 26 | } 27 | 28 | func TestArenaSizeOverflow(t *testing.T) { 29 | a := newArena(math.MaxUint32) 30 | 31 | offset, _, err := a.alloc(math.MaxUint16, 0, 0) 32 | require.Nil(t, err) 33 | require.Equal(t, uint32(1), offset) 34 | require.Equal(t, uint32(math.MaxUint16)+1, a.Size()) 35 | 36 | _, _, err = a.alloc(math.MaxUint32, 0, 0) 37 | require.Equal(t, ErrArenaFull, err) 38 | require.Equal(t, uint32(math.MaxUint32), a.Size()) 39 | 40 | _, _, err = a.alloc(math.MaxUint16, 0, 0) 41 | require.Equal(t, ErrArenaFull, err) 42 | require.Equal(t, uint32(math.MaxUint32), a.Size()) 43 | } 44 | -------------------------------------------------------------------------------- /internal/unsafe2/unsafe.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package unsafe2 16 | 17 | import ( 18 | "unsafe" 19 | ) 20 | 21 | const MaxAllocSize = 0xFFFFFFFFFFFF // 256TB 22 | 23 | func String(b []byte) string { 24 | if len(b) == 0 { 25 | return "" 26 | } 27 | return unsafe.String(&b[0], len(b)) 28 | } 29 | 30 | func ByteSlice(s string) []byte { 31 | return unsafe.Slice(unsafe.StringData(s), len(s)) 32 | } 33 | 34 | func UnsafeAdd(base unsafe.Pointer, offset uintptr) unsafe.Pointer { 35 | return unsafe.Pointer(uintptr(base) + offset) 36 | } 37 | 38 | func UnsafeIndex(base unsafe.Pointer, offset uintptr, elemsz uintptr, n int) unsafe.Pointer { 39 | return unsafe.Pointer(uintptr(base) + offset + uintptr(n)*elemsz) 40 | } 41 | 42 | func UnsafeByteSlice(base unsafe.Pointer, offset uintptr, i, j int) []byte { 43 | return (*[MaxAllocSize]byte)(UnsafeAdd(base, offset))[i:j:j] 44 | } 45 | -------------------------------------------------------------------------------- /internal/cache/lrucache/value_normal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (!invariants && !tracing) || race 16 | // +build !invariants,!tracing race 17 | 18 | package lrucache 19 | 20 | import ( 21 | "unsafe" 22 | 23 | "github.com/zuoyebang/bitalosdb/internal/manual" 24 | ) 25 | 26 | const valueSize = int(unsafe.Sizeof(Value{})) 27 | 28 | func newValue(n int) *Value { 29 | if n == 0 { 30 | return nil 31 | } 32 | 33 | if !cgoEnabled { 34 | v := &Value{buf: make([]byte, n)} 35 | v.ref.init(1) 36 | return v 37 | } 38 | 39 | b := manual.New(valueSize + n) 40 | v := (*Value)(unsafe.Pointer(&b[0])) 41 | v.buf = b[valueSize:] 42 | v.ref.init(1) 43 | return v 44 | } 45 | 46 | func (v *Value) free() { 47 | if !cgoEnabled { 48 | return 49 | } 50 | 51 | n := valueSize + cap(v.buf) 52 | buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(v))[:n:n] 53 | v.buf = nil 54 | manual.Free(buf) 55 | } 56 | -------------------------------------------------------------------------------- /internal/cache/lrucache/value_invariants.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (invariants && !race) || (tracing && !race) 16 | // +build invariants,!race tracing,!race 17 | 18 | package lrucache 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | 24 | "github.com/zuoyebang/bitalosdb/internal/invariants" 25 | "github.com/zuoyebang/bitalosdb/internal/manual" 26 | ) 27 | 28 | func newValue(n int) *Value { 29 | if n == 0 { 30 | return nil 31 | } 32 | b := manual.New(n) 33 | v := &Value{buf: b} 34 | v.ref.init(1) 35 | invariants.SetFinalizer(v, func(obj interface{}) { 36 | v := obj.(*Value) 37 | if v.buf != nil { 38 | fmt.Fprintf(os.Stderr, "%p: cache value was not freed: refs=%d\n%s", 39 | v, v.refs(), v.ref.traces()) 40 | os.Exit(1) 41 | } 42 | }) 43 | return v 44 | } 45 | 46 | func (v *Value) free() { 47 | for i := range v.buf { 48 | v.buf[i] = 0xff 49 | } 50 | manual.Free(v.buf) 51 | v.buf = nil 52 | } 53 | -------------------------------------------------------------------------------- /internal/vfs/file_lock_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build windows 16 | 17 | package vfs 18 | 19 | import ( 20 | "io" 21 | "syscall" 22 | ) 23 | 24 | // lockCloser hides all of an syscall.Handle's methods, except for Close. 25 | type lockCloser struct { 26 | fd syscall.Handle 27 | } 28 | 29 | func (l lockCloser) Close() error { 30 | return syscall.Close(l.fd) 31 | } 32 | 33 | // Lock locks the given file. On Windows, Locking will fail if the file is 34 | // already open by the current process. 35 | func (defaultFS) Lock(name string) (io.Closer, error) { 36 | p, err := syscall.UTF16PtrFromString(name) 37 | if err != nil { 38 | return nil, err 39 | } 40 | fd, err := syscall.CreateFile(p, 41 | syscall.GENERIC_READ|syscall.GENERIC_WRITE, 42 | 0, nil, syscall.CREATE_ALWAYS, 43 | syscall.FILE_ATTRIBUTE_NORMAL, 44 | 0, 45 | ) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return lockCloser{fd: fd}, nil 50 | } 51 | -------------------------------------------------------------------------------- /internal/options/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package options 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/base" 18 | 19 | type IterOptions struct { 20 | LowerBound []byte 21 | UpperBound []byte 22 | Logger base.Logger 23 | SlotId uint32 24 | IsAll bool 25 | DisableCache bool 26 | } 27 | 28 | func (o *IterOptions) GetLowerBound() []byte { 29 | if o == nil { 30 | return nil 31 | } 32 | return o.LowerBound 33 | } 34 | 35 | func (o *IterOptions) GetUpperBound() []byte { 36 | if o == nil { 37 | return nil 38 | } 39 | return o.UpperBound 40 | } 41 | 42 | func (o *IterOptions) GetLogger() base.Logger { 43 | if o == nil || o.Logger == nil { 44 | return base.DefaultLogger 45 | } 46 | return o.Logger 47 | } 48 | 49 | func (o *IterOptions) GetSlotId() uint32 { 50 | if o == nil { 51 | return 0 52 | } 53 | return o.SlotId 54 | } 55 | 56 | func (o *IterOptions) IsGetAll() bool { 57 | return o == nil || o.IsAll 58 | } 59 | -------------------------------------------------------------------------------- /read_state.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | import "sync/atomic" 18 | 19 | type readState struct { 20 | refcnt atomic.Int32 21 | memtables flushableList 22 | } 23 | 24 | func (s *readState) ref() { 25 | s.refcnt.Add(1) 26 | } 27 | 28 | func (s *readState) unref() { 29 | if s.refcnt.Add(-1) != 0 { 30 | return 31 | } 32 | for _, mem := range s.memtables { 33 | mem.readerUnref() 34 | } 35 | } 36 | 37 | func (s *Bitower) loadReadState() *readState { 38 | s.readState.RLock() 39 | state := s.readState.val 40 | state.ref() 41 | s.readState.RUnlock() 42 | return state 43 | } 44 | 45 | func (s *Bitower) updateReadState() { 46 | rs := &readState{ 47 | memtables: s.mu.mem.queue, 48 | } 49 | rs.refcnt.Store(1) 50 | 51 | for _, mem := range rs.memtables { 52 | mem.readerRef() 53 | } 54 | 55 | s.readState.Lock() 56 | old := s.readState.val 57 | s.readState.val = rs 58 | s.readState.Unlock() 59 | 60 | if old != nil { 61 | old.unref() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /internal/utils/format.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package utils 16 | 17 | import ( 18 | "fmt" 19 | "time" 20 | ) 21 | 22 | const ( 23 | B = 1 24 | KB = 1 << 10 25 | MB = 1 << 20 26 | GB = 1 << 30 27 | TB = 1 << 40 28 | EB = 1 << 50 29 | ) 30 | 31 | func FmtSize(size int64) string { 32 | if size < KB { 33 | return fmt.Sprintf("%dB", size) 34 | } else if size < MB { 35 | return fmt.Sprintf("%d.%03dKB", size>>10, size%KB) 36 | } else if size < GB { 37 | return fmt.Sprintf("%d.%03dMB", size>>20, (size>>10)%KB) 38 | } else if size < TB { 39 | return fmt.Sprintf("%d.%03dGB", size>>30, (size>>20)%KB) 40 | } else if size < EB { 41 | return fmt.Sprintf("%d.%03dTB", size>>40, (size>>30)%KB) 42 | } else { 43 | return fmt.Sprintf("%d.%03dEB", size>>50, (size>>40)%KB) 44 | } 45 | } 46 | 47 | func FmtUnixMillTime(ms int64) string { 48 | return time.UnixMilli(ms).Format(time.DateTime) 49 | } 50 | 51 | func FmtUnixTime(s int64) string { 52 | return time.Unix(s, 0).Format(time.DateTime) 53 | } 54 | -------------------------------------------------------------------------------- /internal/cache/lrucache/refcnt_normal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !tracing 16 | // +build !tracing 17 | 18 | package lrucache 19 | 20 | import ( 21 | "fmt" 22 | "sync/atomic" 23 | ) 24 | 25 | type refcnt int32 26 | 27 | func (v *refcnt) init(val int32) { 28 | *v = refcnt(val) 29 | } 30 | 31 | func (v *refcnt) refs() int32 { 32 | return atomic.LoadInt32((*int32)(v)) 33 | } 34 | 35 | func (v *refcnt) acquire() { 36 | switch v := atomic.AddInt32((*int32)(v), 1); { 37 | case v <= 1: 38 | panic(fmt.Sprintf("cache: inconsistent reference count: %d", v)) 39 | } 40 | } 41 | 42 | func (v *refcnt) release() bool { 43 | switch v := atomic.AddInt32((*int32)(v), -1); { 44 | case v < 0: 45 | panic(fmt.Sprintf("cache: inconsistent reference count: %d", v)) 46 | case v == 0: 47 | return true 48 | default: 49 | return false 50 | } 51 | } 52 | 53 | func (v *refcnt) trace(msg string) { 54 | } 55 | 56 | func (v *refcnt) traces() string { 57 | return "" 58 | } 59 | 60 | var _ = (*refcnt)(nil).traces 61 | -------------------------------------------------------------------------------- /bitree/bdb/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import "errors" 18 | 19 | var ( 20 | ErrDatabaseNotOpen = errors.New("database not open") 21 | ErrDatabaseOpen = errors.New("database already open") 22 | ErrInvalid = errors.New("invalid database") 23 | ErrVersionMismatch = errors.New("version mismatch") 24 | ErrChecksum = errors.New("checksum error") 25 | ErrTimeout = errors.New("timeout") 26 | ) 27 | 28 | var ( 29 | ErrTxNotWritable = errors.New("tx not writable") 30 | ErrTxClosed = errors.New("tx closed") 31 | ErrDatabaseReadOnly = errors.New("database is in read-only mode") 32 | ) 33 | 34 | var ( 35 | ErrBucketNotFound = errors.New("bucket not found") 36 | ErrBucketExists = errors.New("bucket already exists") 37 | ErrBucketNameRequired = errors.New("bucket name required") 38 | ErrKeyRequired = errors.New("key required") 39 | ErrKeyTooLarge = errors.New("key too large") 40 | ErrValueTooLarge = errors.New("value too large") 41 | ErrIncompatibleValue = errors.New("incompatible value") 42 | ) 43 | -------------------------------------------------------------------------------- /internal/vfs/fd_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package vfs 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | "time" 21 | 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestFileWrappersHaveFd(t *testing.T) { 26 | // Use the real filesystem so that we can test vfs.Default, which returns 27 | // files with Fd(). 28 | tmpf, err := os.CreateTemp("", "bitalosdb-db-fd-file") 29 | require.NoError(t, err) 30 | filename := tmpf.Name() 31 | defer os.Remove(filename) 32 | 33 | // File wrapper case 1: Check if diskHealthCheckingFile has Fd(). 34 | fs2 := WithDiskHealthChecks(Default, 10*time.Second, func(s string, duration time.Duration) {}) 35 | f2, err := fs2.Open(filename) 36 | require.NoError(t, err) 37 | if _, ok := f2.(fdGetter); !ok { 38 | t.Fatal("expected diskHealthCheckingFile to export Fd() method") 39 | } 40 | // File wrapper case 2: Check if syncingFile has Fd(). 41 | f3 := NewSyncingFile(f2, SyncingFileOptions{BytesPerSync: 8 << 10 /* 8 KB */}) 42 | if _, ok := f3.(fdGetter); !ok { 43 | t.Fatal("expected syncingFile to export Fd() method") 44 | } 45 | require.NoError(t, f2.Close()) 46 | } 47 | -------------------------------------------------------------------------------- /internal/cache/lfucache/merging_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lfucache 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/zuoyebang/bitalosdb/internal/humanize" 22 | ) 23 | 24 | func TestCacheMergingIter(t *testing.T) { 25 | memSize := 40 << 10 26 | maxSize := uint64(10 << 20) 27 | mc := testNewCache() 28 | c := newCache(mc, 0, memSize, maxSize) 29 | num := 1000 30 | for i := 0; i < num; i++ { 31 | key := []byte(fmt.Sprintf("cacheTestKey_%d", i)) 32 | value := []byte(fmt.Sprintf("cacheTestValue_%d", i)) 33 | c.set(key, value) 34 | } 35 | 36 | for i := 0; i < num; i++ { 37 | if i%2 == 0 { 38 | key := []byte(fmt.Sprintf("cacheTestKey_%d", i)) 39 | c.delete(key) 40 | } 41 | } 42 | 43 | var bytesIterated uint64 44 | memNum := len(c.mu.memQueue) 45 | fmt.Println("memNum", memNum) 46 | flushing := c.mu.memQueue[:memNum-1] 47 | 48 | iter := newInputIter(flushing, &bytesIterated) 49 | i := 0 50 | for key, _ := iter.First(); key != nil; key, _ = iter.Next() { 51 | fmt.Println(key.String()) 52 | i++ 53 | } 54 | fmt.Println("end", i, humanize.Uint64(bytesIterated)) 55 | } 56 | -------------------------------------------------------------------------------- /bitpage/flushable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import ( 18 | "fmt" 19 | "sync/atomic" 20 | ) 21 | 22 | type flushable interface { 23 | get([]byte, uint32) ([]byte, bool, internalKeyKind, func()) 24 | newIter(*iterOptions) internalIterator 25 | getKeyStats() (int, int, int) 26 | itemCount() int 27 | inuseBytes() uint64 28 | dataBytes() uint64 29 | readyForFlush() bool 30 | close() error 31 | path() string 32 | idxFilePath() string 33 | empty() bool 34 | getModTime() int64 35 | } 36 | 37 | type flushableEntry struct { 38 | flushable 39 | obsolete bool 40 | fileNum FileNum 41 | readerRefs atomic.Int32 42 | release func() 43 | } 44 | 45 | func (e *flushableEntry) readerRef() { 46 | e.readerRefs.Add(1) 47 | } 48 | 49 | func (e *flushableEntry) readerUnref() { 50 | switch v := e.readerRefs.Add(-1); { 51 | case v < 0: 52 | fmt.Printf("bitpage: inconsistent reference path:%s count:%d\n", e.path(), v) 53 | case v == 0: 54 | if e.release != nil { 55 | e.release() 56 | e.release = nil 57 | } 58 | } 59 | } 60 | 61 | func (e *flushableEntry) setObsolete() { 62 | e.obsolete = true 63 | } 64 | 65 | type flushableList []*flushableEntry 66 | -------------------------------------------------------------------------------- /internal/base/merger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package base 16 | 17 | import "io" 18 | 19 | type Merge func(key, value []byte) (ValueMerger, error) 20 | 21 | type ValueMerger interface { 22 | MergeNewer(value []byte) error 23 | MergeOlder(value []byte) error 24 | Finish(includesBase bool) ([]byte, io.Closer, error) 25 | } 26 | 27 | type Merger struct { 28 | Merge Merge 29 | Name string 30 | } 31 | 32 | type AppendValueMerger struct { 33 | buf []byte 34 | } 35 | 36 | func (a *AppendValueMerger) MergeNewer(value []byte) error { 37 | a.buf = append(a.buf, value...) 38 | return nil 39 | } 40 | 41 | func (a *AppendValueMerger) MergeOlder(value []byte) error { 42 | buf := make([]byte, len(a.buf)+len(value)) 43 | copy(buf, value) 44 | copy(buf[len(value):], a.buf) 45 | a.buf = buf 46 | return nil 47 | } 48 | 49 | func (a *AppendValueMerger) Finish(includesBase bool) ([]byte, io.Closer, error) { 50 | return a.buf, nil, nil 51 | } 52 | 53 | var DefaultMerger = &Merger{ 54 | Merge: func(key, value []byte) (ValueMerger, error) { 55 | res := &AppendValueMerger{} 56 | res.buf = append(res.buf, value...) 57 | return res, nil 58 | }, 59 | 60 | Name: "bitalosdb.concatenate", 61 | } 62 | -------------------------------------------------------------------------------- /internal/os2/dir.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package os2 16 | 17 | import ( 18 | "os" 19 | "os/exec" 20 | "strconv" 21 | "strings" 22 | ) 23 | 24 | func GetDirSize(dir string) int64 { 25 | if _, err := os.Stat(dir); os.IsNotExist(err) { 26 | return 0 27 | } 28 | res := strings.Split(Command("du", "-sb", dir), "\t") 29 | if len(res) < 2 || len(res) > 2 { 30 | return 0 31 | } 32 | size, err := strconv.ParseInt(res[0], 10, 64) 33 | if err != nil { 34 | return 0 35 | } 36 | return size 37 | } 38 | 39 | func Command(key string, arg ...string) string { 40 | cmd := exec.Command(key, arg...) 41 | b, _ := cmd.CombinedOutput() 42 | return strings.TrimSpace(string(b)) 43 | } 44 | 45 | func IsNotExist(name string) bool { 46 | if len(name) == 0 { 47 | return true 48 | } 49 | _, err := os.Stat(name) 50 | return err != nil && os.IsNotExist(err) 51 | } 52 | 53 | func IsExist(name string) bool { 54 | if len(name) == 0 { 55 | return false 56 | } 57 | _, err := os.Stat(name) 58 | return err == nil || !os.IsNotExist(err) 59 | } 60 | 61 | func RemoveDir(dirname string) error { 62 | if IsNotExist(dirname) { 63 | return nil 64 | } 65 | 66 | return os.RemoveAll(dirname) 67 | } 68 | -------------------------------------------------------------------------------- /internal/mmap/mmap_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build darwin || dragonfly || freebsd || linux || openbsd || solaris || netbsd 16 | // +build darwin dragonfly freebsd linux openbsd solaris netbsd 17 | 18 | package mmap 19 | 20 | import ( 21 | "golang.org/x/sys/unix" 22 | ) 23 | 24 | func mmapfd(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { 25 | flags := unix.MAP_SHARED 26 | prot := unix.PROT_READ 27 | switch { 28 | case inprot© != 0: 29 | prot |= unix.PROT_WRITE 30 | flags = unix.MAP_PRIVATE 31 | case inprot&RDWR != 0: 32 | prot |= unix.PROT_WRITE 33 | } 34 | if inprot&EXEC != 0 { 35 | prot |= unix.PROT_EXEC 36 | } 37 | if inflags&ANON != 0 { 38 | flags |= unix.MAP_ANON 39 | } 40 | 41 | b, err := unix.Mmap(int(fd), off, len, prot, flags) 42 | if err != nil { 43 | return nil, err 44 | } 45 | return b, nil 46 | } 47 | 48 | func (m Mbuf) flush() error { 49 | return unix.Msync([]byte(m), unix.MS_SYNC) 50 | } 51 | 52 | func (m Mbuf) lock() error { 53 | return unix.Mlock([]byte(m)) 54 | } 55 | 56 | func (m Mbuf) unlock() error { 57 | return unix.Munlock([]byte(m)) 58 | } 59 | 60 | func (m Mbuf) unmap() error { 61 | return unix.Munmap([]byte(m)) 62 | } 63 | -------------------------------------------------------------------------------- /internal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/base" 18 | 19 | const ( 20 | fileTypeLog = base.FileTypeLog 21 | fileTypeLock = base.FileTypeLock 22 | fileTypeManifest = base.FileTypeManifest 23 | fileTypeMeta = base.FileTypeMeta 24 | fileTypeCurrent = base.FileTypeCurrent 25 | ) 26 | 27 | const ( 28 | InternalKeyKindDelete = base.InternalKeyKindDelete 29 | InternalKeyKindSet = base.InternalKeyKindSet 30 | InternalKeyKindLogData = base.InternalKeyKindLogData 31 | InternalKeyKindPrefixDelete = base.InternalKeyKindPrefixDelete 32 | InternalKeyKindMax = base.InternalKeyKindMax 33 | InternalKeySeqNumMax = base.InternalKeySeqNumMax 34 | ) 35 | 36 | type InternalKeyKind = base.InternalKeyKind 37 | 38 | type InternalKey = base.InternalKey 39 | 40 | type internalIterator = base.InternalIterator 41 | 42 | type FileNum = base.FileNum 43 | 44 | type Compare = base.Compare 45 | 46 | type Equal = base.Equal 47 | 48 | type Split = base.Split 49 | 50 | type Comparer = base.Comparer 51 | 52 | type Logger = base.Logger 53 | 54 | var DefaultComparer = base.DefaultComparer 55 | 56 | var DefaultLogger = base.DefaultLogger 57 | -------------------------------------------------------------------------------- /internal/base/cleaner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package base 16 | 17 | import ( 18 | "github.com/zuoyebang/bitalosdb/internal/vfs" 19 | ) 20 | 21 | type Cleaner interface { 22 | Clean(fs vfs.FS, fileType FileType, path string) error 23 | } 24 | 25 | type NeedsFileContents interface { 26 | needsFileContents() 27 | } 28 | 29 | type DeleteCleaner struct{} 30 | 31 | func (DeleteCleaner) Clean(fs vfs.FS, fileType FileType, path string) error { 32 | return fs.Remove(path) 33 | } 34 | 35 | func (DeleteCleaner) String() string { 36 | return "delete" 37 | } 38 | 39 | type ArchiveCleaner struct{} 40 | 41 | var _ NeedsFileContents = ArchiveCleaner{} 42 | 43 | func (ArchiveCleaner) Clean(fs vfs.FS, fileType FileType, path string) error { 44 | switch fileType { 45 | case FileTypeLog, FileTypeManifest: 46 | destDir := fs.PathJoin(fs.PathDir(path), "archive") 47 | 48 | if err := fs.MkdirAll(destDir, 0755); err != nil { 49 | return err 50 | } 51 | 52 | destPath := fs.PathJoin(destDir, fs.PathBase(path)) 53 | return fs.Rename(path, destPath) 54 | 55 | default: 56 | return fs.Remove(path) 57 | } 58 | } 59 | 60 | func (ArchiveCleaner) String() string { 61 | return "archive" 62 | } 63 | 64 | func (ArchiveCleaner) needsFileContents() { 65 | } 66 | -------------------------------------------------------------------------------- /internal/cache/lfucache/flushable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lfucache 16 | 17 | import ( 18 | "fmt" 19 | "sync/atomic" 20 | ) 21 | 22 | type flushable interface { 23 | get([]byte) ([]byte, bool, internalKeyKind) 24 | newIter(*iterOptions) internalIterator 25 | newFlushIter(*iterOptions, *uint64) internalIterator 26 | inuseBytes() uint64 27 | totalBytes() uint64 28 | readyForFlush() bool 29 | getID() int64 30 | count() int 31 | } 32 | 33 | type flushableEntry struct { 34 | flushable 35 | flushed chan struct{} 36 | readerRefs int32 37 | releaseMemAccounting func() 38 | } 39 | 40 | func (e *flushableEntry) readerRef() { 41 | switch v := atomic.AddInt32(&e.readerRefs, 1); { 42 | case v <= 1: 43 | panic(fmt.Sprintf("mcache: inconsistent reference count: %d", v)) 44 | } 45 | } 46 | 47 | func (e *flushableEntry) readerUnref() { 48 | switch v := atomic.AddInt32(&e.readerRefs, -1); { 49 | case v < 0: 50 | panic(fmt.Sprintf("mcache: inconsistent reference count: %d", v)) 51 | case v == 0: 52 | if e.releaseMemAccounting == nil { 53 | panic("mcache: memtable reservation already released") 54 | } 55 | e.releaseMemAccounting() 56 | e.releaseMemAccounting = nil 57 | } 58 | } 59 | 60 | type flushableList []*flushableEntry 61 | -------------------------------------------------------------------------------- /internal/cache/lrucache/refcnt_tracing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build tracing 16 | // +build tracing 17 | 18 | package lrucache 19 | 20 | import ( 21 | "fmt" 22 | "runtime/debug" 23 | "strings" 24 | "sync" 25 | "sync/atomic" 26 | ) 27 | 28 | type refcnt struct { 29 | val int32 30 | sync.Mutex 31 | msgs []string 32 | } 33 | 34 | func (v *refcnt) init(val int32) { 35 | v.val = val 36 | v.trace("init") 37 | } 38 | 39 | func (v *refcnt) refs() int32 { 40 | return atomic.LoadInt32(&v.val) 41 | } 42 | 43 | func (v *refcnt) acquire() { 44 | switch n := atomic.AddInt32(&v.val, 1); { 45 | case n <= 1: 46 | panic(fmt.Sprintf("cache: inconsistent reference count: %d", n)) 47 | } 48 | v.trace("acquire") 49 | } 50 | 51 | func (v *refcnt) release() bool { 52 | n := atomic.AddInt32(&v.val, -1) 53 | switch { 54 | case n < 0: 55 | panic(fmt.Sprintf("cache: inconsistent reference count: %d", n)) 56 | } 57 | v.trace("release") 58 | return n == 0 59 | } 60 | 61 | func (v *refcnt) trace(msg string) { 62 | s := fmt.Sprintf("%s: refs=%d\n%s", msg, v.refs(), debug.Stack()) 63 | v.Lock() 64 | v.msgs = append(v.msgs, s) 65 | v.Unlock() 66 | } 67 | 68 | func (v *refcnt) traces() string { 69 | v.Lock() 70 | s := strings.Join(v.msgs, "\n") 71 | v.Unlock() 72 | return s 73 | } 74 | -------------------------------------------------------------------------------- /internal/cache/lfucache/read_state.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lfucache 16 | 17 | import "sync/atomic" 18 | 19 | type readState struct { 20 | refcnt int32 21 | memtables flushableList 22 | arrtable *flushableEntry 23 | } 24 | 25 | func (s *readState) ref() { 26 | atomic.AddInt32(&s.refcnt, 1) 27 | } 28 | 29 | func (s *readState) unref() { 30 | if atomic.AddInt32(&s.refcnt, -1) != 0 { 31 | return 32 | } 33 | 34 | if s.arrtable != nil { 35 | s.arrtable.readerUnref() 36 | } 37 | 38 | for _, t := range s.memtables { 39 | t.readerUnref() 40 | } 41 | } 42 | 43 | func (s *shard) loadReadState() *readState { 44 | s.readState.RLock() 45 | state := s.readState.val 46 | state.ref() 47 | s.readState.RUnlock() 48 | return state 49 | } 50 | 51 | func (s *shard) updateReadStateLocked() { 52 | rs := &readState{ 53 | refcnt: 1, 54 | memtables: s.mu.memQueue, 55 | arrtable: s.mu.arrtable, 56 | } 57 | 58 | if rs.arrtable != nil { 59 | rs.arrtable.readerRef() 60 | } 61 | 62 | for _, mem := range rs.memtables { 63 | mem.readerRef() 64 | } 65 | 66 | s.readState.Lock() 67 | old := s.readState.val 68 | s.readState.val = rs 69 | s.readState.Unlock() 70 | 71 | if old != nil { 72 | old.unref() 73 | } 74 | 75 | s.setMemtableNum(int32(len(rs.memtables))) 76 | } 77 | -------------------------------------------------------------------------------- /internal/compress/compress.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package compress 16 | 17 | import ( 18 | "github.com/golang/snappy" 19 | ) 20 | 21 | const ( 22 | CompressTypeNo int = 0 + iota 23 | CompressTypeSnappy 24 | ) 25 | 26 | type Compressor interface { 27 | Encode(dst, src []byte) []byte 28 | Decode(dst, src []byte) ([]byte, error) 29 | Type() int 30 | } 31 | 32 | var ( 33 | NoCompressor noCompressor 34 | SnappyCompressor snappyCompressor 35 | ) 36 | 37 | func SetCompressor(t int) Compressor { 38 | switch t { 39 | case CompressTypeSnappy: 40 | return SnappyCompressor 41 | default: 42 | return NoCompressor 43 | } 44 | } 45 | 46 | type noCompressor struct{} 47 | 48 | func (c noCompressor) Encode(dst, src []byte) []byte { 49 | return src 50 | } 51 | 52 | func (c noCompressor) Decode(dst, src []byte) ([]byte, error) { 53 | return src, nil 54 | } 55 | 56 | func (c noCompressor) Type() int { 57 | return CompressTypeNo 58 | } 59 | 60 | type snappyCompressor struct{} 61 | 62 | func (sc snappyCompressor) Encode(dst, src []byte) []byte { 63 | return snappy.Encode(dst, src) 64 | } 65 | 66 | func (sc snappyCompressor) Decode(dst, src []byte) ([]byte, error) { 67 | return snappy.Decode(dst, src) 68 | } 69 | 70 | func (sc snappyCompressor) Type() int { 71 | return CompressTypeSnappy 72 | } 73 | -------------------------------------------------------------------------------- /bitable/batch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitable 16 | 17 | import ( 18 | bt "github.com/zuoyebang/bitalostable" 19 | ) 20 | 21 | type BitableBatch struct { 22 | batch *bt.Batch 23 | wo *bt.WriteOptions 24 | } 25 | 26 | func (b *Bitable) NewBatch() *BitableBatch { 27 | batch := &BitableBatch{ 28 | batch: b.db.NewBatch(), 29 | wo: b.wo, 30 | } 31 | return batch 32 | } 33 | 34 | func (b *Bitable) NewFlushBatch(n int) *BitableBatch { 35 | batch := &BitableBatch{ 36 | batch: b.db.NewFlushBatch(n), 37 | wo: b.wo, 38 | } 39 | return batch 40 | } 41 | 42 | func (b *BitableBatch) Commit() error { 43 | return b.batch.Commit(b.wo) 44 | } 45 | 46 | func (b *BitableBatch) Set(key []byte, val []byte) error { 47 | return b.batch.Set(key, val, b.wo) 48 | } 49 | 50 | func (b *BitableBatch) Delete(key []byte) error { 51 | return b.batch.Delete(key, b.wo) 52 | } 53 | 54 | func (b *BitableBatch) Size() int { 55 | return b.batch.Len() 56 | } 57 | 58 | func (b *BitableBatch) Empty() bool { 59 | return b.batch.Empty() 60 | } 61 | 62 | func (b *BitableBatch) Reset() { 63 | b.batch.Reset() 64 | } 65 | 66 | func (b *BitableBatch) AllocFree() { 67 | b.batch.AllocFree() 68 | } 69 | 70 | func (b *BitableBatch) Close() error { 71 | err := b.batch.Close() 72 | b.batch = nil 73 | return err 74 | } 75 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | ![bitalos](./docs/bitalos.png) 2 | 3 | ## 简介 4 | 5 | - 高性能KV存储引擎（自研），基于全新IO架构及存储技术，重点解决LSM-Tree的读写放大问题。作为rocksdb的替代品，读写性能均有大幅提升。 6 | 7 | ## 出品 8 | 9 | - 团队：作业帮-平台技术团队 10 | 11 | - 作者：徐锐波(hustxurb@163.com) 12 | 13 | - 贡献者：幸福(wzxingfu@gmail.com)、李景晨(cokin.lee@outlook.com)、卢文伟(422213023@qq.com)、刘方(killcode13@sina.com) 14 | 15 | ## 关键技术 16 | - Bithash（KV分离技术），显著改善写放大；具备O(1)检索效率，可独立完成GC，实现value与index解耦。 17 | 18 | - Bitalostree（高性能压缩索引技术），基本消除读放大；基于超大Page的B+ Tree，运用全新的索引压缩技术，消除B+ Tree的写放大，并将读性能发挥到极致。 19 | 20 | - Bitalostable（冷热数据分离技术），承载冷数据存储，根据数据规模及访问频度，计算相对冷数据，流量低峰时转存至Bitalostable；提升数据压缩率，减少索引内存消耗，实现更合理的资源利用（开源稳定版具备基础功能，企业版支持更全面的冷热分离）。 21 | 22 | ## 性能报告 23 | 24 | - bitalosdb在性能上持续精进，此次性能测试基于bitalosdb v5.0；作为rocksdb的替代品，选取同时期rocksdb稳定版做性能对比。 25 | 26 | ### 硬件 27 | 28 | ``` 29 | CPU: Intel(R) Xeon(R) Platinum 8255C CPU @ 2.50GHz 30 | Memory: 384GB 31 | Disk: 2*3.5TB SSD 32 | ``` 33 | 34 | - Disk(File) IOPS(BW) 35 | 36 | |BlockSize| Write | RandWrite | Read | RandRead 37 | |---------|----------|----------|------|----- 38 | | 4KB | 294K(1150MiB/s) | 232K(905MiB/s) | 446K(1742MiB/s) | 446K(1743MiB/s) 39 | | 8KB | 266K(2080MiB/s) | 244K(1902MiB/s) | 309K(2414MiB/s) | 404K(3159MiB/s) 40 | 41 | ### 程序 42 | 43 | - 压测线程：8 44 | 45 | - CPU限制：8核 46 | 47 | - 对比标准：多核压测QPS换算成单核QPS对比，单核性能更能体现成本优势 48 | 49 | ### 数据 50 | 51 | - 单条数据：key-size：32B、value-size：1KB 52 | 53 | - 对比维度：数据总量（25GB、50GB、100GB） x 读写占比（100%随机写、100%随机读、50%随机写+50%随机读、30%随机写+70%随机读） 54 | 55 | ### 配置 56 | 57 | - rocksdb 58 | 59 | ``` 60 | Memtable：256MB 61 | WAL：enable 62 | Cache：8GB 63 | TargetFileSize：256M 64 | L0CompactTrigger：8 65 | L0StopWritesTrigger：24 66 | ``` 67 | 68 | - bitalosdb 69 | 70 | ``` 71 | Memtable：256MB 72 | WAL：enable 73 | Cache：disable 74 | ``` 75 | 76 | ### 结果 77 | 78 | - QPS ([Horizontal](./docs/benchmark-qps.png)) 79 | 80 | ![benchmark](./docs/benchmark-qps-vertical.png) 81 | 82 | ## 文档 83 | 84 | - 技术架构及文档，参考官网：[bitalos.zuoyebang.com](https://bitalos.zuoyebang.com) -------------------------------------------------------------------------------- /internal/cache/lfucache/internal/arenaskl/skl_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arenaskl 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "testing" 21 | 22 | "github.com/zuoyebang/bitalosdb/internal/cache/lfucache/internal/base" 23 | "github.com/zuoyebang/bitalosdb/internal/manual" 24 | 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestSkl_Rebuild(t *testing.T) { 29 | size := 10 << 20 30 | arenaBuf := manual.New(size) 31 | ar := NewArena(arenaBuf) 32 | skl := NewSkiplist(ar) 33 | 34 | num := 1000 35 | for i := 0; i < num; i++ { 36 | key := []byte(fmt.Sprintf("testSklKey_%d", i)) 37 | value := []byte(fmt.Sprintf("testSklValue_%d", i)) 38 | ikey := base.MakeInternalKey(key, uint64(num), 1) 39 | require.NoError(t, skl.Add(ikey, value)) 40 | } 41 | 42 | newSkl := CloneSkiplist(ar, skl.Height()) 43 | it := newSkl.NewIter(nil, nil) 44 | for i := 0; i < num; i++ { 45 | key := []byte(fmt.Sprintf("testSklKey_%d", i)) 46 | value := []byte(fmt.Sprintf("testSklValue_%d", i)) 47 | ik, v := it.SeekGE(key) 48 | if !bytes.Equal(ik.UserKey, key) { 49 | t.Fatalf("seek key not equal key:%s ik:%s", string(key), ik.String()) 50 | } 51 | if !bytes.Equal(v, value) { 52 | t.Fatalf("seek value not equal key:%s value:%s v:%s", string(key), string(value), string(v)) 53 | } 54 | } 55 | require.NoError(t, it.Close()) 56 | } 57 | -------------------------------------------------------------------------------- /bithash/table_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/zuoyebang/bitalosdb/internal/vfs" 22 | ) 23 | 24 | var testMemFs = vfs.NewMem() 25 | var testFs = vfs.Default 26 | 27 | func TestFooter(t *testing.T) { 28 | f0, err := testMemFs.Create("filename") 29 | if err != nil { 30 | panic(err) 31 | } 32 | var ft = footer{ 33 | metaBH: BlockHandle{5, 4}, 34 | } 35 | buf := make([]byte, bithashFooterLen) 36 | data := ft.encode(buf) 37 | fmt.Println("write data:", data) 38 | f0.Write([]byte("hello")) 39 | n, err := f0.Write(data) 40 | if err != nil { 41 | panic(err) 42 | } 43 | if err := f0.Sync(); err != nil { 44 | panic(err) 45 | } 46 | fmt.Println("n:", n) 47 | if err := f0.Close(); err != nil { 48 | panic(err) 49 | } 50 | f1, err := testMemFs.Open("filename") 51 | if err != nil { 52 | panic(err) 53 | } 54 | ftt, err := readTableFooter(f1) 55 | if err != nil { 56 | panic(err) 57 | } 58 | fmt.Printf("read footer:%+v", ftt) 59 | } 60 | 61 | func TestFooterEncodeDecode(t *testing.T) { 62 | var ft = footer{ 63 | metaBH: BlockHandle{1, 2}, 64 | } 65 | buf := make([]byte, bithashFooterLen) 66 | newbuf := ft.encode(buf) 67 | fmt.Println("encode:", newbuf) 68 | f, err := decodeTableFooter(newbuf, bithashFooterLen) 69 | if err != nil { 70 | panic(err) 71 | } 72 | fmt.Printf("decode result:%+v", f) 73 | } 74 | -------------------------------------------------------------------------------- /internal/vfs/fd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package vfs 16 | 17 | // fdGetter is an interface for a file with an Fd() method. A lot of 18 | // File related optimizations (eg. Prefetch(), WAL recycling) rely on the 19 | // existence of the Fd method to return a raw file descriptor. 20 | type fdGetter interface { 21 | Fd() uintptr 22 | } 23 | 24 | // fdFileWrapper is a File wrapper that also exposes an Fd() method. Used to 25 | // wrap outer (wrapped) Files that could unintentionally hide the Fd() method 26 | // exposed by the inner (unwrapped) File. It effectively lets the Fd() method 27 | // bypass the outer File and go to the inner File. 28 | type fdFileWrapper struct { 29 | File 30 | 31 | // All methods usually pass through to File above, except for Fd(), which 32 | // bypasses it and gets called directly on the inner file. 33 | inner fdGetter 34 | } 35 | 36 | func (f *fdFileWrapper) Fd() uintptr { 37 | return f.inner.Fd() 38 | } 39 | 40 | // WithFd takes an inner (unwrapped) and an outer (wrapped) vfs.File, 41 | // and returns an fdFileWrapper if the inner file has an Fd() method. Use this 42 | // method to fix the hiding of the Fd() method and the subsequent unintentional 43 | // disabling of Fd-related file optimizations. 44 | func WithFd(inner, outer File) File { 45 | if f, ok := inner.(fdGetter); ok { 46 | return &fdFileWrapper{ 47 | File: outer, 48 | inner: f, 49 | } 50 | } 51 | return outer 52 | } 53 | -------------------------------------------------------------------------------- /internal/arenaskl/flush_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arenaskl 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/base" 18 | 19 | type flushIterator struct { 20 | Iterator 21 | bytesIterated *uint64 22 | } 23 | 24 | var _ base.InternalIterator = (*flushIterator)(nil) 25 | 26 | func (it *flushIterator) String() string { 27 | return "memtable" 28 | } 29 | 30 | func (it *flushIterator) SeekGE(key []byte) (*base.InternalKey, []byte) { 31 | panic("bitalosdb: SeekGE unimplemented") 32 | } 33 | 34 | func (it *flushIterator) SeekPrefixGE( 35 | prefix, key []byte, trySeekUsingNext bool, 36 | ) (*base.InternalKey, []byte) { 37 | panic("bitalosdb: SeekPrefixGE unimplemented") 38 | } 39 | 40 | func (it *flushIterator) SeekLT(key []byte) (*base.InternalKey, []byte) { 41 | panic("bitalosdb: SeekLT unimplemented") 42 | } 43 | 44 | func (it *flushIterator) First() (*base.InternalKey, []byte) { 45 | key, val := it.Iterator.First() 46 | if key == nil { 47 | return nil, nil 48 | } 49 | *it.bytesIterated += uint64(it.nd.allocSize) 50 | return key, val 51 | } 52 | 53 | func (it *flushIterator) Next() (*base.InternalKey, []byte) { 54 | it.nd = it.list.getSkipNext(it.nd) 55 | if it.nd == it.list.tail { 56 | return nil, nil 57 | } 58 | it.decodeKey() 59 | *it.bytesIterated += uint64(it.nd.allocSize) 60 | return &it.key, it.value() 61 | } 62 | 63 | func (it *flushIterator) Prev() (*base.InternalKey, []byte) { 64 | panic("bitalosdb: Prev unimplemented") 65 | } 66 | -------------------------------------------------------------------------------- /bitree/bdb/manydbs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "math/rand" 21 | "os" 22 | "path/filepath" 23 | "testing" 24 | ) 25 | 26 | func createDb(t *testing.T) (*DB, func()) { 27 | tempDirName, err := ioutil.TempDir("", "bdbmemtest") 28 | if err != nil { 29 | t.Fatalf("error creating temp dir: %v", err) 30 | } 31 | path := filepath.Join(tempDirName, "testdb.db") 32 | 33 | bdb, err := Open(path, nil) 34 | if err != nil { 35 | t.Fatalf("error creating bdb db: %v", err) 36 | } 37 | 38 | cleanup := func() { 39 | bdb.Close() 40 | os.RemoveAll(tempDirName) 41 | } 42 | 43 | return bdb, cleanup 44 | } 45 | 46 | func createAndPutKeys(t *testing.T) { 47 | t.Parallel() 48 | 49 | db, cleanup := createDb(t) 50 | defer cleanup() 51 | 52 | bucketName := []byte("bucket") 53 | 54 | for i := 0; i < 100; i++ { 55 | err := db.Update(func(tx *Tx) error { 56 | nodes, err := tx.CreateBucketIfNotExists(bucketName) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | var key [16]byte 62 | rand.Read(key[:]) 63 | if err := nodes.Put(key[:], nil); err != nil { 64 | return err 65 | } 66 | 67 | return nil 68 | }) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | } 73 | } 74 | 75 | func TestManyDBs(t *testing.T) { 76 | if testing.Short() { 77 | t.Skip("skipping test in short mode") 78 | } 79 | 80 | for i := 0; i < 100; i++ { 81 | t.Run(fmt.Sprintf("%d", i), createAndPutKeys) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /internal/vfs/disk_usage_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build linux 16 | 17 | package vfs 18 | 19 | import "golang.org/x/sys/unix" 20 | 21 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 22 | stat := unix.Statfs_t{} 23 | if err := unix.Statfs(path, &stat); err != nil { 24 | return DiskUsage{}, err 25 | } 26 | 27 | // We use stat.Frsize here rather than stat.Bsize because on 28 | // Linux Bavail and Bfree are in Frsize units. 29 | // 30 | // On most filesystems Frsize and Bsize will be set to the 31 | // same value, but on some filesystems bsize returns the 32 | // "optimal transfer block size"[1] which may be different 33 | // (typically larger) than the actual block size. 34 | // 35 | // This confusion is cleared up in the statvfs[2] libc function, 36 | // but the statfs system call used above varies across 37 | // platforms. 38 | // 39 | // Frsize is used by GNU coreutils and other libraries, so 40 | // this also helps ensure that we get the same results as one 41 | // would get if they ran `df` on the given path. 42 | // 43 | // [1] https://man7.org/linux/man-pages/man2/statfs.2.html 44 | // [2] https://man7.org/linux/man-pages/man3/statvfs.3.html 45 | freeBytes := uint64(stat.Frsize) * uint64(stat.Bfree) 46 | availBytes := uint64(stat.Frsize) * uint64(stat.Bavail) 47 | totalBytes := uint64(stat.Bsize) * uint64(stat.Blocks) 48 | return DiskUsage{ 49 | AvailBytes: availBytes, 50 | TotalBytes: totalBytes, 51 | UsedBytes: totalBytes - freeBytes, 52 | }, nil 53 | } 54 | -------------------------------------------------------------------------------- /bithash/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import "errors" 18 | 19 | var ( 20 | ErrBhNewReaderNoFile = errors.New("bithash new reader nil file") 21 | ErrBhNewReaderFail = errors.New("bithash: new reader fail") 22 | ErrBhReaderClosed = errors.New("bithash: reader is closed") 23 | ErrBhWriterClosed = errors.New("bithash: writer is closed") 24 | ErrBhIllegalBlockLength = errors.New("bithash: illegal block handle length") 25 | ErrBhCreateTableFile = errors.New("bithash: create table file fail") 26 | ErrBhOpenTableFile = errors.New("bithash: open table file fail") 27 | ErrBhFileNumError = errors.New("bithash: fileNum error") 28 | ErrBhFileNotImmutable = errors.New("bithash: table is not immutable") 29 | ErrBhFileNumZero = errors.New("bithash: fileNum zero") 30 | ErrBhReadRecordNil = errors.New("bithash: read record nil") 31 | ErrBhReadAtIncomplete = errors.New("bithash: readAt incomplete") 32 | ErrBhNotFound = errors.New("bithash: not found") 33 | ErrBhKeyTooLarge = errors.New("bithash: key too large") 34 | ErrBhValueTooLarge = errors.New("bithash: value too large") 35 | ErrBhHashIndexWriteFail = errors.New("bithash: hash_index write fail") 36 | ErrBhHashIndexReadFail = errors.New("bithash: hash_index read fail") 37 | ErrBhFileNumMapCheckFail = errors.New("bithash: check fileNumMap file footer fail") 38 | ErrBhInvalidTableSize = errors.New("bithash: invalid table file size is too small") 39 | ErrBhInvalidTableMeta = errors.New("bithash: invalid table bad metaBH") 40 | ) 41 | -------------------------------------------------------------------------------- /internal/cache/lfucache/internal/arenaskl/flush_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arenaskl 16 | 17 | import ( 18 | base2 "github.com/zuoyebang/bitalosdb/internal/cache/lfucache/internal/base" 19 | ) 20 | 21 | type flushIterator struct { 22 | Iterator 23 | bytesIterated *uint64 24 | } 25 | 26 | var _ base2.InternalIterator = (*flushIterator)(nil) 27 | 28 | func (it *flushIterator) String() string { 29 | return "arenaskl_flushIterator" 30 | } 31 | 32 | func (it *flushIterator) SeekGE(key []byte) (*base2.InternalKey, []byte) { 33 | panic("mcache: arenaskl flushIterator SeekGE unimplemented") 34 | } 35 | 36 | func (it *flushIterator) SeekPrefixGE( 37 | prefix, key []byte, trySeekUsingNext bool, 38 | ) (*base2.InternalKey, []byte) { 39 | panic("mcache: arenaskl flushIterator SeekPrefixGE unimplemented") 40 | } 41 | 42 | func (it *flushIterator) SeekLT(key []byte) (*base2.InternalKey, []byte) { 43 | panic("mcache: arenaskl flushIterator SeekLT unimplemented") 44 | } 45 | 46 | func (it *flushIterator) First() (*base2.InternalKey, []byte) { 47 | key, val := it.Iterator.First() 48 | if key == nil { 49 | return nil, nil 50 | } 51 | *it.bytesIterated += uint64(it.nd.allocSize) 52 | return key, val 53 | } 54 | 55 | func (it *flushIterator) Next() (*base2.InternalKey, []byte) { 56 | it.nd = it.list.getSkipNext(it.nd) 57 | if it.nd == it.list.tail { 58 | return nil, nil 59 | } 60 | it.decodeKey() 61 | *it.bytesIterated += uint64(it.nd.allocSize) 62 | return &it.key, it.value() 63 | } 64 | 65 | func (it *flushIterator) Prev() (*base2.InternalKey, []byte) { 66 | panic("mcache: arenaskl flushIterator Prev unimplemented") 67 | } 68 | -------------------------------------------------------------------------------- /internal/humanize/humanize.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package humanize 16 | 17 | import ( 18 | "fmt" 19 | "math" 20 | 21 | "github.com/cockroachdb/redact" 22 | ) 23 | 24 | func logn(n, b float64) float64 { 25 | return math.Log(n) / math.Log(b) 26 | } 27 | 28 | func humanate(s uint64, base float64, suffixes []string) string { 29 | if s < 10 { 30 | return fmt.Sprintf("%d%s", s, suffixes[0]) 31 | } 32 | e := math.Floor(logn(float64(s), base)) 33 | suffix := suffixes[int(e)] 34 | val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10 35 | f := "%.0f%s" 36 | if val < 10 { 37 | f = "%.1f%s" 38 | } 39 | 40 | return fmt.Sprintf(f, val, suffix) 41 | } 42 | 43 | type config struct { 44 | base float64 45 | suffix []string 46 | } 47 | 48 | var IEC = config{1024, []string{" B", " K", " M", " G", " T", " P", " E"}} 49 | 50 | var SI = config{1000, []string{"", " K", " M", " G", " T", " P", " E"}} 51 | 52 | func (c *config) Int64(s int64) FormattedString { 53 | if s < 0 { 54 | return FormattedString("-" + humanate(uint64(-s), c.base, c.suffix)) 55 | } 56 | return FormattedString(humanate(uint64(s), c.base, c.suffix)) 57 | } 58 | 59 | func (c *config) Uint64(s uint64) FormattedString { 60 | return FormattedString(humanate(s, c.base, c.suffix)) 61 | } 62 | 63 | func Int64(s int64) FormattedString { 64 | return IEC.Int64(s) 65 | } 66 | 67 | func Uint64(s uint64) FormattedString { 68 | return IEC.Uint64(s) 69 | } 70 | 71 | type FormattedString string 72 | 73 | var _ redact.SafeValue = FormattedString("") 74 | 75 | func (fs FormattedString) SafeValue() {} 76 | 77 | func (fs FormattedString) String() string { return string(fs) } 78 | -------------------------------------------------------------------------------- /internal/vfs/file_lock_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris 16 | 17 | package vfs 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "io" 23 | "os" 24 | "sync" 25 | "syscall" 26 | ) 27 | 28 | var lockedFiles struct { 29 | mu struct { 30 | sync.Mutex 31 | files map[string]bool 32 | } 33 | } 34 | 35 | // lockCloser hides all of an os.File's methods, except for Close. 36 | type lockCloser struct { 37 | name string 38 | f *os.File 39 | } 40 | 41 | func (l lockCloser) Close() error { 42 | lockedFiles.mu.Lock() 43 | defer lockedFiles.mu.Unlock() 44 | if !lockedFiles.mu.files[l.name] { 45 | fmt.Printf("panic: lock file %q is not locked\n", l.name) 46 | } 47 | delete(lockedFiles.mu.files, l.name) 48 | 49 | return l.f.Close() 50 | } 51 | 52 | func (defaultFS) Lock(name string) (io.Closer, error) { 53 | lockedFiles.mu.Lock() 54 | defer lockedFiles.mu.Unlock() 55 | if lockedFiles.mu.files == nil { 56 | lockedFiles.mu.files = map[string]bool{} 57 | } 58 | if lockedFiles.mu.files[name] { 59 | return nil, errors.New("lock held by current process") 60 | } 61 | 62 | f, err := os.Create(name) 63 | if err != nil { 64 | return nil, err 65 | } 66 | spec := syscall.Flock_t{ 67 | Type: syscall.F_WRLCK, 68 | Whence: io.SeekStart, 69 | Start: 0, 70 | Len: 0, // 0 means to lock the entire file. 71 | Pid: int32(os.Getpid()), 72 | } 73 | if err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &spec); err != nil { 74 | f.Close() 75 | return nil, err 76 | } 77 | lockedFiles.mu.files[name] = true 78 | return lockCloser{name, f}, nil 79 | } 80 | -------------------------------------------------------------------------------- /flushable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | import ( 18 | "fmt" 19 | "sync/atomic" 20 | 21 | "github.com/zuoyebang/bitalosdb/internal/base" 22 | ) 23 | 24 | type flushable interface { 25 | get(k []byte) ([]byte, bool, base.InternalKeyKind) 26 | newIter(o *IterOptions) internalIterator 27 | newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator 28 | inuseBytes() uint64 29 | totalBytes() uint64 30 | empty() bool 31 | readyForFlush() bool 32 | } 33 | 34 | type flushableEntry struct { 35 | flushable 36 | flushed chan struct{} 37 | flushForced bool 38 | logNum FileNum 39 | logSize uint64 40 | logSeqNum uint64 41 | readerRefs atomic.Int32 42 | releaseMemAccounting func() 43 | } 44 | 45 | func (e *flushableEntry) readerRef() { 46 | e.readerRefs.Add(1) 47 | } 48 | 49 | func (e *flushableEntry) readerUnref() { 50 | switch v := e.readerRefs.Add(-1); { 51 | case v == 0: 52 | if e.releaseMemAccounting == nil { 53 | fmt.Println("panic: flushableEntry readerUnref reservation already released") 54 | return 55 | } 56 | e.releaseMemAccounting() 57 | e.releaseMemAccounting = nil 58 | case v < 0: 59 | fmt.Printf("panic: flushableEntry readerUnref logNum:%d count:%d\n", e.logNum, v) 60 | } 61 | } 62 | 63 | type flushableList []*flushableEntry 64 | 65 | func newFlushableEntry(f flushable, logNum FileNum, logSeqNum uint64) *flushableEntry { 66 | entry := &flushableEntry{ 67 | flushable: f, 68 | flushed: make(chan struct{}), 69 | logNum: logNum, 70 | logSeqNum: logSeqNum, 71 | } 72 | entry.readerRefs.Store(1) 73 | return entry 74 | } 75 | -------------------------------------------------------------------------------- /internal/fastrand/fastrand_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fastrand 16 | 17 | import ( 18 | "fmt" 19 | "sync" 20 | "testing" 21 | "time" 22 | 23 | "golang.org/x/exp/rand" 24 | ) 25 | 26 | type defaultRand struct { 27 | mu sync.Mutex 28 | src rand.PCGSource 29 | } 30 | 31 | func newDefaultRand() *defaultRand { 32 | r := &defaultRand{} 33 | r.src.Seed(uint64(time.Now().UnixNano())) 34 | return r 35 | } 36 | 37 | func (r *defaultRand) Uint32() uint32 { 38 | r.mu.Lock() 39 | i := uint32(r.src.Uint64()) 40 | r.mu.Unlock() 41 | return i 42 | } 43 | 44 | func BenchmarkFastRand(b *testing.B) { 45 | b.RunParallel(func(pb *testing.PB) { 46 | for pb.Next() { 47 | Uint32() 48 | } 49 | }) 50 | } 51 | 52 | func BenchmarkDefaultRand(b *testing.B) { 53 | r := newDefaultRand() 54 | b.RunParallel(func(pb *testing.PB) { 55 | for pb.Next() { 56 | r.Uint32() 57 | } 58 | }) 59 | } 60 | 61 | var xg uint32 62 | 63 | func BenchmarkSTFastRand(b *testing.B) { 64 | var x uint32 65 | for i := 0; i < b.N; i++ { 66 | x = Uint32n(2097152) 67 | } 68 | xg = x 69 | } 70 | 71 | func BenchmarkSTDefaultRand(b *testing.B) { 72 | for _, newPeriod := range []int{0, 10, 100, 1000} { 73 | name := "no-new" 74 | if newPeriod > 0 { 75 | name = fmt.Sprintf("new-period=%d", newPeriod) 76 | } 77 | b.Run(name, func(b *testing.B) { 78 | r := rand.New(rand.NewSource(uint64(time.Now().UnixNano()))) 79 | b.ResetTimer() 80 | var x uint32 81 | for i := 0; i < b.N; i++ { 82 | if newPeriod > 0 && i%newPeriod == 0 { 83 | r = rand.New(rand.NewSource(uint64(time.Now().UnixNano()))) 84 | } 85 | x = uint32(r.Uint64n(2097152)) 86 | } 87 | xg = x 88 | }) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /bitree/bdb/bdb_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build !windows && !plan9 && !solaris && !aix 16 | 17 | package bdb 18 | 19 | import ( 20 | "syscall" 21 | "time" 22 | "unsafe" 23 | 24 | "github.com/zuoyebang/bitalosdb/internal/errors" 25 | "golang.org/x/sys/unix" 26 | ) 27 | 28 | func flock(db *DB, exclusive bool, timeout time.Duration) error { 29 | var t time.Time 30 | if timeout != 0 { 31 | t = time.Now() 32 | } 33 | fd := db.file.Fd() 34 | flag := syscall.LOCK_NB 35 | if exclusive { 36 | flag |= syscall.LOCK_EX 37 | } else { 38 | flag |= syscall.LOCK_SH 39 | } 40 | for { 41 | err := syscall.Flock(int(fd), flag) 42 | if err == nil { 43 | return nil 44 | } else if err != syscall.EWOULDBLOCK { 45 | return err 46 | } 47 | 48 | if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { 49 | return ErrTimeout 50 | } 51 | 52 | time.Sleep(flockRetryTimeout) 53 | } 54 | } 55 | 56 | func funlock(db *DB) error { 57 | return syscall.Flock(int(db.file.Fd()), syscall.LOCK_UN) 58 | } 59 | 60 | func mmap(db *DB, sz int) error { 61 | b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | err = unix.Madvise(b, syscall.MADV_RANDOM) 67 | if err != nil && err != syscall.ENOSYS { 68 | return errors.Wrapf(err, "madvise err") 69 | } 70 | 71 | db.dataref = b 72 | db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) 73 | db.datasz = sz 74 | return nil 75 | } 76 | 77 | func munmap(db *DB) error { 78 | if db.dataref == nil { 79 | return nil 80 | } 81 | 82 | err := unix.Munmap(db.dataref) 83 | db.dataref = nil 84 | db.data = nil 85 | db.datasz = 0 86 | return err 87 | } 88 | -------------------------------------------------------------------------------- /internal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | type internalIterAdapter struct { 18 | internalIterator 19 | key *InternalKey 20 | val []byte 21 | } 22 | 23 | func newInternalIterAdapter(iter internalIterator) *internalIterAdapter { 24 | return &internalIterAdapter{ 25 | internalIterator: iter, 26 | } 27 | } 28 | 29 | func (i *internalIterAdapter) update(key *InternalKey, val []byte) bool { 30 | i.key = key 31 | i.val = val 32 | return i.key != nil 33 | } 34 | 35 | func (i *internalIterAdapter) String() string { 36 | return "internal-iter-adapter" 37 | } 38 | 39 | func (i *internalIterAdapter) SeekGE(key []byte) bool { 40 | return i.update(i.internalIterator.SeekGE(key)) 41 | } 42 | 43 | func (i *internalIterAdapter) SeekPrefixGE(prefix, key []byte, trySeekUsingNext bool) bool { 44 | return i.update(i.internalIterator.SeekPrefixGE(prefix, key, trySeekUsingNext)) 45 | } 46 | 47 | func (i *internalIterAdapter) SeekLT(key []byte) bool { 48 | return i.update(i.internalIterator.SeekLT(key)) 49 | } 50 | 51 | func (i *internalIterAdapter) First() bool { 52 | return i.update(i.internalIterator.First()) 53 | } 54 | 55 | func (i *internalIterAdapter) Last() bool { 56 | return i.update(i.internalIterator.Last()) 57 | } 58 | 59 | func (i *internalIterAdapter) Next() bool { 60 | return i.update(i.internalIterator.Next()) 61 | } 62 | 63 | func (i *internalIterAdapter) Prev() bool { 64 | return i.update(i.internalIterator.Prev()) 65 | } 66 | 67 | func (i *internalIterAdapter) Key() *InternalKey { 68 | return i.key 69 | } 70 | 71 | func (i *internalIterAdapter) Value() []byte { 72 | return i.val 73 | } 74 | 75 | func (i *internalIterAdapter) Valid() bool { 76 | return i.key != nil 77 | } 78 | -------------------------------------------------------------------------------- /internal/arenaskl/arena.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package arenaskl 16 | 17 | import ( 18 | "errors" 19 | "math" 20 | "sync/atomic" 21 | "unsafe" 22 | ) 23 | 24 | type Arena struct { 25 | n uint64 26 | buf []byte 27 | } 28 | 29 | const ( 30 | align4 = 3 31 | ) 32 | 33 | var ( 34 | ErrArenaFull = errors.New("allocation failed because arena is full") 35 | ) 36 | 37 | func NewArena(buf []byte) *Arena { 38 | return &Arena{ 39 | n: 1, 40 | buf: buf, 41 | } 42 | } 43 | 44 | func (a *Arena) Size() uint32 { 45 | s := atomic.LoadUint64(&a.n) 46 | if s > math.MaxUint32 { 47 | return math.MaxUint32 48 | } 49 | return uint32(s) 50 | } 51 | 52 | func (a *Arena) Capacity() uint32 { 53 | return uint32(len(a.buf)) 54 | } 55 | 56 | func (a *Arena) alloc(size, align, overflow uint32) (uint32, uint32, error) { 57 | origSize := atomic.LoadUint64(&a.n) 58 | if int(origSize) > len(a.buf) { 59 | return 0, 0, ErrArenaFull 60 | } 61 | 62 | padded := size + align 63 | 64 | newSize := atomic.AddUint64(&a.n, uint64(padded)) 65 | if int(newSize)+int(overflow) > len(a.buf) { 66 | return 0, 0, ErrArenaFull 67 | } 68 | 69 | offset := (uint32(newSize) - padded + align) & ^align 70 | return offset, padded, nil 71 | } 72 | 73 | func (a *Arena) getBytes(offset uint32, size uint32) []byte { 74 | if offset == 0 { 75 | return nil 76 | } 77 | return a.buf[offset : offset+size : offset+size] 78 | } 79 | 80 | func (a *Arena) getPointer(offset uint32) unsafe.Pointer { 81 | if offset == 0 { 82 | return nil 83 | } 84 | return unsafe.Pointer(&a.buf[offset]) 85 | } 86 | 87 | func (a *Arena) getPointerOffset(ptr unsafe.Pointer) uint32 { 88 | if ptr == nil { 89 | return 0 90 | } 91 | return uint32(uintptr(ptr) - uintptr(unsafe.Pointer(&a.buf[0]))) 92 | } 93 | -------------------------------------------------------------------------------- /bithash/bithash_writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/base" 18 | 19 | type BithashWriter struct { 20 | b *Bithash 21 | wr *Writer 22 | compact bool 23 | } 24 | 25 | func (w *BithashWriter) Add(ikey base.InternalKey, value []byte) (FileNum, error) { 26 | var err error 27 | if w.wr == nil { 28 | return FileNum(0), err 29 | } 30 | 31 | if err = w.wr.Add(ikey, value); err != nil { 32 | return FileNum(0), err 33 | } 34 | 35 | fileNum := w.wr.fileNum 36 | w.b.stats.KeyTotal.Add(1) 37 | 38 | _ = w.maybeSplitTable() 39 | 40 | return fileNum, nil 41 | } 42 | 43 | func (w *BithashWriter) AddIkey(key *InternalKey, value []byte, khash uint32, fileNum FileNum) error { 44 | return w.wr.AddIkey(key.Clone(), value, khash, fileNum) 45 | } 46 | 47 | func (w *BithashWriter) maybeSplitTable() error { 48 | if !w.wr.isWriteFull() { 49 | return nil 50 | } 51 | 52 | if err := w.wr.Flush(); err != nil { 53 | return err 54 | } 55 | 56 | writer, err := NewTableWriter(w.b, w.compact) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | closeWriter := w.wr 62 | w.wr = writer 63 | 64 | w.b.closeTableAsync(closeWriter, false) 65 | 66 | return nil 67 | } 68 | 69 | func (w *BithashWriter) Finish() error { 70 | if w.wr == nil { 71 | return nil 72 | } 73 | 74 | if err := w.wr.Flush(); err != nil { 75 | return err 76 | } 77 | 78 | if w.compact { 79 | return w.b.closeTable(w.wr, true) 80 | } 81 | 82 | if !w.wr.isWriteFull() { 83 | w.b.pushMutableWriters(w.wr) 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func (w *BithashWriter) GetFileNum() FileNum { 90 | return w.wr.fileNum 91 | } 92 | 93 | func (w *BithashWriter) Remove() error { 94 | return w.wr.Remove() 95 | } 96 | -------------------------------------------------------------------------------- /internal/sortedkv/key.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sortedkv 16 | 17 | import ( 18 | "encoding/binary" 19 | "strconv" 20 | 21 | "github.com/zuoyebang/bitalosdb/internal/hash" 22 | "github.com/zuoyebang/bitalosdb/internal/utils" 23 | ) 24 | 25 | func MakeKey(key []byte) []byte { 26 | slotId := uint16(hash.Crc32(key) % 1024) 27 | keyLen := 2 + len(key) 28 | newKey := make([]byte, keyLen) 29 | binary.BigEndian.PutUint16(newKey[0:2], slotId) 30 | copy(newKey[2:keyLen], key) 31 | return newKey 32 | } 33 | 34 | func MakeKey2(key []byte, slotId uint16, version uint64) []byte { 35 | keyLen := 10 + len(key) 36 | newKey := make([]byte, keyLen) 37 | binary.BigEndian.PutUint16(newKey[0:2], slotId) 38 | binary.LittleEndian.PutUint64(newKey[2:10], version) 39 | if key != nil { 40 | copy(newKey[10:keyLen], key) 41 | } 42 | return newKey 43 | } 44 | 45 | func MakeSlotKey(key []byte, slotId uint16) []byte { 46 | keyLen := 2 + len(key) 47 | newKey := make([]byte, keyLen) 48 | binary.BigEndian.PutUint16(newKey[0:2], slotId) 49 | copy(newKey[2:keyLen], key) 50 | return newKey 51 | } 52 | 53 | func MakeSortedKey(n int) []byte { 54 | if n < 0 { 55 | return []byte(sortedKeyPrefix) 56 | } else { 57 | return []byte(sortedKeyPrefix + strconv.Itoa(n)) 58 | } 59 | } 60 | 61 | func MakeSortedSlotKey(n int, slotId uint16) []byte { 62 | if n < 0 { 63 | return MakeSlotKey([]byte(sortedKeyPrefix), slotId) 64 | } else { 65 | return MakeSlotKey([]byte(sortedKeyPrefix+strconv.Itoa(n)), slotId) 66 | } 67 | } 68 | 69 | func MakeSortedKeyForBitrie(n int) []byte { 70 | if n < 0 { 71 | return []byte(sortedKeyPrefix) 72 | } else { 73 | return []byte(sortedKeyPrefix + string(utils.FuncRandBytes(1)) + "_bitalosdb_" + string(utils.FuncRandBytes(32)) + strconv.Itoa(n)) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /bitree/bdb/page_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb 16 | 17 | import ( 18 | "reflect" 19 | "sort" 20 | "testing" 21 | "testing/quick" 22 | ) 23 | 24 | func TestPage_typ(t *testing.T) { 25 | if typ := (&page{flags: branchPageFlag}).typ(); typ != "branch" { 26 | t.Fatalf("exp=branch; got=%v", typ) 27 | } 28 | if typ := (&page{flags: leafPageFlag}).typ(); typ != "leaf" { 29 | t.Fatalf("exp=leaf; got=%v", typ) 30 | } 31 | if typ := (&page{flags: metaPageFlag}).typ(); typ != "meta" { 32 | t.Fatalf("exp=meta; got=%v", typ) 33 | } 34 | if typ := (&page{flags: freelistPageFlag}).typ(); typ != "freelist" { 35 | t.Fatalf("exp=freelist; got=%v", typ) 36 | } 37 | if typ := (&page{flags: 20000}).typ(); typ != "unknown<4e20>" { 38 | t.Fatalf("exp=unknown<4e20>; got=%v", typ) 39 | } 40 | } 41 | 42 | func TestPage_dump(t *testing.T) { 43 | (&page{id: 256}).hexdump(16) 44 | } 45 | 46 | func TestPgids_merge(t *testing.T) { 47 | a := pgids{4, 5, 6, 10, 11, 12, 13, 27} 48 | b := pgids{1, 3, 8, 9, 25, 30} 49 | c := a.merge(b) 50 | if !reflect.DeepEqual(c, pgids{1, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30}) { 51 | t.Errorf("mismatch: %v", c) 52 | } 53 | 54 | a = pgids{4, 5, 6, 10, 11, 12, 13, 27, 35, 36} 55 | b = pgids{8, 9, 25, 30} 56 | c = a.merge(b) 57 | if !reflect.DeepEqual(c, pgids{4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30, 35, 36}) { 58 | t.Errorf("mismatch: %v", c) 59 | } 60 | } 61 | 62 | func TestPgids_merge_quick(t *testing.T) { 63 | if err := quick.Check(func(a, b pgids) bool { 64 | sort.Sort(a) 65 | sort.Sort(b) 66 | 67 | got := a.merge(b) 68 | 69 | exp := append(a, b...) 70 | sort.Sort(exp) 71 | 72 | if !reflect.DeepEqual(exp, got) { 73 | t.Errorf("\nexp=%+v\ngot=%+v\n", exp, got) 74 | return false 75 | } 76 | 77 | return true 78 | }, nil); err != nil { 79 | t.Fatal(err) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /log_recycler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | import ( 18 | "sync" 19 | 20 | "github.com/zuoyebang/bitalosdb/internal/errors" 21 | ) 22 | 23 | type logRecycler struct { 24 | limit int 25 | minRecycleLogNum FileNum 26 | 27 | mu struct { 28 | sync.Mutex 29 | logs []fileInfo 30 | maxLogNum FileNum 31 | } 32 | } 33 | 34 | func (r *logRecycler) add(logInfo fileInfo) bool { 35 | if logInfo.fileNum < r.minRecycleLogNum { 36 | return false 37 | } 38 | 39 | r.mu.Lock() 40 | defer r.mu.Unlock() 41 | 42 | if logInfo.fileNum <= r.mu.maxLogNum { 43 | return true 44 | } 45 | r.mu.maxLogNum = logInfo.fileNum 46 | if len(r.mu.logs) >= r.limit { 47 | return false 48 | } 49 | r.mu.logs = append(r.mu.logs, logInfo) 50 | return true 51 | } 52 | 53 | func (r *logRecycler) peek() (fileInfo, bool) { 54 | r.mu.Lock() 55 | defer r.mu.Unlock() 56 | 57 | if len(r.mu.logs) == 0 { 58 | return fileInfo{}, false 59 | } 60 | return r.mu.logs[0], true 61 | } 62 | 63 | func (r *logRecycler) stats() (count int, size uint64) { 64 | r.mu.Lock() 65 | defer r.mu.Unlock() 66 | count = len(r.mu.logs) 67 | for i := 0; i < count; i++ { 68 | size += r.mu.logs[i].fileSize 69 | } 70 | return count, size 71 | } 72 | 73 | func (r *logRecycler) pop(logNum FileNum) error { 74 | r.mu.Lock() 75 | defer r.mu.Unlock() 76 | 77 | if len(r.mu.logs) == 0 { 78 | return errors.New("bitalosdb: log recycler empty") 79 | } 80 | if r.mu.logs[0].fileNum != logNum { 81 | return errors.Errorf("bitalosdb: log recycler invalid %d vs %v", logNum, fileInfoNums(r.mu.logs)) 82 | } 83 | r.mu.logs = r.mu.logs[1:] 84 | return nil 85 | } 86 | 87 | func fileInfoNums(finfos []fileInfo) []FileNum { 88 | if len(finfos) == 0 { 89 | return nil 90 | } 91 | nums := make([]FileNum, len(finfos)) 92 | for i := range finfos { 93 | nums[i] = finfos[i].fileNum 94 | } 95 | return nums 96 | } 97 | -------------------------------------------------------------------------------- /internal/list2/stack_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package list2 16 | 17 | import "testing" 18 | 19 | func TestStack(t *testing.T) { 20 | stack := NewStack() 21 | stack.Push(1) 22 | stack.Push(2) 23 | stack.Push(3) 24 | stack.Push(4) 25 | 26 | length := stack.Len() 27 | if length != 4 { 28 | t.Errorf("stack.Len() failed. Got %d, expected 4.", length) 29 | } 30 | 31 | value := stack.Peak().(int) 32 | if value != 4 { 33 | t.Errorf("stack.Peak() failed. Got %d, expected 4.", value) 34 | } 35 | 36 | value = stack.Pop().(int) 37 | if value != 4 { 38 | t.Errorf("stack.Pop() failed. Got %d, expected 4.", value) 39 | } 40 | 41 | length = stack.Len() 42 | if length != 3 { 43 | t.Errorf("stack.Len() failed. Got %d, expected 3.", length) 44 | } 45 | 46 | value = stack.Peak().(int) 47 | if value != 3 { 48 | t.Errorf("stack.Peak() failed. Got %d, expected 3.", value) 49 | } 50 | 51 | value = stack.Pop().(int) 52 | if value != 3 { 53 | t.Errorf("stack.Pop() failed. Got %d, expected 3.", value) 54 | } 55 | 56 | value = stack.Pop().(int) 57 | if value != 2 { 58 | t.Errorf("stack.Pop() failed. Got %d, expected 2.", value) 59 | } 60 | 61 | empty := stack.Empty() 62 | if empty { 63 | t.Errorf("stack.Empty() failed. Got %v, expected false.", empty) 64 | } 65 | 66 | value = stack.Pop().(int) 67 | if value != 1 { 68 | t.Errorf("stack.Pop() failed. Got %d, expected 1.", value) 69 | } 70 | 71 | empty = stack.Empty() 72 | if !empty { 73 | t.Errorf("stack.Empty() failed. Got %v, expected true.", empty) 74 | } 75 | 76 | nilValue := stack.Peak() 77 | if nilValue != nil { 78 | t.Errorf("stack.Peak() failed. Got %d, expected nil.", nilValue) 79 | } 80 | 81 | nilValue = stack.Pop() 82 | if nilValue != nil { 83 | t.Errorf("stack.Pop() failed. Got %d, expected nil.", nilValue) 84 | } 85 | 86 | length = stack.Len() 87 | if length != 0 { 88 | t.Errorf("stack.Len() failed. Got %d, expected 0.", length) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /internal/vfs/testdata/vfs: -------------------------------------------------------------------------------- 1 | define 2 | link a b 3 | create a 4 | link a b 5 | link a b 6 | link c d 7 | remove b 8 | link-or-copy a b 9 | remove b 10 | ---- 11 | link: a -> b [file does not exist] 12 | create: a [] 13 | close: a [] 14 | link: a -> b [] 15 | link: a -> b [file already exists] 16 | link: c -> d [file does not exist] 17 | remove: b [] 18 | link: a -> b [] 19 | remove: b [] 20 | 21 | define linkErr=ErrExist 22 | create a 23 | link a b 24 | link-or-copy a b 25 | ---- 26 | create: a [] 27 | close: a [] 28 | link: a -> b [file already exists] 29 | link: a -> b [file already exists] 30 | 31 | define linkErr=ErrNotExist 32 | create a 33 | link a b 34 | link-or-copy a b 35 | ---- 36 | create: a [] 37 | close: a [] 38 | link: a -> b [file does not exist] 39 | link: a -> b [file does not exist] 40 | 41 | define linkErr=ErrPermission 42 | create a 43 | link a b 44 | link-or-copy a b 45 | ---- 46 | create: a [] 47 | close: a [] 48 | link: a -> b [permission denied] 49 | link: a -> b [permission denied] 50 | 51 | define linkErr=random 52 | create a 53 | link a b 54 | link-or-copy a b 55 | ---- 56 | create: a [] 57 | close: a [] 58 | link: a -> b [random] 59 | link: a -> b [random] 60 | open: a [] 61 | create: b [] 62 | sync: b [] 63 | close: b [] 64 | close: a [] 65 | 66 | define 67 | mkdir d 68 | create d/a 69 | mkdir d/b 70 | create d/b/c 71 | ---- 72 | mkdir: d [] 73 | create: d/a [] 74 | close: a [] 75 | mkdir: d/b [] 76 | create: d/b/c [] 77 | close: c [] 78 | 79 | define 80 | clone d e 81 | ---- 82 | open: d [] 83 | close: d [] 84 | mkdir: e [] 85 | open: d/a [] 86 | close: d/a [] 87 | create: e/a [] 88 | close: a [] 89 | open: d/b [] 90 | close: d/b [] 91 | mkdir: e/b [] 92 | open: d/b/c [] 93 | close: d/b/c [] 94 | create: e/b/c [] 95 | close: c [] 96 | 97 | define 98 | list e 99 | ---- 100 | a 101 | b 102 | 103 | define 104 | list e/b 105 | ---- 106 | c 107 | 108 | define 109 | list / 110 | remove e 111 | remove-all e 112 | remove-all e 113 | remove-all e/a/b/c 114 | list / 115 | ---- 116 | a 117 | b 118 | d 119 | e 120 | remove: e [file already exists] 121 | remove-all: e [] 122 | remove-all: e [] 123 | remove-all: e/a/b/c [] 124 | a 125 | b 126 | d 127 | 128 | define 129 | reuseForWrite a b 130 | reuseForWrite x y 131 | ---- 132 | reuseForWrite: a -> b [] 133 | reuseForWrite: x -> y [file does not exist] 134 | -------------------------------------------------------------------------------- /internal/cache/lfucache/metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lfucache 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "sync/atomic" 21 | ) 22 | 23 | func (lfc *LfuCache) MetricsInfo() string { 24 | return lfc.Metrics().String() 25 | } 26 | 27 | func (lfc *LfuCache) Metrics() Metrics { 28 | var m Metrics 29 | m.ShardsMetrics = make([]ShardMetrics, lfc.shardNum) 30 | for i := range lfc.shards { 31 | s := lfc.shards[i] 32 | s.mu.Lock() 33 | atSize := s.arrayTableInuseSize() 34 | memLen := len(s.mu.memQueue) 35 | memSize := s.memTableInuseSize() 36 | size := atSize + memSize 37 | s.mu.Unlock() 38 | m.ShardsMetrics[i] = ShardMetrics{ 39 | Size: size, 40 | InusePercent: int(size * 100 / s.maxSize), 41 | MemLen: memLen, 42 | MemCount: s.memTableCount(), 43 | MemSize: memSize, 44 | AtCount: s.arrayTableCount(), 45 | AtSize: atSize, 46 | } 47 | m.Size += size 48 | m.Hits += atomic.LoadInt64(&s.atomic.hits) 49 | m.Misses += atomic.LoadInt64(&s.atomic.misses) 50 | } 51 | return m 52 | } 53 | 54 | type ShardMetrics struct { 55 | Size uint64 56 | InusePercent int 57 | MemLen int 58 | MemCount int 59 | MemSize uint64 60 | AtCount int 61 | AtSize uint64 62 | } 63 | 64 | type Metrics struct { 65 | Size uint64 66 | Hits int64 67 | Misses int64 68 | ShardsMetrics []ShardMetrics 69 | } 70 | 71 | func (m Metrics) String() string { 72 | var shards bytes.Buffer 73 | for i := range m.ShardsMetrics { 74 | shards.WriteString(fmt.Sprintf("[%d:%d:%d:%d:%d:%d:%d:%d]", 75 | i, 76 | m.ShardsMetrics[i].Size, 77 | m.ShardsMetrics[i].InusePercent, 78 | m.ShardsMetrics[i].MemLen, 79 | m.ShardsMetrics[i].MemCount, 80 | m.ShardsMetrics[i].MemSize, 81 | m.ShardsMetrics[i].AtCount, 82 | m.ShardsMetrics[i].AtSize)) 83 | } 84 | return fmt.Sprintf("size:%d hit:%d mis:%d shards:%s", m.Size, m.Hits, m.Misses, shards.String()) 85 | } 86 | -------------------------------------------------------------------------------- /merging_iter_heap.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | type mergingIterItem struct { 18 | index int 19 | key InternalKey 20 | value []byte 21 | } 22 | 23 | type mergingIterHeap struct { 24 | cmp Compare 25 | reverse bool 26 | items []mergingIterItem 27 | } 28 | 29 | func (h *mergingIterHeap) len() int { 30 | return len(h.items) 31 | } 32 | 33 | func (h *mergingIterHeap) clear() { 34 | h.items = h.items[:0] 35 | } 36 | 37 | func (h *mergingIterHeap) less(i, j int) bool { 38 | ikey, jkey := h.items[i].key, h.items[j].key 39 | if c := h.cmp(ikey.UserKey, jkey.UserKey); c != 0 { 40 | if h.reverse { 41 | return c > 0 42 | } 43 | return c < 0 44 | } 45 | if h.reverse { 46 | return ikey.Trailer < jkey.Trailer 47 | } 48 | return ikey.Trailer > jkey.Trailer 49 | } 50 | 51 | func (h *mergingIterHeap) swap(i, j int) { 52 | h.items[i], h.items[j] = h.items[j], h.items[i] 53 | } 54 | 55 | func (h *mergingIterHeap) init() { 56 | n := h.len() 57 | for i := n/2 - 1; i >= 0; i-- { 58 | h.down(i, n) 59 | } 60 | } 61 | 62 | func (h *mergingIterHeap) fix(i int) { 63 | if !h.down(i, h.len()) { 64 | h.up(i) 65 | } 66 | } 67 | 68 | func (h *mergingIterHeap) pop() *mergingIterItem { 69 | n := h.len() - 1 70 | h.swap(0, n) 71 | h.down(0, n) 72 | item := &h.items[n] 73 | h.items = h.items[:n] 74 | return item 75 | } 76 | 77 | func (h *mergingIterHeap) up(j int) { 78 | for { 79 | i := (j - 1) / 2 80 | if i == j || !h.less(j, i) { 81 | break 82 | } 83 | h.swap(i, j) 84 | j = i 85 | } 86 | } 87 | 88 | func (h *mergingIterHeap) down(i0, n int) bool { 89 | i := i0 90 | for { 91 | j1 := 2*i + 1 92 | if j1 >= n || j1 < 0 { 93 | break 94 | } 95 | j := j1 96 | if j2 := j1 + 1; j2 < n && h.less(j2, j1) { 97 | j = j2 98 | } 99 | if !h.less(j, i) { 100 | break 101 | } 102 | h.swap(i, j) 103 | i = j 104 | } 105 | return i > i0 106 | } 107 | -------------------------------------------------------------------------------- /metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitalosdb 16 | 17 | import ( 18 | "bytes" 19 | "encoding/json" 20 | ) 21 | 22 | type MetricsInfo struct { 23 | FlushMemTime int64 `json:"-"` 24 | BithashFileTotal int `json:"bithash_file_total"` 25 | BithashKeyTotal int `json:"bithash_key_total"` 26 | BithashDelKeyTotal int `json:"bithash_del_key_total"` 27 | } 28 | 29 | func (s MetricsInfo) String() string { 30 | str, err := json.Marshal(s) 31 | if err != nil { 32 | return "" 33 | } else { 34 | return string(str) 35 | } 36 | } 37 | 38 | func (d *DB) statsDiskSize() int64 { 39 | var size int64 40 | for _, bitower := range d.bitowers { 41 | ps := bitower.btree.BitpageStats() 42 | size += ps.Size 43 | } 44 | return size 45 | } 46 | 47 | func (d *DB) MetricsInfo() MetricsInfo { 48 | var stat MetricsInfo 49 | for i := range d.bitowers { 50 | bs := d.bitowers[i].btree.BithashStats() 51 | if bs != nil { 52 | fileTotal := int(bs.FileTotal.Load()) 53 | stat.BithashFileTotal += fileTotal 54 | stat.BithashKeyTotal += int(bs.KeyTotal.Load()) 55 | stat.BithashDelKeyTotal += int(bs.DelKeyTotal.Load()) 56 | } 57 | } 58 | 59 | stat.FlushMemTime = d.flushMemTime.Load() 60 | return stat 61 | } 62 | 63 | func (d *DB) CacheInfo() string { 64 | if d.cache == nil { 65 | return "" 66 | } 67 | return d.cache.MetricsInfo() 68 | } 69 | 70 | func (d *DB) DebugInfo() string { 71 | dataType := d.opts.DataType 72 | if dataType == "" { 73 | dataType = "base" 74 | } 75 | 76 | dinfo := new(bytes.Buffer) 77 | for _, bitower := range d.bitowers { 78 | bpInfo := bitower.btree.BitreeDebugInfo(dataType) 79 | if bpInfo != "" { 80 | dinfo.WriteString(bpInfo) 81 | } 82 | 83 | btInfo := bitower.btree.BitableDebugInfo(dataType) 84 | if btInfo != "" { 85 | dinfo.WriteString(btInfo) 86 | } 87 | 88 | bhInfo := bitower.btree.BithashDebugInfo(dataType) 89 | if bhInfo != "" { 90 | dinfo.WriteString(bhInfo) 91 | } 92 | } 93 | 94 | return dinfo.String() 95 | } 96 | -------------------------------------------------------------------------------- /internal/cache/lfucache/merging_iter_heap.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lfucache 16 | 17 | import "bytes" 18 | 19 | type mergingIterItem struct { 20 | index int 21 | key internalKey 22 | value []byte 23 | } 24 | 25 | type mergingIterHeap struct { 26 | reverse bool 27 | items []mergingIterItem 28 | } 29 | 30 | func (h *mergingIterHeap) len() int { 31 | return len(h.items) 32 | } 33 | 34 | func (h *mergingIterHeap) clear() { 35 | h.items = h.items[:0] 36 | } 37 | 38 | func (h *mergingIterHeap) less(i, j int) bool { 39 | ikey, jkey := h.items[i].key, h.items[j].key 40 | if c := bytes.Compare(ikey.UserKey, jkey.UserKey); c != 0 { 41 | if h.reverse { 42 | return c > 0 43 | } 44 | return c < 0 45 | } 46 | if h.reverse { 47 | return ikey.Trailer < jkey.Trailer 48 | } 49 | return ikey.Trailer > jkey.Trailer 50 | } 51 | 52 | func (h *mergingIterHeap) swap(i, j int) { 53 | h.items[i], h.items[j] = h.items[j], h.items[i] 54 | } 55 | 56 | func (h *mergingIterHeap) init() { 57 | n := h.len() 58 | for i := n/2 - 1; i >= 0; i-- { 59 | h.down(i, n) 60 | } 61 | } 62 | 63 | func (h *mergingIterHeap) fix(i int) { 64 | if !h.down(i, h.len()) { 65 | h.up(i) 66 | } 67 | } 68 | 69 | func (h *mergingIterHeap) pop() *mergingIterItem { 70 | n := h.len() - 1 71 | h.swap(0, n) 72 | h.down(0, n) 73 | item := &h.items[n] 74 | h.items = h.items[:n] 75 | return item 76 | } 77 | 78 | func (h *mergingIterHeap) up(j int) { 79 | for { 80 | i := (j - 1) / 2 81 | if i == j || !h.less(j, i) { 82 | break 83 | } 84 | h.swap(i, j) 85 | j = i 86 | } 87 | } 88 | 89 | func (h *mergingIterHeap) down(i0, n int) bool { 90 | i := i0 91 | for { 92 | j1 := 2*i + 1 93 | if j1 >= n || j1 < 0 { 94 | break 95 | } 96 | j := j1 97 | if j2 := j1 + 1; j2 < n && h.less(j2, j1) { 98 | j = j2 99 | } 100 | if !h.less(j, i) { 101 | break 102 | } 103 | h.swap(i, j) 104 | i = j 105 | } 106 | return i > i0 107 | } 108 | -------------------------------------------------------------------------------- /bitree/bdb/simulation_no_freelist_sync_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bdb_test 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/zuoyebang/bitalosdb/internal/options" 21 | ) 22 | 23 | func TestSimulateNoFreeListSync_1op_1p(t *testing.T) { 24 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 1, 1) 25 | } 26 | func TestSimulateNoFreeListSync_10op_1p(t *testing.T) { 27 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 10, 1) 28 | } 29 | func TestSimulateNoFreeListSync_100op_1p(t *testing.T) { 30 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 100, 1) 31 | } 32 | func TestSimulateNoFreeListSync_1000op_1p(t *testing.T) { 33 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 1000, 1) 34 | } 35 | func TestSimulateNoFreeListSync_10000op_1p(t *testing.T) { 36 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 10000, 1) 37 | } 38 | func TestSimulateNoFreeListSync_10op_10p(t *testing.T) { 39 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 10, 10) 40 | } 41 | func TestSimulateNoFreeListSync_100op_10p(t *testing.T) { 42 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 100, 10) 43 | } 44 | func TestSimulateNoFreeListSync_1000op_10p(t *testing.T) { 45 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 1000, 10) 46 | } 47 | func TestSimulateNoFreeListSync_10000op_10p(t *testing.T) { 48 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 10000, 10) 49 | } 50 | func TestSimulateNoFreeListSync_100op_100p(t *testing.T) { 51 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 100, 100) 52 | } 53 | func TestSimulateNoFreeListSync_1000op_100p(t *testing.T) { 54 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 1000, 100) 55 | } 56 | func TestSimulateNoFreeListSync_10000op_100p(t *testing.T) { 57 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 10000, 100) 58 | } 59 | func TestSimulateNoFreeListSync_10000op_1000p(t *testing.T) { 60 | testSimulate(t, &options.BdbOptions{NoFreelistSync: true}, 8, 10000, 1000) 61 | } 62 | -------------------------------------------------------------------------------- /bitpage/merging_iter_heap.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import "github.com/zuoyebang/bitalosdb/internal/base" 18 | 19 | type mergingIterItem struct { 20 | index int 21 | key internalKey 22 | value []byte 23 | } 24 | 25 | type mergingIterHeap struct { 26 | cmp base.Compare 27 | reverse bool 28 | items []mergingIterItem 29 | } 30 | 31 | func (h *mergingIterHeap) len() int { 32 | return len(h.items) 33 | } 34 | 35 | func (h *mergingIterHeap) clear() { 36 | h.items = h.items[:0] 37 | } 38 | 39 | func (h *mergingIterHeap) less(i, j int) bool { 40 | ikey, jkey := h.items[i].key, h.items[j].key 41 | if c := h.cmp(ikey.UserKey, jkey.UserKey); c != 0 { 42 | if h.reverse { 43 | return c > 0 44 | } 45 | return c < 0 46 | } 47 | if h.reverse { 48 | return ikey.Trailer < jkey.Trailer 49 | } 50 | return ikey.Trailer > jkey.Trailer 51 | } 52 | 53 | func (h *mergingIterHeap) swap(i, j int) { 54 | h.items[i], h.items[j] = h.items[j], h.items[i] 55 | } 56 | 57 | func (h *mergingIterHeap) init() { 58 | n := h.len() 59 | for i := n/2 - 1; i >= 0; i-- { 60 | h.down(i, n) 61 | } 62 | } 63 | 64 | func (h *mergingIterHeap) fix(i int) { 65 | if !h.down(i, h.len()) { 66 | h.up(i) 67 | } 68 | } 69 | 70 | func (h *mergingIterHeap) pop() *mergingIterItem { 71 | n := h.len() - 1 72 | h.swap(0, n) 73 | h.down(0, n) 74 | item := &h.items[n] 75 | h.items = h.items[:n] 76 | return item 77 | } 78 | 79 | func (h *mergingIterHeap) up(j int) { 80 | for { 81 | i := (j - 1) / 2 // parent 82 | if i == j || !h.less(j, i) { 83 | break 84 | } 85 | h.swap(i, j) 86 | j = i 87 | } 88 | } 89 | 90 | func (h *mergingIterHeap) down(i0, n int) bool { 91 | i := i0 92 | for { 93 | j1 := 2*i + 1 94 | if j1 >= n || j1 < 0 { 95 | break 96 | } 97 | j := j1 98 | if j2 := j1 + 1; j2 < n && h.less(j2, j1) { 99 | j = j2 100 | } 101 | if !h.less(j, i) { 102 | break 103 | } 104 | h.swap(i, j) 105 | i = j 106 | } 107 | return i > i0 108 | } 109 | -------------------------------------------------------------------------------- /bithash/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "sync/atomic" 21 | 22 | "github.com/zuoyebang/bitalosdb/internal/utils" 23 | ) 24 | 25 | type Stats struct { 26 | FileTotal atomic.Uint32 27 | KeyTotal atomic.Uint64 28 | DelKeyTotal atomic.Uint64 29 | } 30 | 31 | func (s *Stats) String() string { 32 | return fmt.Sprintf("fileTotal:%d keyTotal:%d delKeyTotal:%d", 33 | s.FileTotal.Load(), s.KeyTotal.Load(), s.DelKeyTotal.Load()) 34 | } 35 | 36 | func (b *Bithash) StatsAddKeyTotal(n int) { 37 | b.stats.KeyTotal.Add(uint64(n)) 38 | } 39 | 40 | func (b *Bithash) StatsSubKeyTotal(n int) { 41 | b.stats.KeyTotal.Add(^uint64(n - 1)) 42 | } 43 | 44 | func (b *Bithash) StatsGetDelKeyTotal() uint64 { 45 | return b.stats.DelKeyTotal.Load() 46 | } 47 | 48 | func (b *Bithash) StatsSubDelKeyTotal(n int) { 49 | b.stats.DelKeyTotal.Add(^uint64(n - 1)) 50 | } 51 | 52 | func (b *Bithash) StatsSetDelKeyTotal(n int) { 53 | b.stats.DelKeyTotal.Store(uint64(n)) 54 | } 55 | 56 | func (b *Bithash) Stats() *Stats { 57 | return b.stats 58 | } 59 | 60 | func (b *Bithash) StatsToString() string { 61 | return b.stats.String() 62 | } 63 | 64 | func (b *Bithash) DebugInfo(dataType string) string { 65 | b.meta.mu.RLock() 66 | defer b.meta.mu.RUnlock() 67 | 68 | buf := new(bytes.Buffer) 69 | fmt.Fprintf(buf, "%s-bithash%d:dirname=%s dirSize=%s files=%d\n", 70 | dataType, b.index, b.dirname, 71 | utils.FmtSize(utils.GetDirSize(b.dirname)), 72 | len(b.meta.mu.filesMeta)) 73 | 74 | for fn, fileMeta := range b.meta.mu.filesMeta { 75 | if fileMeta.state == fileMetaStateImmutable { 76 | var delPercent float64 77 | if fileMeta.keyNum == 0 { 78 | delPercent = 0 79 | } else { 80 | delPercent = float64(fileMeta.delKeyNum) / float64(fileMeta.keyNum) 81 | } 82 | fmt.Fprintf(buf, "%s-bithash%d-%d.bht:%s delPercent=%.2f\n", dataType, b.index, fn, fileMeta.String(), delPercent) 83 | } else { 84 | fmt.Fprintf(buf, "%s-bithash%d-%d.bht:%s\n", dataType, b.index, fn, fileMeta.String()) 85 | } 86 | } 87 | 88 | return buf.String() 89 | } 90 | -------------------------------------------------------------------------------- /bithash/checkpoint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import ( 18 | "os" 19 | "path/filepath" 20 | 21 | "github.com/zuoyebang/bitalosdb/internal/errors" 22 | "github.com/zuoyebang/bitalosdb/internal/os2" 23 | "github.com/zuoyebang/bitalosdb/internal/vfs" 24 | ) 25 | 26 | func (b *Bithash) Checkpoint(fs vfs.FS, destDir string) (err error) { 27 | if os2.IsExist(destDir) { 28 | return errors.Errorf("bithash: checkpoint dir exist %s", destDir) 29 | } 30 | 31 | var dir vfs.File 32 | err = os.Mkdir(destDir, 0755) 33 | if err != nil { 34 | return err 35 | } 36 | dir, err = fs.OpenDir(destDir) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | if err = b.closeMutableWriters(); err != nil { 42 | return err 43 | } 44 | 45 | manifest := MakeFilepath(fs, b.dirname, fileTypeManifest, 0) 46 | destManifest := MakeFilepath(fs, destDir, fileTypeManifest, 0) 47 | err = vfs.Copy(fs, manifest, destManifest) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | fileNumMap := MakeFilepath(fs, b.dirname, fileTypeFileNumMap, 0) 53 | destFileNumMap := MakeFilepath(fs, destDir, fileTypeFileNumMap, 0) 54 | if err = vfs.Copy(fs, fileNumMap, destFileNumMap); err != nil { 55 | return err 56 | } 57 | 58 | compactLog := MakeFilepath(fs, b.dirname, fileTypeCompactLog, 0) 59 | destCompactLog := MakeFilepath(fs, destDir, fileTypeCompactLog, 0) 60 | if err = vfs.Copy(fs, compactLog, destCompactLog); err != nil { 61 | return err 62 | } 63 | 64 | copyTable := func() error { 65 | b.meta.mu.RLock() 66 | defer b.meta.mu.RUnlock() 67 | for fn := range b.meta.mu.filesMeta { 68 | filename := MakeFilepath(fs, b.dirname, fileTypeTable, fn) 69 | if _, err = fs.Stat(filename); err != nil { 70 | return err 71 | } 72 | err = vfs.LinkOrCopy(fs, filename, filepath.Join(destDir, fs.PathBase(filename))) 73 | if err != nil { 74 | return err 75 | } 76 | } 77 | return nil 78 | } 79 | err = copyTable() 80 | if err != nil { 81 | return err 82 | } 83 | 84 | err = dir.Sync() 85 | if err != nil { 86 | return err 87 | } 88 | err = dir.Close() 89 | dir = nil 90 | 91 | return err 92 | } 93 | -------------------------------------------------------------------------------- /internal/base/filenames_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package base 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/zuoyebang/bitalosdb/internal/vfs" 21 | ) 22 | 23 | func TestParseFilename(t *testing.T) { 24 | testCases := map[string]bool{ 25 | "000000.log": true, 26 | "000000.log.zip": false, 27 | "000000..log": false, 28 | "a000000.log": false, 29 | "abcdef.log": false, 30 | "000001ldb": false, 31 | "CURRENT": true, 32 | "CURRaNT": false, 33 | "LOCK": true, 34 | "xLOCK": false, 35 | "x.LOCK": false, 36 | "MANIFEST": false, 37 | "MANIFEST123456": false, 38 | "MANIFEST-": false, 39 | "MANIFEST-123456": true, 40 | "MANIFEST-123456.doc": false, 41 | "OPTIONS": false, 42 | "OPTIONS123456": false, 43 | "OPTIONS-": false, 44 | "OPTIONS-123456.doc": false, 45 | "CURRENT.123456": false, 46 | "CURRENT.dbtmp": false, 47 | } 48 | fs := vfs.NewMem() 49 | for tc, want := range testCases { 50 | _, _, got := ParseFilename(fs, fs.PathJoin("foo", tc)) 51 | if got != want { 52 | t.Errorf("%q: got %v, want %v", tc, got, want) 53 | } 54 | } 55 | } 56 | 57 | func TestFilenameRoundTrip(t *testing.T) { 58 | testCases := map[FileType]bool{ 59 | FileTypeCurrent: false, 60 | FileTypeLock: false, 61 | FileTypeLog: true, 62 | FileTypeManifest: true, 63 | } 64 | fs := vfs.NewMem() 65 | for fileType, numbered := range testCases { 66 | fileNums := []FileNum{0} 67 | if numbered { 68 | fileNums = []FileNum{0, 1, 2, 3, 10, 42, 99, 1001} 69 | } 70 | for _, fileNum := range fileNums { 71 | filename := MakeFilepath(fs, "foo", fileType, fileNum) 72 | gotFT, gotFN, gotOK := ParseFilename(fs, filename) 73 | if !gotOK { 74 | t.Errorf("could not parse %q", filename) 75 | continue 76 | } 77 | if gotFT != fileType || gotFN != fileNum { 78 | t.Errorf("filename=%q: got %v, %v, want %v, %v", filename, gotFT, gotFN, fileType, fileNum) 79 | continue 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /bitpage/checkpoint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import ( 18 | "os" 19 | "path" 20 | 21 | "github.com/zuoyebang/bitalosdb/internal/errors" 22 | "github.com/zuoyebang/bitalosdb/internal/os2" 23 | "github.com/zuoyebang/bitalosdb/internal/utils" 24 | "github.com/zuoyebang/bitalosdb/internal/vfs" 25 | ) 26 | 27 | func (b *Bitpage) Checkpoint(fs vfs.FS, dstDir string) (err error) { 28 | if os2.IsExist(dstDir) { 29 | return errors.Errorf("bitpage: checkpoint dir exist %s", dstDir) 30 | } 31 | 32 | var dir vfs.File 33 | var files []string 34 | 35 | err = os.Mkdir(dstDir, 0755) 36 | if err != nil { 37 | return err 38 | } 39 | dir, err = fs.OpenDir(dstDir) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | b.pages.Range(func(k, v interface{}) bool { 45 | pn := k.(PageNum) 46 | p := v.(*page) 47 | if b.PageSplitted(pn) { 48 | return true 49 | } 50 | 51 | if p.mu.stMutable != nil { 52 | if err = p.makeMutableForWrite(true); err != nil { 53 | return false 54 | } 55 | } 56 | 57 | for i := range p.mu.stQueue { 58 | if !p.mu.stQueue[i].empty() { 59 | files = append(files, p.mu.stQueue[i].path()) 60 | idxPath := p.mu.stQueue[i].idxFilePath() 61 | if utils.IsFileExist(idxPath) { 62 | files = append(files, idxPath) 63 | } 64 | } 65 | } 66 | 67 | if p.mu.arrtable != nil { 68 | files = append(files, p.mu.arrtable.path()) 69 | } 70 | 71 | return true 72 | }) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | for i := range files { 78 | newname := path.Join(dstDir, fs.PathBase(files[i])) 79 | if err = vfs.LinkOrCopy(fs, files[i], newname); err != nil { 80 | return err 81 | } 82 | b.opts.Logger.Infof("bitpage checkpoint save %s to %s", files[i], newname) 83 | } 84 | 85 | metaFile := makeFilepath(b.dirname, fileTypeManifest, 0, 0) 86 | metaNewFile := makeFilepath(dstDir, fileTypeManifest, 0, 0) 87 | if err = vfs.Copy(fs, metaFile, metaNewFile); err != nil { 88 | return err 89 | } 90 | b.opts.Logger.Infof("bitpage checkpoint save %s to %s", metaFile, metaNewFile) 91 | 92 | err = dir.Sync() 93 | if err != nil { 94 | return err 95 | } 96 | return dir.Close() 97 | } 98 | -------------------------------------------------------------------------------- /internal/statemachine/db_state_machine.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package statemachine 16 | 17 | import ( 18 | "sync" 19 | "sync/atomic" 20 | "time" 21 | 22 | "github.com/zuoyebang/bitalosdb/internal/consts" 23 | ) 24 | 25 | type DbStateMachine struct { 26 | TaskLock sync.Mutex // compact-bithash｜checkpoint 27 | BitowerHighPriority [consts.DefaultBitowerNum]atomic.Bool 28 | BitowerWrite [consts.DefaultBitowerNum]sync.Mutex // mem-flush｜bitpage-flush/split ｜compact-bitable｜checkpoint 29 | 30 | bitpage struct { 31 | flushCount atomic.Uint64 32 | splitCount atomic.Uint64 33 | } 34 | } 35 | 36 | func NewDbStateMachine() *DbStateMachine { 37 | return &DbStateMachine{} 38 | } 39 | 40 | func (s *DbStateMachine) LockDbWrite() { 41 | for i := range s.BitowerWrite { 42 | s.BitowerWrite[i].Lock() 43 | } 44 | } 45 | 46 | func (s *DbStateMachine) UnlockDbWrite() { 47 | for i := range s.BitowerWrite { 48 | s.BitowerWrite[i].Unlock() 49 | } 50 | } 51 | 52 | func (s *DbStateMachine) LockBitowerWrite(i int) { 53 | s.BitowerWrite[i].Lock() 54 | } 55 | 56 | func (s *DbStateMachine) UnlockBitowerWrite(i int) { 57 | s.BitowerWrite[i].Unlock() 58 | } 59 | 60 | func (s *DbStateMachine) LockTask() { 61 | s.TaskLock.Lock() 62 | } 63 | 64 | func (s *DbStateMachine) UnlockTask() { 65 | s.TaskLock.Unlock() 66 | } 67 | 68 | func (s *DbStateMachine) AddBitpageFlushCount() { 69 | s.bitpage.flushCount.Add(1) 70 | } 71 | 72 | func (s *DbStateMachine) GetBitpageFlushCount() uint64 { 73 | return s.bitpage.flushCount.Load() 74 | } 75 | 76 | func (s *DbStateMachine) AddBitpageSplitCount() { 77 | s.bitpage.splitCount.Add(1) 78 | } 79 | 80 | func (s *DbStateMachine) GetBitpageSplitCount() uint64 { 81 | return s.bitpage.splitCount.Load() 82 | } 83 | 84 | func (s *DbStateMachine) SetDbHighPriority(v bool) { 85 | for i := range s.BitowerHighPriority { 86 | s.BitowerHighPriority[i].Store(v) 87 | } 88 | } 89 | 90 | func (s *DbStateMachine) SetBitowerHighPriority(i int, v bool) { 91 | s.BitowerHighPriority[i].Store(v) 92 | } 93 | 94 | func (s *DbStateMachine) WaitBitowerHighPriority(i int) { 95 | for s.BitowerHighPriority[i].Load() { 96 | time.Sleep(1 * time.Second) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /internal/cache/lrucache/entry_normal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build (!invariants && !tracing) || race 16 | // +build !invariants,!tracing race 17 | 18 | package lrucache 19 | 20 | import ( 21 | "runtime" 22 | "sync" 23 | "unsafe" 24 | 25 | "github.com/zuoyebang/bitalosdb/internal/invariants" 26 | "github.com/zuoyebang/bitalosdb/internal/manual" 27 | ) 28 | 29 | const ( 30 | entrySize = int(unsafe.Sizeof(entry{})) 31 | entryAllocCacheLimit = 128 32 | entriesGoAllocated = invariants.RaceEnabled || !cgoEnabled 33 | ) 34 | 35 | var entryAllocPool = sync.Pool{ 36 | New: func() interface{} { 37 | return newEntryAllocCache() 38 | }, 39 | } 40 | 41 | func entryAllocNew() *entry { 42 | a := entryAllocPool.Get().(*entryAllocCache) 43 | e := a.alloc() 44 | entryAllocPool.Put(a) 45 | return e 46 | } 47 | 48 | func entryAllocFree(e *entry) { 49 | a := entryAllocPool.Get().(*entryAllocCache) 50 | a.free(e) 51 | entryAllocPool.Put(a) 52 | } 53 | 54 | type entryAllocCache struct { 55 | entries []*entry 56 | } 57 | 58 | func newEntryAllocCache() *entryAllocCache { 59 | c := &entryAllocCache{} 60 | if !entriesGoAllocated { 61 | runtime.SetFinalizer(c, freeEntryAllocCache) 62 | } 63 | return c 64 | } 65 | 66 | func freeEntryAllocCache(obj interface{}) { 67 | c := obj.(*entryAllocCache) 68 | for i, e := range c.entries { 69 | c.dealloc(e) 70 | c.entries[i] = nil 71 | } 72 | } 73 | 74 | func (c *entryAllocCache) alloc() *entry { 75 | n := len(c.entries) 76 | if n == 0 { 77 | if entriesGoAllocated { 78 | return &entry{} 79 | } 80 | b := manual.New(entrySize) 81 | return (*entry)(unsafe.Pointer(&b[0])) 82 | } 83 | e := c.entries[n-1] 84 | c.entries = c.entries[:n-1] 85 | return e 86 | } 87 | 88 | func (c *entryAllocCache) dealloc(e *entry) { 89 | if !entriesGoAllocated { 90 | buf := (*[manual.MaxArrayLen]byte)(unsafe.Pointer(e))[:entrySize:entrySize] 91 | manual.Free(buf) 92 | } 93 | } 94 | 95 | func (c *entryAllocCache) free(e *entry) { 96 | if len(c.entries) == entryAllocCacheLimit { 97 | c.dealloc(e) 98 | return 99 | } 100 | c.entries = append(c.entries, e) 101 | } 102 | -------------------------------------------------------------------------------- /internal/cache/lrucache/entry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lrucache 16 | 17 | type entryType int8 18 | 19 | const ( 20 | etTest entryType = iota 21 | etCold 22 | etHot 23 | ) 24 | 25 | func (p entryType) String() string { 26 | switch p { 27 | case etTest: 28 | return "test" 29 | case etCold: 30 | return "cold" 31 | case etHot: 32 | return "hot" 33 | } 34 | return "unknown" 35 | } 36 | 37 | type entry struct { 38 | key key 39 | val *Value 40 | blockLink struct { 41 | next *entry 42 | prev *entry 43 | } 44 | size int64 45 | ptype entryType 46 | referenced int32 47 | shard *shard 48 | ref refcnt 49 | } 50 | 51 | func newEntry(s *shard, key key, size int64) *entry { 52 | e := entryAllocNew() 53 | *e = entry{ 54 | key: key, 55 | size: size, 56 | ptype: etCold, 57 | shard: s, 58 | } 59 | e.blockLink.next = e 60 | e.blockLink.prev = e 61 | e.ref.init(1) 62 | return e 63 | } 64 | 65 | func (e *entry) free() { 66 | e.setValue(nil) 67 | *e = entry{} 68 | entryAllocFree(e) 69 | } 70 | 71 | func (e *entry) next() *entry { 72 | if e == nil { 73 | return nil 74 | } 75 | return e.blockLink.next 76 | } 77 | 78 | func (e *entry) prev() *entry { 79 | if e == nil { 80 | return nil 81 | } 82 | return e.blockLink.prev 83 | } 84 | 85 | func (e *entry) link(s *entry) { 86 | s.blockLink.prev = e.blockLink.prev 87 | s.blockLink.prev.blockLink.next = s 88 | s.blockLink.next = e 89 | s.blockLink.next.blockLink.prev = s 90 | } 91 | 92 | func (e *entry) unlink() *entry { 93 | next := e.blockLink.next 94 | e.blockLink.prev.blockLink.next = e.blockLink.next 95 | e.blockLink.next.blockLink.prev = e.blockLink.prev 96 | e.blockLink.prev = e 97 | e.blockLink.next = e 98 | return next 99 | } 100 | 101 | func (e *entry) setValue(v *Value) { 102 | if v != nil { 103 | v.acquire() 104 | } 105 | old := e.val 106 | e.val = v 107 | if old != nil { 108 | old.release() 109 | } 110 | } 111 | 112 | func (e *entry) peekValue() *Value { 113 | return e.val 114 | } 115 | 116 | func (e *entry) acquireValue() *Value { 117 | v := e.val 118 | if v != nil { 119 | v.acquire() 120 | } 121 | return v 122 | } 123 | -------------------------------------------------------------------------------- /internal/bitask/memtable_task.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitask 16 | 17 | import ( 18 | "runtime/debug" 19 | "sync" 20 | "sync/atomic" 21 | 22 | "github.com/zuoyebang/bitalosdb/internal/base" 23 | ) 24 | 25 | type MemFlushTaskData struct { 26 | Index int 27 | NeedReport bool 28 | } 29 | 30 | type MemFlushTaskOptions struct { 31 | Size int 32 | DoFunc func(*MemFlushTaskData) 33 | Logger base.Logger 34 | TaskWg *sync.WaitGroup 35 | } 36 | 37 | type MemFlushTask struct { 38 | taskWg *sync.WaitGroup 39 | recvCh chan *MemFlushTaskData 40 | taskCount uint64 41 | closed atomic.Bool 42 | logger base.Logger 43 | doTaskFunc func(*MemFlushTaskData) 44 | } 45 | 46 | func NewMemFlushTask(opts *MemFlushTaskOptions) *MemFlushTask { 47 | task := &MemFlushTask{ 48 | recvCh: make(chan *MemFlushTaskData, opts.Size), 49 | doTaskFunc: opts.DoFunc, 50 | logger: opts.Logger, 51 | taskWg: opts.TaskWg, 52 | } 53 | task.Run() 54 | return task 55 | } 56 | 57 | func (t *MemFlushTask) Run() { 58 | t.taskWg.Add(1) 59 | go func() { 60 | defer func() { 61 | t.taskWg.Done() 62 | t.logger.Infof("memtable flush task background exit...") 63 | }() 64 | 65 | do := func() bool { 66 | defer func() { 67 | if r := recover(); r != nil { 68 | t.logger.Errorf("memtable flush task do panic err:%v stack:%s", r, string(debug.Stack())) 69 | } 70 | }() 71 | 72 | task, ok := <-t.recvCh 73 | if !ok || task == nil { 74 | return true 75 | } 76 | 77 | t.doTaskFunc(task) 78 | return false 79 | } 80 | 81 | var exit bool 82 | for { 83 | if exit = do(); exit { 84 | break 85 | } 86 | } 87 | }() 88 | 89 | t.logger.Infof("memtable flush task background running...") 90 | } 91 | 92 | func (t *MemFlushTask) isClosed() bool { 93 | return t.closed.Load() == true 94 | } 95 | 96 | func (t *MemFlushTask) Count() uint64 { 97 | return t.taskCount 98 | } 99 | 100 | func (t *MemFlushTask) Close() { 101 | t.closed.Store(true) 102 | t.recvCh <- nil 103 | } 104 | 105 | func (t *MemFlushTask) PushTask(task *MemFlushTaskData) { 106 | if !t.isClosed() { 107 | t.taskCount++ 108 | t.recvCh <- task 109 | return 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /bithash/block_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bithash 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "strconv" 21 | "strings" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/require" 25 | "github.com/zuoyebang/bitalosdb/internal/base" 26 | ) 27 | 28 | func TestBlockIter(t *testing.T) { 29 | w := blockWriter{restartInterval: 16} 30 | 31 | makeIkey := func(s string) InternalKey { 32 | j := strings.Index(s, ":") 33 | seqNum, err := strconv.Atoi(s[j+1:]) 34 | if err != nil { 35 | panic(err) 36 | } 37 | return base.MakeInternalKey([]byte(s[:j]), uint64(seqNum), InternalKeyKindSet) 38 | } 39 | 40 | w.add(makeIkey("testkey:1"), nil) 41 | w.add(makeIkey("testkey:2"), nil) 42 | w.add(makeIkey("testkey:3"), nil) 43 | block := w.finish() 44 | 45 | iter, err := newBlockIter(bytes.Compare, block) 46 | require.NoError(t, err) 47 | defer iter.Close() 48 | key := []byte("testkey") 49 | var value []byte 50 | var fk *InternalKey 51 | for k, v := iter.SeekGE(key); iter.Valid(); k, v = iter.Next() { 52 | if bytes.Equal(k.UserKey, key) { 53 | fmt.Println("find Equal value", string(value)) 54 | for ; iter.Valid() && bytes.Equal(k.UserKey, key); k, v = iter.Next() { 55 | value = v 56 | fk = k 57 | } 58 | break 59 | } 60 | } 61 | fmt.Println("find value", string(value), fk.String()) 62 | } 63 | 64 | func TestBlockSameKey(t *testing.T) { 65 | w := blockWriter{restartInterval: 16} 66 | seqNum := uint64(1) 67 | skey := []byte("ip_b_39.144.177.163") 68 | w.add(base.MakeInternalKey(skey, seqNum, InternalKeyKindSet), skey) 69 | seqNum++ 70 | for i := 0; i < 10; i++ { 71 | key := fmt.Sprintf("key_%d", i) 72 | val := fmt.Sprintf("value_%d", i) 73 | w.add(base.MakeInternalKey([]byte(key), seqNum, InternalKeyKindSet), []byte(val)) 74 | seqNum++ 75 | if i == 5 { 76 | w.add(base.MakeInternalKey(skey, seqNum, InternalKeyKindSet), skey) 77 | seqNum++ 78 | } 79 | } 80 | w.add(base.MakeInternalKey(skey, seqNum, InternalKeyKindSet), skey) 81 | seqNum++ 82 | 83 | iter, err := newBlockIter(bytes.Compare, w.finish()) 84 | require.NoError(t, err) 85 | defer iter.Close() 86 | 87 | var v []byte 88 | var k *InternalKey 89 | for k, v = iter.First(); iter.Valid(); k, v = iter.Next() { 90 | fmt.Println("find1 Equal value", k.String(), string(v)) 91 | } 92 | fmt.Println("find end", k, v) 93 | } 94 | -------------------------------------------------------------------------------- /bitpage/super_table_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019-2024 Xu Ruibo (hustxurb@163.com) and Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bitpage 16 | 17 | import ( 18 | "bytes" 19 | ) 20 | 21 | type superTableIterator struct { 22 | st *superTable 23 | indexes stIndexes 24 | indexPos int 25 | iterKey internalKey 26 | iterValue []byte 27 | } 28 | 29 | func (i *superTableIterator) findItem() (*internalKey, []byte) { 30 | if i.indexPos < 0 || i.indexPos >= len(i.indexes) { 31 | return nil, nil 32 | } 33 | 34 | ikey, value := i.st.getItem(i.indexes[i.indexPos]) 35 | if ikey.UserKey == nil { 36 | return nil, nil 37 | } 38 | 39 | i.iterKey = ikey 40 | i.iterValue = value 41 | return &i.iterKey, i.iterValue 42 | } 43 | 44 | func (i *superTableIterator) First() (*internalKey, []byte) { 45 | i.indexPos = 0 46 | return i.findItem() 47 | } 48 | 49 | func (i *superTableIterator) Next() (*internalKey, []byte) { 50 | i.indexPos++ 51 | return i.findItem() 52 | } 53 | 54 | func (i *superTableIterator) Prev() (*internalKey, []byte) { 55 | i.indexPos-- 56 | return i.findItem() 57 | } 58 | 59 | func (i *superTableIterator) Last() (*internalKey, []byte) { 60 | i.indexPos = len(i.indexes) - 1 61 | return i.findItem() 62 | } 63 | 64 | func (i *superTableIterator) SeekGE(key []byte) (*internalKey, []byte) { 65 | i.indexPos = i.st.findKeyIndexPos(i.indexes, key) 66 | return i.findItem() 67 | } 68 | 69 | func (i *superTableIterator) SeekLT(key []byte) (*internalKey, []byte) { 70 | i.indexPos = i.st.findKeyIndexPos(i.indexes, key) 71 | poskey, _ := i.findItem() 72 | if poskey != nil { 73 | return i.Prev() 74 | } 75 | 76 | lastKey, lastValue := i.Last() 77 | if lastKey != nil && bytes.Compare(lastKey.UserKey, key) < 0 { 78 | return lastKey, lastValue 79 | } 80 | return nil, nil 81 | 82 | } 83 | 84 | func (i *superTableIterator) SeekPrefixGE( 85 | prefix, key []byte, trySeekUsingNext bool, 86 | ) (ikey *internalKey, value []byte) { 87 | return i.SeekGE(key) 88 | } 89 | 90 | func (i *superTableIterator) SetBounds(lower, upper []byte) { 91 | } 92 | 93 | func (i *superTableIterator) Error() error { 94 | return nil 95 | } 96 | 97 | func (i *superTableIterator) Close() error { 98 | return nil 99 | } 100 | 101 | func (i *superTableIterator) String() string { 102 | return "superTableIterator" 103 | } 104 | --------------------------------------------------------------------------------