├── .editorconfig ├── .github ├── CODEOWNERS ├── actions │ └── post-issue │ │ └── action.yaml └── workflows │ ├── ci.yaml │ ├── code-cover-gen.yaml │ ├── code-cover-publish.yaml │ ├── instrumented.yaml │ ├── nightlies-25.2.yaml │ ├── nightlies.yaml │ ├── nightly-code-cover.yaml │ ├── s390x.yaml │ ├── stale.yml │ ├── stress.yaml │ └── tests.yaml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── batch.go ├── batch_test.go ├── batchrepr ├── reader.go ├── reader_test.go ├── testdata │ ├── reader │ └── writer ├── writer.go └── writer_test.go ├── bloom ├── bloom.go └── bloom_test.go ├── cache.go ├── checkpoint.go ├── checkpoint_test.go ├── cmd └── pebble │ ├── .gitignore │ ├── db.go │ ├── fsbench.go │ ├── fsbenchlist.go │ ├── main.go │ ├── mvcc.go │ ├── queue.go │ ├── random.go │ ├── replay.go │ ├── replay_test.go │ ├── scan.go │ ├── sync.go │ ├── test.go │ ├── tombstone.go │ ├── util.go │ ├── write_bench.go │ └── ycsb.go ├── cockroachkvs ├── blockproperties.go ├── cockroachkvs.go ├── cockroachkvs_bench_test.go ├── cockroachkvs_test.go ├── key_schema_test.go ├── rowblk_bench_test.go ├── test_utils.go └── testdata │ ├── block_encoding │ ├── comparer_funcs │ ├── key_schema_key_seeker │ ├── key_schema_key_writer │ ├── rand_keys │ ├── seek │ └── suffix_types ├── commit.go ├── commit_test.go ├── compaction.go ├── compaction_picker.go ├── compaction_picker_test.go ├── compaction_scheduler.go ├── compaction_scheduler_test.go ├── compaction_test.go ├── comparer.go ├── data_test.go ├── db.go ├── db_internals.go ├── db_test.go ├── docs ├── RFCS │ ├── 20211018_range_keys.md │ ├── 20220112_pebble_sstable_format_versions.md │ ├── 20220311_pebble_flushable_ingested_sstable.md │ ├── 20221122_virtual_sstable.md │ └── 20240701_tombstone_density_heuristic.md ├── css │ └── app.css ├── index.html ├── io_profiling.md ├── js │ ├── app.js │ ├── d3.v5.min.js │ └── write-throughput.js ├── memory.md ├── range_deletions.md └── rocksdb.md ├── download.go ├── download_test.go ├── error_iter.go ├── error_test.go ├── event.go ├── event_listener_test.go ├── example_test.go ├── excise.go ├── excise_test.go ├── external_iterator.go ├── external_iterator_test.go ├── external_test.go ├── file_cache.go ├── file_cache_test.go ├── filenames_test.go ├── flush_test.go ├── flushable.go ├── flushable_test.go ├── format_major_version.go ├── format_major_version_test.go ├── get_iter.go ├── get_iter_test.go ├── go.mod ├── go.sum ├── ingest.go ├── ingest_test.go ├── internal.go ├── internal ├── ackseq │ └── ackseq.go ├── arenaskl │ ├── LICENSE │ ├── README.md │ ├── arena.go │ ├── arena_test.go │ ├── flush_iterator.go │ ├── iterator.go │ ├── node.go │ ├── race_test.go │ ├── skl.go │ └── skl_test.go ├── base │ ├── cleaner.go │ ├── close_helper.go │ ├── compaction_grant_handle.go │ ├── comparer.go │ ├── comparer_test.go │ ├── error.go │ ├── filenames.go │ ├── filenames_test.go │ ├── internal.go │ ├── internal_test.go │ ├── iterator.go │ ├── iterator_test.go │ ├── key_bounds.go │ ├── key_bounds_test.go │ ├── lazy_value.go │ ├── lazy_value_test.go │ ├── logger.go │ ├── merger.go │ ├── metrics.go │ ├── metrics_test.go │ ├── options.go │ ├── test_utils.go │ └── value.go ├── batchskl │ ├── README.md │ ├── iterator.go │ ├── skl.go │ └── skl_test.go ├── binfmt │ ├── binfmt.go │ ├── hexdump.go │ ├── hexdump_test.go │ └── testdata │ │ └── hexdump ├── bitflip │ └── bitflip.go ├── blobtest │ └── handles.go ├── buildtags │ ├── cgo_off.go │ ├── cgo_on.go │ ├── invariants_off.go │ ├── invariants_on.go │ ├── race_off.go │ ├── race_on.go │ ├── slow_build_off.go │ ├── slow_build_on.go │ ├── tracing_off.go │ └── tracing_on.go ├── bytealloc │ └── bytealloc.go ├── cache │ ├── LICENSE │ ├── block_map.go │ ├── block_map_test.go │ ├── cache.go │ ├── cache_test.go │ ├── clockpro.go │ ├── clockpro_normal.go │ ├── clockpro_test.go │ ├── clockpro_tracing.go │ ├── entry.go │ ├── read_shard.go │ ├── read_shard_test.go │ ├── refcnt_normal.go │ ├── refcnt_tracing.go │ ├── testdata │ │ ├── cache │ │ └── read_shard │ └── value.go ├── compact │ ├── iterator.go │ ├── iterator_test.go │ ├── run.go │ ├── run_test.go │ ├── snapshots.go │ ├── snapshots_test.go │ ├── spans.go │ ├── spans_test.go │ ├── splitting.go │ ├── splitting_test.go │ ├── testdata │ │ ├── frontiers │ │ ├── iter │ │ ├── iter_delete_sized │ │ ├── iter_set_with_del │ │ ├── output_splitter │ │ ├── range_del_compactor │ │ ├── range_key_compactor │ │ ├── split_and_encode_span │ │ ├── table_split_limit │ │ ├── tombstone_elider │ │ ├── tombstone_elision │ │ └── tombstone_elision_setup │ ├── tombstone_elision.go │ └── tombstone_elision_test.go ├── compression │ ├── compression.go │ ├── compression_test.go │ ├── minlz.go │ ├── minlz_test.go │ ├── noop.go │ ├── snappy.go │ ├── zstd_cgo.go │ └── zstd_nocgo.go ├── constants │ ├── constants.go │ └── constants_test.go ├── crc │ └── crc.go ├── datadrivenutil │ └── datadrivenutil.go ├── datatest │ └── datatest.go ├── devtools │ ├── go.mod │ ├── go.sum │ ├── roachvet │ │ ├── forbidden_imports.go │ │ └── main.go │ └── tools.go ├── dsl │ ├── dsl.go │ └── predicates.go ├── genericcache │ ├── cache.go │ ├── cache_test.go │ ├── node.go │ └── shard.go ├── humanize │ ├── humanize.go │ ├── humanize_test.go │ └── testdata │ │ └── humanize ├── intern │ ├── intern.go │ └── intern_test.go ├── invalidating │ └── iter.go ├── invariants │ ├── invariants.go │ ├── off.go │ └── on.go ├── itertest │ ├── datadriven.go │ ├── dsl.go │ ├── probe.go │ ├── probe_test.go │ └── testdata │ │ └── probes ├── keyspan │ ├── assert_iter.go │ ├── assert_iter_test.go │ ├── bounded.go │ ├── bounded_test.go │ ├── defragment.go │ ├── defragment_test.go │ ├── doc.go │ ├── filter.go │ ├── filter_test.go │ ├── fragmenter.go │ ├── fragmenter_test.go │ ├── get.go │ ├── get_test.go │ ├── interleaving_iter.go │ ├── interleaving_iter_test.go │ ├── iter.go │ ├── iter_test.go │ ├── keyspanimpl │ │ ├── doc.go │ │ ├── level_iter.go │ │ ├── level_iter_test.go │ │ ├── merging_iter.go │ │ ├── merging_iter_test.go │ │ └── testdata │ │ │ ├── level_iter │ │ │ └── merging_iter │ ├── logging_iter.go │ ├── logging_iter_test.go │ ├── seek.go │ ├── seek_test.go │ ├── span.go │ ├── span_test.go │ ├── test_utils.go │ ├── testdata │ │ ├── assert_iter │ │ ├── bounded_iter │ │ ├── covers_at │ │ ├── defragmenting_iter │ │ ├── filtering_iter │ │ ├── fragmenter │ │ ├── fragmenter_emit_order │ │ ├── fragmenter_values │ │ ├── get │ │ ├── interleaving_iter │ │ ├── interleaving_iter_masking │ │ ├── iter │ │ ├── logging_iter │ │ ├── seek │ │ ├── truncate │ │ ├── visible │ │ └── visible_at │ ├── transformer.go │ ├── truncate.go │ └── truncate_test.go ├── lint │ ├── lint.go │ └── lint_test.go ├── lsmview │ ├── data.go │ ├── url.go │ └── url_test.go ├── manifest │ ├── annotator.go │ ├── annotator_test.go │ ├── bench_test.go │ ├── blob_metadata.go │ ├── blob_metadata_test.go │ ├── btree.go │ ├── btree_test.go │ ├── doc.go │ ├── l0_sublevels.go │ ├── l0_sublevels_test.go │ ├── layer.go │ ├── layer_test.go │ ├── level_metadata.go │ ├── level_metadata_test.go │ ├── manifest_test.go │ ├── testdata │ │ ├── MANIFEST_import │ │ ├── current_blob_file_set │ │ ├── file_metadata_bounds │ │ ├── l0_sublevels │ │ ├── level_iterator │ │ ├── level_iterator_filtered │ │ ├── overlaps │ │ ├── version_check_ordering │ │ ├── version_edit_apply │ │ ├── version_edit_decode │ │ └── virtual_backings │ ├── version.go │ ├── version_edit.go │ ├── version_edit_test.go │ ├── version_test.go │ ├── virtual_backings.go │ └── virtual_backings_test.go ├── manual │ ├── manual.go │ ├── manual_cgo.go │ └── manual_nocgo.go ├── metamorphic │ ├── .gitignore │ ├── crossversion │ │ └── crossversion_test.go │ ├── doc.go │ ├── meta_test.go │ ├── metaflags │ │ └── meta_flags.go │ ├── metarunner │ │ └── main.go │ └── reduce_test.go ├── mkbench │ ├── main.go │ ├── split.go │ ├── split_test.go │ ├── testdata │ │ ├── README.md │ │ ├── data-symlink │ │ ├── data.js │ │ ├── data │ │ │ ├── 20211027 │ │ │ │ └── pebble │ │ │ │ │ ├── write │ │ │ │ │ └── size=1024 │ │ │ │ │ │ └── run_1 │ │ │ │ │ │ ├── 1.log.gz │ │ │ │ │ │ └── 2.log.gz │ │ │ │ │ └── ycsb │ │ │ │ │ ├── size=1024 │ │ │ │ │ └── run_1 │ │ │ │ │ │ ├── 1.ycsb_A.log.gz │ │ │ │ │ │ ├── 1.ycsb_B.log.gz │ │ │ │ │ │ ├── 1.ycsb_C.log.gz │ │ │ │ │ │ ├── 1.ycsb_D.log.gz │ │ │ │ │ │ ├── 1.ycsb_E.log.gz │ │ │ │ │ │ ├── 1.ycsb_F.log.gz │ │ │ │ │ │ ├── 2.ycsb_A.log.gz │ │ │ │ │ │ ├── 2.ycsb_B.log.gz │ │ │ │ │ │ ├── 2.ycsb_C.log.gz │ │ │ │ │ │ ├── 2.ycsb_D.log.gz │ │ │ │ │ │ ├── 2.ycsb_E.log.gz │ │ │ │ │ │ ├── 2.ycsb_F.log.gz │ │ │ │ │ │ ├── 3.ycsb_A.log.gz │ │ │ │ │ │ ├── 3.ycsb_B.log.gz │ │ │ │ │ │ ├── 3.ycsb_C.log.gz │ │ │ │ │ │ ├── 3.ycsb_D.log.gz │ │ │ │ │ │ ├── 3.ycsb_E.log.gz │ │ │ │ │ │ └── 3.ycsb_F.log.gz │ │ │ │ │ └── size=64 │ │ │ │ │ └── run_1 │ │ │ │ │ ├── 1.ycsb_A.log │ │ │ │ │ ├── 1.ycsb_B.log │ │ │ │ │ ├── 1.ycsb_C.log │ │ │ │ │ ├── 1.ycsb_D.log │ │ │ │ │ ├── 1.ycsb_E.log │ │ │ │ │ ├── 1.ycsb_F.log │ │ │ │ │ ├── 2.ycsb_A.log │ │ │ │ │ ├── 2.ycsb_B.log │ │ │ │ │ ├── 2.ycsb_C.log │ │ │ │ │ ├── 2.ycsb_D.log │ │ │ │ │ ├── 2.ycsb_E.log │ │ │ │ │ ├── 2.ycsb_F.log │ │ │ │ │ ├── 3.ycsb_A.log │ │ │ │ │ ├── 3.ycsb_B.log │ │ │ │ │ ├── 3.ycsb_C.log │ │ │ │ │ ├── 3.ycsb_D.log │ │ │ │ │ ├── 3.ycsb_E.log │ │ │ │ │ └── 3.ycsb_F.log │ │ │ └── 20211028 │ │ │ │ └── pebble │ │ │ │ ├── write │ │ │ │ └── size=1024 │ │ │ │ │ └── run_1 │ │ │ │ │ ├── 1.log.gz │ │ │ │ │ └── 2.log.gz │ │ │ │ └── ycsb │ │ │ │ ├── size=1024 │ │ │ │ └── run_1 │ │ │ │ │ ├── 1.ycsb_A.log.gz │ │ │ │ │ ├── 1.ycsb_B.log.gz │ │ │ │ │ ├── 1.ycsb_C.log.gz │ │ │ │ │ ├── 1.ycsb_D.log.gz │ │ │ │ │ ├── 1.ycsb_E.log.gz │ │ │ │ │ ├── 1.ycsb_F.log.gz │ │ │ │ │ ├── 2.ycsb_A.log.gz │ │ │ │ │ ├── 2.ycsb_B.log.gz │ │ │ │ │ ├── 2.ycsb_C.log.gz │ │ │ │ │ ├── 2.ycsb_D.log.gz │ │ │ │ │ ├── 2.ycsb_E.log.gz │ │ │ │ │ ├── 2.ycsb_F.log.gz │ │ │ │ │ ├── 3.ycsb_A.log.gz │ │ │ │ │ ├── 3.ycsb_B.log.gz │ │ │ │ │ ├── 3.ycsb_C.log.gz │ │ │ │ │ ├── 3.ycsb_D.log.gz │ │ │ │ │ ├── 3.ycsb_E.log.gz │ │ │ │ │ └── 3.ycsb_F.log.gz │ │ │ │ └── size=64 │ │ │ │ └── run_1 │ │ │ │ ├── 1.ycsb_A.log.bz2 │ │ │ │ ├── 1.ycsb_B.log.bz2 │ │ │ │ ├── 1.ycsb_C.log.bz2 │ │ │ │ ├── 1.ycsb_D.log.bz2 │ │ │ │ ├── 1.ycsb_E.log.bz2 │ │ │ │ ├── 1.ycsb_F.log.bz2 │ │ │ │ ├── 2.ycsb_A.log.bz2 │ │ │ │ ├── 2.ycsb_B.log.bz2 │ │ │ │ ├── 2.ycsb_C.log.bz2 │ │ │ │ ├── 2.ycsb_D.log.bz2 │ │ │ │ ├── 2.ycsb_E.log.bz2 │ │ │ │ ├── 2.ycsb_F.log.bz2 │ │ │ │ ├── 3.ycsb_A.log.bz2 │ │ │ │ ├── 3.ycsb_B.log.bz2 │ │ │ │ ├── 3.ycsb_C.log.bz2 │ │ │ │ ├── 3.ycsb_D.log.bz2 │ │ │ │ ├── 3.ycsb_E.log.bz2 │ │ │ │ └── 3.ycsb_F.log.bz2 │ │ └── write-throughput │ │ │ ├── 20211027-pebble-write-size=1024-run_1-summary.json │ │ │ ├── 20211028-pebble-write-size=1024-run_1-summary.json │ │ │ └── summary.json │ ├── testutil.go │ ├── util.go │ ├── util_test.go │ ├── write.go │ ├── write_test.go │ ├── ycsb.go │ └── ycsb_test.go ├── overlap │ ├── checker.go │ ├── checker_test.go │ └── testdata │ │ └── checker ├── pacertoy │ ├── pebble │ │ └── main.go │ └── rocksdb │ │ └── main.go ├── private │ └── batch.go ├── problemspans │ ├── by_level.go │ ├── by_level_test.go │ ├── doc.go │ ├── set.go │ ├── set_test.go │ └── testdata │ │ ├── by_level │ │ └── set ├── randvar │ ├── deck.go │ ├── deck_test.go │ ├── flag.go │ ├── rand.go │ ├── randvar.go │ ├── skewed_latest.go │ ├── skewed_latest_test.go │ ├── uniform.go │ ├── weighted.go │ ├── weighted_test.go │ ├── zipf.go │ └── zipf_test.go ├── rangedel │ └── rangedel.go ├── rangekey │ ├── coalesce.go │ ├── coalesce_test.go │ ├── rangekey.go │ ├── rangekey_test.go │ └── testdata │ │ └── coalesce ├── rangekeystack │ ├── testdata │ │ ├── defragmenting_iter │ │ └── iter │ ├── user_iterator.go │ └── user_iterator_test.go ├── rate │ └── rate.go ├── rawalloc │ ├── rawalloc.go │ ├── rawalloc_32bit.go │ ├── rawalloc_64bit.go │ ├── rawalloc_gccgo.go │ ├── rawalloc_go1.9.go │ ├── rawalloc_mipsall.go │ └── rawalloc_test.go ├── sstableinternal │ └── options.go ├── strparse │ └── strparse.go ├── testkeys │ ├── strconv.go │ ├── testdata │ │ └── divvy │ ├── testkeys.go │ └── testkeys_test.go ├── testutils │ ├── duration.go │ ├── errors.go │ ├── indenttree │ │ ├── indent_tree.go │ │ ├── indent_tree_test.go │ │ └── testdata │ └── rng.go ├── treeprinter │ ├── tree_printer.go │ └── tree_printer_test.go └── treesteps │ ├── data.go │ ├── node_state.go │ ├── testdata │ └── segment_tree │ ├── tree_steps.go │ └── tree_steps_test.go ├── iterator.go ├── iterator_example_test.go ├── iterator_histories_test.go ├── iterator_test.go ├── keyspan_probe_test.go ├── level_checker.go ├── level_checker_test.go ├── level_iter.go ├── level_iter_test.go ├── logger.go ├── lsm_view.go ├── lsm_view_test.go ├── mem_table.go ├── mem_table_test.go ├── merger.go ├── merging_iter.go ├── merging_iter_heap.go ├── merging_iter_heap_test.go ├── merging_iter_test.go ├── metamorphic ├── build.go ├── cockroachkvs.go ├── config.go ├── diagram.go ├── diagram_test.go ├── example_test.go ├── generator.go ├── generator_test.go ├── history.go ├── history_test.go ├── key_manager.go ├── key_manager_test.go ├── meta.go ├── ops.go ├── options.go ├── options_test.go ├── parser.go ├── parser_test.go ├── retryable.go ├── simplify.go ├── simplify_test.go ├── test.go ├── testdata │ ├── diagram │ ├── key_manager │ ├── parser │ ├── reorder_history │ └── simplify ├── testkeys.go └── utils.go ├── metrics.go ├── metrics_test.go ├── objstorage ├── noop_readahead.go ├── objstorage.go ├── objstorageprovider │ ├── objiotracing │ │ ├── obj_io_tracing.go │ │ ├── obj_io_tracing_off.go │ │ ├── obj_io_tracing_on.go │ │ └── obj_io_tracing_test.go │ ├── provider.go │ ├── provider_test.go │ ├── readahead.go │ ├── readahead_test.go │ ├── remote.go │ ├── remote_backing.go │ ├── remote_backing_test.go │ ├── remote_obj_name.go │ ├── remote_obj_name_test.go │ ├── remote_readable.go │ ├── remote_readable_test.go │ ├── remoteobjcat │ │ ├── catalog.go │ │ ├── catalog_test.go │ │ ├── testdata │ │ │ └── catalog │ │ ├── version_edit.go │ │ └── version_edit_test.go │ ├── shared_writable.go │ ├── sharedcache │ │ ├── shared_cache.go │ │ ├── shared_cache_helpers_test.go │ │ ├── shared_cache_internal_test.go │ │ ├── shared_cache_test.go │ │ └── testdata │ │ │ └── cache │ │ │ ├── compaction_reads │ │ │ ├── eof_handling │ │ │ ├── lru │ │ │ ├── read_larger_than_two_cache_shards │ │ │ ├── read_that_hits_two_cache_blocks │ │ │ ├── read_that_hits_two_cache_blocks_with_first_read_at_big_offset │ │ │ ├── read_that_hits_two_cache_blocks_with_first_read_at_offset │ │ │ ├── read_that_hits_two_cache_shards │ │ │ ├── read_that_hits_two_cache_shards_with_first_read_at_offset │ │ │ ├── small_read │ │ │ └── small_read_with_first_read_at_offset │ ├── testdata │ │ ├── provider │ │ │ ├── local │ │ │ ├── local_readahead │ │ │ ├── shared_attach │ │ │ ├── shared_attach_after_unref │ │ │ ├── shared_attach_multi │ │ │ ├── shared_basic │ │ │ ├── shared_no_ref │ │ │ ├── shared_readahead │ │ │ └── shared_remove │ │ ├── readahead │ │ └── remote_read_handle │ ├── vfs.go │ ├── vfs_readable.go │ └── vfs_writable.go ├── remote │ ├── factory.go │ ├── localfs.go │ ├── logging.go │ ├── mem.go │ └── storage.go └── test_utils.go ├── obsolete_files.go ├── obsolete_files_test.go ├── open.go ├── open_test.go ├── options.go ├── options_test.go ├── overlap.go ├── pacer.go ├── pacer_test.go ├── range_del_test.go ├── range_keys.go ├── rangedel └── rangedel.go ├── rangekey └── rangekey.go ├── read_compaction_queue.go ├── read_state.go ├── read_state_test.go ├── record ├── log_writer.go ├── log_writer_test.go ├── record.go ├── record_test.go ├── rotation.go ├── rotation_test.go └── testdata │ ├── rotation │ └── walSync ├── replay ├── replay.go ├── replay_test.go ├── sampled_metric.go ├── sampled_metric_test.go ├── testdata │ ├── collect │ │ ├── clean_before_copy │ │ ├── copy_before_clean │ │ ├── manifest_copying │ │ └── start_stop │ ├── corpus │ │ ├── findManifestStart │ │ ├── findWorkloadFiles │ │ ├── high_read_amp │ │ ├── simple │ │ └── simple_val_sep │ ├── flushed_sstable_keys │ ├── replay │ ├── replay_paced │ ├── replay_val_sep │ └── sampled_metric ├── workload_capture.go └── workload_capture_test.go ├── scan_internal.go ├── scan_internal_test.go ├── scripts ├── changed-go-pkgs.sh ├── check-workspace-clean.sh ├── code-coverage-publish.sh ├── code-coverage.sh ├── pr-codecov-run-tests.sh ├── run-crossversion-meta.sh └── stress.sh ├── snapshot.go ├── snapshot_test.go ├── sstable ├── attributes.go ├── blob │ ├── blob.go │ ├── blob_test.go │ ├── blocks.go │ ├── blocks_test.go │ ├── doc.go │ ├── fetcher.go │ ├── fetcher_test.go │ ├── handle.go │ └── testdata │ │ ├── index_block │ │ ├── value_fetcher │ │ └── writer ├── block │ ├── block.go │ ├── buffer_pool.go │ ├── buffer_pool_test.go │ ├── category_stats.go │ ├── compression.go │ ├── compression_test.go │ ├── compressor.go │ ├── flush_governor.go │ ├── flush_governor_internal_test.go │ ├── flush_governor_test.go │ ├── kv.go │ ├── kv_test.go │ ├── testdata │ │ ├── buffer_pool │ │ └── flush_governor │ ├── transforms.go │ └── transforms_test.go ├── block_property.go ├── block_property_obsolete.go ├── block_property_test.go ├── block_property_test_utils.go ├── colblk │ ├── base.go │ ├── bitmap.go │ ├── bitmap_test.go │ ├── block.go │ ├── block_test.go │ ├── column.go │ ├── data_block.go │ ├── data_block_test.go │ ├── endian.go │ ├── endian_big.go │ ├── endian_little.go │ ├── endian_test.go │ ├── index_block.go │ ├── index_block_test.go │ ├── key_value_block.go │ ├── key_value_block_test.go │ ├── keyspan.go │ ├── keyspan_test.go │ ├── prefix_bytes.go │ ├── prefix_bytes_test.go │ ├── raw_bytes.go │ ├── raw_bytes_test.go │ ├── testdata │ │ ├── bitmap │ │ ├── block_writer │ │ ├── data_block │ │ │ ├── bundle_search │ │ │ ├── external_value │ │ │ ├── finish_without_final_row │ │ │ ├── next_prefix │ │ │ ├── rewrite_suffixes │ │ │ ├── simple │ │ │ └── transforms │ │ ├── index_block │ │ ├── keyspan_block │ │ ├── meta_index_block │ │ ├── prefix_bytes │ │ ├── properties_block │ │ ├── raw_bytes │ │ └── uints │ ├── uints.go │ ├── uints_decode.go │ ├── uints_encode.go │ ├── uints_test.go │ ├── unsafe_uints.go │ └── unsafe_uints_test.go ├── colblk_writer.go ├── colblk_writer_test.go ├── comparer.go ├── compressionanalyzer │ ├── block_analyzer.go │ ├── buckets.go │ ├── buckets_test.go │ ├── file_analyzer.go │ ├── file_analyzer_test.go │ ├── testdata │ │ ├── buckets │ │ └── file_analyzer │ ├── welford.go │ └── welford_test.go ├── copier.go ├── copier_test.go ├── data_test.go ├── filter.go ├── format.go ├── format_test.go ├── internal.go ├── layout.go ├── options.go ├── properties.go ├── properties_test.go ├── random_test.go ├── reader.go ├── reader_common.go ├── reader_iter.go ├── reader_iter_single_lvl.go ├── reader_iter_test.go ├── reader_iter_two_lvl.go ├── reader_test.go ├── rowblk │ ├── rowblk_64bit_test.go │ ├── rowblk_bench_test.go │ ├── rowblk_fragment_iter.go │ ├── rowblk_fragment_iter_test.go │ ├── rowblk_index_iter.go │ ├── rowblk_iter.go │ ├── rowblk_iter_test.go │ ├── rowblk_rewrite.go │ ├── rowblk_writer.go │ ├── rowblk_writer_test.go │ ├── testdata │ │ ├── rowblk_fragment_iter │ │ └── rowblk_iter │ └── unsafe_test.go ├── rowblk_writer.go ├── suffix_rewriter.go ├── suffix_rewriter_test.go ├── table.go ├── table_test.go ├── test_fixtures.go ├── test_utils.go ├── testdata │ ├── .gitignore │ ├── Makefile │ ├── block_properties │ ├── block_properties_boundlimited │ ├── columnar_writer │ │ └── simple_binary │ ├── copy_span │ ├── h-no-compression-sst │ │ └── 000012.sst │ ├── h-no-compression-two-level-index-sst │ │ └── 000003.sst │ ├── h-table-bloom-no-compression-prefix-extractor-no-whole-key-filter-sst │ │ └── 000013.sst │ ├── h-table-bloom-no-compression-sst │ │ └── 000011.sst │ ├── h-table-bloom-sst │ │ └── 000010.sst │ ├── h-zstd-compression-sst │ │ └── 000004.sst │ ├── h.txt │ ├── hamlet-act-1.txt │ ├── hamlet-sst │ │ └── 000002.sst │ ├── hamletreader │ │ └── hamlet_iter │ ├── make-table.go │ ├── prefixreader │ │ ├── bloom │ │ └── iter │ ├── reader │ │ ├── bloom │ │ └── iter │ ├── reader_attributes │ ├── reader_bpf │ │ ├── Pebblev2 │ │ │ └── iter │ │ └── Pebblev3 │ │ │ └── iter │ ├── reader_hide_obsolete │ │ └── iter │ ├── readerstats_LevelDB │ │ └── iter │ ├── readerstats_Pebblev3 │ │ └── iter │ ├── rewriter │ ├── rewriter_v3 │ ├── rewriter_v5 │ ├── rewriter_v6 │ ├── rewriter_v7 │ ├── size_estimate │ ├── virtual_reader_iter │ ├── virtual_reader_props │ ├── writer │ ├── writer_blob_value_handles │ ├── writer_range_keys │ ├── writer_v3 │ ├── writer_v5 │ ├── writer_v6 │ ├── writer_v7 │ └── writer_value_blocks ├── valblk │ ├── reader.go │ ├── valblk.go │ ├── valblk_test.go │ └── writer.go ├── values.go ├── virtual │ └── virtual_reader_params.go ├── write_queue.go ├── writer.go ├── writer_fixture_test.go ├── writer_rangekey_test.go └── writer_test.go ├── table_stats.go ├── table_stats_test.go ├── testdata ├── .gitignore ├── Makefile ├── batch_get ├── batch_range_ops ├── batch_reuse ├── checkpoint ├── checkpoint_shared ├── cleaner ├── compaction │ ├── compaction_cancellation │ ├── file_boundaries_delsized │ ├── multilevel │ ├── range_keys │ ├── score_compaction_picked_before_manual │ ├── set_with_del_sstable_Pebblev4 │ ├── set_with_del_sstable_Pebblev5 │ ├── set_with_del_sstable_Pebblev6 │ ├── set_with_del_sstable_Pebblev7 │ ├── singledel_set_with_del │ └── value_separation ├── compaction_allow_zero_seqnum ├── compaction_check_ordering ├── compaction_corruption ├── compaction_delete_only_hints ├── compaction_error_on_user_key_overlap ├── compaction_expand_inputs ├── compaction_output_file_size ├── compaction_output_level ├── compaction_picker_L0 ├── compaction_picker_concurrency ├── compaction_picker_estimated_debt ├── compaction_picker_level_max_bytes ├── compaction_picker_pick_file ├── compaction_picker_read_triggered ├── compaction_picker_scores ├── compaction_picker_target_level ├── compaction_read_triggered ├── compaction_setup_inputs ├── compaction_setup_inputs_multilevel_dummy ├── compaction_setup_inputs_multilevel_write_amp ├── compaction_tombstones ├── concurrency_limit_scheduler ├── concurrent_excise ├── copy_checkpoint_options ├── db-stage-1 │ ├── 000002.log │ ├── CURRENT │ ├── LOCK │ ├── MANIFEST-000001 │ ├── OPTIONS-000003 │ ├── marker.format-version.000012.013 │ └── marker.manifest.000001.MANIFEST-000001 ├── db-stage-2 │ ├── 000002.log │ ├── CURRENT │ ├── LOCK │ ├── MANIFEST-000001 │ ├── OPTIONS-000003 │ ├── marker.format-version.000012.013 │ └── marker.manifest.000001.MANIFEST-000001 ├── db-stage-3 │ ├── 000004.sst │ ├── 000005.log │ ├── CURRENT │ ├── LOCK │ ├── MANIFEST-000001 │ ├── MANIFEST-000006 │ ├── OPTIONS-000007 │ ├── marker.format-version.000012.013 │ └── marker.manifest.000002.MANIFEST-000006 ├── db-stage-4 │ ├── 000004.sst │ ├── 000005.log │ ├── CURRENT │ ├── LOCK │ ├── MANIFEST-000001 │ ├── MANIFEST-000006 │ ├── OPTIONS-000007 │ ├── marker.format-version.000012.013 │ └── marker.manifest.000002.MANIFEST-000006 ├── delete_range ├── determinism ├── download_cursor ├── download_task ├── event_listener ├── excise ├── excise_bounds ├── external_iterator ├── flushable_batch ├── flushable_ingest ├── indexed_batch_mutation ├── ingest ├── ingest_external ├── ingest_load ├── ingest_memtable_overlaps ├── ingest_shared ├── ingest_shared_lower ├── ingest_sort_and_verify ├── ingest_target_level ├── ingest_update_seqnums ├── ingested_flushable_api ├── internal_iter_bounds ├── internal_iter_next ├── iter_histories │ ├── blob_references │ ├── clone │ ├── errors │ ├── internal_next │ ├── iter_optimizations │ ├── lazy_combined_iteration │ ├── merge │ ├── next_prefix │ ├── prefix_iteration │ ├── range_key_changed │ ├── range_key_masking │ ├── range_keys_simple │ ├── set_options │ ├── skip_point │ ├── stats_no_invariants │ └── with_limit ├── iterator ├── iterator_block_interval_filter ├── iterator_bounds_lifetimes ├── iterator_next_prev ├── iterator_read_sampling ├── iterator_seek_opt ├── iterator_seek_opt_errors ├── iterator_stats ├── iterator_table_filter ├── large_keys ├── level_checker ├── level_iter ├── level_iter_boundaries ├── level_iter_seek ├── lsm_view ├── make-db.go ├── manual_flush ├── marked_for_compaction ├── mem_table ├── merging_iter ├── merging_iter_seek ├── metrics ├── mkdir_all_and_sync_parents ├── open_wal_failover ├── peek ├── point_collapsing_iter ├── range_del ├── read_compaction_queue ├── scan_internal ├── scan_statistics ├── singledel_manual_compaction ├── snapshot ├── sstable_key_compare ├── table_stats ├── table_stats_deletion_iter ├── tracing ├── value_separation_policy ├── version_check_consistency └── version_set ├── tool ├── data │ ├── d3.v5.min.js │ ├── lsm.css │ └── lsm.js ├── data_test.go ├── db.go ├── db_analyze_data.go ├── db_analyze_data_test.go ├── db_io_bench.go ├── db_test.go ├── find.go ├── find_test.go ├── logs │ ├── compaction.go │ ├── compaction_test.go │ ├── testdata │ │ ├── compactions-23-1 │ │ └── compactions-latest │ └── tool.go ├── lsm.go ├── lsm_data.go ├── make_incorrect_manifests.go ├── make_lsm_data.sh ├── make_test_find_db.go ├── make_test_find_db_val_sep.go ├── make_test_remotecat.go ├── make_test_sstables.go ├── manifest.go ├── manifest_test.go ├── remotecat.go ├── remotecat_test.go ├── sstable.go ├── sstable_test.go ├── testdata │ ├── MANIFEST-invalid │ ├── REMOTE-OBJ-CATALOG │ ├── bad-magic-sst │ │ └── 000015.sst │ ├── broken-external-db │ │ ├── 000004.log │ │ ├── 000005.sst │ │ ├── 000006.log │ │ ├── 000007.sst │ │ ├── LOCK │ │ ├── MANIFEST-000001 │ │ ├── OPTIONS-000003 │ │ ├── REMOTE-OBJ-CATALOG-000001 │ │ ├── marker.format-version.000004.017 │ │ ├── marker.manifest.000001.MANIFEST-000001 │ │ └── marker.remote-obj-catalog.000001.REMOTE-OBJ-CATALOG-000001 │ ├── corrupt-options-db │ │ └── OPTIONS-000002 │ ├── corrupted-sst │ │ └── 000003.sst │ ├── cr-schema-db │ │ ├── 000002.log │ │ ├── 000004.log │ │ ├── 000005.sst │ │ ├── LOCK │ │ ├── MANIFEST-000001 │ │ ├── OPTIONS-000003 │ │ ├── marker.format-version.000007.020 │ │ └── marker.manifest.000001.MANIFEST-000001 │ ├── cr-schema-sst │ │ └── 000014.sst │ ├── db_check │ ├── db_checkpoint │ ├── db_excise │ ├── db_get_set │ ├── db_lsm │ ├── db_properties │ ├── db_scan │ ├── db_space │ ├── db_upgrade │ ├── find │ ├── find-db │ │ ├── 000009.log │ │ ├── 000011.sst │ │ ├── CURRENT │ │ ├── LOCK │ │ ├── MANIFEST-000001 │ │ ├── OPTIONS-000003 │ │ ├── archive │ │ │ ├── 000002.log │ │ │ ├── 000004.log │ │ │ ├── 000005.sst │ │ │ ├── 000006.sst │ │ │ ├── 000007.sst │ │ │ ├── 000008.sst │ │ │ └── 000010.sst │ │ ├── marker.format-version.000012.013 │ │ └── marker.manifest.000001.MANIFEST-000001 │ ├── find-mixed │ │ ├── 000001.sst │ │ └── 000002.sst │ ├── find-val-sep-db │ │ ├── 000004.log │ │ ├── 000005.sst │ │ ├── 000006.blob │ │ ├── 000007.log │ │ ├── 000008.sst │ │ ├── 000009.blob │ │ ├── LOCK │ │ ├── MANIFEST-000001 │ │ ├── OPTIONS-000003 │ │ ├── marker.format-version.000009.022 │ │ └── marker.manifest.000001.MANIFEST-000001 │ ├── make-fixtures.go │ ├── manifest_dump │ ├── manifest_summarize │ ├── mixed │ │ ├── 000002.log │ │ ├── 000004.log │ │ ├── 000005.sst │ │ ├── CURRENT │ │ ├── LOCK │ │ ├── MANIFEST-000001 │ │ ├── OPTIONS-000003 │ │ ├── main.go │ │ ├── marker.format-version.000015.016 │ │ └── marker.manifest.000001.MANIFEST-000001 │ ├── out-of-order-sst │ │ └── 000001.sst │ ├── remotecat │ ├── sstable_check │ ├── sstable_layout │ ├── sstable_properties │ ├── sstable_scan │ ├── sstable_space │ └── wal_dump ├── tool.go ├── util.go ├── wal.go └── wal_test.go ├── value_separation.go ├── value_separation_test.go ├── version_set.go ├── version_set_test.go ├── vfs ├── atomicfs │ ├── README.md │ ├── marker.go │ ├── marker_test.go │ └── testdata │ │ └── marker ├── clone.go ├── default_linux.go ├── default_unix.go ├── default_windows.go ├── disk_full.go ├── disk_full_test.go ├── disk_health.go ├── disk_health_test.go ├── disk_usage_linux.go ├── disk_usage_netbsd.go ├── disk_usage_openbsd.go ├── disk_usage_unix.go ├── disk_usage_windows.go ├── errorfs │ ├── dsl.go │ ├── errorfs.go │ ├── errorfs_test.go │ ├── latency.go │ └── testdata │ │ └── errorfs ├── errors_unix.go ├── errors_unix_test.go ├── errors_windows.go ├── fadvise_generic.go ├── fadvise_linux.go ├── fd_test.go ├── file_lock_generic.go ├── file_lock_test.go ├── file_lock_unix.go ├── file_lock_windows.go ├── logging_fs.go ├── mem_fs.go ├── mem_fs_test.go ├── syncing_file.go ├── syncing_file_linux_test.go ├── syncing_file_test.go ├── testdata │ ├── memfs_basics │ ├── memfs_crashable │ ├── memfs_list │ ├── memfs_lock │ └── vfs ├── vfs.go ├── vfs_test.go └── vfstest │ ├── open_files.go │ ├── open_files_test.go │ └── vfstest.go └── wal ├── failover_manager.go ├── failover_manager_test.go ├── failover_writer.go ├── failover_writer_test.go ├── log_recycler.go ├── log_recycler_test.go ├── reader.go ├── reader_test.go ├── standalone_manager.go ├── testdata ├── dir_prober ├── failover_writer │ ├── blocking │ └── errors ├── list ├── manager_failover └── reader └── wal.go /.editorconfig: -------------------------------------------------------------------------------- 1 | # See http://editorconfig.org 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | 8 | # For non-go files, we indent with two spaces. In go files we indent 9 | # with tabs but still set indent_size to control the github web viewer. 10 | indent_size=2 11 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @cockroachdb/storage 2 | -------------------------------------------------------------------------------- /.github/workflows/code-cover-publish.yaml: -------------------------------------------------------------------------------- 1 | name: PR code coverage (publish) 2 | 3 | on: 4 | workflow_run: 5 | workflows: [ "PR code coverage (generate)" ] 6 | types: [ "completed" ] 7 | 8 | 9 | jobs: 10 | # This job downloads the artifacts generated by the code-cover-gen job and 11 | # uploads them to a GCS bucket, from where Reviewable can access them. 12 | code-cover-publish: 13 | runs-on: ubuntu-latest 14 | if: > 15 | github.event.workflow_run.event == 'pull_request' && 16 | github.event.workflow_run.conclusion == 'success' 17 | steps: 18 | - name: 'Download artifact' 19 | uses: actions/download-artifact@v4 20 | with: 21 | name: cover 22 | path: cover 23 | github-token: ${{ github.token }} 24 | run-id: ${{ github.event.workflow_run.id }} 25 | 26 | - name: 'Authenticate to Google Cloud' 27 | uses: 'google-github-actions/auth@v1' 28 | with: 29 | credentials_json: '${{ secrets.CODECOVER_SERVICE_ACCOUNT_KEY }}' 30 | 31 | - name: 'Upload to GCS' 32 | uses: 'google-github-actions/upload-cloud-storage@v1' 33 | with: 34 | path: 'cover' 35 | glob: '**/cover-*.json' 36 | parent: false 37 | destination: 'crl-codecover-public/pr-pebble/' 38 | process_gcloudignore: false 39 | -------------------------------------------------------------------------------- /.github/workflows/nightlies.yaml: -------------------------------------------------------------------------------- 1 | name: Nightlies (master) 2 | 3 | on: 4 | schedule: 5 | - cron: '00 10 * * * ' # 10am UTC daily 6 | workflow_dispatch: 7 | 8 | jobs: 9 | tests: 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | go: [ '1.23', '1.24' ] 14 | uses: ./.github/workflows/tests.yaml 15 | with: 16 | sha: ${{ github.sha }} 17 | file_issue_branch: 'master' 18 | go_version: ${{ matrix.go }} 19 | 20 | s390x: 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | go: [ '1.23', '1.24' ] 25 | uses: ./.github/workflows/s390x.yaml 26 | with: 27 | sha: ${{ github.sha }} 28 | file_issue_branch: 'master' 29 | go_version: ${{ matrix.go }} 30 | 31 | stress: 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | go: [ '1.23', '1.24' ] 36 | uses: ./.github/workflows/stress.yaml 37 | with: 38 | sha: ${{ github.sha }} 39 | file_issue_branch: 'master' 40 | go_version: ${{ matrix.go }} 41 | 42 | instrumented: 43 | strategy: 44 | fail-fast: false 45 | matrix: 46 | go: [ '1.23', '1.24' ] 47 | uses: ./.github/workflows/instrumented.yaml 48 | with: 49 | sha: ${{ github.sha }} 50 | file_issue_branch: 'master' 51 | go_version: ${{ matrix.go }} 52 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Mark stale issues and pull requests 2 | 3 | on: 4 | schedule: 5 | - cron: "0 11 * * 1-4" 6 | workflow_dispatch: 7 | 8 | jobs: 9 | stale: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | issues: write 13 | pull-requests: write 14 | steps: 15 | - uses: actions/stale@v3 16 | with: 17 | operations-per-run: 1000 18 | repo-token: ${{ secrets.GITHUB_TOKEN }} 19 | stale-issue-message: | 20 | We have marked this issue as stale because it has been inactive for 21 | 18 months. If this issue is still relevant, removing the stale label 22 | or adding a comment will keep it active. Otherwise, we'll close it 23 | in 10 days to keep the issue queue tidy. Thank you for your 24 | contribution to Pebble! 25 | stale-pr-message: 'Stale pull request message' 26 | stale-issue-label: 'no-issue-activity' 27 | stale-pr-label: 'no-pr-activity' 28 | close-issue-label: 'X-stale' 29 | close-pr-label: 'X-stale' 30 | # Disable this for PR's, by setting a very high bar 31 | days-before-pr-stale: 99999 32 | days-before-issue-stale: 540 33 | days-before-close: 10 34 | exempt-issue-labels: 'X-nostale' 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Github action artifacts. 2 | artifacts 3 | # Profiling artifacts. 4 | cpu.*.prof 5 | heap.prof 6 | mutex.prof 7 | coverprofile.out 8 | # Testing artifacts 9 | meta.*.test 10 | 11 | # Bazel files, generated with 'make gen-bazel'. 12 | /WORKSPACE 13 | BUILD.bazel 14 | 15 | 16 | -------------------------------------------------------------------------------- /batchrepr/testdata/writer: -------------------------------------------------------------------------------- 1 | init 2 | 0000000000000000 01000000 # Seqnum = 0, Count = 1 3 | 00 0161 # DEL "a" 4 | ---- 5 | 6 | read-header 7 | ---- 8 | [seqNum=0,count=1] 9 | 10 | set-seqnum 1052 11 | ---- 12 | 00-08: x 1c04000000000000 # seqnum=1052 13 | 08-12: x 01000000 # count=1 14 | 12-15: x 00 0161 # DEL "a" 15 | 16 | read-header 17 | ---- 18 | [seqNum=1052,count=1] 19 | 20 | set-count 20 21 | ---- 22 | 00-08: x 1c04000000000000 # seqnum=1052 23 | 08-12: x 14000000 # count=20 24 | 12-15: x 00 0161 # DEL "a" 25 | 26 | read-header 27 | ---- 28 | [seqNum=1052,count=20] 29 | -------------------------------------------------------------------------------- /batchrepr/writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package batchrepr 6 | 7 | import ( 8 | "encoding/binary" 9 | 10 | "github.com/cockroachdb/pebble/internal/base" 11 | ) 12 | 13 | // SetSeqNum mutates the provided batch representation, storing the provided 14 | // sequence number in its header. The provided byte slice must already be at 15 | // least HeaderLen bytes long or else SetSeqNum will panic. 16 | func SetSeqNum(repr []byte, seqNum base.SeqNum) { 17 | binary.LittleEndian.PutUint64(repr[:countOffset], uint64(seqNum)) 18 | } 19 | 20 | // SetCount mutates the provided batch representation, storing the provided 21 | // count in its header. The provided byte slice must already be at least 22 | // HeaderLen bytes long or else SetCount will panic. 23 | func SetCount(repr []byte, count uint32) { 24 | binary.LittleEndian.PutUint32(repr[countOffset:HeaderLen], count) 25 | } 26 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | import "github.com/cockroachdb/pebble/internal/cache" 8 | 9 | // Cache exports the cache.Cache type. 10 | type Cache = cache.Cache 11 | 12 | // NewCache creates a new cache of the specified size. Memory for the cache is 13 | // allocated on demand, not during initialization. The cache is created with a 14 | // reference count of 1. Each DB it is associated with adds a reference, so the 15 | // creator of the cache should usually release their reference after the DB is 16 | // created. 17 | // 18 | // c := pebble.NewCache(...) 19 | // defer c.Unref() 20 | // d, err := pebble.Open(pebble.Options{Cache: c}) 21 | func NewCache(size int64) *cache.Cache { 22 | return cache.New(size) 23 | } 24 | -------------------------------------------------------------------------------- /cmd/pebble/.gitignore: -------------------------------------------------------------------------------- 1 | pebble 2 | -------------------------------------------------------------------------------- /cmd/pebble/fsbenchlist.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/cockroachdb/errors" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var listFsBench = &cobra.Command{ 11 | Use: "list [] [] ...", 12 | Short: "List the available file system benchmarks.", 13 | Long: ` 14 | List the available file system benchmarks. If no is supplied 15 | as an argument, then all the available benchmark names are printed. 16 | If one or more s are supplied as arguments, then the benchmark 17 | descriptions are printed out for those names. 18 | `, 19 | RunE: runListFsBench, 20 | } 21 | 22 | func runListFsBench(_ *cobra.Command, args []string) error { 23 | if len(args) == 0 { 24 | fmt.Println("Available benchmarks:") 25 | for name := range benchmarks { 26 | fmt.Println(name) 27 | } 28 | } else { 29 | for _, v := range args { 30 | benchStruct, ok := benchmarks[v] 31 | if !ok { 32 | return errors.Errorf("trying to print out the description for unknown benchmark: %s", v) 33 | } 34 | fmt.Println("Name:", benchStruct.name) 35 | fmt.Println("Description:", benchStruct.description) 36 | } 37 | } 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /cmd/pebble/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package main 6 | 7 | func encodeUint32Ascending(b []byte, v uint32) []byte { 8 | return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) 9 | } 10 | 11 | func encodeUint64Ascending(b []byte, v uint64) []byte { 12 | return append(b, 13 | byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), 14 | byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) 15 | } 16 | -------------------------------------------------------------------------------- /comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | import "github.com/cockroachdb/pebble/internal/base" 8 | 9 | // Compare exports the base.Compare type. 10 | type Compare = base.Compare 11 | 12 | // Equal exports the base.Equal type. 13 | type Equal = base.Equal 14 | 15 | // AbbreviatedKey exports the base.AbbreviatedKey type. 16 | type AbbreviatedKey = base.AbbreviatedKey 17 | 18 | // Separator exports the base.Separator type. 19 | type Separator = base.Separator 20 | 21 | // Successor exports the base.Successor type. 22 | type Successor = base.Successor 23 | 24 | // Split exports the base.Split type. 25 | type Split = base.Split 26 | 27 | // Comparer exports the base.Comparer type. 28 | type Comparer = base.Comparer 29 | 30 | // DefaultComparer exports the base.DefaultComparer variable. 31 | var DefaultComparer = base.DefaultComparer 32 | 33 | // CheckComparer exports the base.CheckComparer type. 34 | var CheckComparer = base.CheckComparer 35 | -------------------------------------------------------------------------------- /db_internals.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | // JobID identifies a job (like a compaction). Job IDs are passed to event 8 | // listener notifications and act as a mechanism for tying together the events 9 | // and log messages for a single job such as a flush, compaction, or file 10 | // ingestion. Job IDs are not serialized to disk or used for correctness. 11 | type JobID int 12 | 13 | // newJobIDLocked returns a new JobID; DB.mu must be held. 14 | func (d *DB) newJobIDLocked() JobID { 15 | res := d.mu.nextJobID 16 | d.mu.nextJobID++ 17 | return res 18 | } 19 | 20 | func (d *DB) newJobID() JobID { 21 | d.mu.Lock() 22 | defer d.mu.Unlock() 23 | return d.newJobIDLocked() 24 | } 25 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble_test 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | 11 | "github.com/cockroachdb/pebble" 12 | "github.com/cockroachdb/pebble/vfs" 13 | ) 14 | 15 | func Example() { 16 | db, err := pebble.Open("", &pebble.Options{FS: vfs.NewMem()}) 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | key := []byte("hello") 21 | if err := db.Set(key, []byte("world"), pebble.Sync); err != nil { 22 | log.Fatal(err) 23 | } 24 | value, closer, err := db.Get(key) 25 | if err != nil { 26 | log.Fatal(err) 27 | } 28 | fmt.Printf("%s %s\n", key, value) 29 | if err := closer.Close(); err != nil { 30 | log.Fatal(err) 31 | } 32 | if err := db.Close(); err != nil { 33 | log.Fatal(err) 34 | } 35 | // Output: 36 | // hello world 37 | } 38 | -------------------------------------------------------------------------------- /internal/base/close_helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package base 6 | 7 | import "io" 8 | 9 | // CloseHelper wraps an io.Closer in a wrapper that ignores extra calls to 10 | // Close. It is useful to ensure cleanup in error paths (using defer) without 11 | // double-closing. 12 | func CloseHelper(closer io.Closer) io.Closer { 13 | return &closeHelper{ 14 | Closer: closer, 15 | } 16 | } 17 | 18 | type closeHelper struct { 19 | Closer io.Closer 20 | } 21 | 22 | // Close the underlying Closer, unless it was already closed. 23 | func (h *closeHelper) Close() error { 24 | closer := h.Closer 25 | if closer == nil { 26 | return nil 27 | } 28 | h.Closer = nil 29 | return closer.Close() 30 | } 31 | -------------------------------------------------------------------------------- /internal/binfmt/hexdump.go: -------------------------------------------------------------------------------- 1 | package binfmt 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | ) 9 | 10 | // HexDump returns a string representation of the data in a hex dump format. 11 | // The width is the number of bytes per line. 12 | func HexDump(data []byte, width int, includeOffsets bool) string { 13 | var buf bytes.Buffer 14 | FHexDump(&buf, data, width, includeOffsets) 15 | return buf.String() 16 | } 17 | 18 | // FHexDump writes a hex dump of the data to w. 19 | func FHexDump(w io.Writer, data []byte, width int, includeOffsets bool) { 20 | offsetFormatWidth := len(fmt.Sprintf("%x", max(1, len(data)-1))) 21 | offsetFormatStr := "%0" + strconv.Itoa(offsetFormatWidth) + "x" 22 | for i := 0; i < len(data); i += width { 23 | if includeOffsets { 24 | fmt.Fprintf(w, offsetFormatStr+": ", i) 25 | } 26 | for j := 0; j < width; j++ { 27 | if j%4 == 0 { 28 | fmt.Fprint(w, " ") 29 | } 30 | if i+j >= len(data) { 31 | fmt.Fprintf(w, " ") 32 | } else { 33 | fmt.Fprintf(w, "%02x", data[i+j]) 34 | } 35 | } 36 | 37 | fmt.Fprint(w, " | ") 38 | for j := 0; j < width; j++ { 39 | if i+j >= len(data) { 40 | break 41 | } 42 | if j%4 == 0 { 43 | fmt.Fprint(w, " ") 44 | } 45 | if data[i+j] < 32 || data[i+j] > 126 { 46 | fmt.Fprint(w, ".") 47 | } else { 48 | fmt.Fprintf(w, "%c", data[i+j]) 49 | } 50 | } 51 | fmt.Fprintln(w) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /internal/binfmt/hexdump_test.go: -------------------------------------------------------------------------------- 1 | package binfmt 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/cockroachdb/datadriven" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestHexFmt(t *testing.T) { 13 | var width, offset, length int 14 | datadriven.RunTest(t, "testdata/hexdump", func(t *testing.T, d *datadriven.TestData) string { 15 | width = 16 16 | d.ScanArgs(t, "pos", &offset, &length) 17 | d.MaybeScanArgs(t, "width", &width) 18 | switch d.Cmd { 19 | case "read-file": 20 | path := d.CmdArgs[0].String() 21 | data, err := os.ReadFile(path) 22 | require.NoError(t, err) 23 | return HexDump(data[offset:offset+length], width, d.HasArg("include-offsets")) 24 | case "sequential": 25 | var size int 26 | d.ScanArgs(t, "size", &size) 27 | data := make([]byte, size) 28 | for i := 0; i < size; i++ { 29 | data[i] = byte(i) 30 | } 31 | return HexDump(data, width, d.HasArg("include-offsets")) 32 | default: 33 | return fmt.Sprintf("unknown command: %s", d.Cmd) 34 | } 35 | }) 36 | } 37 | -------------------------------------------------------------------------------- /internal/bitflip/bitflip.go: -------------------------------------------------------------------------------- 1 | package bitflip 2 | 3 | // CheckSliceForBitFlip flips bits in data to see if it matches the expected checksum. 4 | // Returns the index and bit if successful. 5 | func CheckSliceForBitFlip( 6 | data []byte, computeChecksum func([]byte) uint32, expectedChecksum uint32, 7 | ) (found bool, indexFound int, bitFound int) { 8 | // TODO(edward) This checking process likely can be made faster. 9 | iterationLimit := 40 * (1 << 10) // 40KB 10 | for i := 0; i < min(len(data), iterationLimit); i++ { 11 | foundFlip, bit := checkByteForFlip(data, i, computeChecksum, expectedChecksum) 12 | if foundFlip { 13 | return true, i, bit 14 | } 15 | } 16 | return false, 0, 0 17 | } 18 | 19 | func checkByteForFlip( 20 | data []byte, i int, computeChecksum func([]byte) uint32, expectedChecksum uint32, 21 | ) (found bool, bit int) { 22 | for bit := 0; bit < 8; bit++ { 23 | data[i] ^= (1 << bit) 24 | var computedChecksum = computeChecksum(data) 25 | data[i] ^= (1 << bit) 26 | if computedChecksum == expectedChecksum { 27 | return true, bit 28 | } 29 | } 30 | return false, 0 31 | } 32 | -------------------------------------------------------------------------------- /internal/buildtags/cgo_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !cgo 6 | 7 | package buildtags 8 | 9 | // Cgo is true if we were built with the "cgo" build tag. 10 | const Cgo = false 11 | -------------------------------------------------------------------------------- /internal/buildtags/cgo_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build cgo 6 | 7 | package buildtags 8 | 9 | // Cgo is true if we were built with the "cgo" build tag. 10 | const Cgo = true 11 | -------------------------------------------------------------------------------- /internal/buildtags/invariants_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !invariants 6 | 7 | package buildtags 8 | 9 | // Invariants indicates if the invariants tag is used. 10 | // See invariants.Enabled. 11 | const Invariants = false 12 | -------------------------------------------------------------------------------- /internal/buildtags/invariants_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build invariants 6 | 7 | package buildtags 8 | 9 | // Invariants indicates if the invariants tag is used. 10 | // See invariants.Enabled. 11 | const Invariants = true 12 | -------------------------------------------------------------------------------- /internal/buildtags/race_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !race 6 | 7 | package buildtags 8 | 9 | // Race is true if we were built with the "race" build tag. 10 | const Race = false 11 | -------------------------------------------------------------------------------- /internal/buildtags/race_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build race 6 | 7 | package buildtags 8 | 9 | // Race is true if we were built with the "race" build tag. 10 | const Race = true 11 | -------------------------------------------------------------------------------- /internal/buildtags/slow_build_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !race && !slowbuild 6 | 7 | package buildtags 8 | 9 | // SlowBuild is true if this is an instrumented testing build that is likely 10 | // to be significantly slower (like race or address sanitizer builds). 11 | // 12 | // Slow builds are either race builds or those built with a `slowbuild` tag. 13 | const SlowBuild = false 14 | -------------------------------------------------------------------------------- /internal/buildtags/slow_build_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build race || slowbuild 6 | 7 | package buildtags 8 | 9 | // SlowBuild is true if this is an instrumented testing build that is likely 10 | // to be significantly slower (like race or address sanitizer builds). 11 | // 12 | // Slow builds are either race builds or those built with a `slowbuild` tag. 13 | const SlowBuild = true 14 | -------------------------------------------------------------------------------- /internal/buildtags/tracing_off.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !tracing 6 | 7 | package buildtags 8 | 9 | // Tracing indicates if the tracing tag is used. 10 | // 11 | // This tag enables low-level tracing code in the block cache. 12 | const Tracing = false 13 | -------------------------------------------------------------------------------- /internal/buildtags/tracing_on.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build tracing 6 | 7 | package buildtags 8 | 9 | // Tracing indicates if the tracing tag is used. 10 | // 11 | // This tag enables low-level tracing code in the block cache. 12 | const Tracing = true 13 | -------------------------------------------------------------------------------- /internal/cache/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2018 Damian Gryski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /internal/cache/clockpro_normal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !tracing 6 | 7 | package cache 8 | 9 | func (c *Cache) trace(_ string, _ int64) {} 10 | -------------------------------------------------------------------------------- /internal/cache/clockpro_tracing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build tracing 6 | 7 | package cache 8 | 9 | import ( 10 | "fmt" 11 | "runtime/debug" 12 | ) 13 | 14 | func (c *Cache) trace(msg string, refs int64) { 15 | s := fmt.Sprintf("%s: refs=%d\n%s", msg, refs, debug.Stack()) 16 | c.tr.Lock() 17 | c.tr.msgs = append(c.tr.msgs, s) 18 | c.tr.Unlock() 19 | } 20 | -------------------------------------------------------------------------------- /internal/compact/testdata/frontiers: -------------------------------------------------------------------------------- 1 | # NB: The empty line in 'init' configures a frontier with no keys. It should 2 | # never be added to the heap. 3 | 4 | init 5 | b e j 6 | a p n z 7 | 8 | f 9 | ---- 10 | 11 | scan 12 | a b c d e f g h j i k l m n o p q r s t u v w x y z 13 | ---- 14 | a : { b: "b", p: "p", f: "f" } 15 | b : { e: "e", p: "p", f: "f" } 16 | c : { e: "e", p: "p", f: "f" } 17 | d : { e: "e", p: "p", f: "f" } 18 | e : { f: "f", p: "p", j: "j" } 19 | f : { j: "j", p: "p" } 20 | g : { j: "j", p: "p" } 21 | h : { j: "j", p: "p" } 22 | j : { p: "p" } 23 | i : { p: "p" } 24 | k : { p: "p" } 25 | l : { p: "p" } 26 | m : { p: "p" } 27 | n : { p: "p" } 28 | o : { p: "p" } 29 | p : { z: "z" } 30 | q : { z: "z" } 31 | r : { z: "z" } 32 | s : { z: "z" } 33 | t : { z: "z" } 34 | u : { z: "z" } 35 | v : { z: "z" } 36 | w : { z: "z" } 37 | x : { z: "z" } 38 | y : { z: "z" } 39 | z : { } 40 | 41 | scan 42 | z 43 | ---- 44 | z : { } 45 | 46 | scan 47 | a z 48 | ---- 49 | a : { b: "b", p: "p", f: "f" } 50 | z : { } 51 | 52 | scan 53 | e 54 | ---- 55 | e : { f: "f", p: "p", j: "j" } 56 | 57 | # Test duplicate user keys within a frontier and across individual frontiers. 58 | 59 | init 60 | b e e g 61 | c e z 62 | ---- 63 | 64 | scan 65 | a c d f z 66 | ---- 67 | a : { b: "b", c: "c" } 68 | c : { e: "e", e: "e" } 69 | d : { e: "e", e: "e" } 70 | f : { g: "g", z: "z" } 71 | z : { } 72 | -------------------------------------------------------------------------------- /internal/compact/testdata/range_del_compactor: -------------------------------------------------------------------------------- 1 | # We retain one key per snapshot plus the top one. 2 | compact snapshots=(5,10,15) in-use-key-ranges=(a-z) 3 | a-c:{(#18,RANGEDEL) (#17,RANGEDEL) (#15,RANGEDEL) (#13,RANGEDEL) (#11,RANGEDEL) (#9,RANGEDEL) (#7,RANGEDEL) (#4,RANGEDEL) (#1,RANGEDEL)} 4 | ---- 5 | a-c:{(#18,RANGEDEL) (#13,RANGEDEL) (#9,RANGEDEL) (#4,RANGEDEL)} 6 | 7 | # Same as before, but elide in the last stripe. 8 | compact snapshots=(5,10,15) 9 | a-c:{(#18,RANGEDEL) (#17,RANGEDEL) (#15,RANGEDEL) (#13,RANGEDEL) (#11,RANGEDEL) (#9,RANGEDEL) (#7,RANGEDEL) (#4,RANGEDEL) (#1,RANGEDEL)} 10 | ---- 11 | a-c:{(#18,RANGEDEL) (#13,RANGEDEL) (#9,RANGEDEL)} 12 | 13 | # No snapshots = retain just the top one. 14 | compact in-use-key-ranges=(a-z) 15 | a-c:{(#4,RANGEDEL) (#2,RANGEDEL)} 16 | ---- 17 | a-c:{(#4,RANGEDEL)} 18 | 19 | # No snapshots and elision: retain nothing. 20 | compact 21 | a-c:{(#4,RANGEDEL) (#2,RANGEDEL)} 22 | ---- 23 | . 24 | -------------------------------------------------------------------------------- /internal/compact/testdata/tombstone_elider: -------------------------------------------------------------------------------- 1 | init elide-nothing 2 | ---- 3 | 4 | points 5 | a 6 | b 7 | ---- 8 | a: don't elide 9 | b: don't elide 10 | 11 | ranges 12 | a-d 13 | a-g 14 | b-f 15 | ---- 16 | a-d: don't elide 17 | a-g: don't elide 18 | b-f: don't elide 19 | 20 | init in-use-ranges 21 | c-e 22 | g-j 23 | u-v 24 | ---- 25 | 26 | points 27 | a 28 | b 29 | c 30 | d 31 | e 32 | h 33 | k 34 | u 35 | w 36 | ---- 37 | a: elide 38 | b: elide 39 | c: don't elide 40 | d: don't elide 41 | e: elide 42 | h: don't elide 43 | k: elide 44 | u: don't elide 45 | w: elide 46 | 47 | ranges 48 | a-b 49 | a-c 50 | a-d 51 | a-e 52 | b-j 53 | b-c 54 | e-f 55 | e-g 56 | f-z 57 | f-g 58 | m-o 59 | m-n 60 | m-u 61 | m-v 62 | ---- 63 | a-b: elide 64 | a-c: elide 65 | a-d: don't elide 66 | a-e: don't elide 67 | b-j: don't elide 68 | b-c: elide 69 | e-f: elide 70 | e-g: elide 71 | f-z: don't elide 72 | f-g: elide 73 | m-o: elide 74 | m-n: elide 75 | m-u: elide 76 | m-v: don't elide 77 | 78 | # No ranges in use (elide everything). 79 | init in-use-ranges 80 | ---- 81 | 82 | points 83 | a 84 | b 85 | z 86 | ---- 87 | a: elide 88 | b: elide 89 | z: elide 90 | 91 | ranges 92 | a-c 93 | b-z 94 | ---- 95 | a-c: elide 96 | b-z: elide 97 | -------------------------------------------------------------------------------- /internal/compression/minlz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package compression 6 | 7 | import ( 8 | "math/rand/v2" 9 | "testing" 10 | 11 | "github.com/minio/minlz" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestMinLZLargeBlock(t *testing.T) { 16 | for _, delta := range []int{-1, 0, 1, 1 << rand.IntN(24)} { 17 | b := make([]byte, minlz.MaxBlockSize+delta) 18 | for i := range b { 19 | b[i] = byte(i) 20 | } 21 | c := GetCompressor(MinLZFastest) 22 | defer c.Close() 23 | compressed := c.Compress(nil, b) 24 | d := GetDecompressor(MinLZ) 25 | decompressed := make([]byte, len(b)) 26 | defer d.Close() 27 | 28 | require.NoError(t, d.DecompressInto(decompressed, compressed)) 29 | require.Equal(t, b, decompressed) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /internal/compression/noop.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package compression 6 | 7 | type noopCompressor struct{} 8 | 9 | var _ Compressor = noopCompressor{} 10 | 11 | func (noopCompressor) Compress(dst, src []byte) []byte { 12 | return append(dst[:0], src...) 13 | } 14 | func (noopCompressor) Close() {} 15 | 16 | type noopDecompressor struct{} 17 | 18 | var _ Decompressor = noopDecompressor{} 19 | 20 | func (noopDecompressor) DecompressInto(dst, src []byte) error { 21 | dst = dst[:len(src)] 22 | copy(dst, src) 23 | return nil 24 | } 25 | 26 | func (noopDecompressor) DecompressedLen(b []byte) (decompressedLen int, err error) { 27 | return len(b), nil 28 | } 29 | 30 | func (noopDecompressor) Close() {} 31 | -------------------------------------------------------------------------------- /internal/compression/snappy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package compression 6 | 7 | import ( 8 | "github.com/cockroachdb/errors" 9 | "github.com/cockroachdb/pebble/internal/base" 10 | "github.com/golang/snappy" 11 | ) 12 | 13 | type snappyCompressor struct{} 14 | 15 | var _ Compressor = snappyCompressor{} 16 | 17 | func (snappyCompressor) Compress(dst, src []byte) []byte { 18 | dst = dst[:cap(dst):cap(dst)] 19 | return snappy.Encode(dst, src) 20 | } 21 | 22 | func (snappyCompressor) Close() {} 23 | 24 | type snappyDecompressor struct{} 25 | 26 | var _ Decompressor = snappyDecompressor{} 27 | 28 | func (snappyDecompressor) DecompressInto(buf, compressed []byte) error { 29 | result, err := snappy.Decode(buf, compressed) 30 | if err != nil { 31 | return err 32 | } 33 | if len(result) != len(buf) || (len(result) > 0 && &result[0] != &buf[0]) { 34 | return base.CorruptionErrorf("pebble: decompressed into unexpected buffer: %p != %p", 35 | errors.Safe(result), errors.Safe(buf)) 36 | } 37 | return nil 38 | } 39 | 40 | func (snappyDecompressor) DecompressedLen(b []byte) (decompressedLen int, err error) { 41 | return snappy.DecodedLen(b) 42 | } 43 | 44 | func (snappyDecompressor) Close() {} 45 | -------------------------------------------------------------------------------- /internal/constants/constants.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package constants 6 | 7 | const ( 8 | // oneIf64Bit is 1 on 64-bit platforms and 0 on 32-bit platforms. 9 | oneIf64Bit = ^uint(0) >> 63 10 | 11 | // MaxUint32OrInt returns min(MaxUint32, MaxInt), i.e 12 | // - MaxUint32 on 64-bit platforms; 13 | // - MaxInt on 32-bit platforms. 14 | // It is used when slices are limited to Uint32 on 64-bit platforms (the 15 | // length limit for slices is naturally MaxInt on 32-bit platforms). 16 | MaxUint32OrInt = (1<<31)< 0 { 28 | t.Fatalf("Bytes allocated %d, want 0", int(n)) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /internal/itertest/probe_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package itertest 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/cockroachdb/datadriven" 13 | "github.com/cockroachdb/pebble/internal/base" 14 | "github.com/cockroachdb/pebble/internal/testkeys" 15 | ) 16 | 17 | func TestProbes(t *testing.T) { 18 | var sb strings.Builder 19 | parser := NewParser() 20 | var iter base.InternalIterator 21 | datadriven.RunTest(t, "testdata/probes", func(t *testing.T, td *datadriven.TestData) string { 22 | sb.Reset() 23 | switch td.Cmd { 24 | case "new": 25 | iter = nil 26 | // new expects each line to describe a probe in the DSL. Each probe 27 | // is used to wrap the iterator resulting from attaching the probe 28 | // on the previous line. 29 | for _, l := range strings.Split(strings.TrimSpace(td.Input), "\n") { 30 | probe, err := parser.Parse(l) 31 | if err != nil { 32 | return fmt.Sprintf("parsing err: %s\n", err) 33 | } 34 | iter = Attach(iter, ProbeState{ 35 | Comparer: testkeys.Comparer, 36 | }, probe) 37 | } 38 | return sb.String() 39 | case "iter": 40 | return RunInternalIterCmd(t, td, iter, Verbose) 41 | default: 42 | return fmt.Sprintf("unrecognized command %q", td.Cmd) 43 | } 44 | }) 45 | } 46 | -------------------------------------------------------------------------------- /internal/keyspan/doc.go: -------------------------------------------------------------------------------- 1 | // Package keyspan provides general facilities for sorting, fragmenting and 2 | // iterating over spans of user keys. 3 | // 4 | // A Span represents a range of user key space with an inclusive start 5 | // key and exclusive end key. A span may hold any number of Keys which are 6 | // applied over the entirety of the span's keyspace. 7 | // 8 | // Spans are used within Pebble as an in-memory representation of range 9 | // deletion tombstones, and range key sets, unsets and deletes. Spans 10 | // are fragmented at overlapping key boundaries by the Fragmenter type. 11 | // This package's various iteration facilities require these 12 | // non-overlapping fragmented spans. 13 | // 14 | // Implementations that are specific to Pebble and use manifest types are 15 | // in the keyspanimpl subpackage. 16 | package keyspan 17 | -------------------------------------------------------------------------------- /internal/keyspan/get.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package keyspan 6 | 7 | import "github.com/cockroachdb/pebble/internal/base" 8 | 9 | // Get returns the newest span that contains the target key. If no span contains 10 | // the target key, an empty span is returned. The iterator must contain 11 | // fragmented spans: no span may overlap another. 12 | // 13 | // If an error occurs while seeking iter, a nil span and non-nil error is 14 | // returned. 15 | func Get(cmp base.Compare, iter FragmentIterator, key []byte) (*Span, error) { 16 | // NB: FragmentIterator.SeekGE moves the iterator to the first span covering 17 | // a key greater than or equal to the given key. This is equivalent to 18 | // seeking to the first span with an end key greater than the given key. 19 | iterSpan, err := iter.SeekGE(key) 20 | switch { 21 | case err != nil: 22 | return nil, err 23 | case iterSpan != nil && cmp(iterSpan.Start, key) > 0: 24 | return nil, nil 25 | default: 26 | return iterSpan, nil 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /internal/keyspan/iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package keyspan 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/cockroachdb/datadriven" 13 | "github.com/cockroachdb/pebble/internal/base" 14 | ) 15 | 16 | func TestIter(t *testing.T) { 17 | var spans []Span 18 | datadriven.RunTest(t, "testdata/iter", func(t *testing.T, d *datadriven.TestData) string { 19 | switch d.Cmd { 20 | case "define": 21 | spans = nil 22 | for _, line := range strings.Split(d.Input, "\n") { 23 | spans = append(spans, ParseSpan(line)) 24 | } 25 | return "" 26 | 27 | case "iter": 28 | iter := NewIter(base.DefaultComparer.Compare, spans) 29 | defer iter.Close() 30 | return RunFragmentIteratorCmd(iter, d.Input, nil) 31 | default: 32 | return fmt.Sprintf("unknown command: %s", d.Cmd) 33 | } 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /internal/keyspan/keyspanimpl/doc.go: -------------------------------------------------------------------------------- 1 | // Package keyspanimpl contains Pebble-specific implementations of keyspan 2 | // fragment iterators. 3 | package keyspanimpl 4 | -------------------------------------------------------------------------------- /internal/keyspan/logging_iter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package keyspan 6 | 7 | import ( 8 | "fmt" 9 | "regexp" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/cockroachdb/datadriven" 14 | "github.com/cockroachdb/pebble/internal/base" 15 | ) 16 | 17 | func TestLoggingIter(t *testing.T) { 18 | var spans []Span 19 | datadriven.RunTest(t, "testdata/logging_iter", func(t *testing.T, d *datadriven.TestData) string { 20 | switch d.Cmd { 21 | case "define": 22 | spans = nil 23 | for _, line := range strings.Split(d.Input, "\n") { 24 | spans = append(spans, ParseSpan(line)) 25 | } 26 | return "" 27 | 28 | case "iter": 29 | l := &base.InMemLogger{} 30 | var iter FragmentIterator 31 | iter = NewIter(base.DefaultComparer.Compare, spans) 32 | // Wrap with an assert as a very simple "stack" example. 33 | iter = Assert(iter, base.DefaultComparer.Compare) 34 | iter = InjectLogging(iter, l) 35 | RunFragmentIteratorCmd(iter, d.Input, nil) 36 | iter.Close() 37 | out := l.String() 38 | // Hide pointer values. 39 | r := regexp.MustCompile(`\(0x[0-9a-f]+\)`) 40 | return r.ReplaceAllString(out, "") 41 | 42 | default: 43 | return fmt.Sprintf("unknown command: %s", d.Cmd) 44 | } 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /internal/keyspan/seek.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package keyspan 6 | 7 | import "github.com/cockroachdb/pebble/internal/base" 8 | 9 | // SeekLE seeks to the span that contains or is before the target key. If an 10 | // error occurs while seeking iter, a nil span and non-nil error is returned. 11 | func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) (*Span, error) { 12 | // Seek to the smallest span that contains a key ≥ key. If some span 13 | // contains the key `key`, SeekGE will return it. 14 | iterSpan, err := iter.SeekGE(key) 15 | if err != nil { 16 | return nil, err 17 | } 18 | if iterSpan != nil && cmp(key, iterSpan.Start) >= 0 { 19 | return iterSpan, nil 20 | } 21 | // No span covers exactly `key`. Step backwards to move onto the largest 22 | // span < key. 23 | return iter.Prev() 24 | } 25 | -------------------------------------------------------------------------------- /internal/keyspan/testdata/assert_iter: -------------------------------------------------------------------------------- 1 | define 2 | a-c:{(#3,RANGEKEYUNSET,@5) (#2,RANGEKEYSET,@5,apples) (#1,RANGEKEYSET,@3,bananas)} 3 | c-d:{(#4,RANGEKEYSET,@3,bananas) (#3,RANGEKEYDEL)} 4 | d-e:{(#4,RANGEKEYSET,@3,bananas) (#4,RANGEKEYSET,@1,pineapple)} 5 | ---- 6 | 7 | assert-userkey-bounds 8 | a 9 | z 10 | ---- 11 | OK 12 | 13 | assert-userkey-bounds 14 | b 15 | z 16 | ---- 17 | lower bound "b" violated by span a-c:{(#3,RANGEKEYUNSET,@5) (#2,RANGEKEYSET,@5,apples) (#1,RANGEKEYSET,@3,bananas)}; wraps *keyspan.Iter 18 | 19 | assert-bounds 20 | a.SET.1 21 | z 22 | ---- 23 | lower bound a#1,SET violated by key a#3,RANGEKEYUNSET; wraps *keyspan.Iter 24 | 25 | assert-bounds 26 | a.SET.5 27 | z 28 | ---- 29 | OK 30 | 31 | assert-bounds 32 | b.SET.1 33 | z 34 | ---- 35 | lower bound "b" violated by span a-c:{(#3,RANGEKEYUNSET,@5) (#2,RANGEKEYSET,@5,apples) (#1,RANGEKEYSET,@3,bananas)}; wraps *keyspan.Iter 36 | 37 | assert-userkey-bounds 38 | a 39 | d 40 | ---- 41 | upper bound "d" violated by span d-e:{(#4,RANGEKEYSET,@3,bananas) (#4,RANGEKEYSET,@1,pineapple)}; wraps *keyspan.Iter 42 | 43 | assert-userkey-bounds 44 | a 45 | e 46 | ---- 47 | OK 48 | -------------------------------------------------------------------------------- /internal/keyspan/testdata/fragmenter_emit_order: -------------------------------------------------------------------------------- 1 | build 2 | a.RANGEKEYSET.5 b 3 | a.RANGEKEYSET.4 b 4 | a.RANGEKEYUNSET.6 b 5 | ---- 6 | a b: #6,RANGEKEYUNSET, #5,RANGEKEYSET, #4,RANGEKEYSET 7 | - 8 | 9 | # Test that keys emitted together that share the same sequence number are 10 | # ordered by key kind, descending. 11 | # NB: RANGEKEYSET > RANGEKEYUNSET > RANGEKEYDEL 12 | 13 | build 14 | b.RANGEKEYSET.5 c 15 | b.RANGEKEYUNSET.5 d 16 | b.RANGEKEYDEL.5 c 17 | ---- 18 | b c: #5,RANGEKEYSET, #5,RANGEKEYUNSET, #5,RANGEKEYDEL 19 | - 20 | c d: #5,RANGEKEYUNSET 21 | - 22 | -------------------------------------------------------------------------------- /internal/keyspan/testdata/iter: -------------------------------------------------------------------------------- 1 | define 2 | a-b:{(#2,SET) (#1,SET)} 3 | b-c:{(#2,SET) (#1,SET)} 4 | c-d:{(#2,SET) (#1,SET)} 5 | ---- 6 | 7 | iter 8 | seek-ge a 9 | seek-ge b 10 | seek-ge c 11 | seek-ge cat 12 | seek-ge d 13 | seek-lt a 14 | seek-lt b 15 | seek-lt c 16 | seek-lt cat 17 | seek-lt d 18 | seek-lt e 19 | ---- 20 | a-b:{(#2,SET) (#1,SET)} 21 | b-c:{(#2,SET) (#1,SET)} 22 | c-d:{(#2,SET) (#1,SET)} 23 | c-d:{(#2,SET) (#1,SET)} 24 | . 25 | . 26 | a-b:{(#2,SET) (#1,SET)} 27 | b-c:{(#2,SET) (#1,SET)} 28 | c-d:{(#2,SET) (#1,SET)} 29 | c-d:{(#2,SET) (#1,SET)} 30 | c-d:{(#2,SET) (#1,SET)} 31 | 32 | iter 33 | first 34 | next 35 | prev 36 | prev 37 | next 38 | next 39 | next 40 | prev 41 | next 42 | next 43 | prev 44 | ---- 45 | a-b:{(#2,SET) (#1,SET)} 46 | b-c:{(#2,SET) (#1,SET)} 47 | a-b:{(#2,SET) (#1,SET)} 48 | . 49 | a-b:{(#2,SET) (#1,SET)} 50 | b-c:{(#2,SET) (#1,SET)} 51 | c-d:{(#2,SET) (#1,SET)} 52 | b-c:{(#2,SET) (#1,SET)} 53 | c-d:{(#2,SET) (#1,SET)} 54 | . 55 | c-d:{(#2,SET) (#1,SET)} 56 | -------------------------------------------------------------------------------- /internal/keyspan/testdata/visible_at: -------------------------------------------------------------------------------- 1 | define 2 | a-b:{(#5,RANGEKEYSET) (#3,RANGEKEYSET)} 3 | ---- 4 | a-b:{(#5,RANGEKEYSET) (#3,RANGEKEYSET)} 5 | 6 | visible-at 7 | 6 8 | 5 9 | 4 10 | 3 11 | 2 12 | 1 13 | ---- 14 | 6 : true 15 | 5 : true 16 | 4 : true 17 | 3 : false 18 | 2 : false 19 | 1 : false 20 | 21 | # NB: #36028797018963996 and #36028797018963995 are sequence numbers with the 22 | # batch bit set. These keys should always be visible. 23 | 24 | define 25 | a-c:{(#36028797018963996,RANGEKEYSET) (#36028797018963995,RANGEKEYSET)} 26 | ---- 27 | a-c:{(#36028797018963996,RANGEKEYSET) (#36028797018963995,RANGEKEYSET)} 28 | 29 | visible-at 30 | 5 31 | 1 32 | ---- 33 | 5 : true 34 | 1 : true 35 | 36 | define 37 | a-c:{(#36028797018963996,RANGEKEYSET) (#36028797018963995,RANGEKEYSET) (#10,RANGEKEYSET) (#9,RANGEKEYSET) (#4,RANGEKEYSET) (#1,RANGEKEYSET)} 38 | ---- 39 | a-c:{(#36028797018963996,RANGEKEYSET) (#36028797018963995,RANGEKEYSET) (#10,RANGEKEYSET) (#9,RANGEKEYSET) (#4,RANGEKEYSET) (#1,RANGEKEYSET)} 40 | 41 | visible-at 42 | 12 43 | 10 44 | 8 45 | 7 46 | 4 47 | 3 48 | 2 49 | 1 50 | ---- 51 | 12: true 52 | 10: true 53 | 8 : true 54 | 7 : true 55 | 4 : true 56 | 3 : true 57 | 2 : true 58 | 1 : true 59 | -------------------------------------------------------------------------------- /internal/lint/lint.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package lint 6 | -------------------------------------------------------------------------------- /internal/lsmview/data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package lsmview 6 | 7 | // Data encodes the data necessary to generate an LSM diagram. 8 | type Data struct { 9 | // LSM levels in newest-to-oldest order. 10 | Levels []Level `json:"levels"` 11 | 12 | // Keys contains all table boundary keys, in sorted key order. 13 | Keys []string `json:"keys"` 14 | } 15 | 16 | // Level contains the data for a level of the LSM. 17 | type Level struct { 18 | Name string `json:"level_name"` 19 | Tables []Table `json:"tables"` 20 | } 21 | 22 | // Table contains the data for a table. 23 | type Table struct { 24 | Label string `json:"label"` 25 | Size uint64 `json:"size"` 26 | // SmallestKey, LargestKey are indexes into the Data.Keys list. 27 | SmallestKey int `json:"smallest_key"` 28 | LargestKey int `json:"largest_key"` 29 | Details []string `json:"details"` 30 | } 31 | -------------------------------------------------------------------------------- /internal/lsmview/url.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package lsmview 6 | 7 | import ( 8 | "bytes" 9 | "compress/zlib" 10 | "encoding/base64" 11 | "encoding/json" 12 | "net/url" 13 | ) 14 | 15 | // GenerateURL generates a URL showing the LSM diagram. The URL contains the 16 | // encoded and compressed data as the URL fragment. 17 | func GenerateURL(data Data) (url.URL, error) { 18 | var jsonBuf bytes.Buffer 19 | if err := json.NewEncoder(&jsonBuf).Encode(data); err != nil { 20 | return url.URL{}, err 21 | } 22 | 23 | var compressed bytes.Buffer 24 | encoder := base64.NewEncoder(base64.URLEncoding, &compressed) 25 | compressor := zlib.NewWriter(encoder) 26 | if _, err := jsonBuf.WriteTo(compressor); err != nil { 27 | return url.URL{}, err 28 | } 29 | if err := compressor.Close(); err != nil { 30 | return url.URL{}, err 31 | } 32 | if err := encoder.Close(); err != nil { 33 | return url.URL{}, err 34 | } 35 | return url.URL{ 36 | Scheme: "https", 37 | Host: "raduberinde.github.io", 38 | Path: "lsmview/decode.html", 39 | Fragment: compressed.String(), 40 | }, nil 41 | } 42 | -------------------------------------------------------------------------------- /internal/manifest/layer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package manifest 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestLayer(t *testing.T) { 14 | testCases := []struct { 15 | layer Layer 16 | expected string 17 | }{ 18 | {Level(0), "L0"}, 19 | {Level(1), "L1"}, 20 | {Level(2), "L2"}, 21 | {Level(3), "L3"}, 22 | {Level(4), "L4"}, 23 | {Level(5), "L5"}, 24 | {Level(6), "L6"}, 25 | 26 | {L0Sublevel(0), "L0.0"}, 27 | {L0Sublevel(1), "L0.1"}, 28 | {L0Sublevel(2), "L0.2"}, 29 | 30 | {FlushableIngestsLayer(), "flushable-ingests"}, 31 | } 32 | 33 | for _, c := range testCases { 34 | t.Run(c.expected, func(t *testing.T) { 35 | s := c.layer.String() 36 | require.EqualValues(t, c.expected, s) 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /internal/manifest/testdata/MANIFEST_import: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/manifest/testdata/MANIFEST_import -------------------------------------------------------------------------------- /internal/manual/manual_nocgo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !cgo 6 | 7 | package manual 8 | 9 | import "unsafe" 10 | 11 | // Provides versions of New and Free when cgo is not available (e.g. cross 12 | // compilation). 13 | 14 | // New allocates a slice of size n. 15 | func New(purpose Purpose, n uintptr) Buf { 16 | if n == 0 { 17 | return Buf{} 18 | } 19 | recordAlloc(purpose, n) 20 | slice := make([]byte, n) 21 | return Buf{ 22 | data: unsafe.Pointer(unsafe.SliceData(slice)), 23 | n: n, 24 | } 25 | } 26 | 27 | // Free frees the specified slice. It has to be exactly the slice that was 28 | // returned by New. 29 | func Free(purpose Purpose, b Buf) { 30 | recordFree(purpose, b.n) 31 | } 32 | -------------------------------------------------------------------------------- /internal/metamorphic/.gitignore: -------------------------------------------------------------------------------- 1 | _meta/ 2 | *.test 3 | -------------------------------------------------------------------------------- /internal/metamorphic/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | // Package metamorphic holds the entrypoint for Pebble's internal metamorphic 6 | // tests. See pebble/metamorphic for the package on which these tests build. 7 | package metamorphic 8 | -------------------------------------------------------------------------------- /internal/mkbench/testdata/README.md: -------------------------------------------------------------------------------- 1 | testdata 2 | ======== 3 | 4 | The files in this directory serve as test fixtures, used for validating that 5 | the `mkbench` command produces the expected output when reading and writing 6 | various Pebble benchmark data. 7 | 8 | The data was generated by downloading a sample of existing raw benchmark log 9 | data from the 10 | [`pebble-benchmarks`](https://s3.console.aws.amazon.com/s3/buckets/pebble-benchmarks) 11 | S3 bucket. The data was trimmed so as to reduce the amount of data checked into 12 | version control, with something akin to: 13 | 14 | ```bash 15 | for _file in $(find "$DOWNLOADED_DATA" -name '*.gz'); do 16 | gunzip --to-stdout "$_file" \ 17 | | grep 'Benchmarkycsb' -B 10 -A 10 \ 18 | | gzip > "$OUTPUT_DIR/$(basename "$_file")" 19 | done 20 | ``` 21 | -------------------------------------------------------------------------------- /internal/mkbench/testdata/data-symlink: -------------------------------------------------------------------------------- 1 | ./data -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/write/size=1024/run_1/1.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/write/size=1024/run_1/1.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/write/size=1024/run_1/2.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/write/size=1024/run_1/2.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_A.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_A.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_B.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_B.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_C.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_C.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_D.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_D.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_E.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_E.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_F.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/1.ycsb_F.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_A.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_A.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_B.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_B.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_C.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_C.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_D.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_D.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_E.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_E.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_F.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/2.ycsb_F.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_A.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_A.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_B.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_B.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_C.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_C.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_D.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_D.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_E.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_E.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_F.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211027/pebble/ycsb/size=1024/run_1/3.ycsb_F.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/write/size=1024/run_1/1.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/write/size=1024/run_1/1.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/write/size=1024/run_1/2.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/write/size=1024/run_1/2.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_A.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_A.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_B.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_B.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_C.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_C.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_D.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_D.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_E.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_E.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_F.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/1.ycsb_F.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_A.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_A.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_B.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_B.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_C.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_C.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_D.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_D.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_E.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_E.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_F.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/2.ycsb_F.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_A.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_A.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_B.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_B.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_C.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_C.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_D.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_D.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_E.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_E.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_F.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=1024/run_1/3.ycsb_F.log.gz -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_A.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_A.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_B.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_B.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_C.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_C.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_D.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_D.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_E.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_E.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_F.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/1.ycsb_F.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_A.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_A.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_B.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_B.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_C.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_C.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_D.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_D.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_E.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_E.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_F.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/2.ycsb_F.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_A.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_A.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_B.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_B.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_C.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_C.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_D.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_D.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_E.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_E.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_F.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/internal/mkbench/testdata/data/20211028/pebble/ycsb/size=64/run_1/3.ycsb_F.log.bz2 -------------------------------------------------------------------------------- /internal/mkbench/testdata/write-throughput/summary.json: -------------------------------------------------------------------------------- 1 | { 2 | "write/values=1024": [ 3 | { 4 | "name": "write/values=1024", 5 | "date": "20211027", 6 | "opsSec": 58825, 7 | "writeAmp": 1.17, 8 | "summaryPath": "20211027-pebble-write-size=1024-run_1-summary.json" 9 | }, 10 | { 11 | "name": "write/values=1024", 12 | "date": "20211028", 13 | "opsSec": 65525, 14 | "writeAmp": 1.39, 15 | "summaryPath": "20211028-pebble-write-size=1024-run_1-summary.json" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /internal/mkbench/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "encoding/json" 9 | "log" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | func prettyJSON(v interface{}) []byte { 15 | data, err := json.MarshalIndent(v, "", "\t") 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | return data 20 | } 21 | 22 | // walkDir recursively walks the given directory (following any symlinks it may 23 | // encounter), running the handleFn on each file. 24 | func walkDir(dir string, handleFn func(path, pathRel string, info os.FileInfo) error) error { 25 | var walkFn filepath.WalkFunc 26 | walkFn = func(path string, info os.FileInfo, err error) error { 27 | if err != nil { 28 | return err 29 | } 30 | if info == nil { 31 | return nil 32 | } 33 | if !info.Mode().IsRegular() && !info.Mode().IsDir() { 34 | info, err = os.Stat(path) 35 | if err == nil && info.Mode().IsDir() { 36 | _ = filepath.Walk(path+string(os.PathSeparator), walkFn) 37 | } 38 | return nil 39 | } 40 | 41 | rel, err := filepath.Rel(dir, path) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | return handleFn(path, rel, info) 47 | } 48 | return filepath.Walk(dir, walkFn) 49 | } 50 | -------------------------------------------------------------------------------- /internal/mkbench/util_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestWalkDir(t *testing.T) { 15 | maybeSkip(t) 16 | 17 | // Number of files and directories in the data fixture directory (and 18 | // therefore the symlinked directory). Generated via: 19 | // find . testdata | wc -l 20 | const wantCount = 97 21 | for _, path := range dataDirPaths { 22 | t.Run(path, func(t *testing.T) { 23 | var paths []string 24 | err := walkDir(path, func(_, pathRel string, info os.FileInfo) error { 25 | paths = append(paths, pathRel) 26 | return nil 27 | }) 28 | require.NoError(t, err) 29 | require.Equal(t, wantCount, len(paths)) 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/private/batch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package private 6 | 7 | import ( 8 | "github.com/cockroachdb/pebble/internal/base" 9 | "github.com/cockroachdb/pebble/internal/keyspan" 10 | ) 11 | 12 | // BatchSort is a hook for constructing iterators over the point and range 13 | // mutations contained in a batch in sorted order. It is intended for testing 14 | // use only. 15 | var BatchSort func(interface{}) ( 16 | points base.InternalIterator, 17 | rangeDels keyspan.FragmentIterator, 18 | rangeKeys keyspan.FragmentIterator, 19 | ) 20 | -------------------------------------------------------------------------------- /internal/problemspans/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | // Package problemspans provides functionality for tracking and managing key 6 | // spans that have been identified as problematic. It allows users to add spans 7 | // with associated expiration times, check if a given key range overlaps any 8 | // active (non-expired) spans, and remove spans when issues are resolved. 9 | // 10 | // This package is designed for efficiently tracking key ranges that may need 11 | // special handling. 12 | // 13 | // Key Attributes: 14 | // 15 | // - **Span Registration:** 16 | // Add spans with specified expiration times so that they automatically 17 | // become inactive after a set duration. 18 | // 19 | // - **Overlap Detection:** 20 | // Quickly check if a key range overlaps with any active problematic spans. 21 | // 22 | // - **Span Excise:** 23 | // Remove or adjust spans to reflect changes as issues are resolved. 24 | // 25 | // - **Level-Based Organization:** 26 | // The package offers a structure to organize and manage problematic spans 27 | // per level, with built-in support for concurrent operations. 28 | package problemspans 29 | -------------------------------------------------------------------------------- /internal/problemspans/testdata/by_level: -------------------------------------------------------------------------------- 1 | is-empty 2 | ---- 3 | empty 4 | ByLevel: 5 | 6 | 7 | add 8 | L2 [c, f) 10s 9 | L4 [d, h) 10s 10 | ---- 11 | ByLevel: 12 | L2: 13 | [c, f) expires in: 10s 14 | L4: 15 | [d, h) expires in: 10s 16 | 17 | is-empty 18 | ---- 19 | not empty 20 | ByLevel: 21 | L2: 22 | [c, f) expires in: 10s 23 | L4: 24 | [d, h) expires in: 10s 25 | 26 | overlap 27 | L1 [a, z) 28 | L3 [a, z] 29 | L2 [a, c) 30 | L2 [d, d] 31 | L2 [f, z] 32 | L4 [a, c] 33 | L4 [g, g] 34 | ---- 35 | [a, z): no overlap 36 | [a, z]: no overlap 37 | [a, c): no overlap 38 | [d, d]: overlap 39 | [f, z]: no overlap 40 | [a, c]: no overlap 41 | [g, g]: overlap 42 | ByLevel: 43 | L2: 44 | [c, f) expires in: 10s 45 | L4: 46 | [d, h) expires in: 10s 47 | 48 | excise 49 | [d, e) 50 | ---- 51 | ByLevel: 52 | L2: 53 | [c, d) expires in: 10s 54 | [e, f) expires in: 10s 55 | L4: 56 | [e, h) expires in: 10s 57 | 58 | overlap 59 | L2 [da, db] 60 | L2 [e, e] 61 | L4 [da, db] 62 | L4 [e, e] 63 | ---- 64 | [da, db]: no overlap 65 | [e, e]: overlap 66 | [da, db]: no overlap 67 | [e, e]: overlap 68 | ByLevel: 69 | L2: 70 | [c, d) expires in: 10s 71 | [e, f) expires in: 10s 72 | L4: 73 | [e, h) expires in: 10s 74 | 75 | overlap now=20s 76 | L2 [a, z] 77 | ---- 78 | [a, z]: no overlap 79 | ByLevel: 80 | 81 | -------------------------------------------------------------------------------- /internal/randvar/deck_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package randvar 6 | 7 | import "testing" 8 | 9 | func TestDeck(t *testing.T) { 10 | d := NewDeck(nil, 10, 20, 20, 0, 30) 11 | 12 | x := make([]int, 10000) 13 | for i := range x { 14 | x[i] = d.Int() 15 | } 16 | 17 | if testing.Verbose() { 18 | dumpSamples(x) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /internal/randvar/rand.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package randvar 6 | 7 | import "math/rand/v2" 8 | 9 | // NewRand creates a new random number generator with a random seed. 10 | func NewRand() *rand.Rand { 11 | return rand.New(rand.NewPCG(0, rand.Uint64())) 12 | } 13 | 14 | func ensureRand(rng *rand.Rand) *rand.Rand { 15 | if rng != nil { 16 | return rng 17 | } 18 | return NewRand() 19 | } 20 | -------------------------------------------------------------------------------- /internal/randvar/randvar.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package randvar 6 | 7 | import "math/rand/v2" 8 | 9 | // Static models a random variable that pulls from a distribution with static 10 | // bounds 11 | type Static interface { 12 | Uint64(rng *rand.Rand) uint64 13 | } 14 | 15 | // Dynamic models a random variable that pulls from a distribution with an 16 | // upper bound that can change dynamically using the IncMax method. 17 | type Dynamic interface { 18 | Static 19 | 20 | // Increment the max value the variable will return. 21 | IncMax(delta uint64) 22 | 23 | // Read the current max value the variable will return. 24 | Max() uint64 25 | } 26 | -------------------------------------------------------------------------------- /internal/randvar/weighted.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package randvar 6 | 7 | import "math/rand/v2" 8 | 9 | // Weighted is a random number generator that generates numbers in the range 10 | // [0,len(weights)-1] where the probability of i is weights(i)/sum(weights). 11 | type Weighted struct { 12 | rng *rand.Rand 13 | sum float64 14 | weights []float64 15 | } 16 | 17 | // NewWeighted returns a new weighted random number generator. 18 | func NewWeighted(rng *rand.Rand, weights ...float64) *Weighted { 19 | var sum float64 20 | for i := range weights { 21 | sum += weights[i] 22 | } 23 | return &Weighted{ 24 | rng: ensureRand(rng), 25 | sum: sum, 26 | weights: weights, 27 | } 28 | } 29 | 30 | // Int returns a random number in the range [0,len(weights)-1] where the 31 | // probability of i is weights(i)/sum(weights). 32 | func (w *Weighted) Int() int { 33 | p := w.rng.Float64() * w.sum 34 | for i, weight := range w.weights { 35 | if p <= weight { 36 | return i 37 | } 38 | p -= weight 39 | } 40 | return len(w.weights) - 1 41 | } 42 | -------------------------------------------------------------------------------- /internal/randvar/weighted_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package randvar 6 | 7 | import "testing" 8 | 9 | func TestWeighted(t *testing.T) { 10 | w := NewWeighted(nil, 1, 2, 2, 0, 3) 11 | 12 | x := make([]int, 10000) 13 | for i := range x { 14 | x[i] = w.Int() 15 | } 16 | 17 | if testing.Verbose() { 18 | dumpSamples(x) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /internal/rangekey/coalesce_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package rangekey 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/cockroachdb/datadriven" 12 | "github.com/cockroachdb/pebble/internal/keyspan" 13 | "github.com/cockroachdb/pebble/internal/testkeys" 14 | ) 15 | 16 | func TestCoalesce(t *testing.T) { 17 | datadriven.RunTest(t, "testdata/coalesce", func(t *testing.T, td *datadriven.TestData) string { 18 | switch td.Cmd { 19 | case "coalesce": 20 | span := keyspan.ParseSpan(td.Input) 21 | coalesced := keyspan.Span{ 22 | Start: span.Start, 23 | End: span.End, 24 | } 25 | Coalesce(testkeys.Comparer.CompareRangeSuffixes, span.Keys, &coalesced.Keys) 26 | return coalesced.String() 27 | 28 | default: 29 | return fmt.Sprintf("unrecognized command %q", td.Cmd) 30 | } 31 | }) 32 | } 33 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cockroach Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | 15 | package rawalloc 16 | 17 | // New returns a new byte slice of the specified length and capacity where the 18 | // backing memory is uninitialized. This differs from make([]byte) which 19 | // guarantees that the backing memory for the slice is initialized to zero. Use 20 | // carefully. 21 | func New(len, cap int) []byte { 22 | ptr := mallocgc(uintptr(cap), nil, false) 23 | return (*[maxArrayLen]byte)(ptr)[:len:cap] 24 | } 25 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_32bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cockroach Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | 15 | //go:build 386 || amd64p32 || arm || armbe || ppc || sparc 16 | 17 | package rawalloc 18 | 19 | const ( 20 | maxArrayLen = 1<<31 - 1 21 | ) 22 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_64bit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cockroach Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | 15 | //go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64 || loong64 16 | 17 | package rawalloc 18 | 19 | const ( 20 | maxArrayLen = 1<<50 - 1 21 | ) 22 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_gccgo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cockroach Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | 15 | //go:build gccgo 16 | 17 | package rawalloc 18 | 19 | import "unsafe" 20 | 21 | //extern runtime.mallocgc 22 | func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer 23 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_go1.9.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Cockroach Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | 15 | //go:build gc && go1.9 16 | 17 | package rawalloc 18 | 19 | import "unsafe" 20 | 21 | // The go:linkname directives provides backdoor access to private functions in 22 | // the runtime. Below we're accessing the mallocgc function. Note that this 23 | // access is necessarily tied to a specific Go release which is why this file 24 | // is protected by a build tag. 25 | 26 | //go:linkname mallocgc runtime.mallocgc 27 | func mallocgc(size uintptr, typ unsafe.Pointer, needzero bool) unsafe.Pointer 28 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_mipsall.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cockroach Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | // implied. See the License for the specific language governing 13 | // permissions and limitations under the License. 14 | 15 | //go:build mips || mipsle || mips64p32 || mips64p32le 16 | 17 | package rawalloc 18 | 19 | const ( 20 | maxArrayLen = 1 << 30 21 | ) 22 | -------------------------------------------------------------------------------- /internal/rawalloc/rawalloc_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package rawalloc 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | var sizes = []int{16, 100, 1024, 1024 * 10, 1024 * 100, 1024 * 1024} 13 | 14 | func BenchmarkRawalloc(b *testing.B) { 15 | for _, size := range sizes { 16 | b.Run(fmt.Sprintf("rawalloc-%d", size), func(b *testing.B) { 17 | for i := 0; i < b.N; i++ { 18 | _ = New(size, size) 19 | } 20 | }) 21 | } 22 | } 23 | 24 | func BenchmarkMake(b *testing.B) { 25 | for _, size := range sizes { 26 | b.Run(fmt.Sprintf("make-%d", size), func(b *testing.B) { 27 | for i := 0; i < b.N; i++ { 28 | _ = make([]byte, size) 29 | } 30 | }) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/sstableinternal/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package sstableinternal 6 | 7 | import ( 8 | "github.com/cockroachdb/pebble/internal/base" 9 | "github.com/cockroachdb/pebble/internal/cache" 10 | ) 11 | 12 | // CacheOptions contains the information needed to interact with the block 13 | // cache. 14 | type CacheOptions struct { 15 | // CacheHandle can be nil, in which case no cache is used. When non-nil, the 16 | // other fields must be set accordingly. 17 | CacheHandle *cache.Handle 18 | FileNum base.DiskFileNum 19 | } 20 | 21 | // ReaderOptions are fields of sstable.ReaderOptions that can only be set from 22 | // within the pebble package. 23 | type ReaderOptions struct { 24 | CacheOpts CacheOptions 25 | } 26 | 27 | // WriterOptions are fields of sstable.ReaderOptions that can only be set from 28 | // within the pebble package. 29 | type WriterOptions struct { 30 | CacheOpts CacheOptions 31 | 32 | // DisableKeyOrderChecks disables the checks that keys are added to an sstable 33 | // in order. It is intended for use only in the construction of invalid 34 | // sstables for testing. See tool/make_test_sstables.go. 35 | DisableKeyOrderChecks bool 36 | } 37 | -------------------------------------------------------------------------------- /internal/testutils/duration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package testutils 6 | 7 | import ( 8 | "runtime" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | // DurationIsAtLeast verifies that the given duration is at least the given 16 | // value. 17 | func DurationIsAtLeast(t testing.TB, d, minValue time.Duration) { 18 | t.Helper() 19 | if runtime.GOOS == "windows" && minValue < 10*time.Millisecond { 20 | // Windows timer precision is coarse (on the order of 1 millisecond) and can 21 | // cause the duration for short operations to be 0. 22 | return 23 | } 24 | require.GreaterOrEqual(t, d, minValue) 25 | } 26 | -------------------------------------------------------------------------------- /internal/testutils/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package testutils 6 | 7 | // CheckErr can be used to simplify test code that expects no errors. 8 | // Instead of: 9 | // 10 | // v, err := SomeFunc() 11 | // if err != nil { .. } 12 | // 13 | // we can use: 14 | // 15 | // v := testutils.CheckErr(someFunc()) 16 | func CheckErr[V any](v V, err error) V { 17 | if err != nil { 18 | panic(err) 19 | } 20 | return v 21 | } 22 | 23 | // CheckErr2 can be used to simplify test code that expects no errors. 24 | // Instead of: 25 | // 26 | // v, w, err := SomeFunc() 27 | // if err != nil { .. } 28 | // 29 | // we can use: 30 | // 31 | // v, w := testutils.CheckErr2(someFunc()) 32 | func CheckErr2[V any, W any](v V, w W, err error) (V, W) { 33 | if err != nil { 34 | panic(err) 35 | } 36 | return v, w 37 | } 38 | -------------------------------------------------------------------------------- /internal/testutils/indenttree/indent_tree_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package indenttree 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/cockroachdb/datadriven" 12 | "github.com/cockroachdb/pebble/internal/treeprinter" 13 | ) 14 | 15 | func TestIndentTree(t *testing.T) { 16 | datadriven.RunTest(t, "testdata", func(t *testing.T, d *datadriven.TestData) string { 17 | switch d.Cmd { 18 | case "parse": 19 | nodes, err := Parse(d.Input) 20 | if err != nil { 21 | return fmt.Sprintf("error: %s", err) 22 | } 23 | tp := treeprinter.New() 24 | root := tp.Child("") 25 | var dfs func(n Node, tp treeprinter.Node) 26 | dfs = func(n Node, parent treeprinter.Node) { 27 | child := parent.Child(n.Value()) 28 | for _, c := range n.Children() { 29 | dfs(c, child) 30 | } 31 | } 32 | for _, c := range nodes { 33 | dfs(c, root) 34 | } 35 | return tp.String() 36 | 37 | default: 38 | t.Fatalf("unknown command: %s", d.Cmd) 39 | return "" 40 | } 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /internal/testutils/rng.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package testutils 6 | 7 | import "math/rand/v2" 8 | 9 | // RandIntInRange returns a value in [min, max) 10 | func RandIntInRange(r *rand.Rand, min, max int) int { 11 | return min + r.IntN(max-min) 12 | } 13 | 14 | var randLetters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 15 | 16 | // RandBytes returns a byte slice of the given length with random 17 | // data. 18 | func RandBytes(r *rand.Rand, size int) []byte { 19 | if size <= 0 { 20 | return nil 21 | } 22 | 23 | arr := make([]byte, size) 24 | for i := 0; i < len(arr); i++ { 25 | arr[i] = randLetters[r.IntN(len(randLetters))] 26 | } 27 | return arr 28 | } 29 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | import "github.com/cockroachdb/pebble/internal/base" 8 | 9 | // Logger defines an interface for writing log messages. 10 | type Logger = base.Logger 11 | 12 | // DefaultLogger logs to the Go stdlib logs. 13 | var DefaultLogger = base.DefaultLogger 14 | 15 | // LoggerAndTracer defines an interface for logging and tracing. 16 | type LoggerAndTracer = base.LoggerAndTracer 17 | -------------------------------------------------------------------------------- /lsm_view_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/cockroachdb/datadriven" 11 | ) 12 | 13 | func TestLSMViewURL(t *testing.T) { 14 | datadriven.RunTest(t, "testdata/lsm_view", 15 | func(t *testing.T, td *datadriven.TestData) string { 16 | switch td.Cmd { 17 | case "define": 18 | d, err := runDBDefineCmd(td, nil /* options */) 19 | if err != nil { 20 | td.Fatalf(t, "error: %s", err) 21 | } 22 | defer d.Close() 23 | return d.LSMViewURL() 24 | 25 | default: 26 | td.Fatalf(t, "unknown command %q", td.Cmd) 27 | return "" 28 | } 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /merger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | import ( 8 | "io" 9 | 10 | "github.com/cockroachdb/pebble/internal/base" 11 | ) 12 | 13 | // Merge exports the base.Merge type. 14 | type Merge = base.Merge 15 | 16 | // Merger exports the base.Merger type. 17 | type Merger = base.Merger 18 | 19 | // ValueMerger exports the base.ValueMerger type. 20 | type ValueMerger = base.ValueMerger 21 | 22 | // DeletableValueMerger exports the base.DeletableValueMerger type. 23 | type DeletableValueMerger = base.DeletableValueMerger 24 | 25 | // DefaultMerger exports the base.DefaultMerger variable. 26 | var DefaultMerger = base.DefaultMerger 27 | 28 | func finishValueMerger( 29 | valueMerger ValueMerger, includesBase bool, 30 | ) (value []byte, needDelete bool, closer io.Closer, err error) { 31 | if valueMerger2, ok := valueMerger.(DeletableValueMerger); ok { 32 | value, needDelete, closer, err = valueMerger2.DeletableFinish(includesBase) 33 | } else { 34 | value, closer, err = valueMerger.Finish(includesBase) 35 | } 36 | return 37 | } 38 | -------------------------------------------------------------------------------- /metamorphic/diagram_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package metamorphic 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/cockroachdb/datadriven" 12 | ) 13 | 14 | func TestDiagram(t *testing.T) { 15 | datadriven.RunTest(t, "testdata/diagram", func(t *testing.T, d *datadriven.TestData) string { 16 | switch d.Cmd { 17 | case "diagram": 18 | res, err := TryToGenerateDiagram(TestkeysKeyFormat, []byte(d.Input)) 19 | if err != nil { 20 | return err.Error() 21 | } 22 | return res 23 | 24 | default: 25 | return fmt.Sprintf("unknown command: %s", d.Cmd) 26 | } 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /metamorphic/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package metamorphic_test 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "math/rand/v2" 11 | 12 | "github.com/cockroachdb/pebble/metamorphic" 13 | ) 14 | 15 | func ExampleExecute() { 16 | const seed = 1698702489658104000 17 | rng := rand.New(rand.NewPCG(0, seed)) 18 | 19 | kf := metamorphic.TestkeysKeyFormat 20 | // Generate a random database by running the metamorphic test. 21 | testOpts := metamorphic.RandomOptions(rng, kf, nil /* custom opt parsers */) 22 | ops := metamorphic.GenerateOps(rng, 10000, kf, metamorphic.DefaultOpConfig()) 23 | test, err := metamorphic.New(ops, testOpts, "" /* dir */, io.Discard) 24 | if err != nil { 25 | panic(err) 26 | } 27 | err = metamorphic.Execute(test) 28 | fmt.Print(err) 29 | // Output: 30 | } 31 | -------------------------------------------------------------------------------- /metamorphic/simplify_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package metamorphic 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/cockroachdb/datadriven" 12 | ) 13 | 14 | func TestSimplifyKeys(t *testing.T) { 15 | datadriven.RunTest(t, "testdata/simplify", func(t *testing.T, d *datadriven.TestData) string { 16 | switch d.Cmd { 17 | case "simplify-keys": 18 | retainSuffixes := d.HasArg("retain-suffixes") 19 | res := TryToSimplifyKeys(TestkeysKeyFormat, []byte(d.Input), retainSuffixes) 20 | return string(res) 21 | 22 | default: 23 | return fmt.Sprintf("unknown command: %s", d.Cmd) 24 | } 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /metamorphic/testdata/parser: -------------------------------------------------------------------------------- 1 | parse 2 | foo 3 | ---- 4 | metamorphic test internal error: 1:1: unknown object type: "foo" 5 | 6 | parse 7 | "foo" 8 | ---- 9 | metamorphic test internal error: 1:1: unexpected token: STRING "\"foo\"" 10 | 11 | parse 12 | db.bar() 13 | ---- 14 | metamorphic test internal error: 1:1: unknown op db1.bar 15 | 16 | parse 17 | db.Apply() 18 | ---- 19 | metamorphic test internal error: 1:10: Apply: not enough arguments 20 | 21 | parse 22 | db.Apply(hello) 23 | ---- 24 | metamorphic test internal error: 1:10: unknown object type: "hello" 25 | 26 | parse 27 | db.NewBatch() 28 | ---- 29 | metamorphic test internal error: 1:1: assignment expected for db1.NewBatch 30 | 31 | parse 32 | batch0 = db.Apply() 33 | ---- 34 | metamorphic test internal error: 1:10: cannot use db1.Apply in assignment 35 | 36 | parse 37 | batch0 = db.NewBatch() 38 | batch0.First() 39 | ---- 40 | metamorphic test internal error: 2:1: batch0.First: First is not a method on batch0 41 | -------------------------------------------------------------------------------- /metamorphic/testdata/simplify: -------------------------------------------------------------------------------- 1 | simplify-keys 2 | db2.RangeKeySet("apple", "raspberry", "", "") 3 | snap9 = db2.NewSnapshot("banana", "durian", "guanabana", "pineapple") 4 | db2.Compact("cranberry", "pear", true /* parallelize */) 5 | ---- 6 | db2.RangeKeySet("a", "h", "", "") 7 | snap9 = db2.NewSnapshot("b", "d", "e", "g") 8 | db2.Compact("c", "f", true /* parallelize */) 9 | 10 | simplify-keys 11 | db2.RangeKeySet("apple", "raspberry", "", "") 12 | snap9 = db2.NewSnapshot("apple", "raspberry") 13 | db2.Compact("apple", "raspberry", true /* parallelize */) 14 | ---- 15 | db2.RangeKeySet("a", "b", "", "") 16 | snap9 = db2.NewSnapshot("a", "b") 17 | db2.Compact("a", "b", true /* parallelize */) 18 | 19 | simplify-keys 20 | db2.RangeKeySet("apple@1", "raspberry", "", "") 21 | snap9 = db2.NewSnapshot("apple@1", "raspberry@4") 22 | db2.Compact("apple@1", "raspberry@4", true /* parallelize */) 23 | ---- 24 | db2.RangeKeySet("a", "b", "", "") 25 | snap9 = db2.NewSnapshot("a", "c") 26 | db2.Compact("a", "c", true /* parallelize */) 27 | 28 | simplify-keys retain-suffixes 29 | db2.RangeKeySet("apple@1", "raspberry", "", "") 30 | snap9 = db2.NewSnapshot("apple@1", "raspberry@4") 31 | db2.Compact("apple@1", "raspberry@4", true /* parallelize */) 32 | ---- 33 | db2.RangeKeySet("a@1", "b", "", "") 34 | snap9 = db2.NewSnapshot("a@1", "b@4") 35 | db2.Compact("a@1", "b@4", true /* parallelize */) 36 | -------------------------------------------------------------------------------- /objstorage/noop_readahead.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package objstorage 6 | 7 | import "context" 8 | 9 | // NoopReadHandle can be used by Readable implementations that don't 10 | // support read-ahead. 11 | type NoopReadHandle struct { 12 | readable Readable 13 | } 14 | 15 | // MakeNoopReadHandle initializes a NoopReadHandle. 16 | func MakeNoopReadHandle(r Readable) NoopReadHandle { 17 | return NoopReadHandle{readable: r} 18 | } 19 | 20 | var _ ReadHandle = (*NoopReadHandle)(nil) 21 | 22 | // ReadAt is part of the ReadHandle interface. 23 | func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error { 24 | return h.readable.ReadAt(ctx, p, off) 25 | } 26 | 27 | // Close is part of the ReadHandle interface. 28 | func (*NoopReadHandle) Close() error { return nil } 29 | 30 | // SetupForCompaction is part of the ReadHandle interface. 31 | func (*NoopReadHandle) SetupForCompaction() {} 32 | 33 | // RecordCacheHit is part of the ReadHandle interface. 34 | func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {} 35 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/shared_cache_helpers_test.go: -------------------------------------------------------------------------------- 1 | package sharedcache 2 | 3 | func (c *Cache) WaitForWritesToComplete() { 4 | close(c.writeWorkers.tasksCh) 5 | c.writeWorkers.doneWaitGroup.Wait() 6 | c.writeWorkers.Start(c, c.writeWorkers.numWorkers) 7 | } 8 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/compaction_reads: -------------------------------------------------------------------------------- 1 | init 2 | ---- 3 | initialized with block-size=32768 size=33554432 num-shards=32 4 | 5 | write size=200000 6 | ---- 7 | 8 | read offset=1024 size=10000 9 | ---- 10 | misses=1 11 | 12 | # This should be in the cache. 13 | read-for-compaction offset=4096 size=2000 14 | ---- 15 | misses=0 16 | 17 | # This should miss the cache. 18 | read-for-compaction offset=4096 size=100000 19 | ---- 20 | misses=1 21 | 22 | # This should miss the cache again - we don't populate the cache when doing 23 | # compaction reads. 24 | read-for-compaction offset=4096 size=100000 25 | ---- 26 | misses=1 27 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/eof_handling: -------------------------------------------------------------------------------- 1 | init 2 | ---- 3 | initialized with block-size=32768 size=33554432 num-shards=32 4 | 5 | write size=40000 6 | ---- 7 | 8 | read offset=35000 size=4000 9 | ---- 10 | misses=1 11 | 12 | read offset=35000 size=4000 13 | ---- 14 | misses=0 15 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/lru: -------------------------------------------------------------------------------- 1 | init num-shards=1 size=1M 2 | ---- 3 | initialized with block-size=32768 size=1048576 num-shards=1 4 | 5 | write size=1500000 6 | ---- 7 | 8 | read offset=0 size=32K 9 | ---- 10 | misses=1 11 | 12 | read offset=32K size=32K 13 | ---- 14 | misses=1 15 | 16 | read offset=64K size=960K 17 | ---- 18 | misses=1 19 | 20 | # The cache should now be full with the first MB. Read a new block. 21 | read offset=1M size=32K 22 | ---- 23 | misses=1 24 | 25 | # The block that was evicted should have been the one at offset 0. 26 | read offset=0 size=32K 27 | ---- 28 | misses=1 29 | 30 | # The block that was evicted should have been the one at offset 32768. 31 | read offset=32K size=32K 32 | ---- 33 | misses=1 34 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/read_larger_than_two_cache_shards: -------------------------------------------------------------------------------- 1 | # Read larger than two cache shards. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=3145728 8 | ---- 9 | 10 | read offset=57 size=3145671 11 | ---- 12 | misses=1 13 | 14 | read offset=57 size=3145671 15 | ---- 16 | misses=0 17 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/read_that_hits_two_cache_blocks: -------------------------------------------------------------------------------- 1 | # Large read that hits two cache blocks. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=32773 8 | ---- 9 | 10 | read offset=0 size=32773 11 | ---- 12 | misses=1 13 | 14 | read offset=0 size=32773 15 | ---- 16 | misses=0 17 | 18 | read offset=57 size=32716 19 | ---- 20 | misses=0 21 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/read_that_hits_two_cache_blocks_with_first_read_at_big_offset: -------------------------------------------------------------------------------- 1 | # Large read that hits two cache blocks, with first read at big offset. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=32773 8 | ---- 9 | 10 | read offset=32768 size=5 11 | ---- 12 | misses=1 13 | 14 | read offset=32768 size=5 15 | ---- 16 | misses=0 17 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/read_that_hits_two_cache_blocks_with_first_read_at_offset: -------------------------------------------------------------------------------- 1 | # Large read that hits two cache blocks, with first read at offset. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=32773 8 | ---- 9 | 10 | read offset=57 size=32716 11 | ---- 12 | misses=1 13 | 14 | read offset=57 size=32716 15 | ---- 16 | misses=0 17 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/read_that_hits_two_cache_shards: -------------------------------------------------------------------------------- 1 | # Large read that hits two cache shards. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=1048776 8 | ---- 9 | 10 | read offset=0 size=1048776 11 | ---- 12 | misses=1 13 | 14 | read offset=0 size=1048776 15 | ---- 16 | misses=0 17 | 18 | read offset=57 size=1048719 19 | ---- 20 | misses=0 21 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/read_that_hits_two_cache_shards_with_first_read_at_offset: -------------------------------------------------------------------------------- 1 | # Large read that hits two cache shards, with first read at offset. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=1048776 8 | ---- 9 | 10 | read offset=57 size=1048719 11 | ---- 12 | misses=1 13 | 14 | read offset=57 size=1048719 15 | ---- 16 | misses=0 17 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/small_read: -------------------------------------------------------------------------------- 1 | # Small read, with one miss then two hits. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=10 8 | ---- 9 | 10 | read offset=0 size=10 11 | ---- 12 | misses=1 13 | 14 | read offset=0 size=10 15 | ---- 16 | misses=0 17 | 18 | read offset=4 size=6 19 | ---- 20 | misses=0 21 | -------------------------------------------------------------------------------- /objstorage/objstorageprovider/sharedcache/testdata/cache/small_read_with_first_read_at_offset: -------------------------------------------------------------------------------- 1 | # Small read, with first read at offset. 2 | 3 | init 4 | ---- 5 | initialized with block-size=32768 size=33554432 num-shards=32 6 | 7 | write size=10 8 | ---- 9 | 10 | read offset=4 size=6 11 | ---- 12 | misses=1 13 | 14 | read offset=4 size=6 15 | ---- 16 | misses=0 17 | 18 | read offset=0 size=10 19 | ---- 20 | misses=0 21 | -------------------------------------------------------------------------------- /objstorage/remote/factory.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package remote 6 | 7 | import "github.com/pkg/errors" 8 | 9 | // MakeSimpleFactory returns a StorageFactory implementation that produces the given 10 | // Storage objects. 11 | func MakeSimpleFactory(m map[Locator]Storage) StorageFactory { 12 | return simpleFactory(m) 13 | } 14 | 15 | type simpleFactory map[Locator]Storage 16 | 17 | var _ StorageFactory = simpleFactory{} 18 | 19 | // CreateStorage is part of the StorageFactory interface. 20 | func (sf simpleFactory) CreateStorage(locator Locator) (Storage, error) { 21 | if s, ok := sf[locator]; ok { 22 | return s, nil 23 | } 24 | return nil, errors.Errorf("unknown locator '%s'", locator) 25 | } 26 | -------------------------------------------------------------------------------- /rangedel/rangedel.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | // Package rangedel provides functionality for working with range deletions. 6 | package rangedel 7 | 8 | import ( 9 | "github.com/cockroachdb/pebble/internal/keyspan" 10 | "github.com/cockroachdb/pebble/internal/rangedel" 11 | "github.com/cockroachdb/pebble/sstable" 12 | ) 13 | 14 | // Fragmenter exports the keyspan.Fragmenter type. 15 | type Fragmenter = keyspan.Fragmenter 16 | 17 | // Key exports the keyspan.Key type. 18 | type Key = keyspan.Key 19 | 20 | // Span exports the keyspan.Span type. 21 | type Span = keyspan.Span 22 | 23 | // Decode decodes an InternalKey representing a range deletion into a Span. If 24 | // keysDst is provided, keys will be appended to keysDst to reduce allocations. 25 | func Decode(ik sstable.InternalKey, val []byte, keysDst []Key) Span { 26 | return rangedel.Decode(ik, val, keysDst) 27 | } 28 | -------------------------------------------------------------------------------- /rangekey/rangekey.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | // Package rangekey provides functionality for working with range keys. 6 | package rangekey 7 | 8 | import ( 9 | "github.com/cockroachdb/pebble/internal/keyspan" 10 | "github.com/cockroachdb/pebble/internal/rangekey" 11 | "github.com/cockroachdb/pebble/sstable" 12 | ) 13 | 14 | // Fragmenter exports the keyspan.Fragmenter type. 15 | type Fragmenter = keyspan.Fragmenter 16 | 17 | // Key exports the keyspan.Key type. 18 | type Key = keyspan.Key 19 | 20 | // Span exports the keyspan.Span type. 21 | type Span = keyspan.Span 22 | 23 | // IsRangeKey returns if this InternalKey is a range key. Alias for 24 | // rangekey.IsRangeKey. 25 | func IsRangeKey(ik sstable.InternalKey) bool { 26 | return rangekey.IsRangeKey(ik.Kind()) 27 | } 28 | 29 | // Decode decodes an InternalKey into a Span, if it is a range key. If 30 | // keysDst is provided, keys will be appended to keysDst to reduce allocations. 31 | func Decode(ik sstable.InternalKey, val []byte, keysDst []Key) (Span, error) { 32 | return rangekey.Decode(ik, val, keysDst) 33 | } 34 | -------------------------------------------------------------------------------- /read_state_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package pebble 6 | 7 | import ( 8 | "fmt" 9 | "math/rand/v2" 10 | "testing" 11 | "time" 12 | 13 | "github.com/cockroachdb/pebble/vfs" 14 | ) 15 | 16 | func BenchmarkReadState(b *testing.B) { 17 | d, err := Open("", &Options{ 18 | FS: vfs.NewMem(), 19 | }) 20 | if err != nil { 21 | b.Fatal(err) 22 | } 23 | 24 | for _, updateFrac := range []float32{0, 0.1, 0.5} { 25 | b.Run(fmt.Sprintf("updates=%.0f", updateFrac*100), func(b *testing.B) { 26 | b.RunParallel(func(pb *testing.PB) { 27 | rng := rand.New(rand.NewPCG(0, uint64(time.Now().UnixNano()))) 28 | 29 | for pb.Next() { 30 | if rng.Float32() < updateFrac { 31 | d.mu.Lock() 32 | d.updateReadStateLocked(nil) 33 | d.mu.Unlock() 34 | } else { 35 | s := d.loadReadState() 36 | s.unref() 37 | } 38 | } 39 | }) 40 | }) 41 | } 42 | 43 | if err := d.Close(); err != nil { 44 | b.Fatal(err) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /record/rotation_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package record 6 | 7 | import ( 8 | "fmt" 9 | "strconv" 10 | "testing" 11 | 12 | "github.com/cockroachdb/datadriven" 13 | ) 14 | 15 | func TestRotation(t *testing.T) { 16 | var rh RotationHelper 17 | datadriven.RunTest(t, "testdata/rotation", func(t *testing.T, td *datadriven.TestData) string { 18 | oneIntArg := func() int64 { 19 | if len(td.CmdArgs) != 1 { 20 | td.Fatalf(t, "expected one integer argument") 21 | } 22 | n, err := strconv.Atoi(td.CmdArgs[0].String()) 23 | if err != nil { 24 | td.Fatalf(t, "expected one integer argument") 25 | } 26 | return int64(n) 27 | } 28 | switch td.Cmd { 29 | case "add": 30 | size := oneIntArg() 31 | rh.AddRecord(size) 32 | 33 | case "should-rotate": 34 | nextSnapshotSize := oneIntArg() 35 | return fmt.Sprint(rh.ShouldRotate(nextSnapshotSize)) 36 | 37 | case "rotate": 38 | snapshotSize := oneIntArg() 39 | rh.Rotate(snapshotSize) 40 | 41 | default: 42 | td.Fatalf(t, "unknown command %s", td.Cmd) 43 | } 44 | 45 | // For commands with no output, show the debug info. 46 | a, b := rh.DebugInfo() 47 | return fmt.Sprintf("last-snapshot-size: %d\nsize-since-last-snapshot: %d", a, b) 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /record/testdata/rotation: -------------------------------------------------------------------------------- 1 | rotate 100 2 | ---- 3 | last-snapshot-size: 100 4 | size-since-last-snapshot: 0 5 | 6 | add 10 7 | ---- 8 | last-snapshot-size: 100 9 | size-since-last-snapshot: 10 10 | 11 | # We should only rotate if the next snapshot is much smaller. 12 | should-rotate 100 13 | ---- 14 | false 15 | 16 | should-rotate 5 17 | ---- 18 | true 19 | 20 | add 50 21 | ---- 22 | last-snapshot-size: 100 23 | size-since-last-snapshot: 60 24 | 25 | add 50 26 | ---- 27 | last-snapshot-size: 100 28 | size-since-last-snapshot: 110 29 | 30 | add 50 31 | ---- 32 | last-snapshot-size: 100 33 | size-since-last-snapshot: 160 34 | 35 | # We exceeded the last snapshot size, we should rotate regardless. 36 | should-rotate 1 37 | ---- 38 | true 39 | 40 | should-rotate 1000 41 | ---- 42 | true 43 | 44 | add 1 45 | ---- 46 | last-snapshot-size: 100 47 | size-since-last-snapshot: 161 48 | 49 | rotate 10 50 | ---- 51 | last-snapshot-size: 10 52 | size-since-last-snapshot: 1 53 | 54 | add 5 55 | ---- 56 | last-snapshot-size: 10 57 | size-since-last-snapshot: 6 58 | 59 | should-rotate 5 60 | ---- 61 | true 62 | 63 | should-rotate 100 64 | ---- 65 | false 66 | -------------------------------------------------------------------------------- /replay/testdata/collect/clean_before_copy: -------------------------------------------------------------------------------- 1 | # This tests a race between Clean and the workload collector. It creates an 2 | # sstable during a flush, and then immediately Cleans the file. The workload 3 | # collector's Cleaner must delay the cleaning of the file until it's been copied 4 | # and then delete it. 5 | 6 | start 7 | ---- 8 | 9 | create-manifest filenum=000001 10 | ---- 11 | 12 | flush 13 | 000002 14 | ---- 15 | created src/000002.sst 16 | [JOB 0] flushed 1 memtable (100B) to L0 [000002] (10KB), in 0.1s (0.1s total), output rate 100KB/s 17 | 18 | clean 19 | src/000002.sst 20 | ---- 21 | 22 | wait 23 | ---- 24 | dst: 25 | 000002.sst 26 | MANIFEST-000001 27 | 28 | # The file should now be both removed from src/ and a copy should be present in 29 | # dst/. 30 | 31 | ls src dst 32 | ---- 33 | src: 34 | MANIFEST-000001 35 | dst: 36 | 000002.sst 37 | MANIFEST-000001 38 | 39 | stop 40 | ---- 41 | -------------------------------------------------------------------------------- /replay/testdata/collect/copy_before_clean: -------------------------------------------------------------------------------- 1 | start 2 | ---- 3 | 4 | create-manifest filenum=000001 5 | ---- 6 | 7 | flush 8 | 000002 9 | ---- 10 | created src/000002.sst 11 | [JOB 0] flushed 1 memtable (100B) to L0 [000002] (10KB), in 0.1s (0.1s total), output rate 100KB/s 12 | 13 | # Wait for 000002.sst to be copied. 14 | 15 | wait 16 | ---- 17 | dst: 18 | 000002.sst 19 | MANIFEST-000001 20 | 21 | # The file 000002.sst should exist in both src and dst. 22 | 23 | ls src dst 24 | ---- 25 | src: 26 | 000002.sst 27 | MANIFEST-000001 28 | dst: 29 | 000002.sst 30 | MANIFEST-000001 31 | 32 | cmp-files src/000002.sst dst/000002.sst 33 | ---- 34 | equal 35 | 36 | # Now that the file has been copied, a clean should immediately delete it. 37 | 38 | clean 39 | src/000002.sst 40 | ---- 41 | 42 | ls src dst 43 | ---- 44 | src: 45 | MANIFEST-000001 46 | dst: 47 | 000002.sst 48 | MANIFEST-000001 49 | 50 | stop 51 | ---- 52 | -------------------------------------------------------------------------------- /replay/testdata/corpus/findWorkloadFiles: -------------------------------------------------------------------------------- 1 | make-file capture log 1 2 | ---- 3 | created 4 | 5 | make-file capture log 2 6 | ---- 7 | created 8 | 9 | make-file capture log 3 10 | ---- 11 | created 12 | 13 | make-file capture table 1 14 | ---- 15 | created 16 | 17 | make-file capture table 4 18 | ---- 19 | created 20 | 21 | make-file capture file totally_not_relevant_file_000001.log 22 | ---- 23 | created 24 | 25 | list-files capture 26 | ---- 27 | capture: 28 | 000001.log 29 | 000001.sst 30 | 000002.log 31 | 000003.log 32 | 000004.sst 33 | totally_not_relevant_file_000001.log 34 | 35 | find-workload-files capture 36 | ---- 37 | manifests 38 | sstables 39 | blob files 40 | error 41 | no manifests found 42 | 43 | make-file capture manifest 1 44 | ---- 45 | created 46 | 47 | make-file capture manifest 2 48 | ---- 49 | created 50 | 51 | make-file capture manifest 3 52 | ---- 53 | created 54 | 55 | 56 | list-files capture 57 | ---- 58 | capture: 59 | 000001.log 60 | 000001.sst 61 | 000002.log 62 | 000003.log 63 | 000004.sst 64 | MANIFEST-000001 65 | MANIFEST-000002 66 | MANIFEST-000003 67 | totally_not_relevant_file_000001.log 68 | 69 | find-workload-files capture 70 | ---- 71 | manifests 72 | MANIFEST-000001 73 | MANIFEST-000002 74 | MANIFEST-000003 75 | sstables 76 | 000001 77 | 000004 78 | blob files 79 | error 80 | -------------------------------------------------------------------------------- /replay/testdata/corpus/simple: -------------------------------------------------------------------------------- 1 | open 2 | ---- 3 | 4 | list-files build 5 | ---- 6 | build: 7 | 000002.log 8 | LOCK 9 | MANIFEST-000001 10 | OPTIONS-000003 11 | marker.format-version.000001.013 12 | marker.manifest.000001.MANIFEST-000001 13 | 14 | commit 15 | set a a 16 | set b b 17 | set c c 18 | ---- 19 | 20 | flush 21 | ---- 22 | 23 | list-files build 24 | ---- 25 | build: 26 | 000002.log 27 | 000004.log 28 | 000005.sst 29 | LOCK 30 | MANIFEST-000001 31 | OPTIONS-000003 32 | marker.format-version.000001.013 33 | marker.manifest.000001.MANIFEST-000001 34 | 35 | start 36 | ---- 37 | started 38 | 39 | list-files simple 40 | ---- 41 | simple: 42 | checkpoint 43 | 44 | list-files simple/checkpoint 45 | ---- 46 | simple/checkpoint: 47 | 000004.log 48 | 000005.sst 49 | MANIFEST-000001 50 | OPTIONS-000003 51 | marker.format-version.000001.013 52 | marker.manifest.000001.MANIFEST-000001 53 | 54 | commit 55 | set d d 56 | set e e 57 | set f f 58 | set i i 59 | set h h 60 | set g g 61 | ---- 62 | 63 | flush 64 | ---- 65 | 66 | stop 67 | ---- 68 | stopped 69 | 70 | list-files simple 71 | ---- 72 | simple: 73 | 000007.sst 74 | MANIFEST-000001 75 | checkpoint 76 | 77 | stat simple/MANIFEST-000001 simple/MANIFEST-000008 simple/000007.sst 78 | ---- 79 | simple/MANIFEST-000001: 80 | size: 133 81 | simple/MANIFEST-000008: stat simple/MANIFEST-000008: file does not exist 82 | simple/000007.sst: 83 | size: 614 84 | -------------------------------------------------------------------------------- /replay/testdata/flushed_sstable_keys: -------------------------------------------------------------------------------- 1 | commit 2 | set a a 3 | set b b 4 | set c c 5 | ---- 6 | 7 | flush 8 | ---- 9 | a.SET: a 10 | b.SET: b 11 | c.SET: c 12 | 13 | # Test that the keys in the batch are in the same order they were originally 14 | # committed, not sorted by user key. 15 | 16 | commit 17 | set c c 18 | set b b 19 | set a a 20 | ---- 21 | 22 | flush 23 | ---- 24 | c.SET: c 25 | b.SET: b 26 | a.SET: a 27 | 28 | # Test that the keys in the batch are in the same order they were originally 29 | # committed, not sorted by user key. 30 | 31 | commit 32 | set c c 33 | del b 34 | del-range d f 35 | singledel a 36 | ---- 37 | 38 | flush 39 | ---- 40 | c.SET: c 41 | b.DEL 42 | d.RANGEDEL-f 43 | a.SINGLEDEL 44 | 45 | commit 46 | set x foo 47 | range-key-del a z 48 | range-key-unset g h @3 49 | range-key-set l m @1 foo 50 | set a bar 51 | del y 52 | ---- 53 | 54 | flush 55 | ---- 56 | x.SET: foo 57 | a.RANGEKEYDEL-g 58 | g.RANGEKEYDEL-h 59 | h.RANGEKEYDEL-l 60 | l.RANGEKEYDEL-m 61 | m.RANGEKEYDEL-z 62 | g.RANGEKEYUNSET-h: @3 63 | l.RANGEKEYSET-m: @1 → foo 64 | a.SET: bar 65 | y.DEL 66 | -------------------------------------------------------------------------------- /scripts/changed-go-pkgs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BASE_SHA="$1" 4 | HEAD_SHA="$2" 5 | 6 | if [ -z "$HEAD_SHA" ];then 7 | echo "Usage: $0 " 8 | exit 1 9 | fi 10 | 11 | git diff --name-only "${BASE_SHA}..${HEAD_SHA}" -- "*.go" \ 12 | | xargs -rn1 dirname \ 13 | | sort -u \ 14 | | grep -v "internal/devtools" \ 15 | | xargs echo 16 | -------------------------------------------------------------------------------- /scripts/check-workspace-clean.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | # The workspace is clean iff `git status --porcelain` produces no output. Any 6 | # output is either an error message or a listing of an untracked/dirty file. 7 | if [[ "$(git status --porcelain 2>&1)" != "" ]]; then 8 | git status >&2 || true 9 | git diff --no-ext-diff -a >&2 || true 10 | echo "" >&2 11 | echo "Error: make generate resulted in changes" >&2 12 | exit 1 13 | fi 14 | 15 | -------------------------------------------------------------------------------- /scripts/pr-codecov-run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script runs unit tests with coverage enabled for a specific list of 4 | # package paths and outputs the coverage to a json file. 5 | # 6 | # Package paths that are not valid in this tree are tolerated. 7 | 8 | set -xeuo pipefail 9 | 10 | output_json_file="$1" 11 | packages="$2" 12 | 13 | # Find the targets. We need to convert from, e.g. 14 | # . objstorage objstorage/objstorageprovider 15 | # to 16 | # . ./objstorage ./objstorage/objstorageprovider 17 | 18 | paths="" 19 | sep="" 20 | 21 | for p in ${packages}; do 22 | # Check that the path exists and contains Go files. 23 | if ls "${p}"/*.go >/dev/null 2>&1; then 24 | if [[ $p != "." ]]; then 25 | p="./$p" 26 | fi 27 | paths="${paths}${sep}${p}" 28 | sep=" " 29 | fi 30 | done 31 | 32 | if [ -z "${paths}" ]; then 33 | echo "Skipping" 34 | touch "${output_json_file}" 35 | exit 0 36 | fi 37 | 38 | tmpfile=$(mktemp --suffix -coverprofile) 39 | trap 'rm -f "${tmpfile}"' EXIT 40 | 41 | make testcoverage COVER_PROFILE="${tmpfile}" PKG="$paths" 42 | go run github.com/cockroachdb/code-cov-utils/gocover2json@v1.0.0 \ 43 | --trim-prefix github.com/cockroachdb/pebble/ \ 44 | "${tmpfile}" "${output_json_file}" 45 | -------------------------------------------------------------------------------- /scripts/run-crossversion-meta.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | BRANCH=$(git symbolic-ref --short HEAD) 6 | 7 | TEMPDIR=(`mktemp -d -t crossversion-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX`) 8 | 9 | VERSIONS="" 10 | for branch in "$@" 11 | do 12 | git checkout "$branch" 13 | sha=`git rev-parse --short HEAD` 14 | 15 | # If the branch name has a "-", pull off the suffix. With the 16 | # crl-release-{XX.X} release branch naming scheme, this will extract the 17 | # {XX.X}. 18 | version=`cut -d- -f3 <<< "$branch"` 19 | 20 | echo "Building $version ($sha)" 21 | go test -c -o "$TEMPDIR/meta.$version.test" ./internal/metamorphic 22 | VERSIONS="$VERSIONS -version $version,$sha,$TEMPDIR/meta.$version.test" 23 | done 24 | 25 | # Return to whence we came. 26 | git checkout $BRANCH 27 | 28 | if [[ -z "${STRESS}" ]]; then 29 | go test ./internal/metamorphic/crossversion \ 30 | -test.v \ 31 | -test.timeout "${TIMEOUT:-30m}" \ 32 | -test.run 'TestMetaCrossVersion$' \ 33 | -seed ${SEED:-0} \ 34 | -factor ${FACTOR:-10} \ 35 | $(echo $VERSIONS) 36 | else 37 | stress -p 1 go test ./internal/metamorphic/crossversion \ 38 | -test.v \ 39 | -test.timeout "${TIMEOUT:-30m}" \ 40 | -test.run 'TestMetaCrossVersion$' \ 41 | -seed ${SEED:-0} \ 42 | -factor ${FACTOR:-10} \ 43 | $(echo $VERSIONS) 44 | fi 45 | 46 | rm -rf $TEMPDIR 47 | -------------------------------------------------------------------------------- /scripts/stress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # Stress all packages, one at a time. This allows for a more useful output. 6 | for p in $(go list ./... | sed 's#github.com/cockroachdb/pebble#.#'); do 7 | echo 8 | echo "" 9 | echo "" 10 | echo "Stressing $p" 11 | echo "" 12 | make stress STRESSFLAGS='-maxtime 5m -maxruns 100' "PKG=$p" 13 | done 14 | -------------------------------------------------------------------------------- /sstable/block/flush_governor_internal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package block 6 | 7 | import "testing" 8 | 9 | func TestFindClosestClass(t *testing.T) { 10 | vals := []int{10, 20, 30, 50} 11 | testCases := [][2]int{ 12 | {0, 10}, 13 | {9, 10}, 14 | {10, 10}, 15 | {11, 10}, 16 | {14, 10}, 17 | {16, 20}, 18 | {18, 20}, 19 | {20, 20}, 20 | {24, 20}, 21 | {26, 30}, 22 | {39, 30}, 23 | {41, 50}, 24 | {60, 50}, 25 | {1000, 50}, 26 | } 27 | for _, tc := range testCases { 28 | res := vals[findClosestClass(vals, tc[0])] 29 | if res != tc[1] { 30 | t.Errorf("expected %d, but got %d", tc[1], res) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /sstable/colblk/uints_decode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2025 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package colblk 6 | 7 | import ( 8 | "unsafe" 9 | 10 | "github.com/cockroachdb/errors" 11 | ) 12 | 13 | // unsafeUint64Decoder is used to access 64-bit unsigned integer backed by a 14 | // buffer in little-ending format. It is desirable to keep this type as small as 15 | // possible since it is embedded multiple times in block decoders. 16 | // 17 | // The At() method is defined in endian_little.go and endian_big.go. 18 | type unsafeUint64Decoder struct { 19 | ptr unsafe.Pointer 20 | } 21 | 22 | func makeUnsafeUint64Decoder(buf []byte, n int) unsafeUint64Decoder { 23 | if n == 0 { 24 | return unsafeUint64Decoder{} 25 | } 26 | ptr := unsafe.Pointer(unsafe.SliceData(buf)) 27 | if align(uintptr(ptr), align64) != uintptr(ptr) { 28 | panic(errors.AssertionFailedf("slice pointer %p not %d-byte aligned", ptr, align64)) 29 | } 30 | if len(buf) < n< h.txt 15 | -------------------------------------------------------------------------------- /sstable/testdata/h-no-compression-sst/000012.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/h-no-compression-sst/000012.sst -------------------------------------------------------------------------------- /sstable/testdata/h-no-compression-two-level-index-sst/000003.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/h-no-compression-two-level-index-sst/000003.sst -------------------------------------------------------------------------------- /sstable/testdata/h-table-bloom-no-compression-prefix-extractor-no-whole-key-filter-sst/000013.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/h-table-bloom-no-compression-prefix-extractor-no-whole-key-filter-sst/000013.sst -------------------------------------------------------------------------------- /sstable/testdata/h-table-bloom-no-compression-sst/000011.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/h-table-bloom-no-compression-sst/000011.sst -------------------------------------------------------------------------------- /sstable/testdata/h-table-bloom-sst/000010.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/h-table-bloom-sst/000010.sst -------------------------------------------------------------------------------- /sstable/testdata/h-zstd-compression-sst/000004.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/h-zstd-compression-sst/000004.sst -------------------------------------------------------------------------------- /sstable/testdata/hamlet-sst/000002.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/sstable/testdata/hamlet-sst/000002.sst -------------------------------------------------------------------------------- /sstable/testdata/make-table.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | 12 | "github.com/cockroachdb/pebble/sstable" 13 | "github.com/cockroachdb/pebble/vfs" 14 | ) 15 | 16 | func main() { 17 | // The test fixture code opens "testdata/h.txt" so we have to move up. 18 | dir, err := os.Getwd() 19 | if err != nil { 20 | panic(err) 21 | } 22 | if filepath.Base(dir) != "testdata" { 23 | panic("This program must be run from sstable/testdata") 24 | } 25 | if err := os.Chdir(filepath.Dir(dir)); err != nil { 26 | panic(err) 27 | } 28 | for _, fixture := range sstable.TestFixtures { 29 | fmt.Printf("Generating %s\n", fixture.Filename) 30 | if err := fixture.Build(vfs.Default, filepath.Join("testdata", fixture.Filename)); err != nil { 31 | panic(err) 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /sstable/testdata/prefixreader/bloom: -------------------------------------------------------------------------------- 1 | build 2 | a.SET.1:A 3 | aa.SET.2:AA 4 | aaa.SET.3:AAA 5 | aaaa.SET.4:AAAA 6 | b.SET.5:B 7 | ---- 8 | 9 | get 10 | b 11 | ab 12 | a 13 | aaa 14 | aa 15 | aaaa 16 | ---- 17 | B 18 | 19 | A 20 | AAA 21 | AA 22 | AAAA 23 | -------------------------------------------------------------------------------- /sstable/testdata/reader/bloom: -------------------------------------------------------------------------------- 1 | build 2 | a.SET.1:A 3 | aa.SET.2:AA 4 | aaa.SET.3:AAA 5 | aaaa.SET.4:AAAA 6 | b.SET.5:B 7 | ---- 8 | 9 | get 10 | b 11 | ab 12 | a 13 | aaa 14 | aa 15 | aaaa 16 | ---- 17 | B 18 | 19 | A 20 | AAA 21 | AA 22 | AAAA 23 | -------------------------------------------------------------------------------- /testdata/.gitignore: -------------------------------------------------------------------------------- 1 | /make-db 2 | -------------------------------------------------------------------------------- /testdata/Makefile: -------------------------------------------------------------------------------- 1 | all: rebuild 2 | 3 | .PHONY: rebuild 4 | rebuild: make-db.go 5 | for stage in 1 2 3 4; do \ 6 | rm -fr db-stage-$$stage && \ 7 | go run make-db.go $$stage; \ 8 | done 9 | -------------------------------------------------------------------------------- /testdata/batch_get: -------------------------------------------------------------------------------- 1 | define 2 | set a 1 3 | ---- 4 | 5 | get a 6 | ---- 7 | 1 8 | 9 | get b 10 | ---- 11 | pebble: not found 12 | 13 | define 14 | set a 1 15 | set a 2 16 | ---- 17 | 18 | get a 19 | ---- 20 | 2 21 | 22 | define 23 | set a 1 24 | set a 2 25 | del a 26 | ---- 27 | 28 | get a 29 | ---- 30 | pebble: not found 31 | 32 | define 33 | set a 1 34 | set a 2 35 | del a 36 | set a 3 37 | ---- 38 | 39 | get a 40 | ---- 41 | 3 42 | 43 | define 44 | merge a 1 45 | merge a 2 46 | merge a 3 47 | ---- 48 | 49 | get a 50 | ---- 51 | 123 52 | 53 | commit 54 | ---- 55 | 56 | define 57 | merge a 4 58 | ---- 59 | 60 | get a 61 | ---- 62 | 1234 63 | 64 | commit 65 | ---- 66 | 67 | define 68 | del a 69 | set a 5 70 | ---- 71 | 72 | get a 73 | ---- 74 | 5 75 | 76 | define 77 | del-range a b 78 | ---- 79 | 80 | get a 81 | ---- 82 | pebble: not found 83 | 84 | commit 85 | ---- 86 | 87 | define 88 | ---- 89 | 90 | get a 91 | ---- 92 | pebble: not found 93 | 94 | define 95 | merge b 1 96 | merge b 2 97 | del-range b c 98 | merge b 3 99 | merge b 4 100 | ---- 101 | 102 | get b 103 | ---- 104 | 34 105 | 106 | commit 107 | ---- 108 | 109 | define 110 | ---- 111 | 112 | get b 113 | ---- 114 | 34 115 | -------------------------------------------------------------------------------- /testdata/compaction_error_on_user_key_overlap: -------------------------------------------------------------------------------- 1 | 2 | error-on-user-key-overlap 3 | a.SET.2-b.SET.3 4 | c.SET.4-d.SET.5 5 | ---- 6 | OK 7 | 8 | # If the end key is the rangedel sentinel key, no error should be returned. 9 | 10 | error-on-user-key-overlap 11 | a.SET.2-c.RANGEDEL.inf 12 | c.SET.4-d.SET.5 13 | ---- 14 | OK 15 | 16 | error-on-user-key-overlap 17 | a.SET.2-c.SET.5 18 | c.SET.4-d.SET.5 19 | ---- 20 | pebble: compaction split user key across two sstables: c#5,SET in 000001 and 000002 21 | -------------------------------------------------------------------------------- /testdata/compaction_expand_inputs: -------------------------------------------------------------------------------- 1 | define 2 | a.SET.1-b.SET.2 3 | ---- 4 | 5 | expand-inputs 0 6 | ---- 7 | 0: a#1,SET-b#2,SET 8 | 9 | define 10 | a.SET.1-b.SET.2 11 | c.SET.3-d.SET.4 12 | e.SET.5-f.SET.6 13 | ---- 14 | 15 | expand-inputs 0 16 | ---- 17 | 0: a#1,SET-b#2,SET 18 | 19 | expand-inputs 1 20 | ---- 21 | 1: c#3,SET-d#4,SET 22 | 23 | expand-inputs 2 24 | ---- 25 | 2: e#5,SET-f#6,SET 26 | 27 | define 28 | a.SET.1-b.RANGEDEL.inf 29 | b.SET.1-d.SET.4 30 | e.SET.2-f.SET.6 31 | ---- 32 | 33 | expand-inputs 0 34 | ---- 35 | 0: a#1,SET-b#inf,RANGEDEL 36 | 37 | expand-inputs 1 38 | ---- 39 | 1: b#1,SET-d#4,SET 40 | -------------------------------------------------------------------------------- /testdata/compaction_setup_inputs_multilevel_dummy: -------------------------------------------------------------------------------- 1 | # init a multi-level compaction with dummy hueristic 2 | setup-inputs a a 3 | L1 4 | a.SET.1-b.SET.2 size=1 5 | L2 6 | a.SET.3-c.SET.4 size=1 7 | L3 8 | c.SET.3-d.SET.2 size=1 9 | ---- 10 | L1 11 | 000001:[a#1,SET-b#2,SET] 12 | L2 13 | 000002:[a#3,SET-c#4,SET] 14 | L3 15 | 000003:[c#3,SET-d#2,SET] 16 | init-multi-level(1,2,3) 17 | Original WriteAmp 2.00; ML WriteAmp 1.50 18 | Original OverlappingRatio 1.00; ML OverlappingRatio 0.50 19 | -------------------------------------------------------------------------------- /testdata/db-stage-1/000002.log: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /testdata/db-stage-1/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000000 2 | -------------------------------------------------------------------------------- /testdata/db-stage-1/LOCK: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-1/LOCK -------------------------------------------------------------------------------- /testdata/db-stage-1/MANIFEST-000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-1/MANIFEST-000001 -------------------------------------------------------------------------------- /testdata/db-stage-1/OPTIONS-000003: -------------------------------------------------------------------------------- 1 | [Version] 2 | pebble_version=0.1 3 | 4 | [Options] 5 | bytes_per_sync=524288 6 | cache_size=8388608 7 | cleaner=delete 8 | compaction_debt_concurrency=1073741824 9 | comparer=leveldb.BytewiseComparator 10 | disable_wal=false 11 | flush_delay_delete_range=0s 12 | flush_delay_range_key=0s 13 | flush_split_bytes=4194304 14 | format_major_version=13 15 | l0_compaction_concurrency=10 16 | l0_compaction_file_threshold=500 17 | l0_compaction_threshold=4 18 | l0_stop_writes_threshold=12 19 | lbase_max_bytes=67108864 20 | max_concurrent_compactions=1 21 | max_manifest_file_size=134217728 22 | max_open_files=1000 23 | mem_table_size=4194304 24 | mem_table_stop_writes_threshold=2 25 | min_deletion_rate=0 26 | merger=pebble.concatenate 27 | read_compaction_rate=16000 28 | read_sampling_multiplier=16 29 | strict_wal_tail=true 30 | table_cache_shards=10 31 | table_property_collectors=[] 32 | validate_on_ingest=false 33 | wal_dir= 34 | wal_bytes_per_sync=0 35 | max_writer_concurrency=0 36 | force_writer_parallelism=false 37 | secondary_cache_size_bytes=0 38 | create_on_shared=0 39 | 40 | [Level "0"] 41 | block_restart_interval=16 42 | block_size=4096 43 | block_size_threshold=90 44 | compression=Snappy 45 | filter_policy=none 46 | filter_type=table 47 | index_block_size=4096 48 | target_file_size=2097152 49 | -------------------------------------------------------------------------------- /testdata/db-stage-1/marker.format-version.000012.013: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-1/marker.format-version.000012.013 -------------------------------------------------------------------------------- /testdata/db-stage-1/marker.manifest.000001.MANIFEST-000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-1/marker.manifest.000001.MANIFEST-000001 -------------------------------------------------------------------------------- /testdata/db-stage-2/000002.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-2/000002.log -------------------------------------------------------------------------------- /testdata/db-stage-2/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000000 2 | -------------------------------------------------------------------------------- /testdata/db-stage-2/LOCK: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-2/LOCK -------------------------------------------------------------------------------- /testdata/db-stage-2/MANIFEST-000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-2/MANIFEST-000001 -------------------------------------------------------------------------------- /testdata/db-stage-2/OPTIONS-000003: -------------------------------------------------------------------------------- 1 | [Version] 2 | pebble_version=0.1 3 | 4 | [Options] 5 | bytes_per_sync=524288 6 | cache_size=8388608 7 | cleaner=delete 8 | compaction_debt_concurrency=1073741824 9 | comparer=leveldb.BytewiseComparator 10 | disable_wal=false 11 | flush_delay_delete_range=0s 12 | flush_delay_range_key=0s 13 | flush_split_bytes=4194304 14 | format_major_version=13 15 | l0_compaction_concurrency=10 16 | l0_compaction_file_threshold=500 17 | l0_compaction_threshold=4 18 | l0_stop_writes_threshold=12 19 | lbase_max_bytes=67108864 20 | max_concurrent_compactions=1 21 | max_manifest_file_size=134217728 22 | max_open_files=1000 23 | mem_table_size=4194304 24 | mem_table_stop_writes_threshold=2 25 | min_deletion_rate=0 26 | merger=pebble.concatenate 27 | read_compaction_rate=16000 28 | read_sampling_multiplier=16 29 | strict_wal_tail=true 30 | table_cache_shards=10 31 | table_property_collectors=[] 32 | validate_on_ingest=false 33 | wal_dir= 34 | wal_bytes_per_sync=0 35 | max_writer_concurrency=0 36 | force_writer_parallelism=false 37 | secondary_cache_size_bytes=0 38 | create_on_shared=0 39 | 40 | [Level "0"] 41 | block_restart_interval=16 42 | block_size=4096 43 | block_size_threshold=90 44 | compression=Snappy 45 | filter_policy=none 46 | filter_type=table 47 | index_block_size=4096 48 | target_file_size=2097152 49 | -------------------------------------------------------------------------------- /testdata/db-stage-2/marker.format-version.000012.013: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-2/marker.format-version.000012.013 -------------------------------------------------------------------------------- /testdata/db-stage-2/marker.manifest.000001.MANIFEST-000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-2/marker.manifest.000001.MANIFEST-000001 -------------------------------------------------------------------------------- /testdata/db-stage-3/000004.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-3/000004.sst -------------------------------------------------------------------------------- /testdata/db-stage-3/000005.log: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /testdata/db-stage-3/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000000 2 | -------------------------------------------------------------------------------- /testdata/db-stage-3/LOCK: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-3/LOCK -------------------------------------------------------------------------------- /testdata/db-stage-3/MANIFEST-000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-3/MANIFEST-000001 -------------------------------------------------------------------------------- /testdata/db-stage-3/MANIFEST-000006: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-3/MANIFEST-000006 -------------------------------------------------------------------------------- /testdata/db-stage-3/OPTIONS-000007: -------------------------------------------------------------------------------- 1 | [Version] 2 | pebble_version=0.1 3 | 4 | [Options] 5 | bytes_per_sync=524288 6 | cache_size=8388608 7 | cleaner=delete 8 | compaction_debt_concurrency=1073741824 9 | comparer=leveldb.BytewiseComparator 10 | disable_wal=false 11 | flush_delay_delete_range=0s 12 | flush_delay_range_key=0s 13 | flush_split_bytes=4194304 14 | format_major_version=13 15 | l0_compaction_concurrency=10 16 | l0_compaction_file_threshold=500 17 | l0_compaction_threshold=4 18 | l0_stop_writes_threshold=12 19 | lbase_max_bytes=67108864 20 | max_concurrent_compactions=1 21 | max_manifest_file_size=134217728 22 | max_open_files=1000 23 | mem_table_size=4194304 24 | mem_table_stop_writes_threshold=2 25 | min_deletion_rate=0 26 | merger=pebble.concatenate 27 | read_compaction_rate=16000 28 | read_sampling_multiplier=16 29 | strict_wal_tail=true 30 | table_cache_shards=10 31 | table_property_collectors=[] 32 | validate_on_ingest=false 33 | wal_dir= 34 | wal_bytes_per_sync=0 35 | max_writer_concurrency=0 36 | force_writer_parallelism=false 37 | secondary_cache_size_bytes=0 38 | create_on_shared=0 39 | 40 | [Level "0"] 41 | block_restart_interval=16 42 | block_size=4096 43 | block_size_threshold=90 44 | compression=Snappy 45 | filter_policy=none 46 | filter_type=table 47 | index_block_size=4096 48 | target_file_size=2097152 49 | -------------------------------------------------------------------------------- /testdata/db-stage-3/marker.format-version.000012.013: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-3/marker.format-version.000012.013 -------------------------------------------------------------------------------- /testdata/db-stage-3/marker.manifest.000002.MANIFEST-000006: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-3/marker.manifest.000002.MANIFEST-000006 -------------------------------------------------------------------------------- /testdata/db-stage-4/000004.sst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/000004.sst -------------------------------------------------------------------------------- /testdata/db-stage-4/000005.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/000005.log -------------------------------------------------------------------------------- /testdata/db-stage-4/CURRENT: -------------------------------------------------------------------------------- 1 | MANIFEST-000000 2 | -------------------------------------------------------------------------------- /testdata/db-stage-4/LOCK: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/LOCK -------------------------------------------------------------------------------- /testdata/db-stage-4/MANIFEST-000001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/MANIFEST-000001 -------------------------------------------------------------------------------- /testdata/db-stage-4/MANIFEST-000006: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/MANIFEST-000006 -------------------------------------------------------------------------------- /testdata/db-stage-4/OPTIONS-000007: -------------------------------------------------------------------------------- 1 | [Version] 2 | pebble_version=0.1 3 | 4 | [Options] 5 | bytes_per_sync=524288 6 | cache_size=8388608 7 | cleaner=delete 8 | compaction_debt_concurrency=1073741824 9 | comparer=leveldb.BytewiseComparator 10 | disable_wal=false 11 | flush_delay_delete_range=0s 12 | flush_delay_range_key=0s 13 | flush_split_bytes=4194304 14 | format_major_version=13 15 | l0_compaction_concurrency=10 16 | l0_compaction_file_threshold=500 17 | l0_compaction_threshold=4 18 | l0_stop_writes_threshold=12 19 | lbase_max_bytes=67108864 20 | max_concurrent_compactions=1 21 | max_manifest_file_size=134217728 22 | max_open_files=1000 23 | mem_table_size=4194304 24 | mem_table_stop_writes_threshold=2 25 | min_deletion_rate=0 26 | merger=pebble.concatenate 27 | read_compaction_rate=16000 28 | read_sampling_multiplier=16 29 | strict_wal_tail=true 30 | table_cache_shards=10 31 | table_property_collectors=[] 32 | validate_on_ingest=false 33 | wal_dir= 34 | wal_bytes_per_sync=0 35 | max_writer_concurrency=0 36 | force_writer_parallelism=false 37 | secondary_cache_size_bytes=0 38 | create_on_shared=0 39 | 40 | [Level "0"] 41 | block_restart_interval=16 42 | block_size=4096 43 | block_size_threshold=90 44 | compression=Snappy 45 | filter_policy=none 46 | filter_type=table 47 | index_block_size=4096 48 | target_file_size=2097152 49 | -------------------------------------------------------------------------------- /testdata/db-stage-4/marker.format-version.000012.013: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/marker.format-version.000012.013 -------------------------------------------------------------------------------- /testdata/db-stage-4/marker.manifest.000002.MANIFEST-000006: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cockroachdb/pebble/809978646f3b007481de67d1c370e528b6546331/testdata/db-stage-4/marker.manifest.000002.MANIFEST-000006 -------------------------------------------------------------------------------- /testdata/delete_range: -------------------------------------------------------------------------------- 1 | define 2 | set a 1 3 | set b 2 4 | merge c 3 5 | del-range a c 6 | del d 7 | ---- 8 | 9 | scan 10 | ---- 11 | a#0,SET:1 12 | b#1,SET:2 13 | c#2,MERGE:3 14 | d#4,DEL: 15 | 16 | scan range-del 17 | ---- 18 | a-c:{(#3,RANGEDEL)} 19 | 20 | clear 21 | ---- 22 | 23 | define 24 | del-range a b 25 | del-range b c 26 | del-range a c 27 | del-range b d 28 | ---- 29 | 30 | scan range-del 31 | ---- 32 | a-b:{(#2,RANGEDEL) (#0,RANGEDEL)} 33 | b-c:{(#3,RANGEDEL) (#2,RANGEDEL) (#1,RANGEDEL)} 34 | c-d:{(#3,RANGEDEL)} 35 | 36 | clear 37 | ---- 38 | 39 | define 40 | del-range a b 41 | ---- 42 | 43 | scan range-del 44 | ---- 45 | a-b:{(#0,RANGEDEL)} 46 | 47 | define 48 | del-range b c 49 | ---- 50 | 51 | scan range-del 52 | ---- 53 | a-b:{(#0,RANGEDEL)} 54 | b-c:{(#1,RANGEDEL)} 55 | 56 | define 57 | del-range a c 58 | ---- 59 | 60 | scan range-del 61 | ---- 62 | a-b:{(#2,RANGEDEL) (#0,RANGEDEL)} 63 | b-c:{(#2,RANGEDEL) (#1,RANGEDEL)} 64 | 65 | define 66 | del-range b d 67 | ---- 68 | 69 | scan range-del 70 | ---- 71 | a-b:{(#2,RANGEDEL) (#0,RANGEDEL)} 72 | b-c:{(#3,RANGEDEL) (#2,RANGEDEL) (#1,RANGEDEL)} 73 | c-d:{(#3,RANGEDEL)} 74 | 75 | clear 76 | ---- 77 | 78 | define 79 | set 1 80 | del-range d 81 | ---- 82 | 83 | scan 84 | ---- 85 | #0,SET:1 86 | 87 | scan range-del 88 | ---- 89 | -d:{(#1,RANGEDEL)} 90 | -------------------------------------------------------------------------------- /testdata/internal_iter_bounds: -------------------------------------------------------------------------------- 1 | define 2 | a.SET.1:1 3 | a.SET.2:2 4 | b.SET.1:1 5 | b.SET.2:2 6 | c.SET.1:1 7 | c.SET.2:2 8 | d.SET.2:2 9 | ---- 10 | 11 | iter lower=b upper=c 12 | seek-ge d 13 | seek-prefix-ge d 14 | seek-lt c 15 | seek-lt b 16 | next 17 | prev 18 | next 19 | next 20 | next 21 | prev 22 | ---- 23 | . 24 | . 25 | b:1 26 | . 27 | b:2 28 | . 29 | b:2 30 | b:1 31 | . 32 | b:1 33 | 34 | # Test changing the bounds. The new bounds should be in effect. 35 | 36 | iter lower=b upper=c 37 | seek-ge d 38 | set-bounds lower=a upper=z 39 | seek-ge d 40 | ---- 41 | . 42 | d:2 43 | 44 | iter lower=b upper=c 45 | seek-lt b 46 | set-bounds lower=a upper=z 47 | seek-lt b 48 | ---- 49 | . 50 | a:1 51 | -------------------------------------------------------------------------------- /testdata/internal_iter_next: -------------------------------------------------------------------------------- 1 | define 2 | a.SET.1:1 3 | a.SET.2:2 4 | b.SET.1:1 5 | b.SET.2:2 6 | c.SET.1:1 7 | c.SET.2:2 8 | ---- 9 | 10 | iter 11 | seek-ge a 12 | seek-ge b 13 | seek-ge c 14 | seek-ge d 15 | seek-lt a 16 | seek-lt b 17 | seek-lt c 18 | seek-lt d 19 | seek-prefix-ge a 20 | seek-prefix-ge b 21 | seek-prefix-ge c 22 | seek-prefix-ge d 23 | ---- 24 | a:2 25 | b:2 26 | c:2 27 | . 28 | . 29 | a:1 30 | b:1 31 | c:1 32 | a:2 33 | b:2 34 | c:2 35 | . 36 | 37 | iter 38 | first 39 | next 40 | next 41 | prev 42 | prev 43 | prev 44 | next 45 | next 46 | next 47 | next 48 | next 49 | next 50 | prev 51 | prev 52 | next 53 | prev 54 | prev 55 | next 56 | next 57 | next 58 | next 59 | prev 60 | ---- 61 | a:2 62 | a:1 63 | b:2 64 | a:1 65 | a:2 66 | . 67 | a:2 68 | a:1 69 | b:2 70 | b:1 71 | c:2 72 | c:1 73 | c:2 74 | b:1 75 | c:2 76 | b:1 77 | b:2 78 | b:1 79 | c:2 80 | c:1 81 | . 82 | c:1 83 | -------------------------------------------------------------------------------- /testdata/iter_histories/merge: -------------------------------------------------------------------------------- 1 | # Test semantics of reading merge keys through the database and through a batch, 2 | # both with an iterator and with Get. 3 | 4 | reset merger=appender 5 | ---- 6 | 7 | batch name=bar 8 | merge k bar 9 | ---- 10 | wrote 1 keys to batch "bar" 11 | 12 | batch commit 13 | merge k foo 14 | ---- 15 | committed 1 keys 16 | 17 | combined-iter 18 | seek-ge k 19 | ---- 20 | k: (foo, .) 21 | 22 | get 23 | k 24 | ---- 25 | k: foo 26 | 27 | combined-iter reader=bar 28 | seek-ge k 29 | seek-prefix-ge k 30 | ---- 31 | k: (foobar, .) 32 | k: (foobar, .) 33 | 34 | get reader=bar 35 | k 36 | ---- 37 | k: foobar 38 | -------------------------------------------------------------------------------- /testdata/iter_histories/set_options: -------------------------------------------------------------------------------- 1 | # Ensure bounds and key-types provided through SetOptions are respected. 2 | 3 | reset 4 | ---- 5 | 6 | batch commit 7 | set a a 8 | set b b 9 | set c c 10 | set d d 11 | set f f 12 | range-key-set a ap @6 foo 13 | range-key-set ap c @5 bar 14 | range-key-set cat zoo @3 bax 15 | ---- 16 | committed 8 keys 17 | 18 | combined-iter lower=b upper=e 19 | first 20 | next 21 | next 22 | next 23 | next 24 | set-options lower=a upper=cat key-types=both 25 | first 26 | next 27 | next 28 | next 29 | set-options lower=a upper=cat key-types=point 30 | first 31 | next 32 | next 33 | next 34 | ---- 35 | b: (b, [b-c) @5=bar UPDATED) 36 | c: (c, . UPDATED) 37 | cat: (., [cat-e) @3=bax UPDATED) 38 | d: (d, [cat-e) @3=bax) 39 | . 40 | . 41 | a: (a, [a-ap) @6=foo UPDATED) 42 | ap: (., [ap-c) @5=bar UPDATED) 43 | b: (b, [ap-c) @5=bar) 44 | c: (c, . UPDATED) 45 | . 46 | a: (a, .) 47 | b: (b, .) 48 | c: (c, .) 49 | . 50 | 51 | flush 52 | ---- 53 | 54 | metrics 55 | ---- 56 | Metrics.Keys.RangeKeySetsCount = 3 57 | -------------------------------------------------------------------------------- /testdata/iterator_seek_opt_errors: -------------------------------------------------------------------------------- 1 | define 2 | a.SET.1:a 3 | b.SET.1:b 4 | c.SET.1:c 5 | d.SET.1:d 6 | ---- 7 | 8 | # Exercise noop optimization with no errors 9 | 10 | iter 11 | seek-ge aa 12 | seek-ge aa 13 | seek-ge aaa 14 | seek-ge b 15 | seek-ge bb 16 | ---- 17 | b: (b, .) 18 | b: (b, .) 19 | b: (b, .) 20 | b: (b, .) 21 | c: (c, .) 22 | 23 | iter 24 | seek-lt ddd 25 | seek-lt ddd 26 | seek-lt dd 27 | seek-lt d 28 | seek-lt c 29 | ---- 30 | d: (d, .) 31 | d: (d, .) 32 | d: (d, .) 33 | c: (c, .) 34 | b: (b, .) 35 | 36 | # Exercise errors which should prevent seek optimizations. 37 | 38 | iter seek-error=(0,1) 39 | seek-ge a 40 | seek-ge b 41 | seek-ge c 42 | seek-ge d 43 | ---- 44 | err=injecting error 45 | err=injecting error 46 | c: (c, .) 47 | d: (d, .) 48 | 49 | iter seek-error=(1) 50 | seek-ge d 51 | seek-ge a 52 | seek-ge b 53 | seek-ge b 54 | ---- 55 | d: (d, .) 56 | err=injecting error 57 | b: (b, .) 58 | b: (b, .) 59 | 60 | iter seek-error=(0,1) 61 | seek-lt e 62 | seek-lt d 63 | seek-lt c 64 | seek-lt b 65 | ---- 66 | err=injecting error 67 | err=injecting error 68 | b: (b, .) 69 | a: (a, .) 70 | 71 | iter seek-error=(1) 72 | seek-lt b 73 | seek-lt e 74 | seek-lt e 75 | ---- 76 | a: (a, .) 77 | err=injecting error 78 | d: (d, .) 79 | 80 | iter seek-error=(1) 81 | seek-prefix-ge d 82 | seek-prefix-ge a 83 | seek-prefix-ge b 84 | ---- 85 | d: (d, .) 86 | err=injecting error 87 | b: (b, .) 88 | -------------------------------------------------------------------------------- /testdata/iterator_table_filter: -------------------------------------------------------------------------------- 1 | define 2 | L0 3 | a.SET.4:4 4 | L1 5 | a.SET.3:3 6 | L2 7 | a.SET.2:2 8 | L3 9 | a.SET.1:1 10 | ---- 11 | L0.0: 12 | 000004:[a#4,SET-a#4,SET] 13 | L1: 14 | 000005:[a#3,SET-a#3,SET] 15 | L2: 16 | 000006:[a#2,SET-a#2,SET] 17 | L3: 18 | 000007:[a#1,SET-a#1,SET] 19 | 20 | iter 21 | first 22 | ---- 23 | a: (4, .) 24 | 25 | # Only scan tables with min-seq-num < filter. 26 | 27 | iter filter=5 28 | first 29 | ---- 30 | a: (4, .) 31 | 32 | iter filter=4 33 | first 34 | ---- 35 | a: (3, .) 36 | 37 | iter filter=3 38 | first 39 | ---- 40 | a: (2, .) 41 | 42 | iter filter=2 43 | first 44 | ---- 45 | a: (1, .) 46 | 47 | iter filter=1 48 | first 49 | ---- 50 | . 51 | -------------------------------------------------------------------------------- /testdata/lsm_view: -------------------------------------------------------------------------------- 1 | define 2 | L0 3 | b.SET.3:3 4 | e.SET.4:4 5 | L0 6 | d.SET.5:5 7 | f.SET.6:6 8 | L0 9 | x.SET.7:7 10 | y.SET.8:8 11 | L3 12 | g.SET.1:1 13 | h.SET.2:2 14 | ---- 15 | https://raduberinde.github.io/lsmview/decode.html#eJyMkr9us0AQxPvvKVZz7fKJfwfWlZHSpUu6CFkg1tgyxokhkUnkd49AkGBA2FRzM1q43yzfyOVT8hLmtZPrIj4IDJ7s_w4YVZzk0sVxIjkMNBjl7ktgdKgZ5SHOcymr9V5qGIeRx6fs9-wxUqniXfsJ2M2jDaVK8_PjC1m0UUGjulca0qF-aA7yXnwcSkOaLArAeDvuiqqcn6TebfXQL450iotMKG0Yo0t04SmmPY_pL2HaI0x3iukbSpTXXVaUv4jpkUX-AHNuknq31UN_jMl_FMESRTCiCKcUgaGzCru71Gq1SBGSRasBxdwk9W6rh_4dy7r6I2dy90buza86XCrJH5WkpyWFhjLldKhb5S6W5JBF7qCkuUnq3VYP_TtK8m-UoG_kwXUeMfZSt7AJGCkYAsYGjAyMLRhnMGpEl38_AQAA___EPTzv 16 | -------------------------------------------------------------------------------- /testdata/manual_flush: -------------------------------------------------------------------------------- 1 | batch 2 | set a 1 3 | set b 2 4 | ---- 5 | 6 | # The first L0 table can have its seqnums zeroed. 7 | flush 8 | ---- 9 | L0.0: 10 | 000005:[a#10,SET-b#11,SET] 11 | 12 | reset 13 | ---- 14 | 15 | batch 16 | set a 1 17 | set b 2 18 | del a 19 | del b 20 | ---- 21 | 22 | flush 23 | ---- 24 | L0.0: 25 | 000005:[a#12,DEL-b#13,DEL] 26 | 27 | batch 28 | set a 3 29 | ---- 30 | 31 | # A second (overlapping) L0 table will have non-zero seqnums. 32 | flush 33 | ---- 34 | L0.1: 35 | 000007:[a#14,SET-a#14,SET] 36 | L0.0: 37 | 000005:[a#12,DEL-b#13,DEL] 38 | 39 | batch 40 | set c 4 41 | ---- 42 | 43 | # A third (non-overlapping) L0 table will have non-zero seqnums. 44 | flush 45 | ---- 46 | L0.1: 47 | 000007:[a#14,SET-a#14,SET] 48 | L0.0: 49 | 000005:[a#12,DEL-b#13,DEL] 50 | 000009:[c#15,SET-c#15,SET] 51 | 52 | reset 53 | ---- 54 | 55 | batch 56 | set a 1 57 | set b 2 58 | del-range a c 59 | ---- 60 | 61 | flush 62 | ---- 63 | L0.0: 64 | 000005:[a#12,RANGEDEL-c#inf,RANGEDEL] 65 | 66 | reset 67 | ---- 68 | 69 | batch 70 | set a 1 71 | set b 2 72 | ---- 73 | 74 | async-flush 75 | ---- 76 | L0.0: 77 | 000005:[a#10,SET-b#11,SET] 78 | -------------------------------------------------------------------------------- /testdata/marked_for_compaction: -------------------------------------------------------------------------------- 1 | define 2 | L0 3 | c.SET.11:foo 4 | L1 5 | c.SET.0:foo 6 | d.SET.0:foo 7 | ---- 8 | L0.0: 9 | 000004:[c#11,SET-c#11,SET] seqnums:[11-11] points:[c#11,SET-c#11,SET] size:744 10 | L1: 11 | 000005:[c#0,SET-d#0,SET] seqnums:[0-0] points:[c#0,SET-d#0,SET] size:755 12 | 13 | mark-for-compaction file=000005 14 | ---- 15 | marked L1.000005 16 | 17 | mark-for-compaction file=000004 18 | ---- 19 | marked L0.000004 20 | 21 | maybe-compact 22 | ---- 23 | [JOB 100] compacted(rewrite) L1 [000005] (755B) Score=0.00 + L1 [] (0B) Score=0.00 -> L1 [000006] (755B), in 1.0s (2.0s total), output rate 755B/s 24 | [JOB 100] compacted(rewrite) L0 [000004] (744B) Score=0.00 + L0 [] (0B) Score=0.00 -> L0 [000007] (744B), in 1.0s (2.0s total), output rate 744B/s 25 | L0.0: 26 | 000007:[c#11,SET-c#11,SET] seqnums:[11-11] points:[c#11,SET-c#11,SET] size:744 27 | L1: 28 | 000006:[c#0,SET-d#0,SET] seqnums:[0-0] points:[c#0,SET-d#0,SET] size:755 29 | -------------------------------------------------------------------------------- /testdata/mem_table: -------------------------------------------------------------------------------- 1 | new size=10000 2 | ---- 3 | 4 | prepare name=batch1 5 | set foo foo 6 | set bar bar 7 | ---- 8 | 8475 of 10000 bytes available 9 | 10 | apply name=batch1 seq=1 11 | ---- 12 | 8475 of 10000 bytes available 13 | 14 | computePossibleOverlaps 15 | a-f 16 | a-bar 17 | a-foo 18 | bar-foo 19 | foo-zoo 20 | goo-zoo 21 | ---- 22 | {a f} 23 | {a foo} 24 | {bar foo} 25 | {foo zoo} 26 | 27 | computePossibleOverlaps stop-after-first 28 | a-f 29 | a-bar 30 | a-foo 31 | bar-foo 32 | foo-zoo 33 | goo-zoo 34 | ---- 35 | {a f} 36 | -------------------------------------------------------------------------------- /testdata/peek: -------------------------------------------------------------------------------- 1 | peek testdata/db-stage-1 2 | ---- 3 | initialized at format major version 013 4 | manifest: testdata/db-stage-1/MANIFEST-000001 5 | options: testdata/db-stage-1/OPTIONS-000003 6 | 7 | peek testdata/db-stage-2 8 | ---- 9 | initialized at format major version 013 10 | manifest: testdata/db-stage-2/MANIFEST-000001 11 | options: testdata/db-stage-2/OPTIONS-000003 12 | 13 | peek testdata/db-stage-3 14 | ---- 15 | initialized at format major version 013 16 | manifest: testdata/db-stage-3/MANIFEST-000006 17 | options: testdata/db-stage-3/OPTIONS-000007 18 | 19 | peek testdata/db-stage-4 20 | ---- 21 | initialized at format major version 013 22 | manifest: testdata/db-stage-4/MANIFEST-000006 23 | options: testdata/db-stage-4/OPTIONS-000007 24 | 25 | peek testdata/db-stage-5 26 | ---- 27 | err=open testdata/db-stage-5: no such file or directory 28 | 29 | peek testdata 30 | ---- 31 | uninitialized 32 | -------------------------------------------------------------------------------- /testdata/point_collapsing_iter: -------------------------------------------------------------------------------- 1 | 2 | define 3 | a.SET.5:foo 4 | b.SET.6:foo 5 | b.DEL.4: 6 | c.SET.7:bar 7 | c.SET.5:foo 8 | ---- 9 | 10 | iter 11 | first 12 | next 13 | next 14 | next 15 | next 16 | ---- 17 | a#5,SET:foo 18 | b#6,SET:foo 19 | c#7,SET:bar 20 | . 21 | . 22 | 23 | # Ensure that we pause at (and return) rangedel start points correctly. 24 | 25 | define 26 | a.RANGEDEL.4:b 27 | a.SET.5:foo 28 | b.RANGEDEL.3:c 29 | b.SET.6:foo 30 | b.DEL.4: 31 | c.SET.7:bar 32 | c.SET.5:foo 33 | ---- 34 | 35 | iter 36 | seek-ge b 37 | next 38 | next 39 | ---- 40 | b#inf,RANGEDEL: 41 | b#6,SET:foo 42 | c#7,SET:bar 43 | 44 | # More rangedel elision tests 45 | 46 | define 47 | a.RANGEDEL.4:b 48 | a.SET.5:foo 49 | b.RANGEDEL.4:c 50 | b.SET.3:foo 51 | b.DEL.2: 52 | c.SET.7:bar 53 | c.SET.5:foo 54 | ---- 55 | 56 | iter 57 | seek-ge a 58 | next 59 | next 60 | next 61 | next 62 | ---- 63 | a#inf,RANGEDEL: 64 | a#5,SET:foo 65 | b#inf,RANGEDEL: 66 | c#7,SET:bar 67 | . 68 | -------------------------------------------------------------------------------- /tool/db_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package tool 6 | 7 | import "testing" 8 | 9 | func TestDB(t *testing.T) { 10 | runTests(t, "testdata/db_*") 11 | } 12 | -------------------------------------------------------------------------------- /tool/find_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package tool 6 | 7 | import "testing" 8 | 9 | func TestFind(t *testing.T) { 10 | runTests(t, "testdata/find") 11 | } 12 | -------------------------------------------------------------------------------- /tool/logs/tool.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package logs 6 | 7 | import ( 8 | "time" 9 | 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | // NewCmd returns a new cobra.Command for parsing logs. 14 | func NewCmd() *cobra.Command { 15 | cmd := &cobra.Command{ 16 | Use: "logs", 17 | Short: "Scan and summarize logs", 18 | } 19 | 20 | compactionCmd := &cobra.Command{ 21 | Use: "compactions", 22 | Short: "Scan and summarize compaction logs", 23 | RunE: runCompactionLogs, 24 | } 25 | compactionCmd.Flags().Duration( 26 | "window", 10*time.Minute, "time window in which to aggregate compactions") 27 | compactionCmd.Flags().Duration( 28 | "long-running-limit", 0, "log compactions with runtime greater than the limit") 29 | 30 | cmd.AddCommand(compactionCmd) 31 | return cmd 32 | } 33 | -------------------------------------------------------------------------------- /tool/make_lsm_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | dest=lsm_data.go 4 | do="DO" 5 | not="NOT" 6 | edit="EDIT" 7 | 8 | cat > ${dest} <> ${dest} 17 | 18 | cat >> ${dest} <> ${dest} 25 | 26 | cat>> ${dest} <= b#0,SET 12 | 13 | sstable check 14 | --key=%x 15 | testdata/out-of-order-sst/000001.sst 16 | ---- 17 | 000001.sst 18 | WARNING: OUT OF ORDER KEYS! 19 | 63#0,SET >= 62#0,SET 20 | 21 | sstable check 22 | --key=pretty 23 | testdata/out-of-order-sst/000001.sst 24 | ---- 25 | 000001.sst 26 | WARNING: OUT OF ORDER KEYS! 27 | c#0,SET >= b#0,SET 28 | 29 | sstable check 30 | --key=pretty:test-comparer 31 | testdata/out-of-order-sst/000001.sst 32 | ---- 33 | 000001.sst 34 | WARNING: OUT OF ORDER KEYS! 35 | test formatter: c#0,SET >= test formatter: b#0,SET 36 | 37 | sstable check 38 | --key=null 39 | testdata/out-of-order-sst/000001.sst 40 | ---- 41 | 000001.sst 42 | WARNING: OUT OF ORDER KEYS! 43 | 44 | sstable check 45 | testdata/corrupted-sst/000003.sst 46 | ---- 47 | 000003.sst: pebble: file 000003: block 87/465: crc32c checksum mismatch c8539ba5 != b972e324 48 | 49 | sstable check 50 | testdata/bad-magic-sst/000015.sst 51 | ---- 52 | 000015.sst: pebble/table: invalid table 000015: (bad magic number: 0xf6cff485b741e288) 53 | -------------------------------------------------------------------------------- /tool/wal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package tool 6 | 7 | import "testing" 8 | 9 | func TestWAL(t *testing.T) { 10 | runTests(t, "testdata/wal_*") 11 | } 12 | -------------------------------------------------------------------------------- /vfs/disk_usage_netbsd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build netbsd 6 | 7 | package vfs 8 | 9 | import "golang.org/x/sys/unix" 10 | 11 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 12 | stat := unix.Statvfs_t{} 13 | if err := unix.Statvfs(path, &stat); err != nil { 14 | return DiskUsage{}, err 15 | } 16 | 17 | freeBytes := uint64(stat.Bsize) * uint64(stat.Bfree) 18 | availBytes := uint64(stat.Bsize) * uint64(stat.Bavail) 19 | totalBytes := uint64(stat.Bsize) * uint64(stat.Blocks) 20 | return DiskUsage{ 21 | AvailBytes: availBytes, 22 | TotalBytes: totalBytes, 23 | UsedBytes: totalBytes - freeBytes, 24 | }, nil 25 | } 26 | -------------------------------------------------------------------------------- /vfs/disk_usage_openbsd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build openbsd 6 | 7 | package vfs 8 | 9 | import "golang.org/x/sys/unix" 10 | 11 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 12 | stat := unix.Statfs_t{} 13 | if err := unix.Statfs(path, &stat); err != nil { 14 | return DiskUsage{}, err 15 | } 16 | 17 | freeBytes := uint64(stat.F_bsize) * uint64(stat.F_bfree) 18 | availBytes := uint64(stat.F_bsize) * uint64(stat.F_bavail) 19 | totalBytes := uint64(stat.F_bsize) * uint64(stat.F_blocks) 20 | return DiskUsage{ 21 | AvailBytes: availBytes, 22 | TotalBytes: totalBytes, 23 | UsedBytes: totalBytes - freeBytes, 24 | }, nil 25 | } 26 | -------------------------------------------------------------------------------- /vfs/disk_usage_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build darwin || dragonfly || freebsd 6 | 7 | package vfs 8 | 9 | import "golang.org/x/sys/unix" 10 | 11 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 12 | stat := unix.Statfs_t{} 13 | if err := unix.Statfs(path, &stat); err != nil { 14 | return DiskUsage{}, err 15 | } 16 | 17 | freeBytes := uint64(stat.Bsize) * uint64(stat.Bfree) 18 | availBytes := uint64(stat.Bsize) * uint64(stat.Bavail) 19 | totalBytes := uint64(stat.Bsize) * uint64(stat.Blocks) 20 | return DiskUsage{ 21 | AvailBytes: availBytes, 22 | TotalBytes: totalBytes, 23 | UsedBytes: totalBytes - freeBytes, 24 | }, nil 25 | } 26 | -------------------------------------------------------------------------------- /vfs/disk_usage_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build windows 6 | 7 | package vfs 8 | 9 | import "golang.org/x/sys/windows" 10 | 11 | func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { 12 | p, err := windows.UTF16PtrFromString(path) 13 | if err != nil { 14 | return DiskUsage{}, err 15 | } 16 | var freeBytes uint64 17 | du := DiskUsage{} 18 | err = windows.GetDiskFreeSpaceEx(p, &du.AvailBytes, &du.TotalBytes, &freeBytes) 19 | du.UsedBytes = du.TotalBytes - freeBytes 20 | return du, err 21 | } 22 | -------------------------------------------------------------------------------- /vfs/errorfs/errorfs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package errorfs 6 | 7 | import ( 8 | "fmt" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/cockroachdb/datadriven" 13 | ) 14 | 15 | func TestErrorFS(t *testing.T) { 16 | var sb strings.Builder 17 | datadriven.RunTest(t, "testdata/errorfs", func(t *testing.T, td *datadriven.TestData) string { 18 | sb.Reset() 19 | switch td.Cmd { 20 | case "parse-dsl": 21 | for _, l := range strings.Split(strings.TrimSpace(td.Input), "\n") { 22 | inj, err := ParseDSL(l) 23 | if err != nil { 24 | fmt.Fprintf(&sb, "parsing err: %s\n", err) 25 | } else { 26 | fmt.Fprintf(&sb, "%s\n", inj.String()) 27 | } 28 | } 29 | return sb.String() 30 | default: 31 | return fmt.Sprintf("unrecognized command %q", td.Cmd) 32 | } 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /vfs/errors_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build darwin || dragonfly || freebsd || linux || openbsd || netbsd 6 | 7 | package vfs 8 | 9 | import ( 10 | "github.com/cockroachdb/errors" 11 | "golang.org/x/sys/unix" 12 | ) 13 | 14 | var errNotEmpty = unix.ENOTEMPTY 15 | 16 | // IsNoSpaceError returns true if the given error indicates that the disk is 17 | // out of space. 18 | func IsNoSpaceError(err error) bool { 19 | return errors.Is(err, unix.ENOSPC) 20 | } 21 | -------------------------------------------------------------------------------- /vfs/errors_unix_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build darwin || dragonfly || freebsd || linux || openbsd || netbsd 6 | 7 | package vfs 8 | 9 | import ( 10 | "testing" 11 | 12 | "github.com/cockroachdb/errors" 13 | "github.com/stretchr/testify/require" 14 | "golang.org/x/sys/unix" 15 | ) 16 | 17 | func TestIsNoSpaceError(t *testing.T) { 18 | err := errors.WithStack(unix.ENOSPC) 19 | require.True(t, IsNoSpaceError(err)) 20 | } 21 | -------------------------------------------------------------------------------- /vfs/errors_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build windows 6 | 7 | package vfs 8 | 9 | import ( 10 | "github.com/cockroachdb/errors" 11 | "golang.org/x/sys/windows" 12 | ) 13 | 14 | var errNotEmpty = windows.ERROR_DIR_NOT_EMPTY 15 | 16 | // IsNoSpaceError returns true if the given error indicates that the disk is 17 | // out of space. 18 | func IsNoSpaceError(err error) bool { 19 | return errors.Is(err, windows.ERROR_DISK_FULL) || 20 | errors.Is(err, windows.ERROR_HANDLE_DISK_FULL) 21 | } 22 | -------------------------------------------------------------------------------- /vfs/fadvise_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !linux 6 | 7 | package vfs 8 | 9 | func fadviseRandom(f uintptr) error { 10 | return nil 11 | } 12 | 13 | func fadviseSequential(f uintptr) error { 14 | return nil 15 | } 16 | -------------------------------------------------------------------------------- /vfs/fadvise_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build linux 6 | 7 | package vfs 8 | 9 | import "golang.org/x/sys/unix" 10 | 11 | // Calls Fadvise with FADV_RANDOM to disable readahead on a file descriptor. 12 | func fadviseRandom(f uintptr) error { 13 | return unix.Fadvise(int(f), 0, 0, unix.FADV_RANDOM) 14 | } 15 | 16 | // Calls Fadvise with FADV_SEQUENTIAL to enable readahead on a file descriptor. 17 | func fadviseSequential(f uintptr) error { 18 | return unix.Fadvise(int(f), 0, 0, unix.FADV_SEQUENTIAL) 19 | } 20 | -------------------------------------------------------------------------------- /vfs/fd_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | package vfs 6 | 7 | import ( 8 | "os" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestFileWrappersHaveFd(t *testing.T) { 16 | // Use the real filesystem so that we can test vfs.Default, which returns 17 | // files with Fd(). 18 | tmpf, err := os.CreateTemp("", "pebble-db-fd-file") 19 | require.NoError(t, err) 20 | filename := tmpf.Name() 21 | defer os.Remove(filename) 22 | 23 | // File wrapper case 1: Check if diskHealthCheckingFile has Fd(). 24 | fs2, closer := WithDiskHealthChecks(Default, 10*time.Second, nil, func(info DiskSlowInfo) {}) 25 | 26 | defer closer.Close() 27 | f2, err := fs2.Open(filename) 28 | require.NoError(t, err) 29 | require.NotZero(t, f2.Fd()) 30 | require.NotEqual(t, f2.Fd(), InvalidFd) 31 | // File wrapper case 2: Check if syncingFile has Fd(). 32 | f3 := NewSyncingFile(f2, SyncingFileOptions{BytesPerSync: 8 << 10 /* 8 KB */}) 33 | require.NotZero(t, f3.Fd()) 34 | require.NotEqual(t, f3.Fd(), InvalidFd) 35 | require.NoError(t, f2.Close()) 36 | } 37 | -------------------------------------------------------------------------------- /vfs/file_lock_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows 6 | 7 | package vfs 8 | 9 | import ( 10 | "io" 11 | "runtime" 12 | 13 | "github.com/cockroachdb/errors" 14 | ) 15 | 16 | func (defFS) Lock(name string) (io.Closer, error) { 17 | return nil, errors.Errorf("pebble: file locking is not implemented on %s/%s", 18 | errors.Safe(runtime.GOOS), errors.Safe(runtime.GOARCH)) 19 | } 20 | -------------------------------------------------------------------------------- /vfs/file_lock_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 | // of this source code is governed by a BSD-style license that can be found in 3 | // the LICENSE file. 4 | 5 | //go:build windows 6 | 7 | package vfs 8 | 9 | import ( 10 | "io" 11 | 12 | "golang.org/x/sys/windows" 13 | ) 14 | 15 | // lockCloser hides all of an windows.Handle's methods, except for Close. 16 | type lockCloser struct { 17 | fd windows.Handle 18 | } 19 | 20 | func (l lockCloser) Close() error { 21 | return windows.Close(l.fd) 22 | } 23 | 24 | // Lock locks the given file. On Windows, Locking will fail if the file is 25 | // already open by the current process. 26 | func (defaultFS) Lock(name string) (io.Closer, error) { 27 | p, err := windows.UTF16PtrFromString(name) 28 | if err != nil { 29 | return nil, err 30 | } 31 | fd, err := windows.CreateFile(p, 32 | windows.GENERIC_READ|windows.GENERIC_WRITE, 33 | 0, nil, windows.CREATE_ALWAYS, 34 | windows.FILE_ATTRIBUTE_NORMAL, 35 | 0, 36 | ) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return lockCloser{fd: fd}, nil 41 | } 42 | -------------------------------------------------------------------------------- /vfs/testdata/memfs_lock: -------------------------------------------------------------------------------- 1 | mkfs A B 2 | ---- 3 | OK 4 | 5 | # 6 | # Locking a path with parents that don't exist should error. 7 | # 8 | 9 | lock fs=A path=a/b/c handle=fsApathABC 10 | ---- 11 | open a/b/c: file does not exist 12 | 13 | # 14 | # If we create the parents, it should succeed. 15 | # 16 | 17 | mkdirall fs=A path=a/b 18 | ---- 19 | OK 20 | 21 | lock fs=A path=a/b/c handle=fsApathABC 22 | ---- 23 | OK 24 | 25 | # 26 | # Locking the same path on the same filesystem should fail with EAGAIN. 27 | # 28 | 29 | lock fs=A path=a/b/c handle=bogus 30 | ---- 31 | resource temporarily unavailable 32 | 33 | # 34 | # Locking the same path on a DIFFERENT filesystem should succeed. 35 | # 36 | 37 | mkdirall fs=B path=a/b 38 | ---- 39 | OK 40 | 41 | lock fs=B path=a/b/c handle=fsBpathABC 42 | ---- 43 | OK 44 | 45 | # 46 | # Releasing the lock on fs A should allow us to reacquire it. 47 | # 48 | 49 | close handle=fsApathABC 50 | ---- 51 | OK 52 | 53 | lock fs=A path=a/b/c handle=fsApathABC 54 | ---- 55 | OK 56 | --------------------------------------------------------------------------------