diff --git a/BUCK b/BUCK index dfabafced8f1..8358a23a0ca3 100644 --- a/BUCK +++ b/BUCK @@ -39,6 +39,11 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[ "db/blob/blob_log_format.cc", "db/blob/blob_log_sequential_reader.cc", "db/blob/blob_log_writer.cc", + "db/blob/blob_reference_census.cc", + "db/blob/blob_reference_metadata.cc", + "db/blob/blob_relocation_file.cc", + "db/blob/blob_relocation_gc.cc", + "db/blob/blob_relocation_manifest.cc", "db/blob/blob_source.cc", "db/blob/blob_write_batch_transformer.cc", "db/blob/prefetch_buffer_collection.cc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e8482391a7e..5fa908171e66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -788,6 +788,11 @@ set(SOURCES db/blob/blob_log_format.cc db/blob/blob_log_sequential_reader.cc db/blob/blob_log_writer.cc + db/blob/blob_relocation_file.cc + db/blob/blob_relocation_gc.cc + db/blob/blob_relocation_manifest.cc + db/blob/blob_reference_census.cc + db/blob/blob_reference_metadata.cc db/blob/blob_source.cc db/blob/blob_write_batch_transformer.cc db/blob/prefetch_buffer_collection.cc diff --git a/c_api_gen/c_generated_option_structs_auto.cc.inc b/c_api_gen/c_generated_option_structs_auto.cc.inc index 54ad0a867a79..4728c4f82327 100644 --- a/c_api_gen/c_generated_option_structs_auto.cc.inc +++ b/c_api_gen/c_generated_option_structs_auto.cc.inc @@ -1349,6 +1349,36 @@ unsigned char rocksdb_options_get_enable_blob_files(rocksdb_options_t* opt) { return opt->rep.enable_blob_files; } +void rocksdb_options_set_enable_blob_indirection(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.enable_blob_indirection = v; +} + +unsigned char rocksdb_options_get_enable_blob_indirection( + rocksdb_options_t* opt) { + return opt->rep.enable_blob_indirection; +} + +void rocksdb_options_set_pin_blob_relocation_location_blocks( + rocksdb_options_t* opt, unsigned char v) { + opt->rep.pin_blob_relocation_location_blocks = v; +} + +unsigned char rocksdb_options_get_pin_blob_relocation_location_blocks( + rocksdb_options_t* opt) { + return opt->rep.pin_blob_relocation_location_blocks; +} + +void rocksdb_options_set_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt, uint64_t v) { + opt->rep.blob_indirection_max_references_per_sst = v; +} + +uint64_t rocksdb_options_get_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt) { + return opt->rep.blob_indirection_max_references_per_sst; +} + void rocksdb_options_set_min_blob_size(rocksdb_options_t* opt, uint64_t v) { opt->rep.min_blob_size = v; } @@ -1410,6 +1440,16 @@ double rocksdb_options_get_blob_gc_force_threshold(rocksdb_options_t* opt) { return opt->rep.blob_garbage_collection_force_threshold; } +void rocksdb_options_set_indirect_blob_gc_read_coalesce_bytes( + rocksdb_options_t* opt, uint64_t v) { + opt->rep.indirect_blob_gc_read_coalesce_bytes = v; +} + +uint64_t rocksdb_options_get_indirect_blob_gc_read_coalesce_bytes( + rocksdb_options_t* opt) { + return opt->rep.indirect_blob_gc_read_coalesce_bytes; +} + void rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, uint64_t v) { opt->rep.blob_compaction_readahead_size = v; diff --git a/c_api_gen/c_generated_option_structs_auto.h.inc b/c_api_gen/c_generated_option_structs_auto.h.inc index 79e04d3a41f2..e47868ed3e93 100644 --- a/c_api_gen/c_generated_option_structs_auto.h.inc +++ b/c_api_gen/c_generated_option_structs_auto.h.inc @@ -431,6 +431,27 @@ rocksdb_options_set_preserve_internal_time_seconds(rocksdb_options_t* opt, extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_get_preserve_internal_time_seconds(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_indirection( + rocksdb_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_blob_indirection(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_pin_blob_relocation_location_blocks(rocksdb_options_t* opt, + unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_pin_blob_relocation_location_blocks(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt, uint64_t v); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt); + extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_writable_file_max_buffer_size( rocksdb_options_t* opt, uint64_t v); @@ -439,6 +460,14 @@ extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_get_blob_file_writable_file_max_buffer_size( rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_indirect_blob_gc_read_coalesce_bytes(rocksdb_options_t* opt, + uint64_t v); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_indirect_blob_gc_read_coalesce_bytes( + rocksdb_options_t* opt); + extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_direct_write( rocksdb_options_t* opt, unsigned char v); diff --git a/c_api_gen/c_generated_roundtrip_tests.c.inc b/c_api_gen/c_generated_roundtrip_tests.c.inc index 86a62acba40a..8fc7b378b3ff 100644 --- a/c_api_gen/c_generated_roundtrip_tests.c.inc +++ b/c_api_gen/c_generated_roundtrip_tests.c.inc @@ -681,6 +681,12 @@ static void rocksdb_roundtrip_test_rocksdb_options(void) { rocksdb_options_set_blob_file_writable_file_max_buffer_size(obj, 0); CheckCondition( rocksdb_options_get_blob_file_writable_file_max_buffer_size(obj) == 0); + rocksdb_options_set_blob_indirection_max_references_per_sst(obj, 1); + CheckCondition( + rocksdb_options_get_blob_indirection_max_references_per_sst(obj) == 1); + rocksdb_options_set_blob_indirection_max_references_per_sst(obj, 0); + CheckCondition( + rocksdb_options_get_blob_indirection_max_references_per_sst(obj) == 0); rocksdb_options_set_block_protection_bytes_per_key(obj, 1); CheckCondition(rocksdb_options_get_block_protection_bytes_per_key(obj) == 1); rocksdb_options_set_block_protection_bytes_per_key(obj, 0); @@ -755,6 +761,10 @@ static void rocksdb_roundtrip_test_rocksdb_options(void) { CheckCondition(rocksdb_options_get_enable_blob_direct_write(obj) == 1); rocksdb_options_set_enable_blob_direct_write(obj, 0); CheckCondition(rocksdb_options_get_enable_blob_direct_write(obj) == 0); + rocksdb_options_set_enable_blob_indirection(obj, 1); + CheckCondition(rocksdb_options_get_enable_blob_indirection(obj) == 1); + rocksdb_options_set_enable_blob_indirection(obj, 0); + CheckCondition(rocksdb_options_get_enable_blob_indirection(obj) == 0); rocksdb_options_set_enable_thread_tracking(obj, 1); CheckCondition(rocksdb_options_get_enable_thread_tracking(obj) == 1); rocksdb_options_set_enable_thread_tracking(obj, 0); @@ -797,6 +807,12 @@ static void rocksdb_roundtrip_test_rocksdb_options(void) { CheckCondition(rocksdb_options_get_force_consistency_checks(obj) == 1); rocksdb_options_set_force_consistency_checks(obj, 0); CheckCondition(rocksdb_options_get_force_consistency_checks(obj) == 0); + rocksdb_options_set_indirect_blob_gc_read_coalesce_bytes(obj, 1); + CheckCondition( + rocksdb_options_get_indirect_blob_gc_read_coalesce_bytes(obj) == 1); + rocksdb_options_set_indirect_blob_gc_read_coalesce_bytes(obj, 0); + CheckCondition( + rocksdb_options_get_indirect_blob_gc_read_coalesce_bytes(obj) == 0); rocksdb_options_set_last_level_temperature(obj, 1); CheckCondition(rocksdb_options_get_last_level_temperature(obj) == 1); rocksdb_options_set_last_level_temperature(obj, 0); @@ -875,6 +891,12 @@ static void rocksdb_roundtrip_test_rocksdb_options(void) { CheckCondition(rocksdb_options_get_persist_user_defined_timestamps(obj) == 1); rocksdb_options_set_persist_user_defined_timestamps(obj, 0); CheckCondition(rocksdb_options_get_persist_user_defined_timestamps(obj) == 0); + rocksdb_options_set_pin_blob_relocation_location_blocks(obj, 1); + CheckCondition(rocksdb_options_get_pin_blob_relocation_location_blocks(obj) == + 1); + rocksdb_options_set_pin_blob_relocation_location_blocks(obj, 0); + CheckCondition(rocksdb_options_get_pin_blob_relocation_location_blocks(obj) == + 0); rocksdb_options_set_preclude_last_level_data_seconds(obj, 1); CheckCondition(rocksdb_options_get_preclude_last_level_data_seconds(obj) == 1); diff --git a/cache/cache_entry_roles.cc b/cache/cache_entry_roles.cc index 6fe19a165c37..0f9eb184df51 100644 --- a/cache/cache_entry_roles.cc +++ b/cache/cache_entry_roles.cc @@ -26,6 +26,9 @@ std::array kCacheEntryRoleToCamelString{{ "BlobValue", "BlobCache", "ReadaheadBuffer", + "BlobReferenceMetadataBuildingBuffer", + "BlobReferenceCensus", + "IndirectBlobGcWorkingMemory", "Misc", }}; @@ -44,6 +47,9 @@ std::array kCacheEntryRoleToHyphenString{{ "blob-value", "blob-cache", "readahead-buffer", + "blob-reference-metadata-building-buffer", + "blob-reference-census", + "indirect-blob-gc-working-memory", "misc", }}; diff --git a/cache/cache_reservation_manager.cc b/cache/cache_reservation_manager.cc index b8d61684ced2..e3e5326bcc77 100644 --- a/cache/cache_reservation_manager.cc +++ b/cache/cache_reservation_manager.cc @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "rocksdb/cache.h" @@ -39,6 +40,18 @@ CacheReservationManagerImpl< s.PermitUncheckedError(); } +template +Status CacheReservationManagerImpl::CacheReservationHandle::Resize( + std::size_t memory_used) { + const std::size_t others = + cache_res_mgr_->GetTotalMemoryUsed() - incremental_memory_used_; + if (memory_used > std::numeric_limits::max() - others) { + return Status::MemoryLimit("Cache reservation size overflow"); + } + incremental_memory_used_ = memory_used; + return cache_res_mgr_->UpdateCacheReservation(others + memory_used); +} + template CacheReservationManagerImpl::CacheReservationManagerImpl( std::shared_ptr cache, bool delayed_decrease) @@ -201,6 +214,12 @@ template class CacheReservationManagerImpl; template class CacheReservationManagerImpl; template class CacheReservationManagerImpl; template class CacheReservationManagerImpl; +template class CacheReservationManagerImpl< + CacheEntryRole::kBlobReferenceMetadataBuildingBuffer>; +template class CacheReservationManagerImpl< + CacheEntryRole::kBlobReferenceCensus>; +template class CacheReservationManagerImpl< + CacheEntryRole::kIndirectBlobGcWorkingMemory>; template class CacheReservationManagerImpl; template class CacheReservationManagerImpl; diff --git a/cache/cache_reservation_manager.h b/cache/cache_reservation_manager.h index 51f9ea9b137c..b196a1de4acc 100644 --- a/cache/cache_reservation_manager.h +++ b/cache/cache_reservation_manager.h @@ -35,6 +35,17 @@ class CacheReservationManager { class CacheReservationHandle { public: virtual ~CacheReservationHandle() = default; + + // Sets the memory that this handle owns to `memory_used` bytes. The other + // handles of the same manager keep the memory they own. The manager's new + // total is its old total minus this handle's old size plus `memory_used`. + // When the cache cannot reserve the new total, Resize() returns an error, + // but the handle still owns `memory_used`, the same as after a failed + // MakeCacheReservation(). A later Resize() or the handle's destructor + // releases it. After an error, the caller must not allocate the memory. + // When the new total would overflow size_t, Resize() returns MemoryLimit + // and the handle keeps its old size. + virtual Status Resize(std::size_t memory_used) = 0; }; virtual ~CacheReservationManager() = default; virtual Status UpdateCacheReservation(std::size_t new_memory_used) = 0; @@ -76,6 +87,7 @@ class CacheReservationManagerImpl std::size_t incremental_memory_used, std::shared_ptr cache_res_mgr); ~CacheReservationHandle() override; + Status Resize(std::size_t memory_used) override; private: std::size_t incremental_memory_used_; @@ -255,6 +267,10 @@ class ConcurrentCacheReservationManager std::lock_guard lock(cache_res_mgr_->cache_res_mgr_mu_); cache_res_handle_.reset(); } + Status Resize(std::size_t memory_used) override { + std::lock_guard lock(cache_res_mgr_->cache_res_mgr_mu_); + return cache_res_handle_->Resize(memory_used); + } private: std::shared_ptr cache_res_mgr_; diff --git a/cache/cache_reservation_manager_test.cc b/cache/cache_reservation_manager_test.cc index c1507350220f..de67925b1f5a 100644 --- a/cache/cache_reservation_manager_test.cc +++ b/cache/cache_reservation_manager_test.cc @@ -16,8 +16,10 @@ #include #include "cache/cache_entry_roles.h" +#include "db/blob/blob_reference_metadata.h" #include "rocksdb/cache.h" #include "rocksdb/slice.h" +#include "rocksdb/table.h" #include "test_util/mock_time_env.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" @@ -43,6 +45,70 @@ class CacheReservationManagerTest : public ::testing::Test { } }; +TEST_F(CacheReservationManagerTest, ResizeKeepsOtherOwnersCharged) { + std::shared_ptr manager = + std::make_shared(test_cache_rev_mng); + std::unique_ptr first; + std::unique_ptr second; + ASSERT_OK(manager->MakeCacheReservation(100, &first)); + ASSERT_OK(manager->MakeCacheReservation(200, &second)); + ASSERT_OK(first->Resize(400)); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 600U); + ASSERT_OK(second->Resize(0)); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 400U); + first.reset(); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 0U); + second.reset(); +} + +TEST_F(CacheReservationManagerTest, BlobSummariesShareOneReservationQuantum) { + // BlockBasedTableFactory makes one manager of this form and gives it to + // every blob reference collector. A BlobReferenceSstSummary takes over the + // handle of its collector, so it uses the same manager. This test checks + // that 1000 handles of 100 bytes each share one dummy cache entry + // (kSizeDummyEntry) instead of reserving one entry each, and that the + // reservation goes back to 0 when all handles are released. + std::shared_ptr first = + std::make_shared( + std::make_shared>(cache)); + std::shared_ptr second = first; + std::vector> + owners; + for (size_t i = 0; i < 1000; ++i) { + std::unique_ptr owner; + ASSERT_OK(first->MakeCacheReservation(100, &owner)); + owners.push_back(std::move(owner)); + } + ASSERT_EQ(first->GetTotalMemoryUsed(), 100000U); + ASSERT_EQ(first->GetTotalReservedCacheSize(), kSizeDummyEntry); + owners.front().reset(); + ASSERT_EQ(second->GetTotalMemoryUsed(), 99900U); + owners.clear(); + ASSERT_EQ(first->GetTotalMemoryUsed(), 0U); + ASSERT_EQ(first->GetTotalReservedCacheSize(), 0U); + first.reset(); + second.reset(); + ASSERT_EQ(cache->GetPinnedUsage(), 0U); +} + +TEST_F(CacheReservationManagerTest, FailedResizeRemainsLocallyOwned) { + cache->SetCapacity(2 * kSizeDummyEntry); + cache->SetStrictCapacityLimit(true); + std::shared_ptr manager = + std::make_shared(test_cache_rev_mng); + std::unique_ptr first; + std::unique_ptr second; + ASSERT_OK(manager->MakeCacheReservation(10, &first)); + ASSERT_OK(manager->MakeCacheReservation(20, &second)); + ASSERT_TRUE(second->Resize(3 * kSizeDummyEntry).IsMemoryLimit()); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 10 + 3 * kSizeDummyEntry); + second.reset(); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 10U); + first.reset(); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 0U); +} + TEST_F(CacheReservationManagerTest, GenerateCacheKey) { std::size_t new_mem_used = 1 * kSizeDummyEntry; Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used); diff --git a/db/blob/blob_constants.h b/db/blob/blob_constants.h index db69d707ea68..1dfc9f3c7047 100644 --- a/db/blob/blob_constants.h +++ b/db/blob/blob_constants.h @@ -13,6 +13,8 @@ namespace ROCKSDB_NAMESPACE { +inline constexpr size_t kBlobGcMaxSelectedFiles = 8; + // WARNING: This value is == kCurrentFileBlobIndexFileNumber. // Use this name only where file number zero means "no valid blob file" or // "current file" is not understood/supported. diff --git a/db/blob/blob_contents.cc b/db/blob/blob_contents.cc index 8b793c5d264e..ed2c7eef2946 100644 --- a/db/blob/blob_contents.cc +++ b/db/blob/blob_contents.cc @@ -5,38 +5,215 @@ #include "db/blob/blob_contents.h" +#include #include +#include +#include #include "cache/cache_entry_roles.h" #include "cache/cache_helpers.h" +#include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_index.h" +#include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" #include "port/malloc.h" +#include "rocksdb/advanced_compression.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/crc32c.h" +#include "util/stop_watch.h" namespace ROCKSDB_NAMESPACE { -size_t BlobContents::ApproximateMemoryUsage() const { - size_t usage = 0; - - if (allocation_) { - MemoryAllocator* const allocator = allocation_.get_deleter().allocator; +namespace { - if (allocator) { - usage += allocator->UsableSize(allocation_.get(), data_.size()); +size_t ApproximateAllocationUsage(const CacheAllocationPtr& allocation, + size_t allocation_size, + [[maybe_unused]] const void* object, + [[maybe_unused]] size_t object_size) { + size_t usage = 0; + if (allocation) { + MemoryAllocator* const allocator = allocation.get_deleter().allocator; + if (allocator != nullptr) { + usage += allocator->UsableSize(allocation.get(), allocation_size); } else { #ifdef ROCKSDB_MALLOC_USABLE_SIZE - usage += malloc_usable_size(allocation_.get()); + usage += malloc_usable_size(allocation.get()); #else - usage += data_.size(); + usage += allocation_size; #endif } } - #ifdef ROCKSDB_MALLOC_USABLE_SIZE - usage += malloc_usable_size(const_cast(this)); + usage += malloc_usable_size(const_cast(object)); #else - usage += sizeof(*this); + usage += object_size; #endif - return usage; } +} // namespace + +size_t BlobContents::ApproximateMemoryUsage() const { + return ApproximateAllocationUsage(allocation_, data_.size(), this, + sizeof(*this)); +} + +Status StableBlobContents::CreateWithVerifiedValue( + const Slice& value, const StableBlobFingerprint& fingerprint, + MemoryAllocator* allocator, std::unique_ptr* output) { + if (output == nullptr) { + return Status::InvalidArgument("Null stable blob cache output"); + } + if (!CompressionTypeSupported(fingerprint.compression) || + (fingerprint.compression == kNoCompression && + fingerprint.stored_value_size != value.size())) { + return Status::InvalidArgument("Invalid stable blob cache metadata"); + } + std::unique_ptr contents(new StableBlobContents()); + contents->allocation_ = AllocateAndCopyBlock(value, allocator); + contents->data_ = Slice(contents->allocation_.get(), value.size()); + contents->fingerprint_ = fingerprint; + contents->valid_ = contents->Validate(/*verify_value=*/false); + if (!contents->valid_) { + return contents->validation_status(); + } + *output = std::move(contents); + return Status::OK(); +} + +Status StableBlobContents::CreateWithVerifiedAllocation( + CacheAllocationPtr&& allocation, size_t value_size, + const StableBlobFingerprint& fingerprint, + std::unique_ptr* output) { + if (output == nullptr || (value_size != 0 && !allocation)) { + return Status::InvalidArgument("Invalid stable blob cache input"); + } + if (!CompressionTypeSupported(fingerprint.compression) || + (fingerprint.compression == kNoCompression && + fingerprint.stored_value_size != value_size)) { + return Status::InvalidArgument("Invalid stable blob cache metadata"); + } + std::unique_ptr contents(new StableBlobContents()); + contents->allocation_ = std::move(allocation); + contents->data_ = Slice(contents->allocation_.get(), value_size); + contents->fingerprint_ = fingerprint; + contents->valid_ = contents->Validate(/*verify_value=*/false); + if (!contents->valid_) { + return contents->validation_status(); + } + *output = std::move(contents); + return Status::OK(); +} + +std::unique_ptr StableBlobContents::CreateFromSaved( + const Slice& saved, MemoryAllocator* allocator) { + std::unique_ptr contents(new StableBlobContents()); + if (saved.size() < kSavedHeaderSize) { + return contents; + } + const char* const header = saved.data(); + contents->fingerprint_.stored_value_size = DecodeFixed64(header); + contents->fingerprint_.stable_checksum = DecodeFixed32(header + 8); + contents->fingerprint_.stored_checksum = DecodeFixed32(header + 12); + contents->fingerprint_.compression = + static_cast(static_cast(header[16])); + if (DecodeFixed32(header + 17) != crc32c::Value(header, 17)) { + return contents; + } + const Slice value(saved.data() + kSavedHeaderSize, + saved.size() - kSavedHeaderSize); + contents->allocation_ = AllocateAndCopyBlock(value, allocator); + contents->data_ = Slice(contents->allocation_.get(), value.size()); + contents->valid_ = contents->Validate(/*verify_value=*/true); + return contents; +} + +bool StableBlobContents::Validate(bool verify_value) const { + const StableBlobFingerprint& f = fingerprint_; + if (!CompressionTypeSupported(f.compression) || + (f.compression == kNoCompression && + (f.stored_value_size != data_.size() || + f.stored_checksum != BlobIndex::ExtendChecksumWithCompression( + f.stable_checksum, f.compression)))) { + return false; + } + return !verify_value || + crc32c::Value(data_.data(), data_.size()) == f.stable_checksum; +} + +void StableBlobContents::EncodeSavedHeader(char* header) const { + EncodeFixed64(header, fingerprint_.stored_value_size); + EncodeFixed32(header + 8, fingerprint_.stable_checksum); + EncodeFixed32(header + 12, fingerprint_.stored_checksum); + header[16] = static_cast(fingerprint_.compression); + EncodeFixed32(header + 17, crc32c::Value(header, 17)); +} + +void StableBlobContents::SaveTo(size_t from_offset, size_t length, + char* out) const { + assert(from_offset + length <= SavedSize()); + if (from_offset < kSavedHeaderSize) { + char header[kSavedHeaderSize]; + EncodeSavedHeader(header); + const size_t n = std::min(length, kSavedHeaderSize - from_offset); + memcpy(out, header + from_offset, n); + out += n; + length -= n; + from_offset = kSavedHeaderSize; + } + if (length > 0) { + memcpy(out, data_.data() + (from_offset - kSavedHeaderSize), length); + } +} + +size_t StableBlobContents::ApproximateMemoryUsage() const { + return ApproximateAllocationUsage(allocation_, data_.size(), this, + sizeof(*this)); +} + +Status UncompressBlobContentsIfNeeded( + const Slice& stored_value, CompressionType compression_type, + Decompressor* decompressor, MemoryAllocator* allocator, SystemClock* clock, + Statistics* statistics, std::unique_ptr* result) { + assert(result != nullptr); + if (!CompressionTypeSupported(compression_type)) { + return Status::Corruption("Unsupported blob compression type"); + } + if (compression_type == kNoCompression) { + BlobContentsCreator::Create(result, nullptr, stored_value, kNoCompression, + allocator); + return Status::OK(); + } + if (decompressor == nullptr) { + return Status::Corruption("Missing blob decompressor"); + } + + Decompressor::Args args; + args.compression_type = compression_type; + args.compressed_data = stored_value; + Status status = decompressor->ExtractUncompressedSize(args); + if (!status.ok()) { + return Status::Corruption(status.ToString()); + } + if (args.uncompressed_size > kMaxSimpleGen2BlobPayloadSize) { + return Status::Corruption("Uncompressed blob value is too large"); + } + + CacheAllocationPtr output = AllocateBlock(args.uncompressed_size, allocator); + { + PERF_TIMER_GUARD(blob_decompress_time); + StopWatch stop_watch(clock, statistics, BLOB_DB_DECOMPRESSION_MICROS); + status = decompressor->DecompressBlock(args, output.get()); + } + TEST_SYNC_POINT_CALLBACK("UncompressBlobContentsIfNeeded:TamperWithResult", + &status); + if (!status.ok()) { + return Status::Corruption(status.ToString()); + } + result->reset(new BlobContents(std::move(output), args.uncompressed_size)); + return Status::OK(); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_contents.h b/db/blob/blob_contents.h index 40b94d51f960..10d0f12ae643 100644 --- a/db/blob/blob_contents.h +++ b/db/blob/blob_contents.h @@ -5,8 +5,11 @@ #pragma once +#include #include +#include "cache/typed_cache.h" +#include "db/blob/blob_index.h" #include "memory/memory_allocator_impl.h" #include "rocksdb/advanced_cache.h" #include "rocksdb/rocksdb_namespace.h" @@ -15,6 +18,10 @@ namespace ROCKSDB_NAMESPACE { +class Decompressor; +class Statistics; +class SystemClock; + // A class representing a single uncompressed value read from a blob file. class BlobContents { public: @@ -34,6 +41,13 @@ class BlobContents { size_t ApproximateMemoryUsage() const; + // Gives up the buffer, so a cache entry can adopt it without a copy. The + // object is empty afterwards. + CacheAllocationPtr ReleaseAllocation() { + data_.clear(); + return std::move(allocation_); + } + // For TypedCacheInterface const Slice& ContentSlice() const { return data_; } static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kBlobValue; @@ -57,4 +71,137 @@ class BlobContentsCreator : public Cache::CreateContext { } }; +// One entry of the stable blob cache. The stable cache key contains the +// value's stable identity (origin, original offset), not the file that holds +// the value, so an entry stays valid when the value moves to a relocation +// file (see BlobSource). An entry holds one uncompressed value plus the +// StableBlobFingerprint (from the BlobIndex) that validated it. +// +// In primary memory the buffer holds only the value, like BlobContents. The +// cache item helper below writes the saved form (header + value) only when +// the entry moves to a secondary cache. Promotion back to primary memory +// checks the header and the whole value again. +class StableBlobContents { + public: + // Saved form: stored value size (8) | logical CRC (4) | stored CRC (4) | + // compression (1) | CRC32C of these 17 bytes (4), then the value. The value + // size is the rest of the saved slice. + static constexpr size_t kSavedHeaderSize = 21; + + StableBlobContents(const StableBlobContents&) = delete; + StableBlobContents& operator=(const StableBlobContents&) = delete; + ~StableBlobContents() = default; + + // Use only to insert into the primary cache. The caller has already + // computed the logical CRC of `value` (for example while it produced the + // value) and matched it against `fingerprint`. This function copies `value` + // and does not compute the CRC of the whole value again. + static Status CreateWithVerifiedValue( + const Slice& value, const StableBlobFingerprint& fingerprint, + MemoryAllocator* allocator, std::unique_ptr* output); + + // Like CreateWithVerifiedValue, but adopts `allocation`, whose first + // `value_size` bytes are the value, instead of copying it. Bytes after the + // value (for example a record trailer) stay unused. + static Status CreateWithVerifiedAllocation( + CacheAllocationPtr&& allocation, size_t value_size, + const StableBlobFingerprint& fingerprint, + std::unique_ptr* output); + + // Promotion from the secondary cache. It does not trust the secondary + // cache: it checks the header CRC, the logical CRC of the whole value and + // that the compression type is supported. It checks the other fingerprint + // fields only when they follow from the value: for kNoCompression, the + // stored size equals the value size and the stored CRC follows from the + // logical CRC. A saved form shorter than kSavedHeaderSize (21 bytes) is + // also rejected. On a failure the returned entry is invalid + // (validation_status() is Corruption) and readers erase it. + static std::unique_ptr CreateFromSaved( + const Slice& saved, MemoryAllocator* allocator); + + Status validation_status() const { + return valid_ ? Status::OK() + : Status::Corruption("Invalid stable blob cache entry"); + } + const Slice& data() const { return data_; } + size_t size() const { return data_.size(); } + const StableBlobFingerprint& fingerprint() const { return fingerprint_; } + + size_t ApproximateMemoryUsage() const; + + // Saved form for the cache item helper. + size_t SavedSize() const { return kSavedHeaderSize + data_.size(); } + void SaveTo(size_t from_offset, size_t length, char* out) const; + + static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kBlobValue; + + private: + StableBlobContents() = default; + + // Checks the fingerprint fields and, if `verify_value`, the value CRC. + bool Validate(bool verify_value) const; + + // Writes the kSavedHeaderSize-byte saved header into `header`. + void EncodeSavedHeader(char* header) const; + + CacheAllocationPtr allocation_; + Slice data_; + StableBlobFingerprint fingerprint_; + bool valid_ = false; +}; + +// Create context type of the stable cache interface. Promotion goes through +// the cache item helper specialization below, so it holds no state. +class StableBlobContentsCreator : public Cache::CreateContext {}; + +// Cache item helper functions for the stable cache. The size and save +// callbacks use the saved form (header + value), not the primary buffer. +template <> +class FullTypedCacheHelperFns + : public BasicTypedCacheHelperFns { + public: + using ObjectPtr = Cache::ObjectPtr; + using CreateContext = Cache::CreateContext; + + protected: + using BasicTypedCacheHelperFns::DownCastValue; + using BasicTypedCacheHelperFns::UpCastValue; + + static size_t Size(ObjectPtr v) { return DownCastValue(v)->SavedSize(); } + + static Status SaveTo(ObjectPtr v, size_t from_offset, size_t length, + char* out) { + DownCastValue(v)->SaveTo(from_offset, length, out); + return Status::OK(); + } + + static Status Create(const Slice& data, CompressionType /*type*/, + CacheTier source, CreateContext* /*context*/, + MemoryAllocator* allocator, ObjectPtr* out_obj, + size_t* out_charge) { + if (source != CacheTier::kVolatileTier) { + return Status::InvalidArgument(); + } + std::unique_ptr value = + StableBlobContents::CreateFromSaved(data, allocator); + if (out_charge != nullptr) { + *out_charge = value->ApproximateMemoryUsage(); + } + *out_obj = UpCastValue(value.release()); + return Status::OK(); + } +}; + +// Decodes the stored bytes of one blob value into an owned uncompressed value +// in `*result`. For kNoCompression it copies the stored bytes. The caller +// passes the decompressor of the file that holds the value. Returns +// Corruption for an unsupported compression type, a null decompressor with a +// compressed value, an uncompressed size above kMaxSimpleGen2BlobPayloadSize +// (checked before allocating; this also applies to legacy blobs), or a +// decompression error. +Status UncompressBlobContentsIfNeeded( + const Slice& stored_value, CompressionType compression_type, + Decompressor* decompressor, MemoryAllocator* allocator, SystemClock* clock, + Statistics* statistics, std::unique_ptr* result); + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_addition.cc b/db/blob/blob_file_addition.cc index 71b1bb7fca10..5faa7ff5fca8 100644 --- a/db/blob/blob_file_addition.cc +++ b/db/blob/blob_file_addition.cc @@ -22,6 +22,11 @@ enum BlobFileAddition::CustomFieldTags : uint32_t { kEndMarker, // Add forward compatible fields here + // An older binary could ignore kIndirectOrigin. This is safe, because a + // column family with indirect-origin blob files also has the per-CF format + // fence, a required tag in the same edit, so an older binary fails on that + // edit and never applies an addition without this field. + kIndirectOrigin, ///////////////////////////////////////////////////////////////////// @@ -42,6 +47,13 @@ void BlobFileAddition::EncodeTo(std::string* output) const { // fields will be ignored during decoding unless they're in the forward // incompatible range. + if (is_indirect_origin_) { + PutVarint32(output, kIndirectOrigin); + std::string encoded; + PutVarint32(&encoded, 1); + PutLengthPrefixedSlice(output, encoded); + } + TEST_SYNC_POINT_CALLBACK("BlobFileAddition::EncodeTo::CustomFields", output); PutVarint32(output, kEndMarker); @@ -73,7 +85,9 @@ Status BlobFileAddition::DecodeFrom(Slice* input) { return Status::Corruption(class_name, "Error decoding checksum value"); } checksum_value_ = checksum_value.ToString(); + is_indirect_origin_ = false; + bool has_indirect_origin = false; while (true) { uint32_t custom_field_tag = 0; if (!GetVarint32(input, &custom_field_tag)) { @@ -94,6 +108,16 @@ Status BlobFileAddition::DecodeFrom(Slice* input) { return Status::Corruption(class_name, "Error decoding custom field value"); } + if (custom_field_tag == kIndirectOrigin) { + uint32_t value = 0; + if (has_indirect_origin || !GetVarint32(&custom_field_value, &value) || + !custom_field_value.empty() || value != 1) { + return Status::Corruption(class_name, + "Error decoding indirect-origin field"); + } + has_indirect_origin = true; + is_indirect_origin_ = true; + } } return Status::OK(); @@ -122,7 +146,8 @@ bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs) { lhs.GetTotalBlobCount() == rhs.GetTotalBlobCount() && lhs.GetTotalBlobBytes() == rhs.GetTotalBlobBytes() && lhs.GetChecksumMethod() == rhs.GetChecksumMethod() && - lhs.GetChecksumValue() == rhs.GetChecksumValue(); + lhs.GetChecksumValue() == rhs.GetChecksumValue() && + lhs.IsIndirectOrigin() == rhs.IsIndirectOrigin(); } bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs) { @@ -136,7 +161,8 @@ std::ostream& operator<<(std::ostream& os, << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes() << " checksum_method: " << blob_file_addition.GetChecksumMethod() << " checksum_value: " - << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true); + << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true) + << " indirect_origin: " << blob_file_addition.IsIndirectOrigin(); return os; } @@ -148,7 +174,8 @@ JSONWriter& operator<<(JSONWriter& jw, << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes() << "ChecksumMethod" << blob_file_addition.GetChecksumMethod() << "ChecksumValue" - << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true); + << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true) + << "IndirectOrigin" << blob_file_addition.IsIndirectOrigin(); return jw; } diff --git a/db/blob/blob_file_addition.h b/db/blob/blob_file_addition.h index 43b1a0bcbe94..2d945ec4857c 100644 --- a/db/blob/blob_file_addition.h +++ b/db/blob/blob_file_addition.h @@ -26,11 +26,19 @@ class BlobFileAddition { BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count, uint64_t total_blob_bytes, std::string checksum_method, std::string checksum_value) + : BlobFileAddition(blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value), + /*is_indirect_origin=*/false) {} + + BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value, bool is_indirect_origin) : blob_file_number_(blob_file_number), total_blob_count_(total_blob_count), total_blob_bytes_(total_blob_bytes), checksum_method_(std::move(checksum_method)), - checksum_value_(std::move(checksum_value)) { + checksum_value_(std::move(checksum_value)), + is_indirect_origin_(is_indirect_origin) { assert(checksum_method_.empty() == checksum_value_.empty()); } @@ -39,6 +47,7 @@ class BlobFileAddition { uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; } const std::string& GetChecksumMethod() const { return checksum_method_; } const std::string& GetChecksumValue() const { return checksum_value_; } + bool IsIndirectOrigin() const { return is_indirect_origin_; } void EncodeTo(std::string* output) const; Status DecodeFrom(Slice* input); @@ -54,6 +63,7 @@ class BlobFileAddition { uint64_t total_blob_bytes_ = 0; std::string checksum_method_; std::string checksum_value_; + bool is_indirect_origin_ = false; }; bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs); diff --git a/db/blob/blob_file_addition_test.cc b/db/blob/blob_file_addition_test.cc index 64cb0a9d6d24..bd039a4dc748 100644 --- a/db/blob/blob_file_addition_test.cc +++ b/db/blob/blob_file_addition_test.cc @@ -37,6 +37,7 @@ TEST_F(BlobFileAdditionTest, Empty) { ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), 0); ASSERT_TRUE(blob_file_addition.GetChecksumMethod().empty()); ASSERT_TRUE(blob_file_addition.GetChecksumValue().empty()); + ASSERT_FALSE(blob_file_addition.IsIndirectOrigin()); TestEncodeDecode(blob_file_addition); } @@ -59,10 +60,20 @@ TEST_F(BlobFileAdditionTest, NonEmpty) { ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), total_blob_bytes); ASSERT_EQ(blob_file_addition.GetChecksumMethod(), checksum_method); ASSERT_EQ(blob_file_addition.GetChecksumValue(), checksum_value); + ASSERT_FALSE(blob_file_addition.IsIndirectOrigin()); TestEncodeDecode(blob_file_addition); } +TEST_F(BlobFileAdditionTest, IndirectOrigin) { + BlobFileAddition blob_file_addition( + /*blob_file_number=*/123, /*total_blob_count=*/2, + /*total_blob_bytes=*/456, "crc32c", "checksum", + /*is_indirect_origin=*/true); + ASSERT_TRUE(blob_file_addition.IsIndirectOrigin()); + TestEncodeDecode(blob_file_addition); +} + TEST_F(BlobFileAdditionTest, DecodeErrors) { std::string str; Slice slice(str); diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 39b9ffca5027..388dfc36eb22 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -29,6 +29,7 @@ #include "test_util/sync_point.h" #include "trace_replay/io_tracer.h" #include "util/compression.h" +#include "util/crc32c.h" namespace ROCKSDB_NAMESPACE { @@ -43,27 +44,27 @@ BlobFileBuilder::BlobFileBuilder( BlobFileCompletionCallback* blob_callback, BlobFileCreationReason creation_reason, std::string blob_output_path, std::vector* output_file_paths, - std::vector* blob_file_additions) + std::vector* blob_file_additions, BlobSource* blob_source) : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs, immutable_options, mutable_cf_options, file_options, write_options, std::move(db_id), std::move(db_session_id), job_id, column_family_id, column_family_name, write_hint, io_tracer, blob_callback, creation_reason, std::move(blob_output_path), output_file_paths, - blob_file_additions) {} + blob_file_additions, blob_source) {} BlobFileBuilder::BlobFileBuilder( std::function file_number_generator, FileSystem* fs, const ImmutableOptions* immutable_options, const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, - const WriteOptions* write_options, std::string db_id, - std::string db_session_id, int job_id, uint32_t column_family_id, + const WriteOptions* write_options, std::string /*db_id*/, + std::string /*db_session_id*/, int job_id, uint32_t column_family_id, const std::string& column_family_name, Env::WriteLifeTimeHint write_hint, const std::shared_ptr& io_tracer, BlobFileCompletionCallback* blob_callback, BlobFileCreationReason creation_reason, std::string blob_output_path, std::vector* output_file_paths, - std::vector* blob_file_additions) + std::vector* blob_file_additions, BlobSource* blob_source) : file_number_generator_(std::move(file_number_generator)), fs_(fs), immutable_options_(immutable_options), @@ -81,10 +82,9 @@ BlobFileBuilder::BlobFileBuilder( ? blob_compressor_->ObtainWorkingArea() : Compressor::ManagedWorkingArea{}), prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache), + enable_blob_indirection_(mutable_cf_options->enable_blob_indirection), file_options_(file_options), write_options_(write_options), - db_id_(std::move(db_id)), - db_session_id_(std::move(db_session_id)), job_id_(job_id), column_family_id_(column_family_id), column_family_name_(column_family_name), @@ -95,6 +95,7 @@ BlobFileBuilder::BlobFileBuilder( blob_output_path_(std::move(blob_output_path)), output_file_paths_(output_file_paths), blob_file_additions_(blob_file_additions), + blob_source_(blob_source), blob_count_(0), blob_bytes_(0) { assert(file_number_generator_); @@ -144,9 +145,23 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value, uint64_t blob_file_number = 0; uint64_t blob_offset = 0; + // The logical CRC and the stored CRC are defined under "Checksums" in + // db/blob/blob_index.h. + // + // With indirection, the CRC32C of the stored (maybe compressed) bytes is + // computed once and used three times: + // - The blob log record CRC covers the key and the stored bytes. It is + // made by combining the key's CRC with this CRC. + // - The stored CRC in the BlobIndex is this CRC extended with the + // compression byte. + // - For an uncompressed value, this CRC is also the logical CRC (the + // value of BlobIndex::stable_checksum()). + const uint32_t stored_bytes_crc = + enable_blob_indirection_ ? crc32c::Value(blob.data(), blob.size()) : 0; { - const Status s = - WriteBlobToFile(key, blob, &blob_file_number, &blob_offset); + const Status s = WriteBlobToFile( + key, blob, enable_blob_indirection_ ? &stored_bytes_crc : nullptr, + &blob_file_number, &blob_offset); if (!s.ok()) { return s; } @@ -159,9 +174,19 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value, } } + const StableBlobFingerprint fingerprint{ + blob.size(), blob_compression_type_, + !enable_blob_indirection_ ? 0 + : blob_compression_type_ == kNoCompression + ? stored_bytes_crc + : crc32c::Value(value.data(), value.size()), + enable_blob_indirection_ ? BlobIndex::ExtendChecksumWithCompression( + stored_bytes_crc, blob_compression_type_) + : 0}; { - const Status s = - PutBlobIntoCacheIfNeeded(value, blob_file_number, blob_offset); + const Status s = PutBlobIntoCacheIfNeeded( + value, blob_file_number, blob_offset, + enable_blob_indirection_ ? &fingerprint : nullptr); if (!s.ok()) { ROCKS_LOG_WARN(immutable_options_->info_log, "Failed to pre-populate the blob into blob cache: %s", @@ -169,8 +194,18 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value, } } - BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(), - blob_compression_type_); + if (enable_blob_indirection_) { + BlobIndex::EncodeUnflattenedIndirectBlob( + blob_index, {.origin_file_number = blob_file_number, + .original_value_offset = blob_offset, + .stored_value_size = blob.size(), + .compression = blob_compression_type_, + .stable_checksum = fingerprint.stable_checksum, + .stored_checksum = fingerprint.stored_checksum}); + } else { + BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, + blob.size(), blob_compression_type_); + } return Status::OK(); } @@ -308,6 +343,7 @@ Status BlobFileBuilder::CompressBlobIfNeeded( } Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob, + const uint32_t* value_crc32c, uint64_t* blob_file_number, uint64_t* blob_offset) { assert(IsBlobFileOpen()); @@ -316,8 +352,12 @@ Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob, uint64_t key_offset = 0; - Status s = - writer_->AddRecord(*write_options_, key, blob, &key_offset, blob_offset); + Status s = value_crc32c != nullptr + ? writer_->AddRecordWithValueCrc(*write_options_, key, blob, + *value_crc32c, &key_offset, + blob_offset) + : writer_->AddRecord(*write_options_, key, blob, &key_offset, + blob_offset); TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s); @@ -361,9 +401,9 @@ Status BlobFileBuilder::CloseBlobFile() { } assert(blob_file_additions_); - blob_file_additions_->emplace_back(blob_file_number, blob_count_, blob_bytes_, - std::move(checksum_method), - std::move(checksum_value)); + blob_file_additions_->emplace_back( + blob_file_number, blob_count_, blob_bytes_, std::move(checksum_method), + std::move(checksum_value), enable_blob_indirection_); assert(immutable_options_); ROCKS_LOG_INFO(immutable_options_->logger, @@ -413,37 +453,17 @@ void BlobFileBuilder::Abandon(const Status& s) { blob_bytes_ = 0; } -Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob, - uint64_t blob_file_number, - uint64_t blob_offset) const { - Status s = Status::OK(); - - BlobSource::SharedCacheInterface blob_cache{immutable_options_->blob_cache}; - auto statistics = immutable_options_->statistics.get(); - bool warm_cache = +Status BlobFileBuilder::PutBlobIntoCacheIfNeeded( + const Slice& blob, uint64_t blob_file_number, uint64_t blob_offset, + const StableBlobFingerprint* fingerprint) const { + const bool warm_cache = prepopulate_blob_cache_ == PrepopulateBlobCache::kFlushOnly && creation_reason_ == BlobFileCreationReason::kFlush; - - if (blob_cache && warm_cache) { - const OffsetableCacheKey base_cache_key(db_id_, db_session_id_, - blob_file_number); - const CacheKey cache_key = base_cache_key.WithOffset(blob_offset); - const Slice key = cache_key.AsSlice(); - - const Cache::Priority priority = Cache::Priority::BOTTOM; - - s = blob_cache.InsertSaved(key, blob, nullptr /*context*/, priority, - immutable_options_->lowest_used_cache_tier); - - if (s.ok()) { - RecordTick(statistics, BLOB_DB_CACHE_ADD); - RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, blob.size()); - } else { - RecordTick(statistics, BLOB_DB_CACHE_ADD_FAILURES); - } + if (blob_source_ == nullptr || !warm_cache) { + return Status::OK(); } - - return s; + return blob_source_->PrepopulateFlushBlob(blob, blob_file_number, blob_offset, + fingerprint); } } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_builder.h b/db/blob/blob_file_builder.h index d5706f0a78a6..d0e61aa9adf2 100644 --- a/db/blob/blob_file_builder.h +++ b/db/blob/blob_file_builder.h @@ -33,6 +33,8 @@ class Slice; class BlobLogWriter; class IOTracer; class BlobFileCompletionCallback; +class BlobSource; +struct StableBlobFingerprint; class BlobFileBuilder { public: @@ -50,7 +52,8 @@ class BlobFileBuilder { BlobFileCreationReason creation_reason, std::string blob_output_path, std::vector* output_file_paths, - std::vector* blob_file_additions); + std::vector* blob_file_additions, + BlobSource* blob_source); BlobFileBuilder(std::function file_number_generator, FileSystem* fs, const ImmutableOptions* immutable_options, @@ -66,7 +69,8 @@ class BlobFileBuilder { BlobFileCreationReason creation_reason, std::string blob_output_path, std::vector* output_file_paths, - std::vector* blob_file_additions); + std::vector* blob_file_additions, + BlobSource* blob_source); BlobFileBuilder(const BlobFileBuilder&) = delete; BlobFileBuilder& operator=(const BlobFileBuilder&) = delete; @@ -85,13 +89,22 @@ class BlobFileBuilder { Status OpenBlobFileIfNeeded(); Status CompressBlobIfNeeded(Slice* blob, GrowableBuffer* compressed_blob) const; + // `value_crc32c`, when non-null, is the CRC32C of `blob`, already computed. Status WriteBlobToFile(const Slice& key, const Slice& blob, + const uint32_t* value_crc32c, uint64_t* blob_file_number, uint64_t* blob_offset); Status CloseBlobFile(); Status CloseBlobFileIfNeeded(); - Status PutBlobIntoCacheIfNeeded(const Slice& blob, uint64_t blob_file_number, - uint64_t blob_offset) const; + // Inserts a value into the blob cache right after flush writes it to the blob + // file, through BlobSource::PrepopulateFlushBlob. Does nothing unless + // prepopulate_blob_cache is kFlushOnly, this builder writes flush output, and + // blob_source_ is set. `fingerprint` is non-null only for a value with an + // indirect BlobIndex (enable_blob_indirection); such a value is cached under + // its stable identity. + Status PutBlobIntoCacheIfNeeded( + const Slice& blob, uint64_t blob_file_number, uint64_t blob_offset, + const StableBlobFingerprint* fingerprint) const; std::function file_number_generator_; FileSystem* fs_; @@ -103,10 +116,9 @@ class BlobFileBuilder { std::unique_ptr blob_compressor_; mutable Compressor::ManagedWorkingArea blob_compressor_wa_; PrepopulateBlobCache prepopulate_blob_cache_; + bool enable_blob_indirection_; const FileOptions* file_options_; const WriteOptions* write_options_; - const std::string db_id_; - const std::string db_session_id_; int job_id_; uint32_t column_family_id_; std::string column_family_name_; @@ -117,6 +129,9 @@ class BlobFileBuilder { const std::string blob_output_path_; std::vector* output_file_paths_; std::vector* blob_file_additions_; + // Used for flush blob cache prepopulation. May be null; then nothing is + // prepopulated. + BlobSource* blob_source_; std::unique_ptr writer_; uint64_t blob_count_; uint64_t blob_bytes_; diff --git a/db/blob/blob_file_builder_test.cc b/db/blob/blob_file_builder_test.cc index 4adf11905d27..e51e1cdf963f 100644 --- a/db/blob/blob_file_builder_test.cc +++ b/db/blob/blob_file_builder_test.cc @@ -25,6 +25,7 @@ #include "test_util/sync_point.h" #include "test_util/testharness.h" #include "util/compression.h" +#include "util/crc32c.h" #include "utilities/fault_injection_env.h" namespace ROCKSDB_NAMESPACE { @@ -149,7 +150,8 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); std::vector> expected_key_value_pairs( number_of_blobs); @@ -194,12 +196,131 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) { ASSERT_EQ( blob_file_addition.GetTotalBlobBytes(), number_of_blobs * (BlobLogRecord::kHeaderSize + key_size + value_size)); + ASSERT_FALSE(blob_file_addition.IsIndirectOrigin()); // Verify the contents of the new blob file as well as the blob references VerifyBlobFile(blob_file_number, blob_file_path, column_family_id, kNoCompression, expected_key_value_pairs, blob_indexes); } +TEST_F(BlobFileBuilderTest, StableIdentityIndex) { + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderTest_StableIdentityIndex"), + 0); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.env = mock_env_.get(); + + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + std::vector blob_file_paths; + std::vector blob_file_additions; + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/, + /*job_id=*/1, /*column_family_id=*/123, "foobar", Env::WLTH_MEDIUM, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); + + const std::string value = "logical uncompressed value"; + std::string encoded; + ASSERT_OK(builder.Add("key", value, &encoded)); + ASSERT_OK(builder.Finish()); + + ASSERT_EQ(blob_file_additions.size(), 1U); + ASSERT_TRUE(blob_file_additions.front().IsIndirectOrigin()); + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + ASSERT_TRUE(blob_index.IsIndirect()); + EXPECT_EQ(blob_index.generation_file_number(), 2); + EXPECT_EQ(blob_index.origin_file_number(), 2); + EXPECT_EQ(blob_index.stable_checksum(), + crc32c::Value(value.data(), value.size())); + EXPECT_EQ(blob_index.stored_checksum(), + BlobIndex::ComputeStoredChecksum(value, kNoCompression)); +} + +TEST_F(BlobFileBuilderTest, StableChecksumsComputedOnceMatchDirectValues) { + // BlobFileBuilder computes the CRC of the stored bytes once. It derives the + // blob log record CRC and the BlobIndex stored CRC from it, and for an + // uncompressed value also the logical CRC. Each CRC must equal the CRC + // computed directly from the bytes, with and without compression. + std::vector compressions{kNoCompression}; + if (Snappy_Supported()) { + compressions.push_back(kSnappyCompression); + } + for (const CompressionType compression : compressions) { + SCOPED_TRACE(CompressionTypeToString(compression)); + Options options; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileBuilderTest_StableChecksums" + + CompressionTypeToString(compression)), + 0); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.blob_compression_type = compression; + options.env = mock_env_.get(); + ImmutableOptions immutable_options(options); + MutableCFOptions mutable_cf_options(options); + std::vector blob_file_paths; + std::vector blob_file_additions; + BlobFileBuilder builder( + TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options, + &file_options_, &write_options_, "" /*db_id*/, "" /*db_session_id*/, + /*job_id=*/1, /*column_family_id=*/123, "foobar", Env::WLTH_MEDIUM, + nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, + BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); + const std::vector> key_values{ + {"k1", std::string(100, 'x')}, + {"key2", "a short value"}, + {"k3", std::string(3000, 'y') + "tail"}}; + std::vector blob_indexes(key_values.size()); + for (size_t i = 0; i < key_values.size(); ++i) { + ASSERT_OK(builder.Add(key_values[i].first, key_values[i].second, + &blob_indexes[i])); + } + ASSERT_OK(builder.Finish()); + ASSERT_EQ(blob_file_paths.size(), 1); + + std::unique_ptr file; + ASSERT_OK(fs_->NewRandomAccessFile(blob_file_paths[0], file_options_, &file, + /*dbg=*/nullptr)); + BlobLogSequentialReader reader( + std::make_unique(std::move(file), + blob_file_paths[0], clock_), + clock_, /*statistics=*/nullptr); + BlobLogHeader header; + ASSERT_OK(reader.ReadHeader(&header)); + for (size_t i = 0; i < key_values.size(); ++i) { + BlobLogRecord record; + uint64_t blob_offset = 0; + ASSERT_OK(reader.ReadRecord( + &record, BlobLogSequentialReader::kReadHeaderKeyBlob, &blob_offset)); + const Slice stored = record.value; + uint32_t record_crc = crc32c::Value(record.key.data(), record.key.size()); + record_crc = crc32c::Extend(record_crc, stored.data(), stored.size()); + EXPECT_EQ(record.blob_crc, crc32c::Mask(record_crc)); + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(blob_indexes[i])); + ASSERT_TRUE(blob_index.IsIndirect()); + EXPECT_EQ(blob_index.stable_checksum(), + crc32c::Value(key_values[i].second.data(), + key_values[i].second.size())); + EXPECT_EQ(blob_index.stored_checksum(), + BlobIndex::ComputeStoredChecksum(stored, compression)); + } + } +} + TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) { // Build multiple blob files: file size limit is set to the size of a single // value, so each blob ends up in a file of its own @@ -234,7 +355,8 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); std::vector> expected_key_value_pairs( number_of_blobs); @@ -322,7 +444,8 @@ TEST_F(BlobFileBuilderTest, InlinedValues) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); for (size_t i = 0; i < number_of_blobs; ++i) { const std::string key = std::to_string(i); @@ -377,7 +500,8 @@ TEST_F(BlobFileBuilderTest, Compression) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); const std::string key("1"); const std::string uncompressed_value(value_size, 'x'); @@ -457,7 +581,8 @@ TEST_F(BlobFileBuilderTest, CompressionError) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); SyncPoint::GetInstance()->SetCallBack( "LegacyForceBuiltinCompression:TamperWithStatus", [](void* arg) { @@ -538,7 +663,8 @@ TEST_F(BlobFileBuilderTest, Checksum) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); const std::string key("1"); const std::string value("deadbeef"); @@ -638,7 +764,8 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) { job_id, column_family_id, column_family_name, write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, - &blob_file_paths, &blob_file_additions); + &blob_file_paths, &blob_file_additions, + /*blob_source=*/nullptr); SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) { Status* const s = static_cast(arg); diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc index 80e4895a372d..e63fe6361401 100644 --- a/db/blob/blob_file_cache.cc +++ b/db/blob/blob_file_cache.cc @@ -9,10 +9,13 @@ #include #include "db/blob/blob_file_reader.h" +#include "db/blob/blob_relocation_file.h" #include "logging/logging.h" #include "options/cf_options.h" #include "rocksdb/cache.h" #include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_factory.h" #include "test_util/sync_point.h" #include "trace_replay/io_tracer.h" #include "util/hash.h" @@ -25,65 +28,132 @@ BlobFileCache::BlobFileCache(Cache* cache, uint32_t column_family_id, HistogramImpl* blob_file_read_hist, const std::shared_ptr& io_tracer) + : BlobFileCache(cache, immutable_options, file_options, column_family_id, + blob_file_read_hist, io_tracer, + /*relocation_file_table_options=*/nullptr, + /*db_session_id=*/std::string(), + /*block_cache_tracer=*/nullptr) {} + +BlobFileCache::BlobFileCache( + Cache* cache, const ImmutableOptions* immutable_options, + const FileOptions* file_options, uint32_t column_family_id, + HistogramImpl* blob_file_read_hist, + const std::shared_ptr& io_tracer, + const BlockBasedTableOptions* relocation_file_table_options, + const std::string& db_session_id, BlockCacheTracer* block_cache_tracer) : cache_(cache), mutex_(kNumberOfMutexStripes), + relocation_mutex_(kNumberOfRelocationMutexStripes), immutable_options_(immutable_options), file_options_(file_options), column_family_id_(column_family_id), blob_file_read_hist_(blob_file_read_hist), - io_tracer_(io_tracer) { + io_tracer_(io_tracer), + db_session_id_(db_session_id), + block_cache_tracer_(block_cache_tracer) { assert(cache_); assert(immutable_options_); assert(file_options_); + if (relocation_file_table_options != nullptr) { + relocation_factory_ = std::make_unique( + MakeBlobRelocationTableOptions(*relocation_file_table_options)); + } } -Status BlobFileCache::GetBlobFileReader( - const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, - CacheHandleGuard* blob_file_reader, - bool allow_footer_skip_retry) { +BlobFileCache::~BlobFileCache() = default; + +port::Mutex* BlobFileCache::GetStripeMutex(FileKind kind, const Slice& key) { + return kind == FileKind::kRelocationFile ? &relocation_mutex_.Get(key) + : &mutex_.Get(key); +} + +template +Status BlobFileCache::GetBlobFileReaderImpl( + const ReadOptions& read_options, uint64_t blob_file_number, FileKind kind, + const ValidateFn& validate, const OpenFn& open, + CacheHandleGuard* blob_file_reader) { assert(blob_file_reader); assert(blob_file_reader->IsEmpty()); - const uint64_t blob_file_number = blob_file.file_number; - // NOTE: sharing same Cache with table_cache const Slice key = GetSliceForKey(&blob_file_number); assert(cache_); - TypedHandle* handle = cache_.Lookup(key); - if (handle) { + const auto use_cached = [&](TypedHandle* handle) { + const Status validation = validate(*cache_.Value(handle)); + if (!validation.ok()) { + cache_.Release(handle); + return validation; + } *blob_file_reader = cache_.Guard(handle); return Status::OK(); + }; + + TypedHandle* handle = cache_.Lookup(key); + if (handle) { + return use_cached(handle); } TEST_SYNC_POINT("BlobFileCache::GetBlobFileReader:DoubleCheck"); // Check again while holding mutex - MutexLock lock(&mutex_.Get(key)); + MutexLock lock(GetStripeMutex(kind, key)); handle = cache_.Lookup(key); if (handle) { - *blob_file_reader = cache_.Guard(handle); - return Status::OK(); + return use_cached(handle); + } + + if (read_options.read_tier == kBlockCacheTier) { + return Status::Incomplete( + "Cannot open blob file reader: no disk I/O allowed"); } assert(immutable_options_); Statistics* const statistics = immutable_options_->stats; + if (kind == FileKind::kRelocationFile) { + MutexLock failed_lock(&failed_relocation_opens_mutex_); + const auto failed = failed_relocation_opens_.find(blob_file_number); + if (failed != failed_relocation_opens_.end()) { + if (immutable_options_->clock->NowMicros() < + failed->second.expires_micros) { + RecordTick(statistics, BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS); + return failed->second.status; + } + failed_relocation_opens_.erase(failed); + } + } RecordTick(statistics, NO_FILE_OPENS); std::unique_ptr reader; { - Status s = OpenBlobFileReader(read_options, blob_file, - allow_footer_skip_retry, &reader); + Status s = open(&reader); if (!s.ok()) { ROCKS_LOG_WARN(immutable_options_->logger, "BlobFileCache open failed for blob file %" PRIu64 " in CF %u: %s", blob_file_number, column_family_id_, s.ToString().c_str()); RecordTick(statistics, NO_FILE_ERRORS); + if (kind == FileKind::kRelocationFile) { + RecordTick(statistics, BLOB_RELOCATION_FILE_OPEN_FAILURES); + } + // Remember only errors that come from the file itself. A missing file is + // NotFound or IOError(PathNotFound). See failed_relocation_opens_ for why + // other errors are not remembered. + if (kind == FileKind::kRelocationFile && + (s.IsCorruption() || s.IsNotFound() || s.IsPathNotFound())) { + MutexLock failed_lock(&failed_relocation_opens_mutex_); + FailedRelocationOpen& failed = + failed_relocation_opens_[blob_file_number]; + failed = + FailedRelocationOpen{s, immutable_options_->clock->NowMicros() + + kFailedRelocationOpenMicros}; + // The kept copy may expire or be forgotten without being returned. + failed.status.PermitUncheckedError(); + } return s; } } @@ -105,18 +175,99 @@ Status BlobFileCache::GetBlobFileReader( return Status::OK(); } +Status BlobFileCache::GetBlobFileReader( + const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, + CacheHandleGuard* blob_file_reader, + bool allow_footer_skip_retry) { + return GetBlobFileReaderImpl( + read_options, blob_file.file_number, FileKind::kBlobFile, + [](const BlobFileReader& reader) { + return reader.relocation() == nullptr + ? Status::OK() + : Status::Corruption("Blob file reader metadata mismatch"); + }, + [&](std::unique_ptr* reader) { + return OpenBlobFileReader(read_options, blob_file, + allow_footer_skip_retry, reader); + }, + blob_file_reader); +} + +Status BlobFileCache::GetRelocationFileReader( + const ReadOptions& read_options, const BlobPhysicalFile& file, + const MutableCFOptions& mutable_cf_options, + CacheHandleGuard* blob_file_reader) { + return GetBlobFileReaderImpl( + read_options, file.open_info.file_number, FileKind::kRelocationFile, + [&](const BlobFileReader& reader) { + const BlobRelocationFileReader* const relocation = reader.relocation(); + if (relocation == nullptr || + relocation->column_family_id() != column_family_id_ || + relocation->file_size() != file.file_size || + relocation->tail_size() != file.tail_size || + relocation->directory_digest() != file.directory_digest) { + return Status::Corruption("Blob file reader metadata mismatch"); + } + return Status::OK(); + }, + [&](std::unique_ptr* reader) { + if (relocation_factory_ == nullptr) { + return Status::NotSupported( + "Blob relocation requires a block-based table factory"); + } + std::unique_ptr relocation; + Status s = BlobRelocationFileReader::Open( + *immutable_options_, read_options, GetFileOptions(file.open_info), + blob_file_read_hist_, column_family_id_, file, db_session_id_, + *relocation_factory_, mutable_cf_options.compression_manager.get(), + mutable_cf_options.block_protection_bytes_per_key, + mutable_cf_options.pin_blob_relocation_location_blocks, io_tracer_, + block_cache_tracer_, &relocation); + if (s.ok()) { + reader->reset(new BlobFileReader(std::move(relocation))); + } + return s; + }, + blob_file_reader); +} + Status BlobFileCache::OpenBlobFileReaderUncached( const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, std::unique_ptr* blob_file_reader, bool allow_footer_skip_retry) { + return OpenBlobFileReaderUncached(read_options, blob_file, blob_file_reader, + allow_footer_skip_retry, + /*prefer_direct_reads=*/false); +} + +Status BlobFileCache::OpenBlobFileReaderUncached( + const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, + std::unique_ptr* blob_file_reader, + bool allow_footer_skip_retry, bool prefer_direct_reads) { assert(blob_file_reader); assert(!*blob_file_reader); Statistics* const statistics = immutable_options_->stats; RecordTick(statistics, NO_FILE_OPENS); - Status s = OpenBlobFileReader(read_options, blob_file, - allow_footer_skip_retry, blob_file_reader); + Status s; + FileOptions file_options = GetFileOptions(blob_file); + // See OpenBlobFileReaderUncached() in blob_file_cache.h. + if (prefer_direct_reads && !file_options.use_mmap_reads) { + file_options.use_direct_reads = true; + s = BlobFileReader::Create( + *immutable_options_, read_options, file_options, column_family_id_, + blob_file_read_hist_, blob_file.file_number, io_tracer_, + /*skip_footer_validation=*/false, blob_file_reader); + if (s.ok()) { + return s; + } + // The file system may not support direct reads; the buffered open below + // reports any real error again. + blob_file_reader->reset(); + } + s = OpenBlobFileReader(read_options, blob_file, allow_footer_skip_retry, + blob_file_reader); if (!s.ok()) { RecordTick(statistics, NO_FILE_ERRORS); } @@ -132,7 +283,7 @@ void BlobFileCache::RegisterBlobFileChecksum( } const Slice key = GetSliceForKey(&blob_file.file_number); - MutexLock cache_lock(&mutex_.Get(key)); + MutexLock cache_lock(GetStripeMutex(FileKind::kBlobFile, key)); { MutexLock checksum_lock(&blob_file_checksums_mutex_); BlobFileChecksum& checksum = blob_file_checksums_[blob_file.file_number]; @@ -150,10 +301,7 @@ void BlobFileCache::UnregisterBlobFileChecksum(uint64_t blob_file_number) { blob_file_checksums_.erase(blob_file_number); } -Status BlobFileCache::OpenBlobFileReader( - const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, - bool allow_footer_skip_retry, - std::unique_ptr* blob_file_reader) { +FileOptions BlobFileCache::GetFileOptions(const BlobFileOpenInfo& blob_file) { assert(file_options_); assert(blob_file.file_checksum.empty() == blob_file.file_checksum_func_name.empty()); @@ -174,7 +322,14 @@ Status BlobFileCache::OpenBlobFileReader( file_options.file_checksum_func_name.clear(); } } + return file_options; +} +Status BlobFileCache::OpenBlobFileReader( + const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, + bool allow_footer_skip_retry, + std::unique_ptr* blob_file_reader) { + const FileOptions file_options = GetFileOptions(blob_file); Status s = BlobFileReader::Create( *immutable_options_, read_options, file_options, column_family_id_, blob_file_read_hist_, blob_file.file_number, io_tracer_, @@ -202,7 +357,7 @@ Status BlobFileCache::InsertBlobFileReader( // Serialize refreshes for the same blob file. The cache API does not expose // a conditional replace, so refresh is intentionally modeled as // Lookup/optional Erase/Insert under this per-key mutex. - MutexLock lock(&mutex_.Get(key)); + MutexLock lock(GetStripeMutex(FileKind::kBlobFile, key)); TypedHandle* handle = cache_.Lookup(key); if (handle) { @@ -235,7 +390,7 @@ Status BlobFileCache::RefreshBlobFileReader( assert(cached_blob_file_reader->IsEmpty()); const Slice key = GetSliceForKey(&blob_file.file_number); - MutexLock lock(&mutex_.Get(key)); + MutexLock lock(GetStripeMutex(FileKind::kBlobFile, key)); TypedHandle* handle = cache_.Lookup(key); @@ -291,8 +446,20 @@ void BlobFileCache::Evict(uint64_t blob_file_number) { assert(cache_); - MutexLock lock(&mutex_.Get(key)); + MutexLock lock(GetStripeMutex(FileKind::kBlobFile, key)); +#ifndef NDEBUG + // Only for normal blob files (see the declaration). + if (TypedHandle* const handle = cache_.Lookup(key)) { + assert(cache_.Value(handle)->relocation() == nullptr); + cache_.Release(handle); + } +#endif // NDEBUG cache_.get()->Erase(key); } +void BlobFileCache::ForgetFailedRelocationOpen(uint64_t file_number) { + MutexLock failed_lock(&failed_relocation_opens_mutex_); + failed_relocation_opens_.erase(file_number); +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h index 6aa44f869804..e87308ec2920 100644 --- a/db/blob/blob_file_cache.h +++ b/db/blob/blob_file_cache.h @@ -6,6 +6,7 @@ #pragma once #include +#include #include #include @@ -13,6 +14,7 @@ #include "db/blob/blob_file_reader.h" #include "db/blob/blob_read_request.h" #include "rocksdb/rocksdb_namespace.h" +#include "util/hash_containers.h" #include "util/mutexlock.h" namespace ROCKSDB_NAMESPACE { @@ -24,36 +26,90 @@ class HistogramImpl; class Status; class Slice; class IOTracer; +struct BlockBasedTableOptions; +struct MutableCFOptions; +class TableFactory; class BlobFileCache { public: + // The base signature. It passes no relocation table options and no DB + // session id, so the cache opens normal blob files only; a relocation file + // read returns NotSupported. Because such a cache opens no relocation file, + // it needs no block cache tracer. BlobFileCache(Cache* cache, const ImmutableOptions* immutable_options, const FileOptions* file_options, uint32_t column_family_id, HistogramImpl* blob_file_read_hist, const std::shared_ptr& io_tracer); + // `relocation_file_table_options`, if not nullptr, are the table options + // that relocation files are opened with. `db_session_id` is passed to the + // readers it opens. `block_cache_tracer` is the DB's block cache tracer (the + // same one the table cache gets), so a block cache trace includes the block + // reads of relocation files. It may be nullptr. + BlobFileCache(Cache* cache, const ImmutableOptions* immutable_options, + const FileOptions* file_options, uint32_t column_family_id, + HistogramImpl* blob_file_read_hist, + const std::shared_ptr& io_tracer, + const BlockBasedTableOptions* relocation_file_table_options, + const std::string& db_session_id, + BlockCacheTracer* block_cache_tracer); + BlobFileCache(const BlobFileCache&) = delete; BlobFileCache& operator=(const BlobFileCache&) = delete; BlobFileCache(BlobFileCache&&) = delete; BlobFileCache& operator=(BlobFileCache&&) = delete; - ~BlobFileCache() = default; + ~BlobFileCache(); // Returns a cached reader for `blob_file`, opening and caching it on miss. // Checksum metadata is forwarded to the FileSystem through FileOptions when // the file is opened. If `allow_footer_skip_retry` is true, a // footer-validation corruption retries once without requiring a footer. + // If the reader is not cached and `read_options.read_tier` is + // kBlockCacheTier, returns Incomplete, because opening the file needs disk + // I/O. If the cached reader of this file number is a relocation file reader, + // returns Corruption. Status GetBlobFileReader(const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, CacheHandleGuard* blob_file_reader, bool allow_footer_skip_retry = false); - // Opens a blob file reader without inserting it into the cache. + // Like GetBlobFileReader, but for relocation file `file` (see + // db/blob/blob_relocation_file.h). The file size, tail size and directory + // digest in `file` are the expected identity of the file. The code that + // installs the file recorded them, and BlobRelocationFileReader::Open checks + // the file against them. + // - Cache hit: if the cached reader is not a relocation file reader of this + // column family, or its identity differs from `file`, returns Corruption. + // - Cache miss: returns Incomplete for a kBlockCacheTier read, like + // GetBlobFileReader. If an open of this file failed less than 1 second ago + // (kFailedRelocationOpenMicros) with an error from the file itself + // (Corruption, NotFound or IOError(PathNotFound)), returns that error again + // without opening the file (see failed_relocation_opens_). Otherwise opens + // the file. The compression manager and block protection of + // `mutable_cf_options` configure the table reader. Returns NotSupported if + // the cache was created without relocation table options. + Status GetRelocationFileReader( + const ReadOptions& read_options, const BlobPhysicalFile& file, + const MutableCFOptions& mutable_cf_options, + CacheHandleGuard* blob_file_reader); + + // The base signature. It passes prefer_direct_reads = false (the file is + // opened as configured). Status OpenBlobFileReaderUncached( const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, std::unique_ptr* blob_file_reader, bool allow_footer_skip_retry = false); + // Opens a blob file reader without inserting it into the cache. With + // `prefer_direct_reads`, the file is opened for direct reads when mmap reads + // are off. If that open fails (for example, the file system does not support + // direct reads), the file is opened as configured. + Status OpenBlobFileReaderUncached( + const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, + std::unique_ptr* blob_file_reader, + bool allow_footer_skip_retry, bool prefer_direct_reads); + // Makes finalized checksum metadata available to direct-write fallback reads // whose pinned Version predates the blob-file addition. Any cached // footer-less reader is evicted as part of the same per-file transition. @@ -81,14 +137,51 @@ class BlobFileCache { // Called when a blob file is obsolete to ensure it is removed from the cache // to avoid effectively leaking the open file and assicated memory + // + // Only for normal blob files: it takes the stripe mutex of a normal blob + // file (mutex_), not relocation_mutex_. PurgeObsoleteFiles() evicts the + // reader of a relocation file with TableCache::Evict(), which takes no + // stripe mutex (this cache shares its Cache with the table cache). void Evict(uint64_t blob_file_number); + // Removes the remembered failed open of relocation file `file_number`, if + // any (see failed_relocation_opens_). The deleter of a Version-owned + // relocation file calls it, with or without the DB mutex (see + // VersionSet::FreeBlobRelocationState()). It takes only + // failed_relocation_opens_mutex_. PurgeObsoleteFiles() evicts the cached + // reader later (see Evict()). + void ForgetFailedRelocationOpen(uint64_t file_number); + // Used to identify cache entries for blob files (not normally useful) static const Cache::CacheItemHelper* GetHelper() { return CacheInterface::GetBasicHelper(); } private: + // The kind of a file in this cache. It picks the stripe mutex set (see + // relocation_mutex_). + enum class FileKind { kBlobFile, kRelocationFile }; + + // The stripe mutex of the file with cache key `key` and kind `kind`. + port::Mutex* GetStripeMutex(FileKind kind, const Slice& key); + + // Shared by GetBlobFileReader and GetRelocationFileReader. On a cache hit, + // returns the cached reader of `blob_file_number` if validate(reader) returns + // OK, and the validate status otherwise. On a miss, returns Incomplete for a + // kBlockCacheTier read. Otherwise it calls open(&reader) under the stripe + // mutex of the file and caches the result. For a relocation file, a recent + // failed open that came from the file itself is returned again without + // opening the file (see failed_relocation_opens_). + template + Status GetBlobFileReaderImpl( + const ReadOptions& read_options, uint64_t blob_file_number, FileKind kind, + const ValidateFn& validate, const OpenFn& open, + CacheHandleGuard* blob_file_reader); + + // Returns the FileOptions to open `blob_file` with, including its checksum + // metadata. + FileOptions GetFileOptions(const BlobFileOpenInfo& blob_file); + Status OpenBlobFileReader(const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, bool allow_footer_skip_retry, @@ -101,11 +194,54 @@ class BlobFileCache { // Note: mutex_ below is used to guard against multiple threads racing to open // the same file. Striped> mutex_; + // The stripe mutexes of relocation files, separate from mutex_ (which normal + // blob files use). A relocation file open can hold its stripe mutex for a + // long time, because with pin_blob_relocation_location_blocks it reads all + // location blocks. Evict() of a normal blob file runs under the DB mutex + // (from the file's deleter), so it must never wait for such an open. + // kNumberOfRelocationMutexStripes (32) is enough because relocation opens + // are rare (a reader stays open after the first open), and a collision of + // two relocation opens only delays one read. Each stripe takes one cache + // line (CACHE_LINE_SIZE, 64 bytes on x86 and 128 bytes on aarch64). So the + // 32 stripes use 2 KB per BlobFileCache on x86, against 8 KB for the 128 + // stripes of mutex_. + Striped> relocation_mutex_; + // Recent failed relocation file opens, keyed by file number. Many reads can + // be routed to one bad relocation file. With this map, they get the same + // error without opening the file again until the entry expires after + // kFailedRelocationOpenMicros (1 second). The value is an estimate, not + // measured: it is long enough to absorb a burst of reads routed to the same + // bad file, and short enough that a repaired or replaced file is tried again + // soon. + // - Kept: errors that come from the file itself. These are Corruption + // (including an identity or directory digest mismatch), NotFound and + // IOError(PathNotFound). + // - Not kept: other IOErrors, TimedOut, Busy and Aborted. They can be + // short-lived or caused by the deadline of one reader. + // ForgetFailedRelocationOpen() also removes the entry. Guarded by + // failed_relocation_opens_mutex_. + static constexpr uint64_t kFailedRelocationOpenMicros = 1000 * 1000; + struct FailedRelocationOpen { + Status status; + uint64_t expires_micros = 0; + }; + port::Mutex failed_relocation_opens_mutex_; + UnorderedMap failed_relocation_opens_; const ImmutableOptions* immutable_options_; const FileOptions* file_options_; uint32_t column_family_id_; HistogramImpl* blob_file_read_hist_; std::shared_ptr io_tracer_; + // The DB session id. Each relocation table reader gets it as + // TableReaderOptions::cur_db_session_id. + const std::string db_session_id_; + // Passed to each relocation table reader as + // TableReaderOptions::block_cache_tracer. May be nullptr. + BlockCacheTracer* const block_cache_tracer_; + // Table factory for relocation files, built once in the constructor from + // MakeBlobRelocationTableOptions(relocation table options) and shared by all + // relocation readers. Null when no relocation table options were given. + std::unique_ptr relocation_factory_; struct BlobFileChecksum { std::string value; @@ -115,6 +251,7 @@ class BlobFileCache { mutable port::Mutex blob_file_checksums_mutex_; static constexpr size_t kNumberOfMutexStripes = 1 << 7; + static constexpr size_t kNumberOfRelocationMutexStripes = 32; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_cache_test.cc b/db/blob/blob_file_cache_test.cc index e4c4eae08042..f40718df7c5a 100644 --- a/db/blob/blob_file_cache_test.cc +++ b/db/blob/blob_file_cache_test.cc @@ -10,6 +10,7 @@ #include "db/blob/blob_log_format.h" #include "db/blob/blob_log_writer.h" +#include "env/composite_env_wrapper.h" #include "env/mock_env.h" #include "file/filename.h" #include "file/read_write_util.h" @@ -20,6 +21,8 @@ #include "rocksdb/file_system.h" #include "rocksdb/options.h" #include "rocksdb/statistics.h" +#include "rocksdb/table.h" +#include "test_util/mock_time_env.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" @@ -326,6 +329,103 @@ TEST_F(BlobFileCacheTest, GetBlobFileReaderIOError) { ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1); } +TEST_F(BlobFileCacheTest, RelocationOpenRemembersOnlyFileErrors) { + // BlobFileCache remembers a failed relocation open for a short time only when + // the error comes from the file itself: Corruption, or a missing file + // reported as IOError(PathNotFound). Other IOErrors and TimedOut can be + // short-lived, so the next read opens the file again. + class OpenFailureFS : public FileSystemWrapper { + public: + explicit OpenFailureFS(const std::shared_ptr& target) + : FileSystemWrapper(target) {} + const char* Name() const override { return "OpenFailureFS"; } + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + if (!next_error.ok()) { + IOStatus error = next_error; + next_error = IOStatus::OK(); + return error; + } + return target()->NewRandomAccessFile(fname, opts, result, dbg); + } + IOStatus next_error; + }; + const auto fs = std::make_shared(mock_env_->GetFileSystem()); + CompositeEnvWrapper env(mock_env_.get(), fs); + Options options; + options.env = &env; + options.statistics = CreateDBStatistics(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileCacheTest_RelocationOpenRemembers"), + 0); + options.enable_blob_files = true; + std::shared_ptr backing_cache = NewLRUCache(10); + ImmutableOptions immutable_options(options); + FileOptions file_options; + BlockBasedTableOptions table_options; + BlobFileCache blob_file_cache( + backing_cache.get(), &immutable_options, &file_options, + /*column_family_id=*/1, /*blob_file_read_hist=*/nullptr, + /*io_tracer=*/nullptr, &table_options, /*db_session_id=*/"session", + /*block_cache_tracer=*/nullptr); + const MutableCFOptions mutable_cf_options(options); + // A file that exists but is not a relocation table. + constexpr uint64_t file_number = 77; + ASSERT_OK(WriteStringToFile( + fs.get(), std::string(4096, 'x'), + BlobFileName(immutable_options.cf_paths.front().path, file_number))); + const BlobPhysicalFile blob_file{ + BlobFileOpenInfo{file_number, Slice(), Slice()}, + /*file_size=*/4096, /*is_relocation=*/true, + /*tail_size=*/1024, + /*directory_digest=*/1}; + const auto open = [&](const ReadOptions& read_options) { + CacheHandleGuard reader; + return blob_file_cache.GetRelocationFileReader(read_options, blob_file, + mutable_cf_options, &reader); + }; + const auto opens = [&]() { + return options.statistics->getTickerCount(NO_FILE_OPENS); + }; + + for (const IOStatus& injected : + {IOStatus::IOError("injected"), IOStatus::TimedOut("injected")}) { + SCOPED_TRACE(injected.ToString()); + fs->next_error = injected; + const uint64_t before = opens(); + const Status first = open(ReadOptions()); + EXPECT_EQ(first.code(), injected.code()) << first.ToString(); + // Not remembered: the next read opens the file again and gets its own + // result (here Corruption, because the file is not a relocation table). + const Status second = open(ReadOptions()); + EXPECT_TRUE(second.IsCorruption()) << second.ToString(); + EXPECT_EQ(opens(), before + 2); + // The Corruption is remembered: no new open within the 1-second window + // (kFailedRelocationOpenMicros). + EXPECT_TRUE(open(ReadOptions()).IsCorruption()); + EXPECT_EQ(opens(), before + 2); + // ForgetFailedRelocationOpen() drops the remembered failure, so the next + // round is clean. + blob_file_cache.ForgetFailedRelocationOpen(file_number); + } + + // A missing file (IOError(PathNotFound)) is remembered like Corruption. + fs->next_error = IOStatus::PathNotFound("injected"); + const uint64_t before = opens(); + const Status missing = open(ReadOptions()); + EXPECT_TRUE(missing.IsPathNotFound()) << missing.ToString(); + EXPECT_EQ(opens(), before + 1); + const Status again = open(ReadOptions()); + EXPECT_TRUE(again.IsPathNotFound()) << again.ToString(); + EXPECT_EQ(opens(), before + 1); + blob_file_cache.ForgetFailedRelocationOpen(file_number); + EXPECT_TRUE(open(ReadOptions()).IsCorruption()); + EXPECT_EQ(opens(), before + 2); +} + TEST_F(BlobFileCacheTest, GetBlobFileReaderCacheFull) { Options options; options.env = mock_env_.get(); @@ -369,6 +469,119 @@ TEST_F(BlobFileCacheTest, GetBlobFileReaderCacheFull) { ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1); } +TEST_F(BlobFileCacheTest, RelocationOpenFailureTickers) { + // Checks the two tickers of failed relocation file opens: + // - BLOB_RELOCATION_FILE_OPEN_FAILURES counts each real open of a relocation + // file that fails. + // - BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS counts each read that gets the + // remembered failure of an earlier open, without a new open. + // A failed open of a normal blob file counts in NO_FILE_ERRORS only. A mock + // clock moves past the 1-second window (kFailedRelocationOpenMicros) without + // a sleep. + class OpenFailureFS : public FileSystemWrapper { + public: + explicit OpenFailureFS(const std::shared_ptr& target) + : FileSystemWrapper(target) {} + const char* Name() const override { return "OpenFailureFS"; } + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + if (!next_error.ok()) { + IOStatus error = next_error; + next_error = IOStatus::OK(); + return error; + } + return target()->NewRandomAccessFile(fname, opts, result, dbg); + } + IOStatus next_error; + }; + const auto fs = std::make_shared(mock_env_->GetFileSystem()); + const auto clock = + std::make_shared(mock_env_->GetSystemClock()); + clock->SetCurrentTime(100); + CompositeEnvWrapper env(mock_env_.get(), fs, clock); + Options options; + options.env = &env; + options.statistics = CreateDBStatistics(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileCacheTest_RelocationOpenFailureTickers"), + 0); + options.enable_blob_files = true; + std::shared_ptr backing_cache = NewLRUCache(10); + ImmutableOptions immutable_options(options); + FileOptions file_options; + BlockBasedTableOptions table_options; + BlobFileCache blob_file_cache( + backing_cache.get(), &immutable_options, &file_options, + /*column_family_id=*/1, /*blob_file_read_hist=*/nullptr, + /*io_tracer=*/nullptr, &table_options, /*db_session_id=*/"session", + /*block_cache_tracer=*/nullptr); + const MutableCFOptions mutable_cf_options(options); + const auto ticker = [&](Tickers t) { + return options.statistics->getTickerCount(t); + }; + + // A normal blob file that does not exist (file 76): the failed open counts + // in NO_FILE_ERRORS, not in the relocation tickers. + { + CacheHandleGuard reader; + const BlobFileOpenInfo normal_file{76, Slice(), Slice()}; + ASSERT_NOK( + blob_file_cache.GetBlobFileReader(ReadOptions(), normal_file, &reader)); + ASSERT_EQ(ticker(NO_FILE_ERRORS), 1); + ASSERT_EQ(ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES), 0); + ASSERT_EQ(ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS), 0); + } + + // For example, relocation file 77 exists but is not a relocation table, so + // its open fails with Corruption. Relocation file 78 does not exist. + ASSERT_OK(WriteStringToFile( + fs.get(), std::string(4096, 'x'), + BlobFileName(immutable_options.cf_paths.front().path, 77))); + const auto open = [&](uint64_t file_number) { + const BlobPhysicalFile file{BlobFileOpenInfo{file_number, Slice(), Slice()}, + /*file_size=*/4096, /*is_relocation=*/true, + /*tail_size=*/1024, + /*directory_digest=*/1}; + CacheHandleGuard reader; + return blob_file_cache.GetRelocationFileReader(ReadOptions(), file, + mutable_cf_options, &reader); + }; + + for (const uint64_t file_number : {uint64_t{77}, uint64_t{78}}) { + SCOPED_TRACE("file " + std::to_string(file_number)); + const uint64_t failures = ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES); + const uint64_t hits = ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS); + // The first read opens the file and fails. + ASSERT_NOK(open(file_number)); + ASSERT_EQ(ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES), failures + 1); + ASSERT_EQ(ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS), hits); + // The next two reads get the remembered failure. + ASSERT_NOK(open(file_number)); + ASSERT_NOK(open(file_number)); + ASSERT_EQ(ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES), failures + 1); + ASSERT_EQ(ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS), hits + 2); + // After the window ends, the next read opens the file again. + clock->SleepForMicroseconds(2 * 1000 * 1000); + ASSERT_NOK(open(file_number)); + ASSERT_EQ(ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES), failures + 2); + ASSERT_EQ(ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS), hits + 2); + blob_file_cache.ForgetFailedRelocationOpen(file_number); + } + + // A short-lived IOError counts as a failed open but is not remembered, so it + // never gives a cache hit. The next read opens file 77 again. + const uint64_t failures = ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES); + const uint64_t hits = ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS); + fs->next_error = IOStatus::IOError("injected"); + ASSERT_TRUE(open(77).IsIOError()); + ASSERT_TRUE(open(77).IsCorruption()); + ASSERT_EQ(ticker(BLOB_RELOCATION_FILE_OPEN_FAILURES), failures + 2); + ASSERT_EQ(ticker(BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS), hits); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/blob/blob_file_completion_callback.h b/db/blob/blob_file_completion_callback.h index d3cae4fcede0..903e19589fec 100644 --- a/db/blob/blob_file_completion_callback.h +++ b/db/blob/blob_file_completion_callback.h @@ -8,6 +8,8 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once +#include + #include "db/error_handler.h" #include "db/event_helpers.h" #include "file/sst_file_manager_impl.h" @@ -98,6 +100,25 @@ class BlobFileCompletionCallback { return space_limit_reached; } + // Indirect blob GC calls this for an output file whose creation started + // (OnBlobFileCreationStarted()) but that does not get its final blob file + // name: the job failed before or during the rename, or the output would not + // reclaim bytes. It reports the failed creation to the listeners, so the + // start event has a matching finish event. It does not register the file with + // the SstFileManager, because the job deletes the file. `status` must not be + // OK. + void OnBlobFileAbandoned(const std::string& file_name, + const std::string& column_family_name, int job_id, + uint64_t file_number, + BlobFileCreationReason creation_reason, + const Status& status) { + assert(!status.ok()); + NotifyCreationFinished(file_name, column_family_name, job_id, file_number, + creation_reason, status, /*checksum_value=*/"", + /*checksum_method=*/"", /*blob_count=*/0, + /*blob_bytes=*/0); + } + void SetBGError(const Status& s) { InstrumentedMutexLock l(mutex_); error_handler_->SetBGError(s, BackgroundErrorReason::kFlush); diff --git a/db/blob/blob_file_meta.cc b/db/blob/blob_file_meta.cc index 4913137e5970..45851ecc1116 100644 --- a/db/blob/blob_file_meta.cc +++ b/db/blob/blob_file_meta.cc @@ -30,7 +30,8 @@ std::ostream& operator<<(std::ostream& os, << " total_blob_bytes: " << shared_meta.GetTotalBlobBytes() << " checksum_method: " << shared_meta.GetChecksumMethod() << " checksum_value: " - << Slice(shared_meta.GetChecksumValue()).ToString(/* hex */ true); + << Slice(shared_meta.GetChecksumValue()).ToString(/* hex */ true) + << " indirect_origin: " << shared_meta.IsIndirectOrigin(); return os; } diff --git a/db/blob/blob_file_meta.h b/db/blob/blob_file_meta.h index 0cd83da00849..7c33e5448d77 100644 --- a/db/blob/blob_file_meta.h +++ b/db/blob/blob_file_meta.h @@ -29,9 +29,19 @@ class SharedBlobFileMetaData { uint64_t blob_file_number, uint64_t total_blob_count, uint64_t total_blob_bytes, std::string checksum_method, std::string checksum_value) { + return Create(blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value), + /*is_indirect_origin=*/false); + } + + static std::shared_ptr Create( + uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value, bool is_indirect_origin) { return std::shared_ptr(new SharedBlobFileMetaData( blob_file_number, total_blob_count, total_blob_bytes, - std::move(checksum_method), std::move(checksum_value))); + std::move(checksum_method), std::move(checksum_value), + is_indirect_origin)); } template @@ -39,11 +49,22 @@ class SharedBlobFileMetaData { uint64_t blob_file_number, uint64_t total_blob_count, uint64_t total_blob_bytes, std::string checksum_method, std::string checksum_value, Deleter deleter) { + return Create(blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value), + /*is_indirect_origin=*/false, std::move(deleter)); + } + + template + static std::shared_ptr Create( + uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value, bool is_indirect_origin, Deleter deleter) { return std::shared_ptr( new SharedBlobFileMetaData(blob_file_number, total_blob_count, total_blob_bytes, std::move(checksum_method), - std::move(checksum_value)), - deleter); + std::move(checksum_value), + is_indirect_origin), + std::move(deleter)); } SharedBlobFileMetaData(const SharedBlobFileMetaData&) = delete; @@ -60,18 +81,20 @@ class SharedBlobFileMetaData { uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; } const std::string& GetChecksumMethod() const { return checksum_method_; } const std::string& GetChecksumValue() const { return checksum_value_; } + bool IsIndirectOrigin() const { return is_indirect_origin_; } std::string DebugString() const; private: SharedBlobFileMetaData(uint64_t blob_file_number, uint64_t total_blob_count, uint64_t total_blob_bytes, std::string checksum_method, - std::string checksum_value) + std::string checksum_value, bool is_indirect_origin) : blob_file_number_(blob_file_number), total_blob_count_(total_blob_count), total_blob_bytes_(total_blob_bytes), checksum_method_(std::move(checksum_method)), - checksum_value_(std::move(checksum_value)) { + checksum_value_(std::move(checksum_value)), + is_indirect_origin_(is_indirect_origin) { assert(checksum_method_.empty() == checksum_value_.empty()); } @@ -80,6 +103,7 @@ class SharedBlobFileMetaData { uint64_t total_blob_bytes_; std::string checksum_method_; std::string checksum_value_; + bool is_indirect_origin_; }; std::ostream& operator<<(std::ostream& os, @@ -143,6 +167,10 @@ class BlobFileMetaData { assert(shared_meta_); return shared_meta_->GetChecksumValue(); } + bool IsIndirectOrigin() const { + assert(shared_meta_); + return shared_meta_->IsIndirectOrigin(); + } const LinkedSsts& GetLinkedSsts() const { return linked_ssts_; } diff --git a/db/blob/blob_file_reader.cc b/db/blob/blob_file_reader.cc index 84e552de7a76..acddda60e25d 100644 --- a/db/blob/blob_file_reader.cc +++ b/db/blob/blob_file_reader.cc @@ -5,28 +5,83 @@ #include "db/blob/blob_file_reader.h" +#include +#include #include +#include +#include #include +#include +#include #include "db/blob/blob_contents.h" +#include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" +#include "db/blob/blob_relocation_file.h" +#include "db/blob/same_file_blob_reader.h" +#include "db/dbformat.h" +#include "db/version_edit.h" #include "file/file_prefetch_buffer.h" #include "file/filename.h" #include "file/read_write_util.h" +#include "logging/logging.h" #include "monitoring/statistics_impl.h" #include "options/cf_options.h" #include "rocksdb/file_system.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/embedded_blob_sst.h" #include "table/format.h" +#include "table/internal_iterator.h" #include "table/multiget_context.h" +#include "table/table_builder.h" +#include "table/table_reader.h" #include "test_util/sync_point.h" #include "util/aligned_buffer.h" +#include "util/cast_util.h" #include "util/compression.h" +#include "util/crc32c.h" #include "util/stop_watch.h" namespace ROCKSDB_NAMESPACE { +namespace { + +// Checks that `blob_index` is a valid indirect index whose stored payload can +// be in a relocation file, and that an uncompressed range fits in the value. +// Returns Corruption for bad metadata or for a range of a compressed value (as +// in base RocksDB), and InvalidArgument for an out-of-bounds range. +Status ValidateRelocationBlobReadRequest(const BlobIndex& blob_index, + uint64_t range_offset, + size_t range_length) { + if (!blob_index.IsIndirect()) { + return Status::Corruption("Invalid stable blob relocation lookup"); + } + Status status = blob_index.Validate(); + if (!status.ok()) { + return status; + } + if (blob_index.size() > kMaxSimpleGen2BlobPayloadSize || + !CompressionTypeSupported(blob_index.compression())) { + return Status::Corruption("Unsupported blob relocation payload"); + } + if (range_length != kWholeBlobLength && + blob_index.compression() != kNoCompression) { + // As in base RocksDB, a range of a compressed value is not supported. + return Status::Corruption("Cannot range-read a compressed blob"); + } + if (range_length != kWholeBlobLength && + (range_offset > blob_index.size() || + range_length > blob_index.size() - range_offset)) { + return Status::InvalidArgument("Blob relocation range is out of bounds"); + } + return Status::OK(); +} + +} // namespace + Status BlobFileReader::Create( const ImmutableOptions& immutable_options, const ReadOptions& read_options, const FileOptions& file_options, uint32_t column_family_id, @@ -43,7 +98,8 @@ Status BlobFileReader::Create( const Status s = OpenFile(immutable_options, file_options, blob_file_read_hist, blob_file_number, io_tracer, &file_size, &file_reader, - /*skip_footer_size_check=*/skip_footer_validation); + /*skip_footer_size_check=*/skip_footer_validation, + /*is_relocation_file=*/false); if (!s.ok()) { return s; } @@ -92,7 +148,7 @@ Status BlobFileReader::OpenFile( HistogramImpl* blob_file_read_hist, uint64_t blob_file_number, const std::shared_ptr& io_tracer, uint64_t* file_size, std::unique_ptr* file_reader, - bool skip_footer_size_check) { + bool skip_footer_size_check, bool is_relocation_file) { assert(file_size); assert(file_reader); @@ -110,7 +166,8 @@ Status BlobFileReader::OpenFile( std::unique_ptr file; FileOptions reader_file_opts = file_opts; - if (skip_footer_size_check && reader_file_opts.use_direct_reads) { + if (skip_footer_size_check && !is_relocation_file && + reader_file_opts.use_direct_reads) { reader_file_opts.use_direct_reads = false; } @@ -136,11 +193,16 @@ Status BlobFileReader::OpenFile( } } - if (!skip_footer_size_check && + if (is_relocation_file && + *file_size < kBlobRelocationFilePrefixSize + Footer::kMinEncodedLength) { + return Status::Corruption("Malformed blob relocation file"); + } + if (!is_relocation_file && !skip_footer_size_check && *file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) { return Status::Corruption("Malformed blob file"); } - if (skip_footer_size_check && *file_size < BlobLogHeader::kSize) { + if (!is_relocation_file && skip_footer_size_check && + *file_size < BlobLogHeader::kSize) { return Status::Corruption("Malformed blob file"); } @@ -323,13 +385,23 @@ BlobFileReader::BlobFileReader( assert(file_reader_); } +BlobFileReader::BlobFileReader( + std::unique_ptr relocation) + : file_size_(relocation->file_size()), + compression_type_(kNoCompression), + clock_(nullptr), + statistics_(nullptr), + has_footer_(true), + relocation_(std::move(relocation)) {} + BlobFileReader::~BlobFileReader() = default; Status BlobFileReader::GetBlob( const ReadOptions& read_options, const Slice& user_key, uint64_t offset, uint64_t value_size, CompressionType compression_type, FilePrefetchBuffer* prefetch_buffer, MemoryAllocator* allocator, - std::unique_ptr* result, uint64_t* bytes_read) const { + std::unique_ptr* result, uint64_t* bytes_read, + std::optional* value_crc32c) const { assert(result); const uint64_t key_size = user_key.size(); @@ -401,7 +473,8 @@ Status BlobFileReader::GetBlob( &record_slice); if (read_options.verify_checksums) { - const Status s = VerifyBlob(record_slice, user_key, value_size); + const Status s = + VerifyBlob(record_slice, user_key, value_size, value_crc32c); if (!s.ok()) { return s; } @@ -410,9 +483,9 @@ Status BlobFileReader::GetBlob( const Slice value_slice(record_slice.data() + adjustment, value_size); { - const Status s = UncompressBlobIfNeeded(value_slice, compression_type, - decompressor_.get(), allocator, - clock_, statistics_, result); + const Status s = UncompressBlobContentsIfNeeded( + value_slice, compression_type, decompressor_.get(), allocator, clock_, + statistics_, result); if (!s.ok()) { return s; } @@ -502,6 +575,165 @@ Status BlobFileReader::GetBlobRange(const ReadOptions& read_options, return Status::OK(); } +Status BlobFileReader::StreamStoredBlobForRelocation( + const ReadOptions& read_options, const BlobIndex& blob_index, + const BlobChunkConsumer& consumer) const { + Status status = blob_index.Validate(); + if (!status.ok()) { + return status; + } + if (!blob_index.IsIndirect()) { + return Status::NotSupported( + "Stored streaming requires a stable blob index"); + } + if (relocation_ != nullptr) { + return Status::InvalidArgument( + "Normal stored-blob stream used on a relocation file"); + } + if (blob_index.compression() != compression_type_ || + !IsValidBlobOffset(blob_index.original_value_offset(), /*key_size=*/0, + blob_index.size(), file_size_, has_footer_)) { + return Status::Corruption("Invalid streaming normal blob reference"); + } + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, blob_index.size()); + PERF_COUNTER_ADD(blob_read_count, 1); + PERF_COUNTER_ADD(blob_read_byte, blob_index.size()); + PERF_TIMER_GUARD(blob_read_time); + return StreamAndVerifyBlobStoredPayload( + read_options, file_reader_.get(), blob_index.original_value_offset(), + blob_index.size(), blob_index.compression(), blob_index.stored_checksum(), + consumer); +} + +void BlobFileReader::MultiGetStoredBlobsForRelocation( + const ReadOptions& read_options, uint64_t coalesce_bytes, + size_t num_requests, StoredNormalBlobReadRequest* requests, + uint64_t max_stored_value_size, CoalescedReadBuffers* buffers, + uint64_t* bytes_read) const { + if (bytes_read != nullptr) { + *bytes_read = 0; + } + if (num_requests == 0 || requests == nullptr) { + return; + } + if (buffers == nullptr) { + for (size_t i = 0; i < num_requests; ++i) { + if (requests[i].status != nullptr) { + *requests[i].status = + Status::InvalidArgument("Missing blob GC read buffers"); + } + } + return; + } + + std::vector request_indexes; + std::vector offsets; + std::vector lengths; + request_indexes.reserve(num_requests); + offsets.reserve(num_requests); + lengths.reserve(num_requests); + uint64_t total_read_size = 0; + for (size_t i = 0; i < num_requests; ++i) { + StoredNormalBlobReadRequest& request = requests[i]; + if (request.status == nullptr) { + continue; + } + Status& request_status = *request.status; + request_status = Status::OK(); + if (request.stored_payload != nullptr) { + *request.stored_payload = Slice(); + } + if (relocation_ != nullptr) { + request_status = Status::InvalidArgument( + "Normal stored-blob read used on a relocation file"); + continue; + } + if (request.stored_value_size > max_stored_value_size) { + request_status = + Status::MemoryLimit("Stored blob exceeds relocation read limit"); + continue; + } + if (request.stored_value_size > std::numeric_limits::max() || + !IsValidBlobOffset(request.offset, /*key_size=*/0, + request.stored_value_size, file_size_, + has_footer_)) { + request_status = + Status::Corruption("Invalid stored blob relocation range"); + continue; + } + if (request.compression != compression_type_) { + request_status = + Status::Corruption("Compression type mismatch when relocating blob"); + continue; + } + if (!offsets.empty()) { + assert(offsets.back() <= request.offset); + } + if (request.stored_value_size > + std::numeric_limits::max() - total_read_size) { + request_status = Status::Corruption("Stored blob read size overflow"); + continue; + } + total_read_size += request.stored_value_size; + request_indexes.push_back(i); + offsets.push_back(request.offset); + lengths.push_back(static_cast(request.stored_value_size)); + } + if (request_indexes.empty()) { + return; + } + + TEST_SYNC_POINT( + "BlobFileReader::MultiGetStoredBlobsForRelocation:ReadFromFile"); + std::vector results(request_indexes.size()); + std::vector statuses(request_indexes.size()); + size_t num_reads = 0; + uint64_t read_bytes = 0; + { + PERF_TIMER_GUARD(blob_read_time); + // Buffer limit: the requested bytes plus at most max_stored_value_size + // bytes of gaps (see ReadCoalescedRanges()). + ReadCoalescedRanges( + read_options, file_reader_.get(), coalesce_bytes, max_stored_value_size, + total_read_size + + std::min(max_stored_value_size, + std::numeric_limits::max() - total_read_size), + request_indexes.size(), offsets.data(), lengths.data(), results.data(), + statuses.data(), buffers, &num_reads, &read_bytes); + } + // Bytes read from the file, merged gaps included. + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, read_bytes); + PERF_COUNTER_ADD(blob_read_count, num_reads); + PERF_COUNTER_ADD(blob_read_byte, read_bytes); + + TEST_SYNC_POINT_CALLBACK( + "BlobFileReader::MultiGetStoredBlobsForRelocation:TamperWithResult", + &statuses); + uint64_t successful_bytes = 0; + for (size_t i = 0; i < request_indexes.size(); ++i) { + StoredNormalBlobReadRequest& request = requests[request_indexes[i]]; + assert(request.status != nullptr); + Status& request_status = *request.status; + request_status = statuses[i]; + if (!request_status.ok()) { + continue; + } + if (BlobIndex::ComputeStoredChecksum(results[i], request.compression) != + request.stored_checksum) { + request_status = + Status::Corruption("Stored checksum mismatch while relocating blob"); + continue; + } + if (request.stored_payload != nullptr) { + *request.stored_payload = results[i]; + } + successful_bytes += results[i].size(); + } + if (bytes_read != nullptr) { + *bytes_read = successful_bytes; + } +} + void BlobFileReader::MultiGetBlob( const ReadOptions& read_options, MemoryAllocator* allocator, autovector>>& @@ -593,6 +825,8 @@ void BlobFileReader::MultiGetBlob( s = file_reader_->PrepareIOOptions(read_options, opts, &dbg); if (s.ok()) { AlignedBufferAllocationContext direct_io_context{&direct_io_buffer}; + TEST_SYNC_POINT_CALLBACK("BlobFileReader::MultiGetBlob:Requests", + &read_reqs); s = file_reader_->MultiRead(opts, read_reqs.data(), read_reqs.size(), &direct_io_context, &dbg); } @@ -644,7 +878,8 @@ void BlobFileReader::MultiGetBlob( // Verify checksums if enabled if (read_options.verify_checksums) { - *req->status = VerifyBlob(record_slice, *req->user_key, req->len); + *req->status = + VerifyBlob(record_slice, *req->user_key, req->len, req->value_crc32c); if (!req->status->ok()) { continue; } @@ -652,7 +887,7 @@ void BlobFileReader::MultiGetBlob( // Uncompress blob if needed Slice value_slice(record_slice.data() + adjustments[j - 1], req->len); - *req->status = UncompressBlobIfNeeded( + *req->status = UncompressBlobContentsIfNeeded( value_slice, compression_type_, decompressor_.get(), allocator, clock_, statistics_, &blob_reqs[i].second); if (req->status->ok()) { @@ -776,6 +1011,8 @@ void BlobFileReader::MultiGetBlobRange( s = file_reader_->PrepareIOOptions(read_options, opts, &dbg); if (s.ok()) { AlignedBufferAllocationContext direct_io_context{&direct_io_buffer}; + TEST_SYNC_POINT_CALLBACK("BlobFileReader::MultiGetBlobRange:Requests", + &read_reqs); s = file_reader_->MultiRead(opts, read_reqs.data(), read_reqs.size(), &direct_io_context, &dbg); } @@ -842,7 +1079,8 @@ void BlobFileReader::MultiGetBlobRange( } Status BlobFileReader::VerifyBlob(const Slice& record_slice, - const Slice& user_key, uint64_t value_size) { + const Slice& user_key, uint64_t value_size, + std::optional* value_crc32c) { PERF_TIMER_GUARD(blob_checksum_time); BlobLogRecord record; @@ -876,56 +1114,601 @@ Status BlobFileReader::VerifyBlob(const Slice& record_slice, TEST_SYNC_POINT_CALLBACK("BlobFileReader::VerifyBlob:CheckBlobCRC", &record); - const Status s = record.CheckBlobCRC(); - if (!s.ok()) { - return s; + if (value_crc32c != nullptr) { + const uint32_t value_crc = + crc32c::Value(record.value.data(), record.value.size()); + const uint32_t blob_crc = crc32c::Mask(crc32c::Crc32cCombine( + crc32c::Value(record.key.data(), record.key.size()), value_crc, + record.value.size())); + if (blob_crc != record.blob_crc) { + return Status::Corruption("Blob CRC mismatch"); + } + *value_crc32c = value_crc; + } else { + const Status s = record.CheckBlobCRC(); + if (!s.ok()) { + return s; + } } } return Status::OK(); } -Status BlobFileReader::UncompressBlobIfNeeded( - const Slice& value_slice, CompressionType compression_type, - Decompressor* decompressor, MemoryAllocator* allocator, SystemClock* clock, - Statistics* statistics, std::unique_ptr* result) { - assert(result); +BlobRelocationFileReader::BlobRelocationFileReader(uint32_t column_family_id, + uint64_t file_size) + : column_family_id_(column_family_id), + file_size_(file_size), + internal_comparator_(BytewiseComparator()) {} + +struct BlobRelocationFileReader::PinnedLocationBlocks { + BlockBasedTable::PinnedDataBlocks blocks; +}; + +BlobRelocationFileReader::~BlobRelocationFileReader() = default; + +size_t BlobRelocationFileReader::pinned_location_block_bytes() const { + return pinned_location_blocks_ != nullptr + ? pinned_location_blocks_->blocks.memory_bytes + : 0; +} + +BlockBasedTable* BlobRelocationFileReader::table() const { + return static_cast_with_check(table_reader_.get()); +} + +Status BlobRelocationFileReader::Open( + const ImmutableOptions& immutable_options, const ReadOptions& read_options, + const FileOptions& file_options, HistogramImpl* blob_file_read_hist, + uint32_t column_family_id, const BlobPhysicalFile& file, + const std::string& db_session_id, const TableFactory& factory, + CompressionManager* compression_manager, + uint8_t block_protection_bytes_per_key, bool pin_location_blocks, + const std::shared_ptr& io_tracer, + BlockCacheTracer* block_cache_tracer, + std::unique_ptr* reader) { + assert(reader != nullptr); + if (!file.is_relocation || file.open_info.file_number == 0 || + file.file_size == 0 || db_session_id.empty() || file.tail_size == 0 || + file.tail_size > file.file_size) { + return Status::InvalidArgument("Invalid blob relocation file metadata"); + } + + uint64_t file_size = 0; + std::unique_ptr file_reader; + Status status = BlobFileReader::OpenFile( + immutable_options, file_options, blob_file_read_hist, + file.open_info.file_number, io_tracer, &file_size, &file_reader, + /*skip_footer_size_check=*/false, /*is_relocation_file=*/true); + if (!status.ok()) { + return status; + } + if (file_size != file.file_size) { + return Status::Corruption("Blob relocation file size mismatch"); + } + // No separate prefix read: `file.is_relocation` already says this is a + // relocation file, and the table footer, file number, tail size and directory + // digest checked below identify it. + + std::unique_ptr relocation( + new BlobRelocationFileReader(column_family_id, file_size)); + const std::shared_ptr no_prefix_extractor; + TableReaderOptions table_reader_options( + immutable_options, no_prefix_extractor, compression_manager, file_options, + relocation->internal_comparator_, block_protection_bytes_per_key, + /*skip_filters=*/true, /*immortal=*/false, + /*force_direct_prefetch=*/false, /*level=*/-1, block_cache_tracer); + table_reader_options.cur_file_num = file.open_info.file_number; + table_reader_options.cur_db_session_id = db_session_id; + table_reader_options.tail_size = file.tail_size; + status = factory.NewTableReader(read_options, table_reader_options, + std::move(file_reader), file_size, + &relocation->table_reader_, + /*prefetch_index_and_filter_in_cache=*/true); + if (!status.ok()) { + return status; + } + + const std::shared_ptr properties = + relocation->table_reader_->GetTableProperties(); + if (properties == nullptr) { + return Status::Corruption("Blob relocation file has no table properties"); + } + relocation->tail_size_ = + FileMetaData::CalculateTailSize(file_size, *properties); + if (relocation->tail_size_ != file.tail_size) { + return Status::Corruption("Blob relocation metadata tail size mismatch"); + } + if (properties->column_family_id != column_family_id) { + return Status::Corruption("Blob relocation column family mismatch"); + } + if (properties->orig_file_number != file.open_info.file_number || + properties->db_session_id.empty()) { + return Status::Corruption("Blob relocation file identity is missing"); + } + const auto directory_it = properties->user_collected_properties.find( + kBlobRelocationDirectoryPropertyName); + const auto stats_it = properties->user_collected_properties.find( + kEmbeddedBlobSstStatsPropertyName); + if (directory_it == properties->user_collected_properties.end() || + stats_it == properties->user_collected_properties.end()) { + return Status::Corruption("Blob relocation file metadata is missing"); + } + // The directory is checked here, then only its totals and digest are kept: + // a lookup proves a value exists by its key match (CheckEntry). + Slice encoded_directory = directory_it->second; + TEST_SYNC_POINT_CALLBACK("BlobRelocationFileReader::Open:EncodedDirectory", + &encoded_directory); + BlobRelocationDirectory directory; + status = BlobRelocationDirectory::DecodeFrom(encoded_directory, &directory); + if (!status.ok()) { + return status; + } + status = directory.ValidateForFileSize(file_size); + if (!status.ok()) { + return status; + } + // Check that every origin and served file is older than the file itself, + // that the directory (a table property) fits in the tail, and that the + // records fit between the prefix and the tail. + if (directory.origins().back().origin_file_number >= + file.open_info.file_number || + directory.served_file_numbers().back() >= file.open_info.file_number) { + return Status::Corruption( + "Blob relocation directory names a file that is not older"); + } + if (directory.encoded_size() > file.tail_size || + file_size - file.tail_size < kBlobRelocationFilePrefixSize || + directory.total_record_bytes() > + file_size - file.tail_size - kBlobRelocationFilePrefixSize) { + return Status::Corruption("Blob relocation directory exceeds its region"); + } + relocation->total_entries_ = directory.total_entries(); + relocation->total_record_bytes_ = directory.total_record_bytes(); + relocation->directory_digest_ = directory.directory_digest(); + if (relocation->directory_digest_ != file.directory_digest) { + return Status::Corruption("Blob relocation directory digest mismatch"); + } + EmbeddedBlobStats stats; + status = DecodeEmbeddedBlobStats(stats_it->second, &stats); + if (!status.ok()) { + return status; + } + const uint64_t entry_count = relocation->total_entries_; + const bool trailer_bytes_overflow = + entry_count > + std::numeric_limits::max() / kSimpleGen2BlobTrailerSize; + const uint64_t trailer_bytes = + trailer_bytes_overflow ? 0 : entry_count * kSimpleGen2BlobTrailerSize; + // One location entry per group of up to kBlobRelocationGroupSize values of + // one origin (see "Location entry" in db/blob/blob_relocation_file.h). The + // directory entry counts give the exact number of groups. The group counts + // inside the entries are checked when a lookup decodes an entry; checking + // their sum here would read every location block at open. + uint64_t group_count = 0; + for (const BlobRelocationOrigin& origin : directory.origins()) { + group_count += (origin.entry_count + kBlobRelocationGroupSize - 1) / + kBlobRelocationGroupSize; + } + if (!stats.HasRecords() || stats.blob_count != entry_count || + properties->num_entries != group_count || trailer_bytes_overflow || + stats.payload_bytes > + std::numeric_limits::max() - trailer_bytes || + stats.payload_bytes + trailer_bytes != relocation->total_record_bytes_) { + return Status::Corruption("Blob relocation file entry totals mismatch"); + } - if (compression_type == kNoCompression) { - BlobContentsCreator::Create(result, nullptr, value_slice, kNoCompression, - allocator); - return Status::OK(); + // The encoded directory in the table properties is not needed after open. + relocation->table()->DropUserCollectedProperty( + kBlobRelocationDirectoryPropertyName); + + if (pin_location_blocks) { + TEST_SYNC_POINT("BlobRelocationFileReader::Open:PinLocationBlocks"); + auto pinned = std::make_unique(); + const Status pin_status = + relocation->table()->PinDataBlocks(read_options, &pinned->blocks); + if (pin_status.ok()) { + relocation->pinned_location_blocks_ = std::move(pinned); + } else { + // Not an open failure: normal lookups read the same blocks. + ROCKS_LOG_WARN(immutable_options.logger, + "Cannot keep the location blocks of blob relocation file " + "#%" PRIu64 " in memory; using normal lookups: %s", + file.open_info.file_number, pin_status.ToString().c_str()); + } } - assert(decompressor); + *reader = std::move(relocation); + return Status::OK(); +} - Decompressor::Args args; - args.compression_type = compression_type; - args.compressed_data = value_slice; +Status BlobRelocationFileReader::PrepareLookup( + const BlobIndex& stable_blob_index, uint64_t range_offset, + size_t range_length) const { + return ValidateRelocationBlobReadRequest(stable_blob_index, range_offset, + range_length); +} - Status s = decompressor->ExtractUncompressedSize(args); - if (!s.ok()) { - return Status::Corruption(s.ToString()); +void BlobRelocationFileReader::EncodeLookupKey( + const BlobIndex& stable_blob_index, char* lookup_key) { + EncodeBlobRelocationKeyTo(stable_blob_index.origin_file_number(), + stable_blob_index.original_value_offset(), + lookup_key); + EncodeFixed64(lookup_key + kBlobRelocationKeySize, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); +} + +Status BlobRelocationFileReader::CheckEntry(const BlobIndex& stable_blob_index, + const Slice& internal_key, + const Slice& value, + BlobIndex* same_file_blob_index, + DecodedGroupCache* cache) { + assert(same_file_blob_index != nullptr); + // The lookup found the first location entry at or after the lookup key. It + // holds the value only if it has the same origin. Its key holds the origin + // and the original offset of the group's last value, so with the same origin + // that offset is at or after the requested offset. + const BlobRelocationGroup* group = nullptr; + BlobRelocationGroup decoded; + if (cache != nullptr && cache->valid && internal_key == cache->internal_key) { + group = &cache->group; + } else { + ParsedInternalKey parsed_key; + Status status = ParseInternalKey(internal_key, &parsed_key, + /*log_err_key=*/false); + if (!status.ok()) { + return status; + } + uint64_t origin_file_number = 0; + uint64_t last_original_value_offset = 0; + status = DecodeBlobRelocationKey(parsed_key.user_key, &origin_file_number, + &last_original_value_offset); + if (!status.ok() || parsed_key.sequence != 0 || + parsed_key.type != kTypeValue) { + return Status::Corruption("Invalid blob relocation entry key"); + } + if (origin_file_number != stable_blob_index.origin_file_number()) { + return Status::Corruption("Blob relocation entry is missing"); + } + TEST_SYNC_POINT("BlobRelocationFileReader::CheckEntry:Decode"); + if (cache != nullptr) { + cache->valid = false; + } + BlobRelocationGroup* const target = + cache != nullptr ? &cache->group : &decoded; + status = + DecodeBlobRelocationGroup(value, last_original_value_offset, target); + if (!status.ok()) { + return status; + } + if (cache != nullptr) { + cache->internal_key.assign(internal_key.data(), internal_key.size()); + cache->origin_file_number = origin_file_number; + cache->valid = true; + } + group = target; + } + const BlobRelocationGroup::Value* const found = + group->Find(stable_blob_index.original_value_offset()); + if (found == nullptr) { + return Status::Corruption("Blob relocation entry is missing"); + } + // A same-file BlobIndex on the stack: type, file number, record offset, + // stored size and compression. The location entry has no compression type: + // the record's compression marker must match the one of + // `stable_blob_index`, and the record read checks that. + char encoded[2 + 2 * kMaxVarint64Length + 1]; + char* end = encoded; + *end++ = static_cast(BlobIndex::Type::kBlob); + end = EncodeVarint64(end, kCurrentFileBlobIndexFileNumber); + end = EncodeVarint64(end, found->record_offset); + end = EncodeVarint64(end, found->stored_value_size); + *end++ = static_cast(stable_blob_index.compression()); + Status status = same_file_blob_index->DecodeFrom( + Slice(encoded, static_cast(end - encoded))); + if (!status.ok()) { + return status; } + if (same_file_blob_index->size() != stable_blob_index.size()) { + return Status::Corruption("Blob relocation entry metadata mismatch"); + } + return Status::OK(); +} - CacheAllocationPtr output = AllocateBlock(args.uncompressed_size, allocator); +void BlobRelocationFileReader::MultiGetBlob(const ReadOptions& read_options, + MemoryAllocator* allocator, + size_t num_reads, + RelocationBlobReadRequest* requests, + uint64_t* bytes_read) const { + if (bytes_read != nullptr) { + *bytes_read = 0; + } + if (num_reads == 0) { + return; + } + assert(requests != nullptr); + + // Checks each request once, then looks up the valid ones in key order. The + // lookup keys and the same-file requests of one batch + // (MultiGetContext::MAX_BATCH_SIZE) live on the stack. The lookup order is on + // the stack while the call has at most one batch of requests. The record + // reads and the cached key of the decoded group can still allocate on the + // heap. + constexpr size_t kBatch = MultiGetContext::MAX_BATCH_SIZE; + autovector lookup_order; + for (size_t i = 0; i < num_reads; ++i) { + RelocationBlobReadRequest& request = requests[i]; + assert(request.blob_index != nullptr); + assert(request.output != nullptr); + assert(request.status != nullptr); + request.output->allocator = allocator; + request.output->contents.reset(); + request.output->gen2 = SimpleGen2ReadExtras(); + *request.status = PrepareLookup(*request.blob_index, request.range_offset, + request.range_length); + if (request.status->ok()) { + lookup_order.push_back(i); + } + } + std::sort(lookup_order.begin(), lookup_order.end(), + [requests](size_t lhs, size_t rhs) { + const BlobIndex& left = *requests[lhs].blob_index; + const BlobIndex& right = *requests[rhs].blob_index; + return std::make_tuple(left.origin_file_number(), + left.original_value_offset()) < + std::make_tuple(right.origin_file_number(), + right.original_value_offset()); + }); + + uint64_t total_bytes_read = 0; + constexpr size_t kLookupKeySize = kBlobRelocationKeySize + kNumInternalBytes; + for (size_t begin = 0; begin < lookup_order.size(); begin += kBatch) { + const size_t count = std::min(lookup_order.size() - begin, kBatch); + // One raw lookup of the batch's keys. Pinned or cached data blocks need no + // read. MultiGetRawEntries() reads the missing blocks, with one batched + // read when two or more are missing. + std::array lookup_key_bytes; + std::array lookup_keys; + for (size_t k = 0; k < count; ++k) { + char* const lookup_key = lookup_key_bytes.data() + k * kLookupKeySize; + EncodeLookupKey(*requests[lookup_order[begin + k]].blob_index, + lookup_key); + lookup_keys[k] = Slice(lookup_key, kLookupKeySize); + } + autovector same_file_blob_indexes; + same_file_blob_indexes.resize(count); + // Keys are sorted, so the keys that fall in one location entry come one + // after another: the entry is decoded once for all of them. + DecodedGroupCache group_cache; + const auto on_entry = [&](size_t k, const Status& lookup_status, + const Slice& internal_key, const Slice& value) { + RelocationBlobReadRequest& request = requests[lookup_order[begin + k]]; + if (lookup_status.IsNotFound()) { + *request.status = + Status::Corruption("Blob relocation entry is missing"); + } else if (!lookup_status.ok()) { + *request.status = lookup_status; + } else { + *request.status = CheckEntry(*request.blob_index, internal_key, value, + &same_file_blob_indexes[k], &group_cache); + } + }; + if (pinned_location_blocks_ != nullptr) { + table()->MultiGetRawEntriesPinned(pinned_location_blocks_->blocks, count, + lookup_keys.data(), on_entry); + } else { + table()->MultiGetRawEntries(read_options, count, lookup_keys.data(), + on_entry); + } - { - PERF_TIMER_GUARD(blob_decompress_time); - StopWatch stop_watch(clock, statistics, BLOB_DB_DECOMPRESSION_MICROS); - s = decompressor->DecompressBlock(args, output.get()); + // The table pins nothing into these; each value goes to + // request.output->contents. + autovector unused_results; + autovector same_file_requests; + for (size_t k = 0; k < count; ++k) { + RelocationBlobReadRequest& request = requests[lookup_order[begin + k]]; + if (!request.status->ok()) { + continue; + } + // Same rule as for normal blob files: check the record checksum only when + // verify_checksums is set and the check needs no extra read (see the + // read table in db/blob/blob_relocation_file.h). + const BlobVerifyPolicy verify_policy = + BlobVerifyPolicy::kVerifyIfNoAmplification; + unused_results.emplace_back(); + SameFileBlobReadRequest same_file_request; + same_file_request.blob_index = &same_file_blob_indexes[k]; + same_file_request.range_offset = request.range_offset; + same_file_request.range_length = request.range_length; + same_file_request.verify_policy = verify_policy; + same_file_request.result = &unused_results.back(); + same_file_request.status = request.status; + same_file_request.standalone = request.output; + same_file_requests.push_back(same_file_request); + } + if (!same_file_requests.empty()) { + // At most kBatch requests, so they are contiguous in the inline storage. + table()->MultiGetSameFileBlob(read_options, same_file_requests.size(), + &same_file_requests[0]); + } + + // Add the record size estimate to `bytes_read`. It does not count the + // index and data block reads. + for (size_t k = 0; k < count; ++k) { + const RelocationBlobReadRequest& request = + requests[lookup_order[begin + k]]; + if (!request.status->ok() || request.output->gen2.copied) { + continue; + } + const BlobIndex& blob_index = *request.blob_index; + total_bytes_read += request.range_length == kWholeBlobLength + ? blob_index.size() + kSimpleGen2BlobTrailerSize + : request.range_length; + } + } + if (bytes_read != nullptr) { + *bytes_read = total_bytes_read; } +} - TEST_SYNC_POINT_CALLBACK( - "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", &s); +std::unique_ptr BlobRelocationFileReader::NewLookupIterator( + ReadOptions* lookup_options) const { + assert(lookup_options != nullptr); + lookup_options->iterate_lower_bound = nullptr; + lookup_options->iterate_upper_bound = nullptr; + lookup_options->async_io = false; + lookup_options->readahead_size = 0; + // Relocation table values are location entries (kTypeValue). With + // kBlobRelocationFileRead no wrapper iterator touches the values before + // FindBlobIndex checks them. + return std::unique_ptr(table_reader_->NewIterator( + *lookup_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/true, TableReaderCaller::kBlobRelocationFileRead, + /*compaction_readahead_size=*/0, + /*allow_unprepared_value=*/false)); +} - if (!s.ok()) { - return Status::Corruption(s.ToString()); +Status BlobRelocationFileReader::FindBlobIndex( + InternalIterator* iterator, const BlobIndex& stable_blob_index, + BlobIndex* same_file_blob_index, DecodedGroupCache* cache) const { + assert(iterator != nullptr); + assert(same_file_blob_index != nullptr); + Status status = + PrepareLookup(stable_blob_index, /*range_offset=*/0, kWholeBlobLength); + if (!status.ok()) { + return status; + } + if (cache != nullptr && cache->valid && + cache->origin_file_number == stable_blob_index.origin_file_number()) { + const BlobRelocationGroup& group = cache->group; + const uint64_t offset = stable_blob_index.original_value_offset(); + if (offset >= group.values[0].original_value_offset && + offset <= group.values[group.count - 1].original_value_offset) { + return CheckEntry(stable_blob_index, Slice(cache->internal_key), Slice(), + same_file_blob_index, cache); + } + } + // The internal lookup key, built on the stack: kBlobRelocationKeySize (16 + // bytes) + kNumInternalBytes (8 bytes). + char lookup_key[kBlobRelocationKeySize + kNumInternalBytes]; + EncodeLookupKey(stable_blob_index, lookup_key); + iterator->Seek(Slice(lookup_key, sizeof(lookup_key))); + if (!iterator->Valid()) { + status = iterator->status(); + return status.ok() ? Status::Corruption("Blob relocation entry is missing") + : status; + } + return CheckEntry(stable_blob_index, iterator->key(), iterator->value(), + same_file_blob_index, cache); +} + +void BlobRelocationFileReader::MultiGetStoredBlobs( + const ReadOptions& read_options, uint64_t coalesce_bytes, + size_t num_requests, StoredRelocationFileReadRequest* requests, + uint64_t max_stored_value_size, CoalescedReadBuffers* buffers, + uint64_t* bytes_read) const { + if (bytes_read != nullptr) { + *bytes_read = 0; + } + if (num_requests == 0) { + return; + } + if (requests == nullptr) { + return; } - result->reset(new BlobContents(std::move(output), args.uncompressed_size)); + using OrderedRequest = std::tuple; + std::vector order; + order.reserve(num_requests); + for (size_t i = 0; i < num_requests; ++i) { + StoredRelocationFileReadRequest& request = requests[i]; + if (request.status == nullptr) { + continue; + } + Status& request_status = *request.status; + request_status = Status::OK(); + if (request.stored_payload != nullptr) { + *request.stored_payload = Slice(); + } + if (request.origin_file_number == 0 || + request.stored_value_size > max_stored_value_size) { + request_status = + Status::MemoryLimit("Invalid or oversized stored relocation request"); + continue; + } + order.emplace_back(request.origin_file_number, + request.original_value_offset, i); + } + std::sort(order.begin(), order.end()); + + ReadOptions lookup_options(read_options); + std::unique_ptr iterator = + NewLookupIterator(&lookup_options); + std::vector same_file_indexes(num_requests); + // Requests are sorted, so the requests of one location entry come one after + // another and share one seek and one decode. + DecodedGroupCache group_cache; + std::vector same_file_requests; + same_file_requests.reserve(order.size()); + for (const OrderedRequest& ordered_request : order) { + const size_t index = std::get<2>(ordered_request); + StoredRelocationFileReadRequest& request = requests[index]; + assert(request.status != nullptr); + Status& request_status = *request.status; + // BlobIndex has no setters, so build the stable index by encoding and + // decoding it. DecodeFrom() checks the identity fields; FindBlobIndex() + // runs the full Validate(). + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, request.origin_file_number, + {.origin_file_number = request.origin_file_number, + .original_value_offset = request.original_value_offset, + .stored_value_size = request.stored_value_size, + .compression = request.compression, + .stable_checksum = request.stable_checksum, + .stored_checksum = request.stored_checksum}); + BlobIndex stable_index; + request_status = stable_index.DecodeFrom(Slice(encoded)); + if (!request_status.ok()) { + continue; + } + request_status = FindBlobIndex(iterator.get(), stable_index, + &same_file_indexes[index], &group_cache); + if (!request_status.ok()) { + continue; + } + same_file_requests.push_back(StoredSameFileBlobReadRequest{ + same_file_indexes[index], request.stored_payload, request_status, + request.stored_checksum}); + } + if (!same_file_requests.empty()) { + table()->MultiGetStoredSameFileBlob( + read_options, coalesce_bytes, same_file_requests.size(), + same_file_requests.data(), max_stored_value_size, buffers, bytes_read); + } +} - return Status::OK(); +Status BlobRelocationFileReader::StreamStoredBlob( + const ReadOptions& read_options, const BlobIndex& blob_index, + const BlobChunkConsumer& consumer) const { + Status status = blob_index.Validate(); + if (!status.ok()) { + return status; + } + if (!blob_index.IsIndirect()) { + return Status::NotSupported( + "Stored streaming requires a stable blob index"); + } + ReadOptions lookup_options(read_options); + std::unique_ptr iterator = + NewLookupIterator(&lookup_options); + BlobIndex physical_index; + status = FindBlobIndex(iterator.get(), blob_index, &physical_index, + /*cache=*/nullptr); + return status.ok() ? table()->StreamStoredSameFileBlob( + read_options, physical_index, + blob_index.stored_checksum(), consumer) + : status; } } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_reader.h b/db/blob/blob_file_reader.h index fde01854944b..736e6ae8b3f4 100644 --- a/db/blob/blob_file_reader.h +++ b/db/blob/blob_file_reader.h @@ -7,12 +7,18 @@ #include #include +#include +#include +#include "db/blob/blob_gen2_format.h" #include "db/blob/blob_read_request.h" +#include "db/blob/blob_relocation_file.h" +#include "db/dbformat.h" #include "file/random_access_file_reader.h" #include "rocksdb/advanced_compression.h" #include "rocksdb/compression_type.h" #include "rocksdb/rocksdb_namespace.h" +#include "table/internal_iterator.h" #include "util/autovector.h" namespace ROCKSDB_NAMESPACE { @@ -26,7 +32,68 @@ class Slice; class FilePrefetchBuffer; class BlobContents; class Statistics; +class BlobIndex; +class BlockBasedTable; +class BlockCacheTracer; +class BlobRelocationFileReader; +class TableReader; +// One value that indirect blob GC reads from a normal blob file with +// MultiGetStoredBlobsForRelocation(). GC fills the input fields from a +// BlobRelocationLiveValue of its live set (see db/blob/blob_relocation_gc.h). +// For a value in a relocation file, see StoredRelocationFileReadRequest. +struct StoredNormalBlobReadRequest { + // Offset of the stored value in the blob file (not of its record header). + uint64_t offset = 0; + uint64_t stored_value_size = 0; + CompressionType compression = kNoCompression; + + // Output: set to the stored bytes, which live in the call's + // CoalescedReadBuffers. If null, the value is read and checked with the + // stored CRC, but its bytes are not kept. Indirect blob GC always passes a + // non-null pointer. + Slice* stored_payload = nullptr; + + // Output: the result of this request. A null `status` skips the request. + Status* status = nullptr; + + // Input: the expected stored CRC of the value. The read fails with + // Corruption if the stored bytes do not match it. + uint32_t stored_checksum = 0; +}; + +// One value to read from a relocation file in +// BlobRelocationFileReader::MultiGetStoredBlobs(). The read finds the value by +// its stable identity (origin, original offset). +struct StoredRelocationFileReadRequest { + uint64_t origin_file_number = 0; + uint64_t original_value_offset = 0; + uint64_t stored_value_size = 0; + CompressionType compression = kNoCompression; + + // The logical CRC. The read needs it only to build the BlobIndex that it + // looks up. It is not checked, because the value is not decompressed. + uint32_t stable_checksum = 0; + + // Output: set to the stored bytes, which live in the call's + // CoalescedReadBuffers. If null, the value is only checked. + Slice* stored_payload = nullptr; + + // Output: the result of this request. A null `status` skips the request. + Status* status = nullptr; + + // Input: the expected stored CRC of the value (see "Stored CRC" in the + // overview of db/blob/blob_relocation_gc.h). The read fails with Corruption + // if the stored bytes do not match it. + uint32_t stored_checksum = 0; +}; + +// Reads one blob file of either kind. BlobFileCache stores both kinds under one +// cache entry type. +// - Blob log file: GetBlob(), GetBlobRange(), MultiGetBlob() and +// MultiGetBlobRange() read it. +// - Relocation file: relocation() is not null, and only relocation() may be +// used. The four read functions above are for blob log files only. class BlobFileReader { public: static Status Create(const ImmutableOptions& immutable_options, @@ -51,6 +118,9 @@ class BlobFileReader { uint64_t blob_file_number, const std::shared_ptr& io_tracer, bool skip_footer_validation, std::unique_ptr* reader); + // Wraps an open relocation file; see BlobRelocationFileReader. + explicit BlobFileReader(std::unique_ptr relocation); + BlobFileReader(const BlobFileReader&) = delete; BlobFileReader& operator=(const BlobFileReader&) = delete; BlobFileReader(BlobFileReader&&) = delete; @@ -63,8 +133,8 @@ class BlobFileReader { CompressionType compression_type, FilePrefetchBuffer* prefetch_buffer, MemoryAllocator* allocator, - std::unique_ptr* result, - uint64_t* bytes_read) const; + std::unique_ptr* result, uint64_t* bytes_read, + std::optional* value_crc32c = nullptr) const; // Reads a byte sub-range [range_offset, range_offset + range_length) of an // *uncompressed* blob's value directly from the file, reading only those @@ -83,6 +153,39 @@ class BlobFileReader { std::unique_ptr* result, uint64_t* bytes_read) const; + // Streams the exact stored bytes of a value in a normal blob file to + // `consumer` in bounded chunks, and checks them with the stored CRC. It never + // decompresses and never fills a cache. The consumer's output is valid only + // if the final status is OK. For relocation files, see + // BlobRelocationFileReader::StreamStoredBlob. + Status StreamStoredBlobForRelocation(const ReadOptions& read_options, + const BlobIndex& blob_index, + const BlobChunkConsumer& consumer) const; + + // Reads the exact stored payload bytes of normal indirect-origin records and + // checks them with the stored checksum, without decompressing. The user key + // is not needed, because the kIndirectBlob index carries the checksums. + // - Order: requests must be sorted by offset. + // - Value limit: a request whose stored size is larger than + // `max_stored_value_size` gets MemoryLimit. + // - Merging: neighbor values share one read when the gap between them is + // at most `coalesce_bytes`, and one read is at most + // `max_stored_value_size` (see ReadCoalescedRanges()). + // - Buffer limit: the requested bytes plus `max_stored_value_size`. A + // request whose read would go past it is not read and gets + // Status::Incomplete. The caller retries it in a later call. + // - Null payload: a request with a null stored_payload is only checked; + // its stored bytes are not returned. Other payloads point into `buffers`. + // - Counters: BLOB_DB_BLOB_FILE_BYTES_READ and blob_read_byte count the + // bytes read, merged gaps included. + void MultiGetStoredBlobsForRelocation(const ReadOptions& read_options, + uint64_t coalesce_bytes, + size_t num_requests, + StoredNormalBlobReadRequest* requests, + uint64_t max_stored_value_size, + CoalescedReadBuffers* buffers, + uint64_t* bytes_read) const; + // offsets must be sorted in ascending order by caller. void MultiGetBlob( const ReadOptions& read_options, MemoryAllocator* allocator, @@ -105,6 +208,11 @@ class BlobFileReader { std::unique_ptr>>& blob_reqs, uint64_t* bytes_read) const; + // The relocation file reader, or null for a blob log file. + const BlobRelocationFileReader* relocation() const { + return relocation_.get(); + } + CompressionType GetCompressionType() const { return compression_type_; } uint64_t GetFileSize() const { return file_size_; } @@ -116,6 +224,9 @@ class BlobFileReader { std::shared_ptr decompressor, SystemClock* clock, Statistics* statistics, bool has_footer); + // BlobRelocationFileReader::Open() uses OpenFile() to open the file. + friend class BlobRelocationFileReader; + // `skip_footer_size_check` is used for direct-write files that are still // missing their footer at open time. static Status OpenFile(const ImmutableOptions& immutable_options, @@ -125,7 +236,7 @@ class BlobFileReader { const std::shared_ptr& io_tracer, uint64_t* file_size, std::unique_ptr* file_reader, - bool skip_footer_size_check); + bool skip_footer_size_check, bool is_relocation_file); static Status ReadHeader(const RandomAccessFileReader* file_reader, const ReadOptions& read_options, @@ -144,16 +255,12 @@ class BlobFileReader { Statistics* statistics, Slice* slice, Buffer* buf, AlignedBuffer* direct_io_buffer); + // Checks the record header and the CRC over key + value. With a non-null + // `value_crc32c`, computes the key and value CRCs separately, combines them + // for the record check, and returns the value CRC. static Status VerifyBlob(const Slice& record_slice, const Slice& user_key, - uint64_t value_size); - - static Status UncompressBlobIfNeeded(const Slice& value_slice, - CompressionType compression_type, - Decompressor* decompressor, - MemoryAllocator* allocator, - SystemClock* clock, - Statistics* statistics, - std::unique_ptr* result); + uint64_t value_size, + std::optional* value_crc32c = nullptr); std::unique_ptr file_reader_; uint64_t file_size_; @@ -163,6 +270,225 @@ class BlobFileReader { Statistics* statistics_; // False when the reader was opened before the blob file footer was written. bool has_footer_; + std::unique_ptr relocation_; +}; + +// Reads one relocation file (format in db/blob/blob_relocation_file.h). +// BlobSource uses it to read the values of indirect BlobIndexes that resolve +// to this file. It opens the file as a block-based table behind the 8-byte +// prefix. It is immutable after Open(), so several threads can read with it. +// +// How a read works (MultiGetBlob()): +// 1. BlockBasedTable::MultiGetRawEntries() finds the location entry of each +// requested value, using the lookup key (origin, original offset). When +// pin_blob_relocation_location_blocks is on and the reader keeps its +// location blocks in memory, MultiGetRawEntriesPinned() does it instead. +// 2. CheckEntry() decodes the entry and turns the place of the value into a +// same-file BlobIndex (record offset and stored size). +// 3. BlockBasedTable::MultiGetSameFileBlob() reads the record. Normal SSTs +// use the same function to read embedded blobs. Here it runs without a +// BlobSource. A record has the same format as an embedded blob (the +// SimpleGen2Blob format, see db/blob/blob_gen2_format.h). Which +// checksums a read checks is in the table under "Reads" in +// db/blob/blob_relocation_file.h. +class BlobRelocationFileReader { + public: + // Opens relocation file `file` of `column_family_id`. The caller passes + // the expected file size, tail size and directory digest in `file` (the + // file's identity, see BlobPhysicalFile). + // + // Errors: + // - InvalidArgument: `file` is not marked as a relocation file, or its + // file number, size or tail size is missing, the tail is larger than + // the file, or `db_session_id` is empty. + // - Corruption: the file does not match `file` (size, tail size, file + // number or directory digest) or `column_family_id`, the table + // properties have no db session id, the table properties or the + // directory are missing or bad, the directory totals do not match the + // embedded blob stats property of the table + // (kEmbeddedBlobSstStatsPropertyName in table/embedded_blob_sst.h: record + // count and payload bytes), or the table's entry count (num_entries) is + // not the number of location entries that the directory implies. + // + // `factory` is built from MakeBlobRelocationTableOptions() and is shared by + // the owning BlobFileCache. `compression_manager` and + // `block_protection_bytes_per_key` configure the table reader. + // `block_cache_tracer` (may be nullptr) records the block reads of the + // table reader in a block cache trace, with caller kBlobRelocationFileRead. + // + // With `pin_location_blocks` (the column family option + // pin_blob_relocation_location_blocks), Open reads every location block and + // keeps it until the reader is closed, so later lookups need no index or + // block cache lookup. With a block cache, the blocks are held through their + // cache handles, so they are charged and not evicted. Without one, they are + // kept in the reader's memory. If the blocks cannot be kept (the block cache + // cannot charge them, or a read fails), Open logs a warning and still + // succeeds, and the reader does normal lookups. + static Status Open( + const ImmutableOptions& immutable_options, + const ReadOptions& read_options, const FileOptions& file_options, + HistogramImpl* blob_file_read_hist, uint32_t column_family_id, + const BlobPhysicalFile& file, const std::string& db_session_id, + const TableFactory& factory, CompressionManager* compression_manager, + uint8_t block_protection_bytes_per_key, bool pin_location_blocks, + const std::shared_ptr& io_tracer, + BlockCacheTracer* block_cache_tracer, + std::unique_ptr* reader); + + BlobRelocationFileReader(const BlobRelocationFileReader&) = delete; + BlobRelocationFileReader& operator=(const BlobRelocationFileReader&) = delete; + + ~BlobRelocationFileReader(); + + // Reads the values of indirect BlobIndexes from this file. A request reads + // the whole value when range_length == kWholeBlobLength, and decompresses a + // compressed value. Otherwise it reads the range of an uncompressed value. A + // range of a compressed value is not supported and gets Corruption. + // + // Steps: + // 1. Check each request (BlobIndex and range). A bad request gets + // Corruption, or InvalidArgument for an out-of-bounds range, and is + // skipped. + // 2. Sort the valid requests in key order (origin, original offset). + // 3. For each batch of up to MultiGetContext::MAX_BATCH_SIZE (32) + // requests, look up all keys of the batch. A data block in the block + // cache needs no read. The missing data blocks are read together: one + // plain read when one block is missing, or else one batched read. Then + // check each location entry found and read the records. + // + // It does not check the logical CRC. The caller (BlobSource) does. Each + // request gets its own status: Corruption when the entry is missing or does + // not match. With read_tier == kBlockCacheTier, a request whose data block + // is not cached gets Incomplete, also when its entry is missing. + // + // `*bytes_read` is an estimate. For each request that succeeded and was not + // copied from a duplicate, it adds the record bytes (stored size + + // kSimpleGen2BlobTrailerSize) for a whole read, and the range length for a + // range read. It does not count the index and data block reads. Values are + // owned buffers allocated from `allocator` (may be null). + void MultiGetBlob(const ReadOptions& read_options, MemoryAllocator* allocator, + size_t num_reads, RelocationBlobReadRequest* requests, + uint64_t* bytes_read) const; + + // Reads values by their stable identity (origin, original offset). When + // stored_payload is set, it returns their exact stored bytes, for indirect + // blob GC to copy into a new relocation file. A request with a null + // stored_payload is only checked. Returned payloads point into `buffers`. + // - Order: requests can be in any order; the read sorts them. + // - Records: each read covers whole records, trailer included. Neighbor + // records share one read when the gap between them is at most + // `coalesce_bytes` (see ReadCoalescedRanges()). + // - Limits: `max_stored_value_size` has more uses here than in + // BlobFileReader::MultiGetStoredBlobsForRelocation(). A request whose + // stored size is larger than it gets MemoryLimit, and so does a request + // that would bring the total stored bytes of the call past it. It is + // also the limit of one read and sets the buffer limit (see + // BlockBasedTable::MultiGetStoredSameFileBlob(), where it is named + // `max_stored_bytes`). A request past the buffer limit gets + // Status::Incomplete, and the caller retries it in a later call. + // - Counters: unlike the normal-file read, these reads update no blob read + // ticker (BLOB_DB_BLOB_FILE_BYTES_READ) and no blob_read_* perf counter. + void MultiGetStoredBlobs(const ReadOptions& read_options, + uint64_t coalesce_bytes, size_t num_requests, + StoredRelocationFileReadRequest* requests, + uint64_t max_stored_value_size, + CoalescedReadBuffers* buffers, + uint64_t* bytes_read) const; + + // Like BlobFileReader::StreamStoredBlobForRelocation, for the value of + // indirect `blob_index` in this file. + Status StreamStoredBlob(const ReadOptions& read_options, + const BlobIndex& blob_index, + const BlobChunkConsumer& consumer) const; + + uint32_t column_family_id() const { return column_family_id_; } + uint64_t file_size() const { return file_size_; } + uint64_t tail_size() const { return tail_size_; } + uint32_t directory_digest() const { return directory_digest_; } + uint64_t total_entries() const { return total_entries_; } + uint64_t total_record_bytes() const { return total_record_bytes_; } + // The memory of the kept location blocks, or 0 when they are not kept (see + // `pin_location_blocks` in Open()). + size_t pinned_location_block_bytes() const; + + private: + BlobRelocationFileReader(uint32_t column_family_id, uint64_t file_size); + // The location blocks that Open() keeps; defined in the .cc file. + struct PinnedLocationBlocks; + + // Returns an iterator over the table that yields the raw location entries + // (packed groups, see db/blob/blob_relocation_file.h). Adjusts + // `*lookup_options` (no bounds, no readahead, no async I/O). + // `*lookup_options` must outlive the iterator. + std::unique_ptr NewLookupIterator( + ReadOptions* lookup_options) const; + + // The last location entry that CheckEntry() decoded, so that a batch of + // sorted lookups decodes each entry once. + struct DecodedGroupCache { + bool valid = false; + std::string internal_key; + // The origin of the entry, from its key. + uint64_t origin_file_number = 0; + BlobRelocationGroup group; + }; + + // Finds the location entry of `stable_blob_index` and builds the same-file + // BlobIndex of its record in `*same_file_blob_index` (see CheckEntry()). + // When `cache` holds an entry of the same origin whose offsets cover the + // requested one, it uses that entry and does not seek. Returns the + // PrepareLookup() error when the request is not valid. Returns Corruption + // when the value is not in the file ("Blob relocation entry is missing"), + // when the entry is malformed, or when its stored size does not match. + // Returns the iterator status on an I/O error. + Status FindBlobIndex(InternalIterator* iterator, + const BlobIndex& stable_blob_index, + BlobIndex* same_file_blob_index, + DecodedGroupCache* cache) const; + + BlockBasedTable* table() const; + + // Checks before the lookup: the request (BlobIndex and range) is valid. The + // location entry check after the lookup (CheckEntry()) proves the value is + // in this file. + Status PrepareLookup(const BlobIndex& stable_blob_index, + uint64_t range_offset, size_t range_length) const; + + // Writes the internal lookup key of `stable_blob_index` into `lookup_key`: + // the 16-byte relocation key (origin, original offset) plus the 8-byte + // internal key footer of a seek (kMaxSequenceNumber, kValueTypeForSeek), 24 + // bytes in total. The stored entries use sequence 0 and kTypeValue, so the + // seek lands on the first entry whose relocation key is at or after the + // lookup key. + static void EncodeLookupKey(const BlobIndex& stable_blob_index, + char* lookup_key); + + // Checks after the lookup: the location entry found (the first entry at or + // after the lookup key) has the requested origin and holds the requested + // original offset, and the value's stored size matches `stable_blob_index`. + // Returns Corruption ("Blob relocation entry is missing") when the origin or + // the offset is not there. Builds the same-file BlobIndex of the record in + // `*same_file_blob_index`, with the compression of `stable_blob_index`. The + // location entry stores no compression type, so the record read checks the + // compression against the record's compression marker. `cache` may be null. + static Status CheckEntry(const BlobIndex& stable_blob_index, + const Slice& internal_key, const Slice& value, + BlobIndex* same_file_blob_index, + DecodedGroupCache* cache); + + const uint32_t column_family_id_; + const uint64_t file_size_; + uint64_t tail_size_ = 0; + // Must outlive table_reader_. + InternalKeyComparator internal_comparator_; + std::unique_ptr table_reader_; + // Set only when Open() kept the location blocks. Declared after + // table_reader_, so it is released first. + std::unique_ptr pinned_location_blocks_; + // Directory totals and digest; the directory itself is not kept. + uint64_t total_entries_ = 0; + uint64_t total_record_bytes_ = 0; + uint32_t directory_digest_ = 0; }; } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_file_reader_test.cc b/db/blob/blob_file_reader_test.cc index 7782e9730a94..13a1740dd6e9 100644 --- a/db/blob/blob_file_reader_test.cc +++ b/db/blob/blob_file_reader_test.cc @@ -5,25 +5,60 @@ #include "db/blob/blob_file_reader.h" +#include +#include #include +#include +#include +#include +#include +#include #include #include #include "db/blob/blob_contents.h" +#include "db/blob/blob_file_cache.h" +#include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" #include "db/blob/blob_log_writer.h" +#include "db/blob/blob_relocation_file.h" +#include "db/blob/blob_relocation_gc.h" +#include "db/blob/blob_source.h" +#include "db/blob/same_file_blob_reader.h" +#include "db/dbformat.h" +#include "db/table_cache.h" #include "env/composite_env_wrapper.h" +#include "env/emulated_clock.h" #include "env/mock_env.h" #include "file/filename.h" +#include "file/random_access_file_reader.h" #include "file/read_write_util.h" #include "file/writable_file_writer.h" #include "options/cf_options.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" #include "rocksdb/options.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/perf_level.h" +#include "rocksdb/sst_file_writer.h" +#include "rocksdb/table.h" +#include "rocksdb/trace_reader_writer.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/format.h" +#include "table/meta_blocks.h" +#include "table/table_builder.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" +#include "test_util/testutil.h" +#include "trace_replay/block_cache_tracer.h" +#include "util/cast_util.h" +#include "util/coding.h" #include "util/compression.h" +#include "util/crc32c.h" +#include "util/defer.h" +#include "util/mutexlock.h" +#include "util/random.h" #include "utilities/fault_injection_env.h" namespace ROCKSDB_NAMESPACE { @@ -130,14 +165,2737 @@ void WriteBlobFile(const ImmutableOptions& immutable_options, } } +struct RelocationTestRecord { + uint64_t origin_file_number; + uint64_t original_value_offset; + Slice stored_payload; + CompressionType compression; +}; + +void WriteRelocationFile(const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options, + const BlockBasedTableOptions& table_options, + uint32_t column_family_id, uint64_t blob_file_number, + const std::vector& records, + BlobRelocationFileMetadata* metadata, + uint64_t* file_size) { + assert(metadata != nullptr); + assert(file_size != nullptr); + const std::string file_path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + std::unique_ptr file; + ASSERT_OK(immutable_options.fs->NewWritableFile(file_path, FileOptions(), + &file, /*dbg=*/nullptr)); + std::unique_ptr writer = + std::make_unique( + std::move(file), file_path, FileOptions(), immutable_options.clock); + + const InternalKeyComparator comparator(BytewiseComparator()); + std::unique_ptr builder; + BlobRelocationTableFileInfo file_info; + file_info.column_family_id = column_family_id; + file_info.column_family_name = kDefaultColumnFamilyName; + file_info.db_id = "test-db"; + file_info.db_session_id = "test-session"; + file_info.file_number = blob_file_number; + ASSERT_OK(NewBlobRelocationTableBuilder( + immutable_options, mutable_cf_options, table_options, ReadOptions(), + WriteOptions(), comparator, file_info, writer.get(), &builder)); + for (const RelocationTestRecord& record : records) { + ASSERT_OK(builder->AddBlobRelocation( + record.origin_file_number, record.original_value_offset, + record.stored_payload, record.compression)); + } + // The file serves the file numbers of its origins, each added once. + std::set served; + for (const RelocationTestRecord& record : records) { + served.insert(record.origin_file_number); + } + for (uint64_t file_number : served) { + ASSERT_OK(builder->AddBlobRelocationServedFile(file_number)); + } + ASSERT_OK(builder->Finish()); + const TableProperties properties = builder->GetTableProperties(); + ASSERT_EQ(properties.comparator_name, BytewiseComparator()->Name()); + ASSERT_FALSE(properties.user_defined_timestamps_persisted); + ASSERT_OK(builder->GetBlobRelocationFileMetadata(metadata)); + ASSERT_OK(writer->Close(IOOptions(), /*dbg=*/nullptr)); + ASSERT_OK(immutable_options.fs->GetFileSize(file_path, IOOptions(), file_size, + /*dbg=*/nullptr)); +} + +// The decoded directory of a relocation file written by WriteRelocationFile. +BlobRelocationDirectory DecodeDirectory( + const BlobRelocationFileMetadata& metadata) { + BlobRelocationDirectory directory; + EXPECT_OK(BlobRelocationDirectory::DecodeFrom(metadata.encoded_directory, + &directory)); + return directory; +} + +// The expected identity (BlobPhysicalFile) of a relocation file written by +// WriteRelocationFile. +BlobPhysicalFile RelocationPhysicalFile( + uint64_t file_number, uint64_t file_size, + const BlobRelocationFileMetadata& metadata) { + return BlobPhysicalFile{BlobFileOpenInfo{file_number, Slice(), Slice()}, + file_size, /*is_relocation=*/true, metadata.tail_size, + DecodeDirectory(metadata).directory_digest()}; +} + +// Sets a callback on `sync_point`, which passes the batched file reads +// (std::vector*). The callback counts the calls in `*calls` and +// checks that the reads are sorted and do not overlap. +void ExpectSortedNonOverlappingReads(const char* sync_point, + std::atomic* calls) { + SyncPoint::GetInstance()->SetCallBack(sync_point, [calls](void* arg) { + const auto* const reads = static_cast*>(arg); + ++*calls; + for (size_t i = 1; i < reads->size(); ++i) { + EXPECT_LE((*reads)[i - 1].offset + (*reads)[i - 1].len, + (*reads)[i].offset); + } + }); +} + +// Opens `file` like BlobFileCache does, with a factory built from +// `table_options`. +Status OpenRelocationReader( + const ImmutableOptions& immutable_options, const ReadOptions& read_options, + const FileOptions& file_options, uint32_t column_family_id, + const BlobPhysicalFile& file, const std::string& db_session_id, + const BlockBasedTableOptions& table_options, + CompressionManager* compression_manager, + std::unique_ptr* reader, bool pin_location_blocks = false) { + const BlockBasedTableFactory factory( + MakeBlobRelocationTableOptions(table_options)); + std::unique_ptr relocation; + const Status s = BlobRelocationFileReader::Open( + immutable_options, read_options, file_options, + /*blob_file_read_hist=*/nullptr, column_family_id, file, db_session_id, + factory, compression_manager, /*block_protection_bytes_per_key=*/0, + pin_location_blocks, /*io_tracer=*/nullptr, + /*block_cache_tracer=*/nullptr, &relocation); + if (s.ok()) { + reader->reset(new BlobFileReader(std::move(relocation))); + } + return s; +} + +// Builds one relocation read request with named fields. +RelocationBlobReadRequest MakeRelocationRequest( + const BlobIndex* blob_index, uint64_t range_offset, size_t range_length, + Status* status, SameFileBlobStandaloneRead* output) { + RelocationBlobReadRequest request; + request.blob_index = blob_index; + request.range_offset = range_offset; + request.range_length = range_length; + request.status = status; + request.output = output; + return request; +} + +// Reads one value from a relocation file reader: a batch of one. +Status GetFromRelocationFile(const BlobFileReader& reader, + const ReadOptions& read_options, + const BlobIndex& blob_index, uint64_t range_offset, + size_t range_length, MemoryAllocator* allocator, + std::unique_ptr* result, + uint64_t* bytes_read) { + Status status; + SameFileBlobStandaloneRead output; + RelocationBlobReadRequest request = MakeRelocationRequest( + &blob_index, range_offset, range_length, &status, &output); + reader.relocation()->MultiGetBlob(read_options, allocator, /*num_reads=*/1, + &request, bytes_read); + *result = std::move(output.contents); + return status; +} + } // anonymous namespace +TEST(BlobRelocationDirectoryTest, ValidatesOrderingAndChecksums) { + BlobRelocationDirectoryBuilder builder; + ASSERT_OK(builder.Add(/*origin_file_number=*/10, + /*original_value_offset=*/2, + /*stored_value_size=*/3, kNoCompression)); + ASSERT_OK(builder.Add(/*origin_file_number=*/10, + /*original_value_offset=*/7, + /*stored_value_size=*/4, kNoCompression)); + ASSERT_OK(builder.Add(/*origin_file_number=*/20, + /*original_value_offset=*/1, + /*stored_value_size=*/5, kNoCompression)); + ASSERT_OK(builder.AddServedFile(/*file_number=*/30)); + ASSERT_TRUE(builder.AddServedFile(/*file_number=*/0).IsInvalidArgument()); + ASSERT_OK(builder.AddServedFile(/*file_number=*/10)); + ASSERT_OK(builder.AddServedFile(/*file_number=*/30)); + // Values of one origin must not overlap: 7 + 4 = 11 is the end of the last + // value, so a value at 10 overlaps it. A value that starts exactly at the + // end is fine (see the end-based gap in blob_relocation_file.h). + { + BlobRelocationDirectoryBuilder overlap; + ASSERT_OK(overlap.Add(10, 7, 4, kNoCompression)); + ASSERT_TRUE(overlap.Add(10, 10, 1, kNoCompression).IsInvalidArgument()); + BlobRelocationDirectoryBuilder adjacent; + ASSERT_OK(adjacent.Add(10, 7, 4, kNoCompression)); + ASSERT_OK(adjacent.Add(10, 11, 1, kNoCompression)); + } + BlobRelocationDirectory directory; + ASSERT_OK(builder.Finish(&directory)); + // Sorted and deduplicated. + ASSERT_EQ(directory.served_file_numbers(), std::vector({10, 30})); + ASSERT_EQ(directory.origins().size(), 2U); + ASSERT_EQ(directory.total_entries(), 3U); + ASSERT_EQ(directory.total_record_bytes(), + 12U + 3U * kSimpleGen2BlobTrailerSize); + ASSERT_NE(directory.Find(10), nullptr); + ASSERT_EQ(directory.Find(99), nullptr); + std::string encoded; + directory.EncodeTo(&encoded); + // ValidateForFileSize: the record bytes must fit in the file after the + // prefix. With one 1000-byte record, the record bytes (1000 + + // kSimpleGen2BlobTrailerSize = 1005) are larger than the encoded directory + // (96 bytes), so the record-bytes check decides. + BlobRelocationDirectoryBuilder large; + ASSERT_OK(large.Add(10, 2, 1000, kNoCompression)); + ASSERT_OK(large.AddServedFile(10)); + BlobRelocationDirectory large_directory; + ASSERT_OK(large.Finish(&large_directory)); + ASSERT_OK(large_directory.ValidateForFileSize( + kBlobRelocationFilePrefixSize + large_directory.total_record_bytes())); + ASSERT_TRUE(large_directory + .ValidateForFileSize(kBlobRelocationFilePrefixSize + + large_directory.total_record_bytes() - 1) + .IsCorruption()); + ASSERT_OK(directory.ValidateForFileSize(encoded.size())); + BlobRelocationDirectory decoded; + ASSERT_OK(BlobRelocationDirectory::DecodeFrom(encoded, &decoded)); + ASSERT_EQ(decoded.directory_digest(), directory.directory_digest()); + ASSERT_EQ(decoded.served_file_numbers(), std::vector({10, 30})); + // The encoded directory must also fit in the file. + ASSERT_TRUE(decoded.ValidateForFileSize(encoded.size() - 1).IsCorruption()); + + // Re-seals an edited directory with valid digest and header CRC. + constexpr size_t kDirectoryHeaderSize = 48; + constexpr size_t kDirectoryDigestOffset = 40; + constexpr size_t kHeaderChecksumOffset = 44; + // The digest covers header bytes 8..39 (counts and totals) and the body. + constexpr size_t kDigestedHeaderOffset = 8; + const auto seal_header = [&](std::string* edited) { + EncodeFixed32(edited->data() + kHeaderChecksumOffset, + crc32c::Value(edited->data(), kHeaderChecksumOffset)); + }; + const auto reseal = [&](std::string* edited) { + const uint32_t header_crc = + crc32c::Value(edited->data() + kDigestedHeaderOffset, + kDirectoryDigestOffset - kDigestedHeaderOffset); + EncodeFixed32( + edited->data() + kDirectoryDigestOffset, + crc32c::Extend(header_crc, edited->data() + kDirectoryHeaderSize, + edited->size() - kDirectoryHeaderSize)); + seal_header(edited); + }; + // The digest covers the totals: a changed total with a valid header CRC + // but the old digest is rejected. + { + constexpr size_t kTotalEntriesOffset = 24; + std::string other_totals = encoded; + EncodeFixed64(other_totals.data() + kTotalEntriesOffset, + directory.total_entries() + 1); + seal_header(&other_totals); + const Status status = + BlobRelocationDirectory::DecodeFrom(other_totals, &decoded); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("checksum mismatch"), std::string::npos) + << status.ToString(); + } + // Origin 10 has two entries, at offsets 2 and 7. Setting its last offset to + // 2, equal to its first offset, leaves room for only one offset, so the + // decoder rejects the entry. + std::string impossible_cardinality = encoded; + constexpr size_t kLastOffsetField = 16; + EncodeFixed64( + impossible_cardinality.data() + kDirectoryHeaderSize + kLastOffsetField, + /*last_original_offset=*/2); + reseal(&impossible_cardinality); + ASSERT_TRUE( + BlobRelocationDirectory::DecodeFrom(impossible_cardinality, &decoded) + .IsCorruption()); + + // A file must serve at least one old file number. + BlobRelocationDirectoryBuilder no_source; + ASSERT_OK(no_source.Add(10, 2, 3, kNoCompression)); + BlobRelocationDirectory unused; + ASSERT_TRUE(no_source.Finish(&unused).IsInvalidArgument()); + BlobRelocationDirectoryBuilder one_source; + ASSERT_OK(one_source.Add(10, 2, 3, kNoCompression)); + ASSERT_OK(one_source.AddServedFile(10)); + BlobRelocationDirectory single; + ASSERT_OK(one_source.Finish(&single)); + std::string no_served; + single.EncodeTo(&no_served); + // Drop the only served file and set the served count to zero. + constexpr size_t kServedCountOffset = 16; + no_served.resize(no_served.size() - sizeof(uint64_t)); + EncodeFixed64(no_served.data() + kServedCountOffset, 0); + reseal(&no_served); + ASSERT_TRUE( + BlobRelocationDirectory::DecodeFrom(no_served, &decoded).IsCorruption()); + // The size must match the counts exactly. + std::string extra_bytes; + single.EncodeTo(&extra_bytes); + extra_bytes.append(sizeof(uint64_t), '\0'); + reseal(&extra_bytes); + ASSERT_TRUE(BlobRelocationDirectory::DecodeFrom(extra_bytes, &decoded) + .IsCorruption()); + // Many origins, more than 1 MB of directory: there is no fixed limit. + BlobRelocationDirectoryBuilder many_origins; + for (uint64_t origin = 1; origin <= 30000; ++origin) { + ASSERT_OK(many_origins.Add(origin, 0, 1, kNoCompression)); + } + ASSERT_OK(many_origins.AddServedFile(1)); + BlobRelocationDirectory many; + ASSERT_OK(many_origins.Finish(&many)); + ASSERT_EQ(many.origins().size(), 30000U); + ASSERT_GT(many.encoded_size(), 1024U * 1024U); + + std::string checksum_corruption = encoded; + checksum_corruption.back() ^= 1; + ASSERT_TRUE(BlobRelocationDirectory::DecodeFrom(checksum_corruption, &decoded) + .IsCorruption()); + + BlobRelocationDirectoryBuilder duplicate; + ASSERT_OK(duplicate.Add(10, 2, 3, kNoCompression)); + ASSERT_TRUE(duplicate.Add(10, 2, 4, kNoCompression).IsInvalidArgument()); + BlobRelocationDirectoryBuilder unsorted; + ASSERT_OK(unsorted.Add(20, 1, 3, kNoCompression)); + ASSERT_TRUE(unsorted.Add(10, 2, 4, kNoCompression).IsInvalidArgument()); + BlobRelocationDirectoryBuilder unsupported; + ASSERT_TRUE( + unsupported.Add(10, 2, 3, kCustomCompression80).IsInvalidArgument()); + + // A rejected overflowing addition must not change the current origin. + BlobRelocationDirectoryBuilder overflow; + ASSERT_OK(overflow.Add( + 10, 2, std::numeric_limits::max() - kSimpleGen2BlobTrailerSize, + kNoCompression)); + ASSERT_TRUE(overflow.Add(10, 7, 1, kNoCompression).IsInvalidArgument()); + ASSERT_OK(overflow.AddServedFile(10)); + BlobRelocationDirectory after_overflow; + ASSERT_OK(overflow.Finish(&after_overflow)); + ASSERT_EQ(after_overflow.total_entries(), 1U); + + // Header fields, at their byte offsets. + constexpr size_t kOriginCountOffset = 8; + // Counts that the input cannot hold, including a pair whose encoded size + // would overflow. + for (const auto& counts : std::vector>{ + {uint64_t{1} << 61, 1}, + {1, uint64_t{1} << 61}, + {std::numeric_limits::max() / 40 + 1, + std::numeric_limits::max() / 8 + 1}}) { + std::string huge = encoded; + EncodeFixed64(huge.data() + kOriginCountOffset, counts.first); + EncodeFixed64(huge.data() + kServedCountOffset, counts.second); + reseal(&huge); + ASSERT_TRUE( + BlobRelocationDirectory::DecodeFrom(huge, &decoded).IsCorruption()); + } + // Fewer bytes than the counts say. + std::string short_body = encoded; + short_body.resize(short_body.size() - sizeof(uint64_t)); + reseal(&short_body); + ASSERT_TRUE( + BlobRelocationDirectory::DecodeFrom(short_body, &decoded).IsCorruption()); + + // Decode-level order checks, on directories edited after encoding: origins + // must be increasing and served files increasing and non-zero. + const auto with_served = [&](const std::vector& served) { + std::string edited = encoded; // origins 10, 20; served 10, 30 + edited.resize(kDirectoryHeaderSize + 2 * 40); + for (uint64_t file_number : served) { + PutFixed64(&edited, file_number); + } + EncodeFixed64(edited.data() + kServedCountOffset, served.size()); + reseal(&edited); + return edited; + }; + ASSERT_OK( + BlobRelocationDirectory::DecodeFrom(with_served({10, 30}), &decoded)); + for (const auto& served : + std::vector>{{30, 10}, {10, 10}, {0, 10}}) { + ASSERT_TRUE( + BlobRelocationDirectory::DecodeFrom(with_served(served), &decoded) + .IsCorruption()); + } + for (const uint64_t second_origin : {uint64_t{10}, uint64_t{5}}) { + std::string edited = encoded; + EncodeFixed64(edited.data() + kDirectoryHeaderSize + 40, second_origin); + reseal(&edited); + ASSERT_TRUE( + BlobRelocationDirectory::DecodeFrom(edited, &decoded).IsCorruption()); + } + + EXPECT_LT(EncodeBlobRelocationKey(10, 99), EncodeBlobRelocationKey(11, 1)); + uint64_t origin_file_number = 0; + uint64_t original_value_offset = 0; + ASSERT_OK(DecodeBlobRelocationKey(EncodeBlobRelocationKey(10, 99), + &origin_file_number, + &original_value_offset)); + EXPECT_EQ(origin_file_number, 10U); + EXPECT_EQ(original_value_offset, 99U); +} + +TEST(BlobRelocationLiveSetTest, DeduplicatesAndRejectsConflicts) { + const auto make_reference = [](uint64_t offset, uint64_t stored_value_size, + uint64_t record_bytes, + uint32_t stable_checksum) { + BlobReference reference; + reference.encoding_class = BlobReferenceEncodingClass::kIndirectUnflattened; + reference.generation_file_number = 10; + reference.origin_file_number = 10; + reference.original_value_offset = offset; + reference.stored_value_size = stored_value_size; + reference.record_bytes = record_bytes; + reference.compression = kNoCompression; + reference.stable_checksum = stable_checksum; + reference.stored_checksum = BlobIndex::ExtendChecksumWithCompression( + stable_checksum, kNoCompression); + return reference; + }; + + const BlobReference first = make_reference(/*offset=*/100, + /*stored_value_size=*/20, + /*record_bytes=*/40, + /*stable_checksum=*/1); + const BlobReference second = make_reference(/*offset=*/200, + /*stored_value_size=*/30, + /*record_bytes=*/50, + /*stable_checksum=*/2); + BlobRelocationLiveSet live_set{std::shared_ptr(), + /*generation_numbers=*/{10}}; + ASSERT_OK(live_set.Add({second, first})); + ASSERT_OK(live_set.Add({first})); + ASSERT_OK(live_set.Finalize()); + ASSERT_EQ(live_set.references().size(), 2U); + EXPECT_EQ(live_set.references()[0].original_value_offset, 100U); + EXPECT_EQ(live_set.references()[1].original_value_offset, 200U); + EXPECT_EQ(live_set.live_stored_value_bytes(), 50U); + EXPECT_EQ(live_set.live_record_bytes(), 90U); + ASSERT_EQ(live_set.origin_runs().size(), 1U); + EXPECT_EQ(live_set.origin_runs()[0].origin_file_number, 10U); + + BlobReference conflict = first; + ++conflict.stable_checksum; + BlobRelocationLiveSet conflicting_set{ + std::shared_ptr(), /*generation_numbers=*/{10}}; + ASSERT_OK(conflicting_set.Add({first, conflict})); + ASSERT_TRUE(conflicting_set.Finalize().IsCorruption()); + + // Copies that differ only in the stored CRC also conflict. + BlobReference stored_conflict = first; + ++stored_conflict.stored_checksum; + BlobRelocationLiveSet stored_conflict_set{ + std::shared_ptr(), /*generation_numbers=*/{10}}; + ASSERT_OK(stored_conflict_set.Add({stored_conflict, first})); + ASSERT_TRUE(stored_conflict_set.Finalize().IsCorruption()); + + // The set's generation numbers are the ones it was read for. Each reference + // also has an encoding class: unflattened when generation == origin, + // flattened otherwise. Add() rejects a reference under another generation + // number, or one whose class does not match its generation number. + BlobRelocationLiveSet checked_set{std::shared_ptr(), + /*generation_numbers=*/{10}}; + BlobReference other_name = first; + other_name.origin_file_number = other_name.generation_file_number = 11; + ASSERT_TRUE(checked_set.Add({other_name}).IsCorruption()); + + BlobReference wrong_class = first; + wrong_class.encoding_class = BlobReferenceEncodingClass::kIndirectFlattened; + ASSERT_TRUE(checked_set.Add({wrong_class}).IsCorruption()); + + // A relocation input 20 has two generation numbers: 20 itself (flattened + // references of origins 5 and 10) and old file 10, which is routed to it + // (unflattened references). A value referenced under both generation numbers + // is kept once. The set is sorted by (origin, original offset) and has one + // run per origin. + BlobRelocationLiveSet input_set{std::shared_ptr(), + /*generation_numbers=*/{10, 20}}; + BlobReference flattened = first; + flattened.encoding_class = BlobReferenceEncodingClass::kIndirectFlattened; + flattened.generation_file_number = 20; + flattened.record_bytes = flattened.stored_value_size + 8; + BlobReference other_origin = flattened; + other_origin.origin_file_number = 5; + other_origin.original_value_offset = 300; + ASSERT_OK(input_set.Add({flattened, second, other_origin, first})); + ASSERT_OK(input_set.Finalize()); + ASSERT_EQ(input_set.references().size(), 3U); + EXPECT_EQ(input_set.references()[0].origin_file_number, 5U); + EXPECT_EQ(input_set.references()[1].original_value_offset, 100U); + EXPECT_EQ(input_set.references()[2].original_value_offset, 200U); + ASSERT_EQ(input_set.origin_runs().size(), 2U); + EXPECT_EQ(input_set.origin_runs()[0].origin_file_number, 5U); + EXPECT_EQ(input_set.origin_runs()[0].end, 1U); + EXPECT_EQ(input_set.origin_runs()[1].origin_file_number, 10U); + EXPECT_EQ(input_set.origin_runs()[1].begin, 1U); + EXPECT_EQ(input_set.origin_runs()[1].end, 3U); + EXPECT_LE(sizeof(BlobRelocationLiveValue), 40U); +} + +TEST(BlobRelocationLiveSetTest, ReservesExactCapacity) { + BlobRelocationLiveSet live_set{std::shared_ptr(), + /*generation_numbers=*/{10}}; + ASSERT_OK(live_set.Reserve(1000)); + EXPECT_EQ(live_set.TEST_ReservedMemory(), + 1000 * sizeof(BlobRelocationLiveValue)); +} + +TEST(BlobRelocationPackingTest, FirstFitDecreasingIsDeterministicAndWhole) { + const std::vector items{ + {/*input_file_number=*/300, /*estimated_bytes=*/40, {30}}, + {/*input_file_number=*/100, /*estimated_bytes=*/60, {10}}, + {/*input_file_number=*/200, /*estimated_bytes=*/40, {20}}, + {/*input_file_number=*/400, /*estimated_bytes=*/30, {40, 41}}}; + std::vector bins; + ASSERT_OK(PlanBlobRelocationPacking(items, /*target_output_bytes=*/100, + /*max_outputs=*/3, &bins)); + ASSERT_EQ(bins.size(), 2U); + EXPECT_EQ(bins[0].item_indexes, (std::vector{1, 2})); + EXPECT_EQ(bins[0].estimated_bytes, 100U); + EXPECT_EQ(bins[1].item_indexes, (std::vector{0, 3})); + EXPECT_EQ(bins[1].estimated_bytes, 70U); + + // Two inputs that hold the same origin (this can happen after RepairDB) never + // share an output, because an output holds one entry per (origin, original + // offset). + const std::vector shared_origin{ + {/*input_file_number=*/100, /*estimated_bytes=*/30, {10, 11}}, + {/*input_file_number=*/200, /*estimated_bytes=*/20, {11, 12}}, + {/*input_file_number=*/300, /*estimated_bytes=*/10, {13}}}; + ASSERT_OK(PlanBlobRelocationPacking(shared_origin, + /*target_output_bytes=*/100, + /*max_outputs=*/3, &bins)); + ASSERT_EQ(bins.size(), 2U); + EXPECT_EQ(bins[0].item_indexes, (std::vector{0, 2})); + EXPECT_EQ(bins[1].item_indexes, (std::vector{1})); + + // An input larger than the target gets an output of its own. + const std::vector oversized{ + {/*input_file_number=*/100, /*estimated_bytes=*/250, {10}}, + {/*input_file_number=*/200, /*estimated_bytes=*/20, {20}}}; + ASSERT_OK(PlanBlobRelocationPacking(oversized, /*target_output_bytes=*/100, + /*max_outputs=*/3, &bins)); + ASSERT_EQ(bins.size(), 2U); + EXPECT_EQ(bins[0].item_indexes, (std::vector{0})); + EXPECT_EQ(bins[0].estimated_bytes, 250U); + + // Inputs that need more than max_outputs outputs get MemoryLimit. A zero + // target output size is InvalidArgument. + ASSERT_TRUE(PlanBlobRelocationPacking(items, /*target_output_bytes=*/50, + /*max_outputs=*/2, &bins) + .IsMemoryLimit()); + ASSERT_TRUE(PlanBlobRelocationPacking(items, /*target_output_bytes=*/0, + /*max_outputs=*/2, &bins) + .IsInvalidArgument()); +} + class BlobFileReaderTest : public testing::Test { protected: BlobFileReaderTest() { mock_env_.reset(MockEnv::Create(Env::Default())); } std::unique_ptr mock_env_; }; +TEST_F(BlobFileReaderTest, RelocationTableBuilderChecksServedFilesAndOrigins) { + // A relocation file serves at least one old file number, and every origin + // and every served file number is older than the file. + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_RelocationBuilderChecks"), + 0); + BlockBasedTableOptions table_options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint64_t kFileNumber = 300; + enum class Mode { kNoServedFile, kServesItself, kNewerOrigin }; + for (const Mode mode : + {Mode::kNoServedFile, Mode::kServesItself, Mode::kNewerOrigin}) { + SCOPED_TRACE(static_cast(mode)); + const std::string file_path = + BlobFileName(immutable_options.cf_paths.front().path, kFileNumber); + std::unique_ptr file; + ASSERT_OK(immutable_options.fs->NewWritableFile(file_path, FileOptions(), + &file, /*dbg=*/nullptr)); + WritableFileWriter writer(std::move(file), file_path, FileOptions(), + immutable_options.clock); + const InternalKeyComparator comparator(BytewiseComparator()); + std::unique_ptr builder; + BlobRelocationTableFileInfo file_info; + file_info.column_family_id = 1; + file_info.column_family_name = kDefaultColumnFamilyName; + file_info.db_id = "test-db"; + file_info.db_session_id = "test-session"; + file_info.file_number = kFileNumber; + ASSERT_OK(NewBlobRelocationTableBuilder( + immutable_options, mutable_cf_options, table_options, ReadOptions(), + WriteOptions(), comparator, file_info, &writer, &builder)); + const uint64_t origin = mode == Mode::kNewerOrigin ? 400 : 100; + const Status added = + builder->AddBlobRelocation(origin, 10, "value", kNoCompression); + if (mode == Mode::kNewerOrigin) { + ASSERT_TRUE(added.IsInvalidArgument()) << added.ToString(); + builder->Abandon(); + continue; + } + ASSERT_OK(added); + if (mode == Mode::kServesItself) { + ASSERT_TRUE(builder->AddBlobRelocationServedFile(kFileNumber) + .IsInvalidArgument()); + builder->Abandon(); + continue; + } + ASSERT_FALSE(builder->Finish().ok()); + BlobRelocationFileMetadata metadata; + ASSERT_FALSE(builder->GetBlobRelocationFileMetadata(&metadata).ok()); + } +} + +TEST(BlobRelocationKeyTest, BufferEncodingMatchesStringEncoding) { + for (const auto& [origin, offset] : + std::vector>{ + {1, 0}, {42, 123456}, {0x3fffffffffffffff, ~uint64_t{0}}}) { + char buffer[kBlobRelocationKeySize]; + EncodeBlobRelocationKeyTo(origin, offset, buffer); + EXPECT_EQ(Slice(buffer, sizeof(buffer)), + Slice(EncodeBlobRelocationKey(origin, offset))); + } +} + +TEST(BlobRelocationGroupTest, RoundTripAndFind) { + // Values at original offsets 10, 25 and 1000, with record offsets that + // follow each other (stored size + 5 trailer bytes). + const std::array values{ + {{10, 7}, {25, 100}, {1000, 1}}}; + std::string encoded; + EncodeBlobRelocationGroup(/*first_record_offset=*/8, values.data(), + values.size(), &encoded); + BlobRelocationGroup group; + ASSERT_OK(DecodeBlobRelocationGroup(encoded, 1000, &group)); + ASSERT_EQ(group.count, 3U); + const uint64_t expected_record_offsets[] = {8, 8 + 7 + 5, + 8 + 7 + 5 + 100 + 5}; + for (size_t i = 0; i < values.size(); ++i) { + const BlobRelocationGroup::Value* found = + group.Find(values[i].original_value_offset); + ASSERT_NE(found, nullptr) << i; + EXPECT_EQ(found->stored_value_size, values[i].stored_value_size); + EXPECT_EQ(found->record_offset, expected_record_offsets[i]); + } + for (uint64_t absent : {0, 9, 11, 26, 999, 1001}) { + EXPECT_EQ(group.Find(absent), nullptr) << absent; + } + // The key must hold the last value's offset. + EXPECT_TRUE(DecodeBlobRelocationGroup(encoded, 25, &group).IsCorruption()); + + // The gap is counted from the end of the value before. A value at offset 0, + // an adjacent value (gap 0) and a large gap round-trip: + // - offset 0, size 7: gap 0 (the end before the first value is 0); + // - offset 7, size 3: gap 0 (the first value ends at 7); + // - offset 1000000, size 1: gap 1000000 - 10 (the second value ends at 10). + const std::array edge_values{ + {{0, 7}, {7, 3}, {1000000, 1}}}; + EncodeBlobRelocationGroup(/*first_record_offset=*/8, edge_values.data(), + edge_values.size(), &encoded); + std::string expected; + PutVarint32(&expected, 3); + PutVarint64(&expected, 8); + for (const auto& [gap, size] : std::vector>{ + {0, 7}, {0, 3}, {1000000 - 10, 1}}) { + PutVarint64(&expected, gap); + PutVarint64(&expected, size); + } + EXPECT_EQ(encoded, expected); + ASSERT_OK(DecodeBlobRelocationGroup(encoded, 1000000, &group)); + ASSERT_EQ(group.count, 3U); + for (const BlobRelocationGroupValue& value : edge_values) { + const BlobRelocationGroup::Value* found = + group.Find(value.original_value_offset); + ASSERT_NE(found, nullptr) << value.original_value_offset; + EXPECT_EQ(found->stored_value_size, value.stored_value_size); + } + + // A full group of kBlobRelocationGroupSize values. + std::vector full; + for (size_t i = 0; i < kBlobRelocationGroupSize; ++i) { + full.push_back({100 + i * 50, 40}); + } + EncodeBlobRelocationGroup(8, full.data(), full.size(), &encoded); + ASSERT_OK(DecodeBlobRelocationGroup( + encoded, full.back().original_value_offset, &group)); + EXPECT_EQ(group.count, kBlobRelocationGroupSize); +} + +TEST(BlobRelocationGroupTest, RejectsMalformedEntries) { + // Each case builds a raw entry value; all must fail with Corruption. + const auto raw = [](uint32_t count, uint64_t first_record_offset, + const std::vector>& pairs, + const std::string& extra = std::string()) { + std::string value; + PutVarint32(&value, count); + PutVarint64(&value, first_record_offset); + for (const auto& pair : pairs) { + PutVarint64(&value, pair.first); + PutVarint64(&value, pair.second); + } + value.append(extra); + return value; + }; + constexpr uint64_t kMax = std::numeric_limits::max(); + struct Case { + const char* name; + std::string value; + uint64_t last_offset; + }; + std::vector> too_many; + for (size_t i = 0; i <= kBlobRelocationGroupSize; ++i) { + too_many.emplace_back(1, 1); + } + const std::vector cases{ + {"empty", std::string(), 10}, + {"count 0", raw(0, 8, {}), 0}, + {"count 33", raw(kBlobRelocationGroupSize + 1, 8, too_many), + kBlobRelocationGroupSize + 1}, + {"value end overflow", raw(2, 8, {{kMax, 1}, {1, 1}}), 0}, + {"gap overflow", raw(2, 8, {{10, 1}, {kMax, 1}}), 0}, + {"size 0", raw(1, 8, {{10, 0}}), 10}, + {"size too large", raw(1, 8, {{10, kMaxSimpleGen2BlobPayloadSize + 1}}), + 10}, + {"record offset overflow", raw(2, kMax - 3, {{10, 1}, {5, 1}}), 16}, + {"truncated", raw(2, 8, {{10, 1}}), 10}, + {"extra bytes", raw(1, 8, {{10, 1}}, "x"), 10}, + {"last offset mismatch", raw(1, 8, {{10, 1}}), 11}, + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + BlobRelocationGroup group; + EXPECT_TRUE(DecodeBlobRelocationGroup(c.value, c.last_offset, &group) + .IsCorruption()); + } +} + +TEST(BlobRelocationGroupTest, DecoderFuzz) { + // Random truncations and byte flips of valid entries must give OK or + // Corruption, never a crash, and an OK result keeps the decoder rules. + constexpr uint32_t kSeed = 301; + SCOPED_TRACE(kSeed); + Random random(kSeed); + for (int round = 0; round < 2000; ++round) { + std::vector values; + const size_t count = 1 + random.Uniform(kBlobRelocationGroupSize); + // Each value after the first starts 0..63 bytes after the end of the value + // before. + uint64_t end = random.Uniform(1000); + for (size_t i = 0; i < count; ++i) { + const uint64_t offset = end + random.Uniform(64); + const uint64_t size = 1 + random.Uniform(4096); + values.push_back({offset, size}); + end = offset + size; + } + std::string encoded; + EncodeBlobRelocationGroup(8 + random.Uniform(1 << 20), values.data(), + values.size(), &encoded); + if (random.OneIn(2)) { + encoded.resize(random.Uniform(static_cast(encoded.size()))); + } else { + encoded[random.Uniform(static_cast(encoded.size()))] ^= + static_cast(1 + random.Uniform(255)); + } + BlobRelocationGroup group; + const Status status = DecodeBlobRelocationGroup( + encoded, values.back().original_value_offset, &group); + if (status.ok()) { + ASSERT_GE(group.count, 1U); + ASSERT_LE(group.count, kBlobRelocationGroupSize); + for (size_t i = 0; i < group.count; ++i) { + ASSERT_GE(group.values[i].stored_value_size, 1U); + ASSERT_TRUE(i == 0 || group.values[i].original_value_offset > + group.values[i - 1].original_value_offset); + } + } else { + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + } + } +} + +TEST_F(BlobFileReaderTest, RelocationLocationEntriesCoverGroupsOfOneOrigin) { + // Origin 100 has 70 values (groups of 32, 32 and 6), origin 101 has 32 (one + // full group) and origin 102 has 1. Every value reads back, offsets that + // are not in the file give "entry is missing", the table has one entry per + // group, and a batch decodes each location entry once. + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "RelocationLocationEntries"), 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + table_options.format_version = 7; + table_options.checksum = kCRC32c; + table_options.block_cache = NewLRUCache(8 << 20); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t relocation_file_number = 400; + const std::vector> origins{ + {100, 70}, {101, 32}, {102, 1}}; + std::vector payloads; + std::vector records; + for (const auto& origin : origins) { + for (int i = 0; i < origin.second; ++i) { + payloads.push_back("value-" + std::to_string(origin.first) + "-" + + std::to_string(i)); + } + } + size_t next = 0; + for (const auto& origin : origins) { + for (int i = 0; i < origin.second; ++i) { + records.push_back({origin.first, static_cast(10 + i * 100), + payloads[next++], kNoCompression}); + } + } + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + column_family_id, relocation_file_number, records, + &metadata, &file_size); + std::unique_ptr reader; + ASSERT_OK(OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), column_family_id, + RelocationPhysicalFile(relocation_file_number, file_size, metadata), + /*db_session_id=*/"reader", table_options, + mutable_cf_options.compression_manager.get(), &reader)); + ASSERT_EQ(reader->relocation()->total_entries(), records.size()); + + const auto make_index = [](uint64_t origin, uint64_t offset, + const std::string& payload, BlobIndex* index) { + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = offset, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + ASSERT_OK(index->DecodeFrom(encoded)); + }; + std::vector indexes(records.size()); + for (size_t i = 0; i < records.size(); ++i) { + make_index(records[i].origin_file_number, records[i].original_value_offset, + payloads[i], &indexes[i]); + } + std::atomic decodes{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlobRelocationFileReader::CheckEntry:Decode", + [&](void*) { decodes.fetch_add(1); }); + SyncPoint::GetInstance()->EnableProcessing(); + const auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + // The first 32 values of origin 100 are one group: one decode for the + // batch. + { + std::vector outputs(32); + std::vector statuses(32); + std::vector requests; + for (size_t i = 0; i < 32; ++i) { + requests.push_back(MakeRelocationRequest(&indexes[i], 0, kWholeBlobLength, + &statuses[i], &outputs[i])); + } + reader->relocation()->MultiGetBlob(ReadOptions(), /*allocator=*/nullptr, + requests.size(), requests.data(), + /*bytes_read=*/nullptr); + for (size_t i = 0; i < 32; ++i) { + ASSERT_OK(statuses[i]) << i; + EXPECT_EQ(outputs[i].contents->data(), payloads[i]) << i; + } + EXPECT_EQ(decodes.load(), 1); + } + for (size_t i = 0; i < records.size(); ++i) { + std::unique_ptr value; + ASSERT_OK(GetFromRelocationFile(*reader, ReadOptions(), indexes[i], + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, + /*bytes_read=*/nullptr)) + << i; + EXPECT_EQ(value->data(), payloads[i]) << i; + } + // Absent offsets: inside a group, between groups, after the last value of + // an origin, before the first value, and an origin the file does not hold. + const std::vector> absent{ + {100, 15}, {100, 3115}, {100, 6915}, {101, 5}, {102, 11}, {103, 10}}; + for (const auto& missing : absent) { + SCOPED_TRACE(std::to_string(missing.first) + ":" + + std::to_string(missing.second)); + BlobIndex index; + make_index(missing.first, missing.second, payloads[0], &index); + std::unique_ptr value; + const Status status = GetFromRelocationFile( + *reader, ReadOptions(), index, /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, /*bytes_read=*/nullptr); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_NE(status.ToString().find("entry is missing"), std::string::npos) + << status.ToString(); + } + // One location entry per group: 3 + 1 + 1. + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + immutable_options.fs, + BlobFileName(immutable_options.cf_paths.front().path, + relocation_file_number), + FileOptions(), &file_reader, /*dbg=*/nullptr)); + std::unique_ptr properties; + ASSERT_OK(ReadTableProperties(file_reader.get(), file_size, + kBlockBasedTableMagicNumber, immutable_options, + ReadOptions(), &properties)); + EXPECT_EQ(properties->num_entries, 5U); +} + +TEST_F(BlobFileReaderTest, RelocationLocationEntryNeedsValueTypeAndSequence0) { + // A location entry key must have type kTypeValue and sequence 0. A file + // whose entry has type kTypeBlobIndex, or sequence 5, fails the lookup with + // Corruption. + for (const auto& footer : {PackSequenceAndType(0, kTypeBlobIndex), + PackSequenceAndType(5, kTypeValue)}) { + SCOPED_TRACE(footer); + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath( + mock_env_.get(), "RelocationEntryKeyType" + std::to_string(footer)), + 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + table_options.format_version = 7; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + const std::string payload = "value"; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::FinishBlobRelocationGroup:InternalKey", + [footer](void* arg) { + EncodeFixed64(static_cast(arg) + kBlobRelocationKeySize, + footer); + }); + SyncPoint::GetInstance()->EnableProcessing(); + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + /*column_family_id=*/1, /*blob_file_number=*/400, + {{100, 10, payload, kNoCompression}}, &metadata, + &file_size); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + std::unique_ptr reader; + ASSERT_OK(OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), /*column_family_id=*/1, + RelocationPhysicalFile(400, file_size, metadata), + /*db_session_id=*/"reader", table_options, + mutable_cf_options.compression_manager.get(), &reader)); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, 100, + {.origin_file_number = 100, + .original_value_offset = 10, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + BlobIndex index; + ASSERT_OK(index.DecodeFrom(encoded)); + std::unique_ptr value; + const Status status = GetFromRelocationFile( + *reader, ReadOptions(), index, /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, /*bytes_read=*/nullptr); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_NE(status.ToString().find("Invalid blob relocation entry key"), + std::string::npos) + << status.ToString(); + } +} + +TEST_F(BlobFileReaderTest, CompressedRelocationLocationBlocksReadBack) { + // Location blocks use the column family's compression. Round 0 reads some + // values one at a time, and round 1 reads all values in one MultiGetBlob + // batch. Each round uses a new block cache, so the first read of each block + // in a round misses the cache and reads the compressed block from the file. + CompressionType compression = kNoCompression; + for (CompressionType candidate : + {kLZ4Compression, kZSTD, kSnappyCompression}) { + if (CompressionTypeSupported(candidate)) { + compression = candidate; + break; + } + } + if (compression == kNoCompression) { + ROCKSDB_GTEST_SKIP("No supported compression type"); + return; + } + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "CompressedRelocationBlocks"), 0); + options.enable_blob_files = true; + options.compression = compression; + BlockBasedTableOptions table_options; + table_options.format_version = 7; + table_options.block_cache = NewLRUCache(8 << 20); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint64_t kOrigin = 100; + constexpr int kRecords = 5000; + std::vector payloads; + for (int i = 0; i < kRecords; ++i) { + payloads.push_back("value-" + std::to_string(i)); + } + std::vector records; + for (int i = 0; i < kRecords; ++i) { + records.push_back({kOrigin, static_cast(10 + i * 100), + payloads[i], kNoCompression}); + } + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + /*column_family_id=*/1, /*blob_file_number=*/400, records, + &metadata, &file_size); + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + immutable_options.fs, + BlobFileName(immutable_options.cf_paths.front().path, 400), FileOptions(), + &file_reader, /*dbg=*/nullptr)); + std::unique_ptr properties; + ASSERT_OK(ReadTableProperties(file_reader.get(), file_size, + kBlockBasedTableMagicNumber, immutable_options, + ReadOptions(), &properties)); + // format_version 7 writes the compression name as + // ";;". + char expected_name[32]; + snprintf(expected_name, sizeof(expected_name), "BuiltinV2;%02X;", + static_cast(compression)); + EXPECT_EQ(properties->compression_name, expected_name); + + for (int round = 0; round < 2; ++round) { + SCOPED_TRACE(round); + // A new block cache each round, so the first read of each block misses the + // cache. + BlockBasedTableOptions reader_options = table_options; + reader_options.block_cache = NewLRUCache(8 << 20); + std::unique_ptr reader; + ASSERT_OK(OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), /*column_family_id=*/1, + RelocationPhysicalFile(400, file_size, metadata), + /*db_session_id=*/"reader", reader_options, + mutable_cf_options.compression_manager.get(), &reader)); + std::vector indexes(kRecords); + for (int i = 0; i < kRecords; ++i) { + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, kOrigin, + {.origin_file_number = kOrigin, + .original_value_offset = records[i].original_value_offset, + .stored_value_size = payloads[i].size(), + .compression = kNoCompression, + .stable_checksum = + crc32c::Value(payloads[i].data(), payloads[i].size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payloads[i], kNoCompression)}); + ASSERT_OK(indexes[i].DecodeFrom(encoded)); + } + if (round == 0) { + for (int i = 0; i < kRecords; i += 97) { + std::unique_ptr value; + ASSERT_OK(GetFromRelocationFile(*reader, ReadOptions(), indexes[i], 0, + kWholeBlobLength, nullptr, &value, + nullptr)) + << i; + EXPECT_EQ(value->data(), payloads[i]); + } + } else { + std::vector outputs(kRecords); + std::vector statuses(kRecords); + std::vector requests; + for (int i = 0; i < kRecords; ++i) { + requests.push_back(MakeRelocationRequest( + &indexes[i], 0, kWholeBlobLength, &statuses[i], &outputs[i])); + } + reader->relocation()->MultiGetBlob( + ReadOptions(), nullptr, requests.size(), requests.data(), nullptr); + for (int i = 0; i < kRecords; ++i) { + ASSERT_OK(statuses[i]) << i; + EXPECT_EQ(outputs[i].contents->data(), payloads[i]); + } + } + } +} + +TEST_F(BlobFileReaderTest, RelocationTableOptionsUseStandardIndexOnly) { + // Relocation files never carry a user-defined index, so a CF configured + // with a custom index mode must still produce valid relocation options. + for (const auto mode : + {BlockBasedTableOptions::IndexMode::kCustomDefault, + BlockBasedTableOptions::IndexMode::kCustomOnly, + BlockBasedTableOptions::IndexMode::kStandardRequired}) { + BlockBasedTableOptions source; + source.index_mode = mode; + source.use_udi_as_primary_index = true; + const BlockBasedTableOptions relocation = + MakeBlobRelocationTableOptions(source); + ASSERT_EQ(relocation.GetEffectiveIndexMode(), + BlockBasedTableOptions::IndexMode::kStandardOnly); + ASSERT_EQ(relocation.user_defined_index_factory, nullptr); + BlockBasedTableFactory factory(relocation); + ASSERT_OK(factory.ValidateOptions(DBOptions(), ColumnFamilyOptions())); + } +} + +TEST_F(BlobFileReaderTest, RelocationTableOptionsForcePointLookupLayout) { + // MakeBlobRelocationTableOptions forces the relocation table layout (no block + // alignment, binary search data blocks with no hash index, no index key + // shortening, 4 KB blocks, format_version >= 6), whatever the column family + // sets. + BlockBasedTableOptions source; + source.block_align = true; + source.super_block_alignment_size = 1 << 20; + source.block_restart_interval = 1; + source.data_block_index_type = BlockBasedTableOptions::kDataBlockBinarySearch; + source.block_size = 64 * 1024; + const BlockBasedTableOptions relocation = + MakeBlobRelocationTableOptions(source); + EXPECT_FALSE(relocation.block_align); + EXPECT_EQ(relocation.super_block_alignment_size, 0U); + // The CF restart interval is kept. + EXPECT_EQ(relocation.block_restart_interval, 1); + // Lookups land between location entry keys: no hash index, and the index + // key of a block is its last key. + EXPECT_EQ(relocation.data_block_index_type, + BlockBasedTableOptions::kDataBlockBinarySearch); + EXPECT_EQ(relocation.index_shortening, + BlockBasedTableOptions::IndexShorteningMode::kNoShortening); + EXPECT_EQ(relocation.block_size, 4U * 1024); + EXPECT_GE(relocation.format_version, 6U); +} + +TEST_F(BlobFileReaderTest, RelocationIndexPinningFollowsTableOptions) { + // With cache_index_and_filter_blocks, the index block of a relocation file + // lives in the block cache. By default it is not pinned, as for a non-L0 + // SST, so each read looks it up in the block cache. With + // unpartitioned_pinning = kAll it is pinned, so reads do not look it up. + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_RelocationIndexPinning"), + 0); + options.enable_blob_files = true; + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.format_version = 7; + table_options.checksum = kCRC32c; + table_options.cache_index_and_filter_blocks = true; + table_options.block_cache = NewLRUCache(8 << 20); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t relocation_file_number = 400; + constexpr uint64_t origin = 100; + const std::string payload = "pinned-index-value"; + const std::vector records{ + {origin, /*original_value_offset=*/10, payload, kNoCompression}}; + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + column_family_id, relocation_file_number, records, + &metadata, &file_size); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = 10, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + BlobIndex index; + ASSERT_OK(index.DecodeFrom(encoded)); + Statistics* const stats = options.statistics.get(); + for (const bool pin_all : {false, true}) { + SCOPED_TRACE(pin_all); + BlockBasedTableOptions reader_options = table_options; + reader_options.block_cache = NewLRUCache(8 << 20); + if (pin_all) { + reader_options.metadata_cache_options.unpartitioned_pinning = + PinningTier::kAll; + } + std::unique_ptr reader; + ASSERT_OK(OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), column_family_id, + RelocationPhysicalFile(relocation_file_number, file_size, metadata), + /*db_session_id=*/"reader", reader_options, + mutable_cf_options.compression_manager.get(), &reader)); + std::unique_ptr value; + // The first read warms the data block. + ASSERT_OK(GetFromRelocationFile(*reader, ReadOptions(), index, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, + /*bytes_read=*/nullptr)); + const uint64_t index_lookups = + stats->getTickerCount(BLOCK_CACHE_INDEX_HIT) + + stats->getTickerCount(BLOCK_CACHE_INDEX_MISS); + value.reset(); + ASSERT_OK(GetFromRelocationFile(*reader, ReadOptions(), index, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, + /*bytes_read=*/nullptr)); + ASSERT_EQ(value->data(), payload); + const uint64_t delta = stats->getTickerCount(BLOCK_CACHE_INDEX_HIT) + + stats->getTickerCount(BLOCK_CACHE_INDEX_MISS) - + index_lookups; + EXPECT_EQ(delta, pin_all ? 0U : 1U); + } +} + +TEST_F(BlobFileReaderTest, RelocationMultiGetReadsColdDataBlocksInOneBatch) { + // Relocation MultiGet looks up all keys of a batch first. It reads the data + // blocks that are not in the block cache with one batched read. Blocks that + // are already in the block cache need no read. Keys in the same block, and + // duplicate keys, share one block read. + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_RelocationMultiGetColdBlocks"), + 0); + options.enable_blob_files = true; + options.statistics = CreateDBStatistics(); + BlockBasedTableOptions table_options; + table_options.format_version = 7; + table_options.checksum = kCRC32c; + table_options.block_cache = NewLRUCache(8 << 20); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t relocation_file_number = 400; + constexpr uint64_t origin = 100; + // A location entry holds 32 small values (about 85 bytes with its key, an + // estimate), so 40000 values (1250 entries) fill about 25 data blocks of + // 4 KB. + constexpr int kRecords = 40000; + std::vector payloads; + std::vector records; + payloads.reserve(kRecords); + for (int i = 0; i < kRecords; ++i) { + payloads.push_back("value-" + std::to_string(1000000 + i)); + } + for (int i = 0; i < kRecords; ++i) { + records.push_back({origin, static_cast(10 + i * 100), payloads[i], + kNoCompression}); + } + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + column_family_id, relocation_file_number, records, + &metadata, &file_size); + const auto open_reader = [&](const BlockBasedTableOptions& reader_options, + std::unique_ptr* reader) { + ASSERT_OK(OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), column_family_id, + RelocationPhysicalFile(relocation_file_number, file_size, metadata), + /*db_session_id=*/"reader", reader_options, + mutable_cf_options.compression_manager.get(), reader)); + }; + std::unique_ptr reader; + open_reader(table_options, &reader); + + // 30 keys spread over the file, two keys next to each other (one block) + // and one duplicate. + std::vector picks; + for (int i = 0; i < 30; ++i) { + picks.push_back(i * 1333); + } + picks.push_back(1); + picks.push_back(1333); + std::vector indexes(picks.size()); + for (size_t i = 0; i < picks.size(); ++i) { + const std::string& payload = payloads[picks[i]]; + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = records[picks[i]].original_value_offset, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + ASSERT_OK(indexes[i].DecodeFrom(encoded)); + } + std::vector outputs(picks.size()); + std::vector statuses(picks.size()); + std::vector requests(picks.size()); + const auto reset_requests = [&]() { + for (size_t i = 0; i < picks.size(); ++i) { + outputs[i].contents.reset(); + statuses[i] = Status::OK(); + requests[i] = MakeRelocationRequest(&indexes[i], 0, kWholeBlobLength, + &statuses[i], &outputs[i]); + } + }; + size_t batched_reads = 0; + size_t blocks_read = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::MultiGetRawEntries:RetrieveMultipleBlocks", + [&](void* arg) { + ++batched_reads; + for (const BlockHandle& handle : + *static_cast< + autovector*>( + arg)) { + blocks_read += handle.IsNull() ? 0 : 1; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + ReadOptions read_options; + read_options.verify_checksums = true; + reset_requests(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + requests.size(), requests.data(), + /*bytes_read=*/nullptr); + for (size_t i = 0; i < picks.size(); ++i) { + ASSERT_OK(statuses[i]) << i; + EXPECT_EQ(outputs[i].contents->data(), payloads[picks[i]]) << i; + } + // Cold: one batched read of many distinct blocks (the duplicate and the + // neighbor need no extra block). + EXPECT_EQ(batched_reads, 1U); + EXPECT_GT(blocks_read, 10U); + EXPECT_LT(blocks_read, picks.size() - 1); + + // Warm: the blocks are in the block cache, so no block read. + batched_reads = 0; + reset_requests(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + requests.size(), requests.data(), + /*bytes_read=*/nullptr); + for (size_t i = 0; i < picks.size(); ++i) { + ASSERT_OK(statuses[i]) << i; + } + EXPECT_EQ(batched_reads, 0U); + + // An offset inside the directory range of its origin but not in the table: + // the location entry found does not hold it. + { + const std::string& payload = payloads[picks[3]]; + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = records[picks[3]].original_value_offset + 1, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + BlobIndex absent; + ASSERT_OK(absent.DecodeFrom(encoded)); + SameFileBlobStandaloneRead absent_output; + Status absent_status; + RelocationBlobReadRequest absent_request = MakeRelocationRequest( + &absent, 0, kWholeBlobLength, &absent_status, &absent_output); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + /*num_reads=*/1, &absent_request, + /*bytes_read=*/nullptr); + EXPECT_TRUE(absent_status.IsCorruption()) << absent_status.ToString(); + EXPECT_NE(absent_status.ToString().find("entry is missing"), + std::string::npos) + << absent_status.ToString(); + EXPECT_EQ(absent_output.contents, nullptr); + } + + // A reader with a cold block cache and kBlockCacheTier: Incomplete. + BlockBasedTableOptions cold_options = table_options; + cold_options.block_cache = NewLRUCache(8 << 20); + std::unique_ptr cold_reader; + open_reader(cold_options, &cold_reader); + ReadOptions no_io = read_options; + no_io.read_tier = kBlockCacheTier; + reset_requests(); + cold_reader->relocation()->MultiGetBlob(no_io, /*allocator=*/nullptr, + requests.size(), requests.data(), + /*bytes_read=*/nullptr); + for (size_t i = 0; i < picks.size(); ++i) { + EXPECT_TRUE(statuses[i].IsIncomplete()) << i << statuses[i].ToString(); + } + + // A single relocated Get with a cold cache does one data block read on the + // normal block path and one plain positional record read, and no batched + // block read. The block cache counts exactly one miss, so the read path + // does not look the block up a second time. + { + BlockBasedTableOptions single_options = table_options; + single_options.block_cache = NewLRUCache(8 << 20); + std::unique_ptr single_reader; + open_reader(single_options, &single_reader); + size_t single_reads = 0; + size_t multi_reads = 0; + SyncPoint::GetInstance()->SetCallBack("MultiReadGen2:SingleRead", + [&](void*) { ++single_reads; }); + SyncPoint::GetInstance()->SetCallBack("MultiReadGen2:Requests", + [&](void*) { ++multi_reads; }); + batched_reads = 0; + Statistics* const stats = options.statistics.get(); + const uint64_t misses = stats->getTickerCount(BLOCK_CACHE_MISS); + const uint64_t data_misses = stats->getTickerCount(BLOCK_CACHE_DATA_MISS); + // Block cache misses are counted per level in the perf context. + SetPerfLevel(PerfLevel::kEnableCount); + get_perf_context()->Reset(); + get_perf_context()->EnablePerLevelPerfContext(); + std::unique_ptr value; + ASSERT_OK(GetFromRelocationFile(*single_reader, read_options, indexes[3], + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, + /*bytes_read=*/nullptr)); + uint64_t perf_misses = 0; + for (const auto& level : *get_perf_context()->level_to_perf_context) { + perf_misses += level.second.block_cache_miss_count; + } + get_perf_context()->ClearPerLevelPerfContext(); + get_perf_context()->DisablePerLevelPerfContext(); + SetPerfLevel(PerfLevel::kDisable); + EXPECT_EQ(value->data(), payloads[picks[3]]); + EXPECT_EQ(batched_reads, 0U); + EXPECT_EQ(multi_reads, 1U); + EXPECT_EQ(single_reads, 1U); + EXPECT_EQ(stats->getTickerCount(BLOCK_CACHE_MISS), misses + 1); + EXPECT_EQ(stats->getTickerCount(BLOCK_CACHE_DATA_MISS), data_misses + 1); + EXPECT_EQ(perf_misses, 1U); + SyncPoint::GetInstance()->ClearCallBack("MultiReadGen2:SingleRead"); + SyncPoint::GetInstance()->ClearCallBack("MultiReadGen2:Requests"); + } + + // A corrupt data block fails only the keys in that block. + size_t verified_blocks = 0; + SyncPoint::GetInstance()->SetCallBack( + "RetrieveMultipleBlocks:VerifyChecksum", [&](void* arg) { + if (verified_blocks++ == 1) { + *static_cast(arg) = Status::Corruption("injected"); + } + }); + reset_requests(); + cold_reader->relocation()->MultiGetBlob(read_options, + /*allocator=*/nullptr, + requests.size(), requests.data(), + /*bytes_read=*/nullptr); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + size_t failed = 0; + for (size_t i = 0; i < picks.size(); ++i) { + if (statuses[i].ok()) { + EXPECT_EQ(outputs[i].contents->data(), payloads[picks[i]]) << i; + } else { + EXPECT_TRUE(statuses[i].IsCorruption()) << i; + ++failed; + } + } + EXPECT_GE(failed, 1U); + EXPECT_LT(failed, 4U); + + // A damaged record fails its checksum on the single-read path (one Get) + // and on the batched path (two records), and only for that record. + const std::string file_path = BlobFileName( + immutable_options.cf_paths.front().path, relocation_file_number); + std::string file_contents; + ASSERT_OK( + ReadFileToString(immutable_options.fs.get(), file_path, &file_contents)); + const size_t payload_pos = file_contents.find(payloads[picks[3]]); + ASSERT_NE(payload_pos, std::string::npos); + file_contents[payload_pos + 2] ^= 0x1; + ASSERT_OK( + WriteStringToFile(immutable_options.fs.get(), file_contents, file_path)); + BlockBasedTableOptions damaged_options = table_options; + damaged_options.block_cache = NewLRUCache(8 << 20); + std::unique_ptr damaged_reader; + open_reader(damaged_options, &damaged_reader); + std::unique_ptr damaged_value; + EXPECT_TRUE(GetFromRelocationFile(*damaged_reader, read_options, indexes[3], + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &damaged_value, + /*bytes_read=*/nullptr) + .IsCorruption()); + std::array pair_outputs; + std::array pair_statuses; + std::array pair_requests{ + MakeRelocationRequest(&indexes[3], 0, kWholeBlobLength, &pair_statuses[0], + &pair_outputs[0]), + MakeRelocationRequest(&indexes[4], 0, kWholeBlobLength, &pair_statuses[1], + &pair_outputs[1])}; + damaged_reader->relocation()->MultiGetBlob( + read_options, /*allocator=*/nullptr, pair_requests.size(), + pair_requests.data(), /*bytes_read=*/nullptr); + EXPECT_TRUE(pair_statuses[0].IsCorruption()) << pair_statuses[0].ToString(); + ASSERT_OK(pair_statuses[1]); + EXPECT_EQ(pair_outputs[1].contents->data(), payloads[picks[4]]); + + // Callers send the internal range functions only strict parts of a value (see + // ReadPathBlobResolver::ResolveColumnRange), and these functions do not turn + // a range into a whole read. So a range read checks no checksum, even when + // the test passes a range that covers the whole uncompressed value. On the + // damaged record, a range of the whole value and a strict sub-range both + // return OK, for one request and for a batch (both use + // BlockBasedTable::MultiGetSameFileBlob()). A range read counts only the + // range bytes in `bytes_read`. + const size_t damaged_size = payloads[picks[3]].size(); + ASSERT_OK(GetFromRelocationFile(*damaged_reader, read_options, indexes[3], + /*range_offset=*/0, damaged_size, + /*allocator=*/nullptr, &damaged_value, + /*bytes_read=*/nullptr)); + EXPECT_NE(damaged_value->data(), payloads[picks[3]]); + uint64_t range_bytes_read = 0; + ASSERT_OK(GetFromRelocationFile(*damaged_reader, read_options, indexes[3], + /*range_offset=*/0, damaged_size - 1, + /*allocator=*/nullptr, &damaged_value, + &range_bytes_read)); + EXPECT_EQ(range_bytes_read, damaged_size - 1); + std::unique_ptr whole_range_value; + uint64_t whole_range_bytes_read = 0; + ASSERT_OK(GetFromRelocationFile(*damaged_reader, read_options, indexes[4], + /*range_offset=*/0, payloads[picks[4]].size(), + /*allocator=*/nullptr, &whole_range_value, + &whole_range_bytes_read)); + EXPECT_EQ(whole_range_value->data(), payloads[picks[4]]); + EXPECT_EQ(whole_range_bytes_read, payloads[picks[4]].size()); + std::array range_outputs; + std::array range_statuses; + std::array range_requests{ + MakeRelocationRequest(&indexes[3], 0, damaged_size, &range_statuses[0], + &range_outputs[0]), + MakeRelocationRequest(&indexes[4], 0, payloads[picks[4]].size(), + &range_statuses[1], &range_outputs[1]), + MakeRelocationRequest(&indexes[3], 0, damaged_size - 1, + &range_statuses[2], &range_outputs[2])}; + damaged_reader->relocation()->MultiGetBlob( + read_options, /*allocator=*/nullptr, range_requests.size(), + range_requests.data(), /*bytes_read=*/nullptr); + ASSERT_OK(range_statuses[0]); + EXPECT_NE(range_outputs[0].contents->data(), payloads[picks[3]]); + ASSERT_OK(range_statuses[1]); + EXPECT_EQ(range_outputs[1].contents->data(), payloads[picks[4]]); + ASSERT_OK(range_statuses[2]); + EXPECT_EQ(range_outputs[2].contents->size(), damaged_size - 1); +} + +enum class RawLookupMode { + kMmap, + kDirect, + kNoBlockCache, + kNoFillCache, + kBlockCacheTier +}; + +class RelocationRawLookupTest + : public testing::TestWithParam> {}; + +TEST_P(RelocationRawLookupTest, ReadsCorrectValues) { + // Relocation MultiGet in each I/O and cache mode, with more keys than one + // lookup batch (MultiGetContext::MAX_BATCH_SIZE) and a duplicate key, or + // with a single key (the single block read path). + const RawLookupMode mode = std::get<0>(GetParam()); + const bool single = std::get<1>(GetParam()); + Env* const env = Env::Default(); + Options options; + options.env = env; + const std::string path = test::PerThreadDBPath( + env, "RelocationRawLookup" + std::to_string(static_cast(mode)) + + (single ? "s" : "b")); + ASSERT_OK(DestroyDir(env, path)); + ASSERT_OK(env->CreateDirIfMissing(path)); + options.cf_paths.emplace_back(path, 0); + options.enable_blob_files = true; + // Uncompressed location blocks: an mmap read then uses the mapping and never + // fills the block cache, which the batched-read counts below expect. + options.compression = kNoCompression; + options.allow_mmap_reads = mode == RawLookupMode::kMmap; + options.use_direct_reads = mode == RawLookupMode::kDirect; + BlockBasedTableOptions table_options; + table_options.format_version = 7; + table_options.checksum = kCRC32c; + if (mode == RawLookupMode::kNoBlockCache) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = NewLRUCache(8 << 20); + } + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t relocation_file_number = 500; + constexpr uint64_t origin = 100; + // A location entry holds 32 small values (about 85 bytes with its key, an + // estimate), so 40000 values (1250 entries) fill about 25 data blocks of + // 4 KB. + constexpr int kRecords = 40000; + std::vector payloads; + std::vector records; + payloads.reserve(kRecords); + for (int i = 0; i < kRecords; ++i) { + payloads.push_back("value-" + std::to_string(1000000 + i)); + } + for (int i = 0; i < kRecords; ++i) { + records.push_back({origin, static_cast(10 + i * 100), payloads[i], + kNoCompression}); + } + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + column_family_id, relocation_file_number, records, + &metadata, &file_size); + std::unique_ptr reader; + const Status open_status = OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(DBOptions(options)), + column_family_id, + RelocationPhysicalFile(relocation_file_number, file_size, metadata), + /*db_session_id=*/"reader", table_options, + mutable_cf_options.compression_manager.get(), &reader); + if (mode == RawLookupMode::kDirect && !open_status.ok()) { + ROCKSDB_GTEST_SKIP("Direct I/O is not supported here"); + return; + } + ASSERT_OK(open_status); + + std::vector picks; + if (single) { + picks.push_back(997); + } else { + for (int i = 0; i < 40; ++i) { + picks.push_back(i * 997); + } + picks.push_back(997); + } + std::vector indexes(picks.size()); + for (size_t i = 0; i < picks.size(); ++i) { + const std::string& payload = payloads[picks[i]]; + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = records[picks[i]].original_value_offset, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + ASSERT_OK(indexes[i].DecodeFrom(encoded)); + } + ReadOptions read_options; + read_options.verify_checksums = true; + read_options.fill_cache = mode != RawLookupMode::kNoFillCache; + if (mode == RawLookupMode::kBlockCacheTier) { + read_options.read_tier = kBlockCacheTier; + } + size_t batched_reads = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::MultiGetRawEntries:RetrieveMultipleBlocks", + [&](void*) { ++batched_reads; }); + SyncPoint::GetInstance()->EnableProcessing(); + for (int round = 0; round < 2; ++round) { + SCOPED_TRACE(round); + batched_reads = 0; + std::vector outputs(picks.size()); + std::vector statuses(picks.size()); + std::vector requests(picks.size()); + for (size_t i = 0; i < picks.size(); ++i) { + requests[i] = MakeRelocationRequest(&indexes[i], 0, kWholeBlobLength, + &statuses[i], &outputs[i]); + } + SetPerfLevel(PerfLevel::kEnableCount); + get_perf_context()->Reset(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + requests.size(), requests.data(), + /*bytes_read=*/nullptr); + const uint64_t block_reads = get_perf_context()->block_read_count; + SetPerfLevel(PerfLevel::kDisable); + if (mode == RawLookupMode::kBlockCacheTier) { + // The block cache is cold and no I/O is allowed, also for one key. + for (size_t i = 0; i < picks.size(); ++i) { + EXPECT_TRUE(statuses[i].IsIncomplete()) << i << statuses[i].ToString(); + } + EXPECT_EQ(batched_reads, 0U); + EXPECT_EQ(block_reads, 0U); + continue; + } + for (size_t i = 0; i < picks.size(); ++i) { + ASSERT_OK(statuses[i]) << i; + EXPECT_EQ(outputs[i].contents->data(), payloads[picks[i]]) << i; + } + // 41 keys are looked up in two batches. Blocks stay in the block cache + // only with a block cache and fill_cache, and not with mmap reads (as in + // the normal MultiGet path, mmap blocks are read from the mapping). A + // single key never takes the batched read. + const bool cached = mode == RawLookupMode::kDirect; + if (single || (round > 0 && cached)) { + EXPECT_EQ(batched_reads, 0U); + } else { + EXPECT_EQ(batched_reads, 2U); + } + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(DestroyDir(env, path)); +} + +INSTANTIATE_TEST_CASE_P( + BlobFileReaderTest, RelocationRawLookupTest, + ::testing::Combine(::testing::Values(RawLookupMode::kMmap, + RawLookupMode::kDirect, + RawLookupMode::kNoBlockCache, + RawLookupMode::kNoFillCache, + RawLookupMode::kBlockCacheTier), + ::testing::Bool())); + +// Writes one relocation file of 40000 small values of one origin for the +// pinned location block tests. A location entry holds 32 values (about 85 +// bytes with its key, an estimate), so the file has about 25 data blocks of +// 4 KB, and lookups cross block boundaries. +struct PinnedBlocksTestFile { + static constexpr uint32_t kColumnFamilyId = 1; + static constexpr uint64_t kFileNumber = 600; + static constexpr uint64_t kOrigin = 100; + static constexpr int kRecords = 40000; + + // With `mmap`, the file is read through allow_mmap_reads and its blocks are + // not compressed, so the data blocks point into the mapped file. + PinnedBlocksTestFile(const std::string& name, std::shared_ptr cache, + bool mmap = false) { + Env* const env = Env::Default(); + options.env = env; + path = test::PerThreadDBPath(env, name); + EXPECT_OK(DestroyDir(env, path)); + EXPECT_OK(env->CreateDirIfMissing(path)); + options.cf_paths.emplace_back(path, 0); + options.enable_blob_files = true; + options.allow_mmap_reads = mmap; + if (mmap) { + options.compression = kNoCompression; + } + table_options.format_version = 7; + table_options.checksum = kCRC32c; + if (cache == nullptr) { + table_options.no_block_cache = true; + } else { + table_options.block_cache = std::move(cache); + } + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + immutable_options = std::make_unique(options); + file_options = FileOptions(DBOptions(options)); + const MutableCFOptions mutable_cf_options(options); + for (int i = 0; i < kRecords; ++i) { + payloads.push_back("value-" + std::to_string(1000000 + i)); + } + for (int i = 0; i < kRecords; ++i) { + records.push_back({kOrigin, static_cast(10 + i * 100), + payloads[i], kNoCompression}); + } + WriteRelocationFile(*immutable_options, mutable_cf_options, table_options, + kColumnFamilyId, kFileNumber, records, &metadata, + &file_size); + } + + Status Open(bool pin, std::unique_ptr* reader, + const ReadOptions& read_options = ReadOptions()) const { + return OpenRelocationReader( + *immutable_options, read_options, FileOptions(DBOptions(options)), + kColumnFamilyId, + RelocationPhysicalFile(kFileNumber, file_size, metadata), + /*db_session_id=*/"reader", table_options, + /*compression_manager=*/nullptr, reader, pin); + } + + // Looks up every value, plus one offset after each value that the file does + // not hold, in batches of 32. Returns one line per lookup: the status, and + // the value when the lookup succeeded. + std::vector LookUpAll(const BlobFileReader& reader) const { + std::vector indexes; + for (int i = 0; i < kRecords; ++i) { + for (uint64_t delta : {uint64_t{0}, uint64_t{1}}) { + const std::string& payload = payloads[i]; + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, kOrigin, + {.origin_file_number = kOrigin, + .original_value_offset = records[i].original_value_offset + delta, + .stored_value_size = payload.size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payload.data(), payload.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression)}); + indexes.emplace_back(); + EXPECT_OK(indexes.back().DecodeFrom(encoded)); + } + } + std::vector results; + constexpr size_t kBatch = MultiGetContext::MAX_BATCH_SIZE; + for (size_t begin = 0; begin < indexes.size(); begin += kBatch) { + const size_t count = std::min(kBatch, indexes.size() - begin); + std::vector statuses(count); + std::vector outputs(count); + std::vector requests; + for (size_t k = 0; k < count; ++k) { + requests.push_back(MakeRelocationRequest(&indexes[begin + k], 0, + kWholeBlobLength, &statuses[k], + &outputs[k])); + } + reader.relocation()->MultiGetBlob(ReadOptions(), /*allocator=*/nullptr, + count, requests.data(), + /*bytes_read=*/nullptr); + for (size_t k = 0; k < count; ++k) { + results.push_back(statuses[k].ToString() + ":" + + (statuses[k].ok() + ? outputs[k].contents->data().ToString() + : std::string())); + } + } + return results; + } + + Options options; + BlockBasedTableOptions table_options; + std::unique_ptr immutable_options; + FileOptions file_options; + std::string path; + std::vector payloads; + std::vector records; + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; +}; + +TEST(BlobRelocationPinnedBlocksTest, SameResultsAsNormalLookups) { + // With and without a block cache, a reader that keeps its location blocks + // returns the same values and errors as a normal reader, for every value of + // a file with about 25 data blocks and for offsets the file does not hold. + for (bool with_cache : {true, false}) { + SCOPED_TRACE(with_cache); + const PinnedBlocksTestFile file( + with_cache ? "PinnedSameResultsCache" : "PinnedSameResultsNoCache", + with_cache ? NewLRUCache(8 << 20) : nullptr); + std::unique_ptr normal; + std::unique_ptr pinned; + ASSERT_OK(file.Open(/*pin=*/false, &normal)); + ASSERT_OK(file.Open(/*pin=*/true, &pinned)); + EXPECT_EQ(normal->relocation()->pinned_location_block_bytes(), 0U); + EXPECT_GT(pinned->relocation()->pinned_location_block_bytes(), 0U); + const std::vector expected = file.LookUpAll(*normal); + ASSERT_EQ(expected.size(), 2U * PinnedBlocksTestFile::kRecords); + EXPECT_EQ(expected[0], "OK:" + file.payloads[0]); + EXPECT_EQ(expected[1].rfind("Corruption", 0), 0U) << expected[1]; + EXPECT_EQ(file.LookUpAll(*pinned), expected); + } +} + +TEST(BlobRelocationPinnedBlocksTest, PinsWhenOpenReadDoesNotFillCache) { + // The pin reads with its own options. An open whose ReadOptions has + // fill_cache == false still keeps every location block in the block cache, + // and later lookups read no data block. The cold open reads the blocks with + // batched reads (up to MultiGetContext::MAX_BATCH_SIZE (32) blocks per + // read). + std::shared_ptr cache = NewLRUCache(8 << 20); + const PinnedBlocksTestFile file("PinnedNoFillCache", cache); + ReadOptions open_options; + open_options.fill_cache = false; + std::atomic batched_reads{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::MultiGetRawEntries:RetrieveMultipleBlocks", + [&](void*) { ++batched_reads; }); + SyncPoint::GetInstance()->EnableProcessing(); + std::unique_ptr pinned; + const Status open_status = file.Open(/*pin=*/true, &pinned, open_options); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(open_status); + EXPECT_GT(batched_reads.load(), 0); + ASSERT_GT(pinned->relocation()->pinned_location_block_bytes(), 0U); + EXPECT_GT(cache->GetPinnedUsage(), 0U); + SetPerfLevel(PerfLevel::kEnableCount); + get_perf_context()->Reset(); + const std::vector results = file.LookUpAll(*pinned); + EXPECT_EQ(get_perf_context()->block_read_count, 0U); + SetPerfLevel(PerfLevel::kDisable); + EXPECT_EQ(results[0], "OK:" + file.payloads[0]); +} + +TEST(BlobRelocationPinnedBlocksTest, KeepsMemoryMappedBlocks) { + // With allow_mmap_reads and no compression, a data block points into the + // mapped file, and the block cache does not take it. The reader still keeps + // the blocks (their bytes are in the mapped file, not on the heap) and + // returns the same results as a normal reader. + std::shared_ptr cache = NewLRUCache(8 << 20); + const PinnedBlocksTestFile file("PinnedMmap", cache, /*mmap=*/true); + std::unique_ptr normal; + std::unique_ptr pinned; + ASSERT_OK(file.Open(/*pin=*/false, &normal)); + ASSERT_OK(file.Open(/*pin=*/true, &pinned)); + ASSERT_GT(pinned->relocation()->pinned_location_block_bytes(), 0U); + EXPECT_EQ(cache->GetPinnedUsage(), 0U); + EXPECT_EQ(file.LookUpAll(*pinned), file.LookUpAll(*normal)); +} + +TEST(BlobRelocationPinnedBlocksTest, PinDoesNotBlockBlobFileEvict) { + // A relocation file open holds a stripe mutex of the BlobFileCache while it + // pins its location blocks. Relocation files use their own stripe mutexes, + // so Evict() of a normal blob file never waits for that open. The test picks + // a normal blob file number that would share the stripe of the relocation + // file if both kinds used one set of 128 stripes, and evicts it while the + // open waits inside the pin. + std::shared_ptr cache = NewLRUCache(8 << 20); + const PinnedBlocksTestFile file("PinnedStripes", cache); + // The readers get their own cache. A reader holds the table options, which + // hold the block cache, so a reader in the block cache would keep that cache + // alive forever. + std::shared_ptr file_cache = NewLRUCache(1024); + BlobFileCache blob_file_cache( + file_cache.get(), file.immutable_options.get(), &file.file_options, + PinnedBlocksTestFile::kColumnFamilyId, /*blob_file_read_hist=*/nullptr, + /*io_tracer=*/nullptr, &file.table_options, /*db_session_id=*/"session", + /*block_cache_tracer=*/nullptr); + // The same Striped type, size and hash as one shared set of 128 stripes. + Striped> single_set(128); + uint64_t relocation_number = PinnedBlocksTestFile::kFileNumber; + port::Mutex* const relocation_stripe = + &single_set.Get(GetSliceForKey(&relocation_number)); + uint64_t blob_file_number = relocation_number + 1; + while (&single_set.Get(GetSliceForKey(&blob_file_number)) != + relocation_stripe) { + ++blob_file_number; + } + + std::promise pin_started; + std::promise evict_done; + std::shared_future evict_done_future = evict_done.get_future(); + SyncPoint::GetInstance()->SetCallBack( + "BlobRelocationFileReader::Open:PinLocationBlocks", [&](void*) { + pin_started.set_value(); + evict_done_future.wait(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + MutableCFOptions mutable_cf_options(file.options); + mutable_cf_options.pin_blob_relocation_location_blocks = true; + Status open_status; + std::thread opener([&]() { + CacheHandleGuard reader; + open_status = blob_file_cache.GetRelocationFileReader( + ReadOptions(), + RelocationPhysicalFile(PinnedBlocksTestFile::kFileNumber, + file.file_size, file.metadata), + mutable_cf_options, &reader); + }); + pin_started.get_future().wait(); + // With one shared set of stripes, this would wait for the open forever. + blob_file_cache.Evict(blob_file_number); + evict_done.set_value(); + opener.join(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(open_status); +} + +TEST(BlobRelocationPinnedBlocksTest, KeptBlocksSurviveCachePressure) { + // The kept blocks are charged to the block cache and are not evicted. After + // other entries fill the cache (and evict each other), a lookup reads no + // data block. + std::shared_ptr cache = NewLRUCache(1 << 20, /*num_shard_bits=*/0); + const PinnedBlocksTestFile file("PinnedCachePressure", cache); + std::unique_ptr pinned; + ASSERT_OK(file.Open(/*pin=*/true, &pinned)); + ASSERT_GT(pinned->relocation()->pinned_location_block_bytes(), 0U); + EXPECT_GT(cache->GetPinnedUsage(), 0U); + // 64 entries of 32 KB (2 MB) do not fit in the 1 MB cache. + static const Cache::CacheItemHelper kFillerHelper{ + CacheEntryRole::kMisc, [](Cache::ObjectPtr, MemoryAllocator*) {}}; + static char filler_object; + for (int i = 0; i < 64; ++i) { + ASSERT_OK(cache->Insert("filler" + std::to_string(i), &filler_object, + &kFillerHelper, 32 << 10)); + } + Cache::Handle* const first_filler = cache->Lookup("filler0"); + EXPECT_EQ(first_filler, nullptr); + if (first_filler != nullptr) { + cache->Release(first_filler); + } + SetPerfLevel(PerfLevel::kEnableCount); + get_perf_context()->Reset(); + const std::vector results = file.LookUpAll(*pinned); + EXPECT_EQ(get_perf_context()->block_read_count, 0U); + SetPerfLevel(PerfLevel::kDisable); + EXPECT_EQ(results[0], "OK:" + file.payloads[0]); +} + +TEST(BlobRelocationPinnedBlocksTest, FallsBackWhenCacheCannotCharge) { + // A block cache with strict_capacity_limit that is too small to hold the + // blocks: Open still succeeds, keeps nothing, and lookups work normally. + LRUCacheOptions cache_options; + cache_options.capacity = 16 << 10; + cache_options.num_shard_bits = 0; + cache_options.strict_capacity_limit = true; + const PinnedBlocksTestFile file("PinnedStrictCapacity", + NewLRUCache(cache_options)); + std::unique_ptr pinned; + ASSERT_OK(file.Open(/*pin=*/true, &pinned)); + EXPECT_EQ(pinned->relocation()->pinned_location_block_bytes(), 0U); + const std::vector results = file.LookUpAll(*pinned); + EXPECT_EQ(results[0], "OK:" + file.payloads[0]); + EXPECT_EQ(results[2 * (PinnedBlocksTestFile::kRecords - 1)], + "OK:" + file.payloads.back()); +} + +TEST(BlobRelocationPinnedBlocksTest, CloseReleasesKeptBlocks) { + // Closing the reader releases every cache handle it held, so the pinned + // usage of the block cache returns to where it was before the open. + std::shared_ptr cache = NewLRUCache(8 << 20); + const PinnedBlocksTestFile file("PinnedCloseReleases", cache); + const size_t pinned_before = cache->GetPinnedUsage(); + std::unique_ptr pinned; + ASSERT_OK(file.Open(/*pin=*/true, &pinned)); + ASSERT_GT(pinned->relocation()->pinned_location_block_bytes(), 0U); + EXPECT_GT(cache->GetPinnedUsage(), pinned_before); + pinned.reset(); + EXPECT_EQ(cache->GetPinnedUsage(), pinned_before); +} + +TEST_F(BlobFileReaderTest, RejectsOversizedCompressedBlobBeforeAllocation) { + if (!CompressionTypeSupported(kZlibCompression)) { + ROCKSDB_GTEST_SKIP("Zlib is required for this test"); + return; + } + std::string stored_value; + PutVarint64(&stored_value, std::numeric_limits::max()); + const std::shared_ptr decompressor = + GetBuiltinV2CompressionManager()->GetDecompressorOptimizeFor( + kZlibCompression); + std::unique_ptr result; + ASSERT_TRUE(UncompressBlobContentsIfNeeded( + stored_value, kZlibCompression, decompressor.get(), + /*allocator=*/nullptr, mock_env_->GetSystemClock().get(), + /*statistics=*/nullptr, &result) + .IsCorruption()); + ASSERT_EQ(result, nullptr); +} + +TEST_F(BlobFileReaderTest, + RejectsUnsupportedRelocationComparatorBeforeWritingPrefix) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath( + mock_env_.get(), + "BlobFileReaderTest_RejectsUnsupportedRelocationComparator"), + 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + + const auto verify_rejected = [&](const Comparator* user_comparator, + const std::string& db_id, + const std::string& db_session_id, + uint64_t file_number) { + const std::string file_path = + BlobFileName(immutable_options.cf_paths.front().path, file_number); + std::unique_ptr file; + ASSERT_OK(immutable_options.fs->NewWritableFile(file_path, FileOptions(), + &file, /*dbg=*/nullptr)); + std::unique_ptr writer = + std::make_unique( + std::move(file), file_path, FileOptions(), immutable_options.clock); + const InternalKeyComparator comparator(user_comparator); + std::unique_ptr builder; + BlobRelocationTableFileInfo file_info; + file_info.column_family_id = 1; + file_info.column_family_name = kDefaultColumnFamilyName; + file_info.db_id = db_id; + file_info.db_session_id = db_session_id; + file_info.file_number = file_number; + ASSERT_TRUE(NewBlobRelocationTableBuilder( + immutable_options, mutable_cf_options, table_options, + ReadOptions(), WriteOptions(), comparator, file_info, + writer.get(), &builder) + .IsInvalidArgument()); + ASSERT_EQ(builder, nullptr); + ASSERT_OK(writer->Close(IOOptions(), /*dbg=*/nullptr)); + uint64_t file_size = 1; + ASSERT_OK(immutable_options.fs->GetFileSize(file_path, IOOptions(), + &file_size, /*dbg=*/nullptr)); + EXPECT_EQ(file_size, 0U); + }; + + verify_rejected(ReverseBytewiseComparator(), "test-db", "test-session", + /*file_number=*/301); + verify_rejected(test::BytewiseComparatorWithU64TsWrapper(), "test-db", + "test-session", + /*file_number=*/302); + verify_rejected(BytewiseComparator(), "test-db", /*db_session_id=*/"", + /*file_number=*/303); +} + +TEST_F(BlobFileReaderTest, + RelocationFilePersistsItsDedicatedComparatorMetadata) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath( + mock_env_.get(), + "BlobFileReaderTest_RelocationFileDedicatedComparatorMetadata"), + 0); + options.enable_blob_files = true; + options.comparator = test::BytewiseComparatorWithU64TsWrapper(); + options.persist_user_defined_timestamps = true; + BlockBasedTableOptions table_options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + const std::string value = "value"; + const std::vector records{{/*origin_file_number=*/100, + /*original_value_offset=*/7, + value, kNoCompression}}; + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + /*column_family_id=*/1, /*blob_file_number=*/304, records, + &metadata, &file_size); + EXPECT_GT(file_size, 0U); +} + +TEST_F(BlobFileReaderTest, RelocationFileForcesContextChecksums) { + // A relocation file built from a CF with format_version < 6 still uses + // context checksums, so its record checksums depend on the record offset. + BlockBasedTableOptions newer; + newer.format_version = 7; + ASSERT_EQ(MakeBlobRelocationTableOptions(newer).format_version, 7U); + + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_RelocationContextChecksums"), + 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + table_options.format_version = 5; + table_options.checksum = kCRC32c; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ASSERT_EQ(MakeBlobRelocationTableOptions(table_options).format_version, 6U); + + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint64_t relocation_file_number = 301; + const std::vector records{{/*origin_file_number=*/100, + /*original_value_offset=*/10, + "value", kNoCompression}}; + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + ASSERT_NO_FATAL_FAILURE( + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + /*column_family_id=*/1, relocation_file_number, + records, &metadata, &file_size)); + + const std::string path = BlobFileName(immutable_options.cf_paths.front().path, + relocation_file_number); + std::unique_ptr file; + ASSERT_OK(immutable_options.fs->NewRandomAccessFile(path, FileOptions(), + &file, nullptr)); + RandomAccessFileReader reader(std::move(file), path); + Footer footer; + ASSERT_OK(ReadFooterFromFile(IOOptions(), &reader, *immutable_options.fs, + /*prefetch_buffer=*/nullptr, file_size, + &footer)); + EXPECT_GE(footer.format_version(), 6U); + EXPECT_NE(footer.base_context_checksum(), 0U); +} + +TEST_F(BlobFileReaderTest, RelocationFileMultiOriginCompressedRoundTrip) { + if (!CompressionTypeSupported(kZSTD)) { + ROCKSDB_GTEST_SKIP("ZSTD is required for this test"); + return; + } + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath( + mock_env_.get(), + "BlobFileReaderTest_RelocationFileMultiOriginCompressedRoundTrip"), + 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + table_options.format_version = 7; + table_options.checksum = kCRC32c; + table_options.block_cache = NewLRUCache(1 << 20); + options.blob_cache = NewLRUCache(1 << 20); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t relocation_file_number = 300; + constexpr uint64_t first_origin = 100; + constexpr uint64_t second_origin = 200; + const std::string first_value = "uncompressed-value"; + const std::string second_value(4096, 'z'); + const std::string third_value = "other-origin"; + + std::unique_ptr compressor = + GetBuiltinV2CompressionManager()->GetCompressor({}, kZSTD); + GrowableBuffer compressed_second_buffer; + ASSERT_OK(LegacyForceBuiltinCompression(*compressor, /*working_area=*/nullptr, + second_value, + &compressed_second_buffer)); + const std::string compressed_second = + compressed_second_buffer.AsSlice().ToString(); + // Values of one origin do not overlap: the second one starts after the end + // of the first one, with a 40-byte gap like a record header and a key. + const uint64_t second_offset = 10 + first_value.size() + 40; + const std::vector records{ + {first_origin, 10, first_value, kNoCompression}, + {first_origin, second_offset, compressed_second, kZSTD}, + {second_origin, 7, third_value, kNoCompression}}; + + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + column_family_id, relocation_file_number, records, + &metadata, &file_size); + const BlobRelocationDirectory directory = DecodeDirectory(metadata); + ASSERT_EQ(directory.origins().size(), 2U); + ASSERT_EQ(directory.total_entries(), records.size()); + ASSERT_EQ(directory.total_record_bytes(), + first_value.size() + compressed_second.size() + third_value.size() + + records.size() * kSimpleGen2BlobTrailerSize); + const BlobPhysicalFile physical_file = + RelocationPhysicalFile(relocation_file_number, file_size, metadata); + + std::unique_ptr reader; + ReadOptions read_options; + read_options.verify_checksums = true; + ASSERT_OK(OpenRelocationReader( + immutable_options, read_options, FileOptions(), column_family_id, + physical_file, /*db_session_id=*/"reader", table_options, + mutable_cf_options.compression_manager.get(), &reader)); + ASSERT_NE(reader, nullptr); + ASSERT_NE(reader->relocation(), nullptr); + + const auto open_invalid = [&](uint32_t cf_id, const BlobPhysicalFile& file, + const std::string& db_session_id) { + std::unique_ptr invalid_reader; + return OpenRelocationReader(immutable_options, read_options, FileOptions(), + cf_id, file, db_session_id, table_options, + mutable_cf_options.compression_manager.get(), + &invalid_reader); + }; + ASSERT_TRUE( + open_invalid(column_family_id, physical_file, /*db_session_id=*/"") + .IsInvalidArgument()); + BlobPhysicalFile wrong_digest = physical_file; + wrong_digest.directory_digest ^= 1; + ASSERT_TRUE( + open_invalid(column_family_id, wrong_digest, "reader").IsCorruption()); + BlobPhysicalFile wrong_size = physical_file; + ++wrong_size.file_size; + ASSERT_TRUE( + open_invalid(column_family_id, wrong_size, "reader").IsCorruption()); + ASSERT_TRUE(open_invalid(column_family_id + 1, physical_file, "reader") + .IsCorruption()); + + // A directory that names a served file or an origin that is not older + // than the file itself is rejected at open, even with valid checksums. + const auto open_crafted = [&](const std::function& edit) { + std::string crafted = metadata.encoded_directory; + edit(&crafted); + constexpr size_t kDigestedHeaderOffset = 8; + constexpr size_t kDirectoryDigestOffset = 40; + constexpr size_t kHeaderChecksumOffset = 44; + constexpr size_t kDirectoryHeaderSize = 48; + const uint32_t header_crc = + crc32c::Value(crafted.data() + kDigestedHeaderOffset, + kDirectoryDigestOffset - kDigestedHeaderOffset); + EncodeFixed32( + crafted.data() + kDirectoryDigestOffset, + crc32c::Extend(header_crc, crafted.data() + kDirectoryHeaderSize, + crafted.size() - kDirectoryHeaderSize)); + EncodeFixed32(crafted.data() + kHeaderChecksumOffset, + crc32c::Value(crafted.data(), kHeaderChecksumOffset)); + BlobRelocationDirectory crafted_directory; + Status status = + BlobRelocationDirectory::DecodeFrom(crafted, &crafted_directory); + if (!status.ok()) { + return status; + } + SyncPoint::GetInstance()->SetCallBack( + "BlobRelocationFileReader::Open:EncodedDirectory", + [&](void* arg) { *static_cast(arg) = crafted; }); + SyncPoint::GetInstance()->EnableProcessing(); + BlobPhysicalFile crafted_file = physical_file; + crafted_file.directory_digest = crafted_directory.directory_digest(); + status = open_invalid(column_family_id, crafted_file, "reader"); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + return status; + }; + { + // The last served file is the file itself. + const Status status = open_crafted([&](std::string* crafted) { + EncodeFixed64(crafted->data() + crafted->size() - sizeof(uint64_t), + relocation_file_number); + }); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("not older"), std::string::npos) + << status.ToString(); + } + { + // The last origin is the file itself. + const Status status = open_crafted([&](std::string* crafted) { + constexpr size_t kOriginCountOffset = 8; + constexpr size_t kDirectoryHeaderSize = 48; + constexpr size_t kOriginEntrySize = 40; + const uint64_t origin_count = + DecodeFixed64(crafted->data() + kOriginCountOffset); + EncodeFixed64(crafted->data() + kDirectoryHeaderSize + + (origin_count - 1) * kOriginEntrySize, + relocation_file_number); + }); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("not older"), std::string::npos) + << status.ToString(); + } + + // The stored CRC always covers `stored_value`; `stored_size` may differ to + // build invalid indexes. + const auto make_index = [](uint64_t origin, uint64_t original_offset, + uint64_t stored_size, CompressionType compression, + const Slice& logical_value, + const Slice& stored_value) { + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = original_offset, + .stored_value_size = stored_size, + .compression = compression, + .stable_checksum = + crc32c::Value(logical_value.data(), logical_value.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(stored_value, compression)}); + BlobIndex index; + EXPECT_OK(index.DecodeFrom(encoded)); + return index; + }; + const BlobIndex first_index = + make_index(first_origin, 10, first_value.size(), kNoCompression, + first_value, first_value); + const BlobIndex second_index = + make_index(first_origin, second_offset, compressed_second.size(), kZSTD, + second_value, compressed_second); + const BlobIndex third_index = + make_index(second_origin, 7, third_value.size(), kNoCompression, + third_value, third_value); + + // The reader keeps only the directory totals and digest. + EXPECT_EQ(reader->relocation()->total_entries(), records.size()); + EXPECT_EQ(reader->relocation()->total_record_bytes(), + directory.total_record_bytes()); + EXPECT_EQ(reader->relocation()->directory_digest(), + directory.directory_digest()); + // A value of an origin the file does not hold (below, between and above its + // origins), or a missing offset of a held origin: the key lookup misses. + for (const auto& missing : std::vector>{ + {50, 10}, {150, 10}, {250, 10}, {first_origin, 15}}) { + SCOPED_TRACE(std::to_string(missing.first) + ":" + + std::to_string(missing.second)); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, relocation_file_number, + {.origin_file_number = missing.first, + .original_value_offset = missing.second, + .stored_value_size = first_value.size(), + .compression = kNoCompression, + .stable_checksum = + crc32c::Value(first_value.data(), first_value.size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(first_value, kNoCompression)}); + BlobIndex missing_index; + ASSERT_OK(missing_index.DecodeFrom(encoded)); + std::unique_ptr missing_value; + uint64_t missing_bytes = 0; + const Status status = GetFromRelocationFile( + *reader, read_options, missing_index, /*range_offset=*/0, + kWholeBlobLength, /*allocator=*/nullptr, &missing_value, + &missing_bytes); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("entry is missing"), std::string::npos) + << status.ToString(); + } + + std::array stored_payloads; + std::array stored_statuses; + const auto stored_request = + [](uint64_t origin, uint64_t original_offset, const Slice& stored_value, + CompressionType compression, const Slice& logical_value, + Slice* payload, Status* status) { + return StoredRelocationFileReadRequest{ + origin, + original_offset, + stored_value.size(), + compression, + crc32c::Value(logical_value.data(), logical_value.size()), + payload, + status, + BlobIndex::ComputeStoredChecksum(stored_value, compression)}; + }; + std::array stored_requests{ + stored_request(first_origin, 10, first_value, kNoCompression, first_value, + &stored_payloads[0], &stored_statuses[0]), + stored_request(first_origin, second_offset, compressed_second, kZSTD, + second_value, &stored_payloads[1], &stored_statuses[1]), + stored_request(second_origin, 7, third_value, kNoCompression, third_value, + &stored_payloads[2], &stored_statuses[2])}; + // A wrong stored checksum fails only that request. + stored_requests[2].stored_checksum ^= 1; + uint64_t stored_bytes_read = 0; + CoalescedReadBuffers stored_buffers; + reader->relocation()->MultiGetStoredBlobs( + read_options, /*coalesce_bytes=*/4096, stored_requests.size(), + stored_requests.data(), /*max_stored_value_size=*/1 << 20, + &stored_buffers, &stored_bytes_read); + ASSERT_OK(stored_statuses[0]); + ASSERT_OK(stored_statuses[1]); + ASSERT_TRUE(stored_statuses[2].IsCorruption()); + EXPECT_EQ(stored_payloads[0], first_value); + EXPECT_EQ(stored_payloads[1], Slice(compressed_second)); + EXPECT_TRUE(stored_payloads[2].empty()); + EXPECT_EQ(stored_bytes_read, first_value.size() + compressed_second.size() + + 2 * kSimpleGen2BlobTrailerSize); + + stored_requests[2].stored_checksum ^= 1; + for (Slice& payload : stored_payloads) { + payload.clear(); + } + stored_requests[1].stored_payload = nullptr; + stored_bytes_read = 0; + CoalescedReadBuffers validate_buffers; + reader->relocation()->MultiGetStoredBlobs( + read_options, /*coalesce_bytes=*/4096, stored_requests.size(), + stored_requests.data(), /*max_stored_value_size=*/1 << 20, + &validate_buffers, &stored_bytes_read); + ASSERT_OK(stored_statuses[0]); + ASSERT_OK(stored_statuses[1]); + ASSERT_OK(stored_statuses[2]); + EXPECT_EQ(stored_payloads[0], first_value); + EXPECT_TRUE(stored_payloads[1].empty()); + EXPECT_EQ(stored_payloads[2], third_value); + EXPECT_EQ(stored_bytes_read, directory.total_record_bytes()); + stored_requests[1].stored_payload = &stored_payloads[1]; + + std::unique_ptr value; + uint64_t bytes_read = 0; + ASSERT_OK(GetFromRelocationFile(*reader, read_options, first_index, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, &bytes_read)); + EXPECT_EQ(value->data(), first_value); + EXPECT_EQ(bytes_read, first_value.size() + kSimpleGen2BlobTrailerSize); + + value.reset(); + ASSERT_OK(GetFromRelocationFile(*reader, read_options, first_index, + /*range_offset=*/2, /*range_length=*/5, + /*allocator=*/nullptr, &value, &bytes_read)); + EXPECT_EQ(value->data(), Slice(first_value.data() + 2, 5)); + EXPECT_EQ(bytes_read, 5U); + + // As in base RocksDB, a range of a compressed value is not supported: a + // byte range cannot be decompressed alone. A whole read of the value works. + value.reset(); + EXPECT_TRUE(GetFromRelocationFile(*reader, read_options, second_index, + /*range_offset=*/3, /*range_length=*/7, + /*allocator=*/nullptr, &value, &bytes_read) + .IsCorruption()); + value.reset(); + ASSERT_OK(GetFromRelocationFile(*reader, read_options, second_index, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, &bytes_read)); + EXPECT_EQ(value->data(), second_value); + EXPECT_EQ(bytes_read, compressed_second.size() + kSimpleGen2BlobTrailerSize); + + value.reset(); + EXPECT_TRUE(GetFromRelocationFile( + *reader, read_options, first_index, + /*range_offset=*/first_value.size(), + /*range_length=*/std::numeric_limits::max() - 1, + /*allocator=*/nullptr, &value, &bytes_read) + .IsInvalidArgument()); + + const BlobIndex unsupported_index = + make_index(first_origin, 10, first_value.size(), kFirstCustomCompression, + first_value, first_value); + EXPECT_TRUE(GetFromRelocationFile(*reader, read_options, unsupported_index, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, &bytes_read) + .IsCorruption()); + const BlobIndex oversized_index = + make_index(first_origin, 10, kMaxSimpleGen2BlobPayloadSize + 1, + kNoCompression, first_value, first_value); + EXPECT_TRUE(GetFromRelocationFile(*reader, read_options, oversized_index, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, &bytes_read) + .IsCorruption()); + + std::array invalid_outputs; + std::array invalid_statuses; + std::array invalid_requests{ + MakeRelocationRequest(&first_index, first_value.size(), + std::numeric_limits::max() - 1, + &invalid_statuses[0], &invalid_outputs[0]), + MakeRelocationRequest(&unsupported_index, 0, kWholeBlobLength, + &invalid_statuses[1], &invalid_outputs[1]), + MakeRelocationRequest(&oversized_index, 0, kWholeBlobLength, + &invalid_statuses[2], &invalid_outputs[2])}; + bytes_read = std::numeric_limits::max(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + invalid_requests.size(), + invalid_requests.data(), &bytes_read); + EXPECT_TRUE(invalid_statuses[0].IsInvalidArgument()); + EXPECT_TRUE(invalid_statuses[1].IsCorruption()); + EXPECT_TRUE(invalid_statuses[2].IsCorruption()); + EXPECT_EQ(bytes_read, 0U); + + std::array outputs; + std::array statuses; + std::array requests; + const std::array indexes{&third_index, &first_index, + &second_index}; + for (size_t i = 0; i < requests.size(); ++i) { + requests[i].blob_index = indexes[i]; + requests[i].status = &statuses[i]; + requests[i].output = &outputs[i]; + } + std::atomic relocation_multiread_calls{0}; + std::atomic relocation_multiread_requests{0}; + SyncPoint::GetInstance()->SetCallBack( + "MultiReadGen2:Requests", [&](void* arg) { + const std::vector* const aligned_reqs = + static_cast*>(arg); + ++relocation_multiread_calls; + relocation_multiread_requests += aligned_reqs->size(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + requests.size(), requests.data(), + &bytes_read); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(statuses[0]); + ASSERT_OK(statuses[1]); + ASSERT_OK(statuses[2]); + EXPECT_EQ(outputs[0].contents->data(), third_value); + EXPECT_EQ(outputs[1].contents->data(), first_value); + EXPECT_EQ(outputs[2].contents->data(), second_value); + EXPECT_EQ(bytes_read, directory.total_record_bytes()); + EXPECT_EQ(relocation_multiread_calls.load(), 1U); + EXPECT_EQ(relocation_multiread_requests.load(), requests.size()); + + std::array duplicate_outputs; + std::array duplicate_statuses; + std::array duplicate_requests{ + MakeRelocationRequest(&first_index, 0, kWholeBlobLength, + &duplicate_statuses[0], &duplicate_outputs[0]), + MakeRelocationRequest(&first_index, 0, kWholeBlobLength, + &duplicate_statuses[1], &duplicate_outputs[1])}; + std::atomic duplicate_multiread_calls{0}; + ExpectSortedNonOverlappingReads("MultiReadGen2:Requests", + &duplicate_multiread_calls); + SyncPoint::GetInstance()->EnableProcessing(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + duplicate_requests.size(), + duplicate_requests.data(), &bytes_read); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(duplicate_statuses[0]); + ASSERT_OK(duplicate_statuses[1]); + EXPECT_EQ(duplicate_outputs[0].contents->data(), first_value); + EXPECT_EQ(duplicate_outputs[1].contents->data(), first_value); + // An exact duplicate is read once and shared, and counted once. + EXPECT_EQ(duplicate_multiread_calls.load(), 1U); + EXPECT_EQ(duplicate_outputs[0].gen2.copied + duplicate_outputs[1].gen2.copied, + 1); + EXPECT_EQ(bytes_read, first_value.size() + kSimpleGen2BlobTrailerSize); + + std::array overlapping_outputs; + std::array overlapping_statuses; + std::array overlapping_requests{ + MakeRelocationRequest(&first_index, 0, 5, &overlapping_statuses[0], + &overlapping_outputs[0]), + MakeRelocationRequest(&first_index, 3, 5, &overlapping_statuses[1], + &overlapping_outputs[1])}; + std::atomic overlapping_multiread_calls{0}; + ExpectSortedNonOverlappingReads("MultiReadGen2:Requests", + &overlapping_multiread_calls); + SyncPoint::GetInstance()->EnableProcessing(); + reader->relocation()->MultiGetBlob(read_options, /*allocator=*/nullptr, + overlapping_requests.size(), + overlapping_requests.data(), &bytes_read); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(overlapping_statuses[0]); + ASSERT_OK(overlapping_statuses[1]); + EXPECT_EQ(overlapping_outputs[0].contents->data(), + Slice(first_value.data(), 5)); + EXPECT_EQ(overlapping_outputs[1].contents->data(), + Slice(first_value.data() + 3, 5)); + EXPECT_EQ(overlapping_multiread_calls.load(), 2U); + + for (SameFileBlobStandaloneRead& overlapping_output : overlapping_outputs) { + overlapping_output.contents.reset(); + } + overlapping_statuses = {}; + ReadOptions deadline_options(read_options); + deadline_options.deadline = + std::chrono::microseconds(mock_env_->NowMicros() + 1000); + EmulatedSystemClock* const mock_clock = + static_cast_with_check( + mock_env_->GetSystemClock().get()); + std::atomic deadline_multiread_calls{0}; + SyncPoint::GetInstance()->SetCallBack( + "MultiReadGen2:Requests", [&](void* /*arg*/) { + if (++deadline_multiread_calls == 1) { + mock_clock->MockSleepForMicroseconds(2000); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + reader->relocation()->MultiGetBlob(deadline_options, /*allocator=*/nullptr, + overlapping_requests.size(), + overlapping_requests.data(), &bytes_read); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(overlapping_statuses[0]); + EXPECT_TRUE(overlapping_statuses[1].IsTimedOut()); + EXPECT_EQ(overlapping_outputs[0].contents->data(), + Slice(first_value.data(), 5)); + EXPECT_EQ(overlapping_outputs[1].contents, nullptr); + EXPECT_EQ(deadline_multiread_calls.load(), 1U); + + const BlobIndex missing = + make_index(/*origin=*/999, /*original_offset=*/1, first_value.size(), + kNoCompression, first_value, first_value); + value.reset(); + EXPECT_TRUE(GetFromRelocationFile(*reader, read_options, missing, + /*range_offset=*/0, kWholeBlobLength, + /*allocator=*/nullptr, &value, + /*bytes_read=*/nullptr) + .IsCorruption()); + + std::shared_ptr file_cache = NewLRUCache(1024); + FileOptions file_options; + const std::string db_id = "relocation-test-db"; + const std::string db_session_id = "relocation-test-session"; + BlobFileCache blob_file_cache( + file_cache.get(), &immutable_options, &file_options, column_family_id, + /*blob_file_read_hist=*/nullptr, /*io_tracer=*/nullptr, &table_options, + db_session_id, /*block_cache_tracer=*/nullptr); + BlobSource blob_source(immutable_options, mutable_cf_options, db_id, + db_session_id, &blob_file_cache); + { + // A duplicate in a BlobSource batch is read once and counted once. + ReadOptions no_fill = read_options; + no_fill.fill_cache = false; + std::array dup_values; + std::array dup_statuses; + std::array dup_requests; + for (size_t i = 0; i < dup_requests.size(); ++i) { + dup_requests[i].user_key = "unused"; + dup_requests[i].blob_index = &first_index; + dup_requests[i].result = &dup_values[i]; + dup_requests[i].status = &dup_statuses[i]; + } + blob_source.MultiGetIndirectBlobFromOneFile( + no_fill, physical_file, mutable_cf_options, dup_requests.size(), + dup_requests.data(), &bytes_read); + ASSERT_OK(dup_statuses[0]); + ASSERT_OK(dup_statuses[1]); + EXPECT_EQ(dup_values[0], first_value); + EXPECT_EQ(dup_values[1], first_value); + EXPECT_EQ(bytes_read, first_value.size() + kSimpleGen2BlobTrailerSize); + } + std::array source_values; + std::array source_statuses; + std::array source_requests; + const std::array source_indexes{ + &second_index, &third_index, &first_index}; + for (size_t i = 0; i < source_requests.size(); ++i) { + source_requests[i].user_key = "unused"; + source_requests[i].blob_index = source_indexes[i]; + source_requests[i].result = &source_values[i]; + source_requests[i].status = &source_statuses[i]; + } + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, physical_file, mutable_cf_options, source_requests.size(), + source_requests.data(), &bytes_read); + ASSERT_OK(source_statuses[0]); + ASSERT_OK(source_statuses[1]); + ASSERT_OK(source_statuses[2]); + EXPECT_EQ(source_values[0], second_value); + EXPECT_EQ(source_values[1], third_value); + EXPECT_EQ(source_values[2], first_value); + EXPECT_EQ(bytes_read, directory.total_record_bytes()); + + { + // A cache without a session id cannot open relocation files. + BlobFileCache no_session_cache( + file_cache.get(), &immutable_options, &file_options, column_family_id, + /*blob_file_read_hist=*/nullptr, /*io_tracer=*/nullptr, &table_options, + /*db_session_id=*/"", /*block_cache_tracer=*/nullptr); + // BlobFileCache::Evict() is for normal blob files only. + TableCache::Evict(file_cache.get(), relocation_file_number); + CacheHandleGuard invalid_cached_reader; + EXPECT_TRUE(no_session_cache + .GetRelocationFileReader(read_options, physical_file, + mutable_cf_options, + &invalid_cached_reader) + .IsInvalidArgument()); + } + + for (PinnableSlice& source_value : source_values) { + source_value.Reset(); + } + read_options.read_tier = kBlockCacheTier; + BlobPhysicalFile unopened_file = physical_file; + unopened_file.open_info.file_number = 999; + unopened_file.directory_digest ^= 1; + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, unopened_file, mutable_cf_options, source_requests.size(), + source_requests.data(), &bytes_read); + ASSERT_OK(source_statuses[0]); + ASSERT_OK(source_statuses[1]); + ASSERT_OK(source_statuses[2]); + EXPECT_EQ(source_values[0], second_value); + EXPECT_EQ(source_values[1], third_value); + EXPECT_EQ(source_values[2], first_value); + EXPECT_EQ(bytes_read, 0U); +} + namespace { // Returns a stale path-level size for one target blob file while leaving the @@ -381,6 +3139,22 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) { total_size += blob_sizes[i]; } ASSERT_EQ(bytes_read, total_size); + + bytes_read = 0; + Status stored_status; + StoredNormalBlobReadRequest stored_request{ + blob_offsets[0], + blob_sizes[0], + kNoCompression, + /*stored_payload=*/nullptr, + &stored_status, + BlobIndex::ComputeStoredChecksum(blob_strs[0], kNoCompression)}; + CoalescedReadBuffers buffers; + reader->MultiGetStoredBlobsForRelocation( + read_options, /*coalesce_bytes=*/0, 1, &stored_request, + /*max_stored_value_size=*/1 << 20, &buffers, &bytes_read); + ASSERT_OK(stored_status); + EXPECT_EQ(bytes_read, blob_sizes[0]); } read_options.verify_checksums = true; @@ -591,6 +3365,414 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) { } } +TEST_F(BlobFileReaderTest, + MultiGetStoredBlobsForRelocationPreservesCompressedBytes) { + if (!CompressionTypeSupported(kZSTD)) { + ROCKSDB_GTEST_SKIP("ZSTD is required for this test"); + return; + } + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath( + mock_env_.get(), + "BlobFileReaderTest_MultiGetStoredBlobsForRelocation"), + 0); + options.enable_blob_files = true; + const ImmutableOptions immutable_options(options); + + constexpr uint32_t column_family_id = 1; + constexpr uint64_t blob_file_number = 1; + constexpr ExpirationRange expiration_range; + const std::array key_strings{{"key1", "key2", "key3"}}; + const std::array blob_strings{ + {std::string(4096, 'a'), std::string(4096, 'b'), std::string(4096, 'c')}}; + const std::vector keys{key_strings[0], key_strings[1], key_strings[2]}; + const std::vector blobs{blob_strings[0], blob_strings[1], + blob_strings[2]}; + std::vector blob_offsets(keys.size()); + std::vector blob_sizes(keys.size()); + WriteBlobFile(immutable_options, column_family_id, + /*has_ttl=*/false, expiration_range, expiration_range, + blob_file_number, keys, blobs, kZSTD, blob_offsets, blob_sizes); + + std::unique_ptr reader; + const ReadOptions read_options; + ASSERT_OK(BlobFileReader::Create( + immutable_options, read_options, FileOptions(), column_family_id, + /*blob_file_read_hist=*/nullptr, blob_file_number, nullptr /*IOTracer*/, + &reader)); + + // Read the raw file to get the stored (compressed) bytes of each value. The + // stored CRC covers these exact bytes. + std::string file_contents; + ASSERT_OK(ReadFileToString( + mock_env_.get(), + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number), + &file_contents)); + + std::array stored_payloads; + std::array statuses; + std::array requests; + for (size_t i = 0; i < requests.size(); ++i) { + const Slice stored(file_contents.data() + blob_offsets[i], blob_sizes[i]); + requests[i] = StoredNormalBlobReadRequest{ + blob_offsets[i], blob_sizes[i], + kZSTD, &stored_payloads[i], + &statuses[i], BlobIndex::ComputeStoredChecksum(stored, kZSTD)}; + } + ++requests.back().stored_checksum; + + uint64_t bytes_read = 0; + CoalescedReadBuffers buffers; + reader->MultiGetStoredBlobsForRelocation( + read_options, /*coalesce_bytes=*/0, requests.size(), requests.data(), + /*max_stored_value_size=*/1 << 20, &buffers, &bytes_read); + ASSERT_OK(statuses[0]); + ASSERT_OK(statuses[1]); + ASSERT_TRUE(statuses[2].IsCorruption()); + EXPECT_EQ(stored_payloads[0].size(), blob_sizes[0]); + EXPECT_EQ(stored_payloads[1].size(), blob_sizes[1]); + EXPECT_TRUE(stored_payloads[2].empty()); + EXPECT_NE(stored_payloads[0], blob_strings[0]); + EXPECT_EQ(stored_payloads[0], + file_contents.substr(blob_offsets[0], blob_sizes[0])); + EXPECT_EQ(bytes_read, blob_sizes[0] + blob_sizes[1]); + --requests.back().stored_checksum; + + // The gap between two values is the next record's header and key. When + // coalesce_bytes is at least the gap, the three values come back from one + // read. With a smaller coalesce_bytes, or 0, each value takes its own read. + // The returned bytes are the same in each case. + SetPerfLevel(PerfLevel::kEnableCount); + for (const auto& [coalesce_bytes, expected_reads] : + std::vector>{{0, 3}, {1, 3}, {4096, 1}}) { + SCOPED_TRACE(coalesce_bytes); + get_perf_context()->Reset(); + CoalescedReadBuffers coalesced_buffers; + reader->MultiGetStoredBlobsForRelocation( + read_options, coalesce_bytes, requests.size(), requests.data(), + /*max_stored_value_size=*/1 << 20, &coalesced_buffers, &bytes_read); + EXPECT_EQ(get_perf_context()->blob_read_count, expected_reads); + for (size_t i = 0; i < requests.size(); ++i) { + ASSERT_OK(statuses[i]); + EXPECT_EQ(stored_payloads[i], + file_contents.substr(blob_offsets[i], blob_sizes[i])); + } + } + // One read is at most max_stored_value_size bytes. Here that is the size of + // the first two values without their gap, so each value takes its own read. + get_perf_context()->Reset(); + CoalescedReadBuffers bounded_buffers; + reader->MultiGetStoredBlobsForRelocation( + read_options, /*coalesce_bytes=*/4096, requests.size(), requests.data(), + /*max_stored_value_size=*/blob_sizes[0] + blob_sizes[1], &bounded_buffers, + &bytes_read); + EXPECT_EQ(get_perf_context()->blob_read_count, 3U); + SetPerfLevel(PerfLevel::kDisable); + + for (Slice& payload : stored_payloads) { + payload.clear(); + } + requests[1].stored_payload = nullptr; + bytes_read = 0; + CoalescedReadBuffers validate_buffers; + reader->MultiGetStoredBlobsForRelocation( + read_options, /*coalesce_bytes=*/0, requests.size(), requests.data(), + /*max_stored_value_size=*/1 << 20, &validate_buffers, &bytes_read); + for (const Status& status : statuses) { + ASSERT_OK(status); + } + EXPECT_FALSE(stored_payloads[0].empty()); + EXPECT_TRUE(stored_payloads[1].empty()); + EXPECT_FALSE(stored_payloads[2].empty()); + EXPECT_EQ(bytes_read, blob_sizes[0] + blob_sizes[1] + blob_sizes[2]); + + requests[1].stored_payload = &stored_payloads[1]; + for (Slice& payload : stored_payloads) { + payload.clear(); + } + bytes_read = 0; + CoalescedReadBuffers limited_buffers; + reader->MultiGetStoredBlobsForRelocation( + read_options, /*coalesce_bytes=*/0, requests.size(), requests.data(), + /*max_stored_value_size=*/1, &limited_buffers, &bytes_read); + for (size_t i = 0; i < requests.size(); ++i) { + EXPECT_TRUE(statuses[i].IsMemoryLimit()); + EXPECT_TRUE(stored_payloads[i].empty()); + } + EXPECT_EQ(bytes_read, 0U); +} + +TEST_F(BlobFileReaderTest, StoredRelocationReadDoesNotDecompress) { + if (!CompressionTypeSupported(kZSTD)) { + ROCKSDB_GTEST_SKIP("ZSTD is required for this test"); + return; + } + Options options; + options.env = mock_env_.get(); + options.enable_blob_files = true; + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), "StoredRelocationRead"), 0); + const ImmutableOptions immutable_options(options); + const std::string key = "key"; + const std::string logical(4096, 'x'); + GrowableBuffer compressed; + std::unique_ptr compressor = + GetBuiltinV2CompressionManager()->GetCompressor({}, kZSTD); + ASSERT_OK(LegacyForceBuiltinCompression(*compressor, nullptr, logical, + &compressed)); + uint64_t offset = 0; + uint64_t size = 0; + const ExpirationRange expiration_range; + WriteBlobFile(immutable_options, 0, false, expiration_range, expiration_range, + 1, key, logical, kZSTD, &offset, &size); + std::unique_ptr reader; + ASSERT_OK(BlobFileReader::Create(immutable_options, ReadOptions(), + FileOptions(), 0, nullptr, 1, nullptr, + &reader)); + const uint32_t logical_crc = crc32c::Value(logical.data(), logical.size()); + const uint32_t stored_crc = + BlobIndex::ComputeStoredChecksum(compressed.AsSlice(), kZSTD); + size_t decompressions = 0; + SyncPoint::GetInstance()->SetCallBack( + "UncompressBlobContentsIfNeeded:TamperWithResult", + [&](void*) { ++decompressions; }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + Slice stored; + CoalescedReadBuffers stored_buffers; + const auto read_stored = [&](uint32_t expected_stored_crc) { + Status status; + StoredNormalBlobReadRequest request{offset, size, kZSTD, + &stored, &status, expected_stored_crc}; + stored_buffers = CoalescedReadBuffers(); + reader->MultiGetStoredBlobsForRelocation( + ReadOptions(), /*coalesce_bytes=*/0, 1, &request, + /*max_stored_value_size=*/size, &stored_buffers, + /*bytes_read=*/nullptr); + return status; + }; + ASSERT_OK(read_stored(stored_crc)); + ASSERT_EQ(Slice(stored), Slice(compressed)); + ASSERT_EQ(decompressions, 0U); + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 1, + .original_value_offset = offset, + .stored_value_size = size, + .compression = kZSTD, + .stable_checksum = logical_crc, + .stored_checksum = stored_crc}); + BlobIndex index; + ASSERT_OK(index.DecodeFrom(encoded)); + std::string streamed; + ASSERT_OK(reader->StreamStoredBlobForRelocation( + ReadOptions(), index, [&](const Slice& chunk) { + EXPECT_LE(chunk.size(), kBlobRelocationStreamChunkSize); + streamed.append(chunk.data(), chunk.size()); + return Status::OK(); + })); + ASSERT_EQ(streamed, stored); + ASSERT_EQ(decompressions, 0U); + ASSERT_TRUE(read_stored(stored_crc ^ 1).IsCorruption()); + ASSERT_EQ(decompressions, 0U); +} + +TEST(BlobStoredStreamTest, ChecksLengthAndPreservesSourceFailures) { + const std::string payload = "stored payload"; + const uint32_t checksum = + BlobIndex::ComputeStoredChecksum(payload, kNoCompression); + for (int mode = 0; mode < 4; ++mode) { + SCOPED_TRACE(mode); + std::unique_ptr sink = + std::make_unique(); + test::StringSink* const sink_ptr = sink.get(); + WritableFileWriter writer(std::move(sink), "stream", FileOptions()); + const Status result = WriteSimpleGen2BlobRecordStream( + &writer, WriteOptions(), kCRC32c, /*base_context_checksum=*/123, + /*record_offset=*/0, payload.size(), kNoCompression, checksum, + [&](const BlobChunkConsumer& consume) { + if (mode == 0) { + return consume(payload); + } + if (mode == 1) { + return consume(Slice(payload.data(), payload.size() - 1)); + } + if (mode == 2) { + Status status = consume(payload + "extra"); + EXPECT_TRUE(status.IsCorruption()); + // A buggy source cannot swallow a failed consumer and publish. + return Status::OK(); + } + Status status = consume(payload); + return status.ok() ? Status::IOError("injected source I/O") : status; + }); + if (mode == 0) { + ASSERT_OK(result); + } else if (mode == 3) { + ASSERT_TRUE(result.IsIOError()); + } else { + ASSERT_TRUE(result.IsCorruption()); + } + ASSERT_OK(writer.Flush(IOOptions())); + ASSERT_EQ(sink_ptr->contents().size(), + mode == 0 ? payload.size() + kSimpleGen2BlobTrailerSize + : (mode == 1 ? payload.size() - 1 + : (mode == 2 ? 0 : payload.size()))); + if (mode == 0) { + const Slice trailer(sink_ptr->contents().data() + payload.size(), + kSimpleGen2BlobTrailerSize); + ASSERT_EQ(DecodeFixed32(trailer.data() + 1), + crc32c::Mask(checksum) + ChecksumModifierForContext(123, 0)); + } + ASSERT_OK(writer.Close(IOOptions(), nullptr)); + } +} + +TEST_F(BlobFileReaderTest, CoalescedStoredReadsAreMonotoneAndDeferPastLimit) { + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "BlobFileReaderTest_CoalescedStoredReads"), + 0); + options.enable_blob_files = true; + const ImmutableOptions immutable_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t blob_file_number = 1; + constexpr ExpirationRange expiration_range; + const uint32_t seed = static_cast( + std::chrono::steady_clock::now().time_since_epoch().count()); + SCOPED_TRACE("seed=" + std::to_string(seed)); + Random rnd(seed); + constexpr size_t kValues = 64; + std::vector key_strings(kValues); + std::vector blob_strings(kValues); + std::vector keys(kValues); + std::vector blobs(kValues); + for (size_t i = 0; i < kValues; ++i) { + key_strings[i] = "key" + std::to_string(1000 + i); + blob_strings[i] = rnd.RandomString(50 + rnd.Uniform(3000)); + keys[i] = key_strings[i]; + blobs[i] = blob_strings[i]; + } + std::vector blob_offsets(kValues); + std::vector blob_sizes(kValues); + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, blob_file_number, keys, + blobs, kNoCompression, blob_offsets, blob_sizes); + std::unique_ptr reader; + ASSERT_OK(BlobFileReader::Create( + immutable_options, ReadOptions(), FileOptions(), column_family_id, + /*blob_file_read_hist=*/nullptr, blob_file_number, nullptr /*IOTracer*/, + &reader)); + + // Reads the values in `live` with these limits and stores the read count in + // `reads`. Returns how many values came back before the first deferred one, + // and checks the bytes of each of them. + const auto read_live = [&](const std::vector& live, + uint64_t coalesce_bytes, uint64_t max_read_bytes, + uint64_t* reads) { + std::vector payloads(live.size()); + std::vector statuses(live.size()); + std::vector requests(live.size()); + for (size_t i = 0; i < live.size(); ++i) { + const size_t value = live[i]; + requests[i] = StoredNormalBlobReadRequest{ + blob_offsets[value], + blob_sizes[value], + kNoCompression, + &payloads[i], + &statuses[i], + BlobIndex::ComputeStoredChecksum(blobs[value], kNoCompression)}; + } + SetPerfLevel(PerfLevel::kEnableCount); + get_perf_context()->Reset(); + CoalescedReadBuffers buffers; + reader->MultiGetStoredBlobsForRelocation( + ReadOptions(), coalesce_bytes, requests.size(), requests.data(), + max_read_bytes, &buffers, /*bytes_read=*/nullptr); + *reads = get_perf_context()->blob_read_count; + SetPerfLevel(PerfLevel::kDisable); + size_t done = 0; + while (done < live.size() && statuses[done].ok()) { + EXPECT_EQ(payloads[done], blobs[live[done]]); + ++done; + } + for (size_t i = done; i < live.size(); ++i) { + EXPECT_TRUE(statuses[i].IsIncomplete()) << statuses[i].ToString(); + } + return done; + }; + + // Reads the whole live set as indirect blob GC does: call again from the + // first deferred value until every value is back. Returns the reads made in + // total. + const auto read_all = [&](std::vector live, uint64_t coalesce_bytes, + uint64_t max_read_bytes) { + uint64_t total_reads = 0; + while (!live.empty()) { + uint64_t reads = 0; + const size_t done = + read_live(live, coalesce_bytes, max_read_bytes, &reads); + EXPECT_GT(done, 0U); + if (done == 0) { + break; + } + total_reads += reads; + live.erase(live.begin(), live.begin() + done); + } + return total_reads; + }; + + // For the same live set, a larger coalesce_bytes never needs more reads, also + // when the buffer limit defers reads. + for (int round = 0; round < 20; ++round) { + std::vector live; + for (size_t i = 0; i < kValues; ++i) { + if (rnd.OneIn(2)) { + live.push_back(i); + } + } + if (live.empty()) { + live.push_back(0); + } + for (uint64_t max_read_bytes : {uint64_t{1} << 20, uint64_t{6000}}) { + uint64_t previous_reads = std::numeric_limits::max(); + for (uint64_t coalesce_bytes : {0, 1, 64, 512, 4096, 65536}) { + SCOPED_TRACE(std::to_string(max_read_bytes) + "/" + + std::to_string(coalesce_bytes)); + const uint64_t reads = read_all(live, coalesce_bytes, max_read_bytes); + EXPECT_LE(reads, previous_reads); + previous_reads = reads; + } + } + } + + // Every other value is dead, so each coalesced read also reads a dead gap. + // The buffer holds the requested bytes plus at most max_read_bytes of gaps. + // Reads past that are deferred, and reading the rest again in later calls + // returns every value. + std::vector remaining; + for (size_t i = 0; i < kValues; i += 2) { + remaining.push_back(i); + } + size_t calls = 0; + while (!remaining.empty()) { + uint64_t reads = 0; + const size_t done = read_live(remaining, /*coalesce_bytes=*/4096, + /*max_read_bytes=*/8000, &reads); + ASSERT_GT(done, 0U); + remaining.erase(remaining.begin(), remaining.begin() + done); + ++calls; + } + EXPECT_GT(calls, 1U); +} + // Verifies Create() prefers the file size reported by the opened handle over a // stale path-level size query by returning a readable reader even when the path // stat intentionally reports 0 for the blob file. @@ -1002,6 +4184,57 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) { .IsCorruption()); } +TEST_F(BlobFileReaderTest, RelocationFileOfAnotherColumnFamilyIsRejected) { + // A relocation file records the ID of the column family that wrote it in its + // table properties. The reader compares it with the ID of the column family + // that opens the file, so a wrong MANIFEST record cannot make one column + // family read another column family's relocation file. + Options options; + options.env = mock_env_.get(); + options.cf_paths.emplace_back( + test::PerThreadDBPath(mock_env_.get(), + "RelocationFileOfAnotherColumnFamily"), + 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const MutableCFOptions mutable_cf_options(options); + constexpr uint32_t column_family_id = 1; + constexpr uint32_t other_column_family_id = 2; + constexpr uint64_t relocation_file_number = 400; + const std::string payload = "relocated-value"; + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + column_family_id, relocation_file_number, + {{/*origin_file_number=*/100, + /*original_value_offset=*/10, payload, kNoCompression}}, + &metadata, &file_size); + const BlobPhysicalFile file = + RelocationPhysicalFile(relocation_file_number, file_size, metadata); + + // The column family that wrote the file opens it. + { + std::unique_ptr reader; + ASSERT_OK(OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), column_family_id, file, + /*db_session_id=*/"reader", table_options, + mutable_cf_options.compression_manager.get(), &reader)); + } + // Another column family gets Corruption. + std::unique_ptr reader; + const Status status = OpenRelocationReader( + immutable_options, ReadOptions(), FileOptions(), other_column_family_id, + file, /*db_session_id=*/"reader", table_options, + mutable_cf_options.compression_manager.get(), &reader); + ASSERT_EQ(status.code(), Status::kCorruption) << status.ToString(); + ASSERT_NE(status.ToString().find("Blob relocation column family mismatch"), + std::string::npos) + << status.ToString(); + ASSERT_EQ(reader, nullptr); +} + TEST_F(BlobFileReaderTest, BlobCRCError) { Options options; options.env = mock_env_.get(); @@ -1173,7 +4406,7 @@ TEST_F(BlobFileReaderTest, UncompressionError) { blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader)); SyncPoint::GetInstance()->SetCallBack( - "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) { + "UncompressBlobContentsIfNeeded:TamperWithResult", [](void* arg) { auto* result = static_cast(arg); assert(result); @@ -1464,6 +4697,206 @@ TEST_F(BlobFileReaderTest, MultiGetBlobWithFailedValidation) { ASSERT_EQ(blob_reqs[2].second->data(), blobs[2]); } +TEST(BlobRelocationTracerTest, RelocationReadsReachBlockCacheTrace) { + // BlobFileCache passes the DB's block cache tracer to the relocation file + // readers it opens. So, while a block cache trace runs, the block reads of a + // relocation file are in the trace with the caller kBlobRelocationFileRead: + // cache misses, cache hits, and each block of a batched read of several + // missing blocks. Without a tracer (nullptr), the same read works and + // nothing is traced. + // + // For example, relocation file 700 holds three values of origin blob file + // 100. + constexpr uint32_t kColumnFamilyId = 1; + constexpr uint64_t kFileNumber = 700; + constexpr uint64_t kOrigin = 100; + Env* const env = Env::Default(); + const std::string path = + test::PerThreadDBPath(env, "BlobRelocationTracerTest"); + ASSERT_OK(DestroyDir(env, path)); + ASSERT_OK(env->CreateDirIfMissing(path)); + Options options; + options.env = env; + options.cf_paths.emplace_back(path, 0); + options.enable_blob_files = true; + BlockBasedTableOptions table_options; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(options); + const FileOptions file_options{DBOptions(options)}; + const MutableCFOptions mutable_cf_options(options); + const std::vector payloads{"value-a", "value-b", "value-c"}; + std::vector records; + for (size_t i = 0; i < payloads.size(); ++i) { + records.push_back( + {kOrigin, 10 + i * 100, Slice(payloads[i]), kNoCompression}); + } + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + kColumnFamilyId, kFileNumber, records, &metadata, + &file_size); + const BlobPhysicalFile physical_file = + RelocationPhysicalFile(kFileNumber, file_size, metadata); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, kOrigin, + {.origin_file_number = kOrigin, + .original_value_offset = records[1].original_value_offset, + .stored_value_size = payloads[1].size(), + .compression = kNoCompression, + .stable_checksum = crc32c::Value(payloads[1].data(), payloads[1].size()), + .stored_checksum = + BlobIndex::ComputeStoredChecksum(payloads[1], kNoCompression)}); + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + // Opens `file` through a new BlobFileCache with `tracer`, then reads the + // second value `reads` times. Each call gets a new, empty block cache, so + // the first read misses in the cache and later reads hit. The table reader + // traces only block cache lookups, and the index block is kept in the block + // cache so that its lookup is traced too. With `pin`, the open reads and + // keeps all location blocks of the file + // (pin_blob_relocation_location_blocks). + const auto read = [&](BlockCacheTracer* tracer, const BlobPhysicalFile& file, + int reads, bool pin) { + BlockBasedTableOptions read_table_options = table_options; + read_table_options.block_cache = NewLRUCache(8 << 20); + read_table_options.cache_index_and_filter_blocks = true; + MutableCFOptions read_cf_options = mutable_cf_options; + read_cf_options.pin_blob_relocation_location_blocks = pin; + // The readers get their own cache. A reader holds the table options, + // which hold the block cache, so a reader in the block cache would keep + // that cache alive forever. + std::shared_ptr file_cache = NewLRUCache(1024); + BlobFileCache blob_file_cache( + file_cache.get(), &immutable_options, &file_options, kColumnFamilyId, + /*blob_file_read_hist=*/nullptr, /*io_tracer=*/nullptr, + &read_table_options, /*db_session_id=*/"session", tracer); + CacheHandleGuard reader; + ASSERT_OK(blob_file_cache.GetRelocationFileReader( + ReadOptions(), file, read_cf_options, &reader)); + for (int i = 0; i < reads; ++i) { + std::unique_ptr value; + ASSERT_OK(GetFromRelocationFile(*reader.GetValue(), ReadOptions(), + blob_index, /*range_offset=*/0, + kWholeBlobLength, /*allocator=*/nullptr, + &value, /*bytes_read=*/nullptr)); + ASSERT_EQ(value->data(), payloads[1]); + } + }; + // Runs `fn` with a tracer that traces, and returns the trace records with + // the caller kBlobRelocationFileRead. + const auto traced = [&](const std::function& fn) { + const std::string trace_path = path + "/block_cache_trace"; + std::vector result; + { + std::unique_ptr trace_writer; + EXPECT_OK( + NewFileTraceWriter(env, EnvOptions(), trace_path, &trace_writer)); + BlockCacheTracer tracer; + EXPECT_OK(tracer.StartTrace( + BlockCacheTraceOptions(), + NewBlockCacheTraceWriter(env->GetSystemClock().get(), + BlockCacheTraceWriterOptions(), + std::move(trace_writer)))); + fn(&tracer); + tracer.EndTrace(); + } + std::unique_ptr trace_reader; + EXPECT_OK(NewFileTraceReader(env, EnvOptions(), trace_path, &trace_reader)); + BlockCacheTraceReader reader(std::move(trace_reader)); + BlockCacheTraceHeader header; + EXPECT_OK(reader.ReadHeader(&header)); + BlockCacheTraceRecord record; + while (reader.ReadAccess(&record).ok()) { + if (record.caller == TableReaderCaller::kBlobRelocationFileRead) { + result.push_back(record); + } + } + EXPECT_OK(env->DeleteFile(trace_path)); + return result; + }; + const auto is_data_block = [](const BlockCacheTraceRecord& record) { + return record.block_type == TraceType::kBlockTraceDataBlock; + }; + + // No tracer: the read works. + read(/*tracer=*/nullptr, physical_file, /*reads=*/1, /*pin=*/false); + + // One cold read: the trace gets kBlobRelocationFileRead records. + { + const std::vector records_read = + traced([&](BlockCacheTracer* tracer) { + read(tracer, physical_file, /*reads=*/1, /*pin=*/false); + }); + ASSERT_GT(records_read.size(), 0U); + } + + // A second read of the same value finds the data block in the block cache. + // The cache hit is traced too. + { + const std::vector records_read = + traced([&](BlockCacheTracer* tracer) { + read(tracer, physical_file, /*reads=*/2, /*pin=*/false); + }); + ASSERT_TRUE(std::any_of(records_read.begin(), records_read.end(), + [&](const BlockCacheTraceRecord& record) { + return is_data_block(record) && + record.is_cache_hit; + })); + } + + // Several cold data blocks in one call: an open with pinned location blocks + // reads every data block of the file in batches. Each data block gets one + // trace record, a cache miss. For example, relocation file 701 holds 5000 + // values of origin 100, so its location entries fill more than one 4 KB + // data block. + { + constexpr uint64_t kLargeFileNumber = 701; + std::vector large_payloads; + std::vector large_records; + for (int i = 0; i < 5000; ++i) { + large_payloads.push_back("value-" + std::to_string(1000000 + i)); + } + for (int i = 0; i < 5000; ++i) { + large_records.push_back({kOrigin, static_cast(10 + i * 100), + Slice(large_payloads[i]), kNoCompression}); + } + BlobRelocationFileMetadata large_metadata; + uint64_t large_file_size = 0; + WriteRelocationFile(immutable_options, mutable_cf_options, table_options, + kColumnFamilyId, kLargeFileNumber, large_records, + &large_metadata, &large_file_size); + std::unique_ptr file_reader; + ASSERT_OK(RandomAccessFileReader::Create( + immutable_options.fs, BlobFileName(path, kLargeFileNumber), + FileOptions(), &file_reader, /*dbg=*/nullptr)); + std::unique_ptr properties; + ASSERT_OK(ReadTableProperties( + file_reader.get(), large_file_size, kBlockBasedTableMagicNumber, + immutable_options, ReadOptions(), &properties)); + ASSERT_GE(properties->num_data_blocks, 2U); + // The open alone reads all data blocks, so this case reads no value. + const BlobPhysicalFile large_file = RelocationPhysicalFile( + kLargeFileNumber, large_file_size, large_metadata); + const std::vector records_read = + traced([&](BlockCacheTracer* tracer) { + read(tracer, large_file, /*reads=*/0, /*pin=*/true); + }); + std::set data_blocks; + size_t data_block_records = 0; + for (const BlockCacheTraceRecord& record : records_read) { + if (is_data_block(record)) { + EXPECT_FALSE(record.is_cache_hit); + data_blocks.insert(record.block_key); + ++data_block_records; + } + } + ASSERT_EQ(data_block_records, properties->num_data_blocks); + ASSERT_EQ(data_blocks.size(), properties->num_data_blocks); + } + ASSERT_OK(DestroyDir(env, path)); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/blob/blob_garbage_meter.cc b/db/blob/blob_garbage_meter.cc index 73bd7af15e87..f0ead69f3d09 100644 --- a/db/blob/blob_garbage_meter.cc +++ b/db/blob/blob_garbage_meter.cc @@ -13,11 +13,18 @@ namespace ROCKSDB_NAMESPACE { Status BlobGarbageMeter::ProcessInFlow(const Slice& key, const Slice& value) { - return ProcessFlow(key, value, /*is_inflow=*/true); + return ProcessFlow(key, value, /*is_inflow=*/true, + /*decoded_blob_index=*/nullptr); } Status BlobGarbageMeter::ProcessOutFlow(const Slice& key, const Slice& value) { - return ProcessFlow(key, value, /*is_inflow=*/false); + return ProcessFlow(key, value, /*is_inflow=*/false, + /*decoded_blob_index=*/nullptr); +} + +Status BlobGarbageMeter::ProcessOutFlow(const Slice& key, const Slice& value, + const BlobIndex* decoded_blob_index) { + return ProcessFlow(key, value, /*is_inflow=*/false, decoded_blob_index); } Status BlobGarbageMeter::GetBlobReferenceDetails(const ParsedInternalKey& ikey, @@ -44,10 +51,10 @@ Status BlobGarbageMeter::GetBlobReferenceDetails(const ParsedInternalKey& ikey, return Status::OK(); } -Status BlobGarbageMeter::ParseBlobIndexReference(const ParsedInternalKey& ikey, - const Slice& value, - uint64_t* blob_file_number, - uint64_t* bytes) { +Status BlobGarbageMeter::ParseBlobIndexReference( + const ParsedInternalKey& ikey, const Slice& value, + const BlobIndex* decoded_blob_index, uint64_t* blob_file_number, + uint64_t* bytes) { assert(blob_file_number); assert(*blob_file_number == kInvalidBlobFileNumber); assert(bytes); @@ -57,6 +64,11 @@ Status BlobGarbageMeter::ParseBlobIndexReference(const ParsedInternalKey& ikey, return Status::OK(); } + if (decoded_blob_index != nullptr) { + return GetBlobReferenceDetails(ikey, *decoded_blob_index, blob_file_number, + bytes); + } + BlobIndex blob_index; { @@ -86,7 +98,8 @@ void BlobGarbageMeter::AddFlow(uint64_t blob_file_number, uint64_t bytes, } Status BlobGarbageMeter::ProcessFlow(const Slice& key, const Slice& value, - bool is_inflow) { + bool is_inflow, + const BlobIndex* decoded_blob_index) { ParsedInternalKey ikey; { @@ -99,8 +112,8 @@ Status BlobGarbageMeter::ProcessFlow(const Slice& key, const Slice& value, uint64_t blob_file_number = kInvalidBlobFileNumber; uint64_t bytes = 0; - if (Status s = - ParseBlobIndexReference(ikey, value, &blob_file_number, &bytes); + if (Status s = ParseBlobIndexReference(ikey, value, decoded_blob_index, + &blob_file_number, &bytes); !s.ok()) { return s; } @@ -119,7 +132,7 @@ Status BlobGarbageMeter::ProcessEntityBlobReferences( return Status::OK(); } - return WideColumnSerialization::ForEachBlobFileNumber( + return WideColumnSerialization::ForEachBlobIndex( value, [&](const BlobIndex& blob_index) -> Status { uint64_t file_number = kInvalidBlobFileNumber; uint64_t blob_bytes = 0; diff --git a/db/blob/blob_garbage_meter.h b/db/blob/blob_garbage_meter.h index 1f7ff29194af..a4881530e98b 100644 --- a/db/blob/blob_garbage_meter.h +++ b/db/blob/blob_garbage_meter.h @@ -97,6 +97,10 @@ class BlobGarbageMeter { Status ProcessInFlow(const Slice& key, const Slice& value); Status ProcessOutFlow(const Slice& key, const Slice& value); + // Same as above. `decoded_blob_index`, if not nullptr, is the caller's + // decode of a kTypeBlobIndex `value`. It saves a second decode. + Status ProcessOutFlow(const Slice& key, const Slice& value, + const BlobIndex* decoded_blob_index); void ProcessOutFlow(const BlobGarbageMeter& inflow) { for (const auto& [file_number, flow] : inflow.flows()) { @@ -124,11 +128,13 @@ class BlobGarbageMeter { uint64_t* bytes); static Status ParseBlobIndexReference(const ParsedInternalKey& ikey, const Slice& value, + const BlobIndex* decoded_blob_index, uint64_t* blob_file_number, uint64_t* bytes); void AddFlow(uint64_t blob_file_number, uint64_t bytes, bool is_inflow); - Status ProcessFlow(const Slice& key, const Slice& value, bool is_inflow); + Status ProcessFlow(const Slice& key, const Slice& value, bool is_inflow, + const BlobIndex* decoded_blob_index); Status ProcessEntityBlobReferences(const ParsedInternalKey& ikey, const Slice& value, bool is_inflow); diff --git a/db/blob/blob_garbage_meter_test.cc b/db/blob/blob_garbage_meter_test.cc index 9a4eaa3fc5d4..aa420a6b1c54 100644 --- a/db/blob/blob_garbage_meter_test.cc +++ b/db/blob/blob_garbage_meter_test.cc @@ -134,6 +134,71 @@ TEST(BlobGarbageMeterTest, MeasureGarbage) { } } +// Compaction passes its own decode of each output BlobIndex to +// ProcessOutFlow(). The status and the flows must be the same as when the +// meter decodes the value itself. +TEST(BlobGarbageMeterTest, OutFlowWithDecodedBlobIndex) { + std::string legacy; + BlobIndex::EncodeBlob(&legacy, /*file_number=*/4, /*offset=*/1234, + /*size=*/555, kLZ4Compression); + std::string ttl; + BlobIndex::EncodeBlobTTL(&ttl, /*expiration=*/9876543210, + /*file_number=*/5, /*offset=*/100, /*size=*/200, + kNoCompression); + std::string same_file; + BlobIndex::EncodeBlob(&same_file, kCurrentFileBlobIndexFileNumber, + /*offset=*/10, /*size=*/20, kNoCompression); + std::string unflattened; + BlobIndex::EncodeUnflattenedIndirectBlob(&unflattened, + {.origin_file_number = 20, + .original_value_offset = 200, + .stored_value_size = 60, + .compression = kNoCompression, + .stable_checksum = 1234, + .stored_checksum = 0}); + std::string flattened; + BlobIndex::EncodeIndirectBlob(&flattened, /*generation=*/40, + {.origin_file_number = 30, + .original_value_offset = 300, + .stored_value_size = 70, + .compression = kZSTD, + .stable_checksum = 5678, + .stored_checksum = 8765}); + + const InternalKey key("user_key", /*seq=*/123, kTypeBlobIndex); + for (const std::string& value : + {legacy, ttl, same_file, unflattened, flattened}) { + BlobIndex decoded; + ASSERT_OK(decoded.DecodeFrom(value)); + SCOPED_TRACE(decoded.DebugString(/*output_hex=*/true)); + + BlobGarbageMeter self_decoded; + BlobGarbageMeter caller_decoded; + const Status in_status = self_decoded.ProcessInFlow(key.Encode(), value); + ASSERT_EQ(caller_decoded.ProcessInFlow(key.Encode(), value).ToString(), + in_status.ToString()); + const Status out_status = self_decoded.ProcessOutFlow(key.Encode(), value); + EXPECT_EQ( + caller_decoded.ProcessOutFlow(key.Encode(), value, &decoded).ToString(), + out_status.ToString()); + + ASSERT_EQ(caller_decoded.flows().size(), self_decoded.flows().size()); + for (const auto& [file_number, flow] : self_decoded.flows()) { + SCOPED_TRACE("blob file " + std::to_string(file_number)); + const auto other = caller_decoded.flows().find(file_number); + ASSERT_NE(other, caller_decoded.flows().end()); + EXPECT_EQ(other->second.GetInFlow().GetCount(), + flow.GetInFlow().GetCount()); + EXPECT_EQ(other->second.GetInFlow().GetBytes(), + flow.GetInFlow().GetBytes()); + EXPECT_EQ(other->second.GetOutFlow().GetCount(), + flow.GetOutFlow().GetCount()); + EXPECT_EQ(other->second.GetOutFlow().GetBytes(), + flow.GetOutFlow().GetBytes()); + } + } +} + TEST(BlobGarbageMeterTest, PlainValue) { constexpr char user_key[] = "user_key"; constexpr SequenceNumber seq = 123; diff --git a/db/blob/blob_gen2_format.cc b/db/blob/blob_gen2_format.cc index f43c316e4ad4..3294b9d17dc7 100644 --- a/db/blob/blob_gen2_format.cc +++ b/db/blob/blob_gen2_format.cc @@ -8,14 +8,19 @@ #include #include #include +#include +#include +#include #include #include +#include "db/blob/blob_index.h" #include "file/random_access_file_reader.h" #include "file/writable_file_writer.h" #include "rocksdb/options.h" #include "rocksdb/slice.h" #include "table/format.h" +#include "test_util/sync_point.h" #include "util/cast_util.h" #include "util/coding.h" #include "util/crc32c.h" @@ -34,17 +39,33 @@ namespace { // format, since a SimpleGen2Blob record's trailer uses the same // builtin-checksum scheme; shared by the scalar and batched readers so both // report the same diagnostics. +// +// With kCRC32c and a non-null `payload_crc32c`, a passing check also sets +// *payload_crc32c to the CRC32C of the payload alone. It is computed in the +// same pass, so a caller that checks the logical CRC of an uncompressed value +// does not read the bytes again. Status VerifySimpleGen2BlobChecksum(ChecksumType checksum_type, uint32_t base_context_checksum, const char* record, size_t payload_size, uint64_t record_offset, - const std::string& file_name) { + const std::string& file_name, + std::optional* payload_crc32c) { uint32_t stored = DecodeFixed32(record + payload_size + 1); const uint32_t modifier = ChecksumModifierForContext(base_context_checksum, record_offset); stored -= modifier; - const uint32_t computed = ComputeBuiltinChecksumWithLastByte( - checksum_type, record, payload_size, record[payload_size]); + uint32_t computed = 0; + if (checksum_type == kCRC32c && payload_crc32c != nullptr) { + const uint32_t payload_crc = crc32c::Value(record, payload_size); + computed = crc32c::Mask( + crc32c::Extend(payload_crc, &record[payload_size], /*size=*/1)); + if (stored == computed) { + *payload_crc32c = payload_crc; + } + } else { + computed = ComputeBuiltinChecksumWithLastByte( + checksum_type, record, payload_size, record[payload_size]); + } if (stored == computed) { return Status::OK(); } @@ -115,24 +136,12 @@ Status ReadAndVerifySimpleGen2BlobRecord( if (read_options.verify_checksums) { const Status checksum_status = VerifySimpleGen2BlobChecksum( checksum_type, base_context_checksum, record, payload_size, - record_offset, file->file_name()); + record_offset, file->file_name(), /*payload_crc32c=*/nullptr); if (!checksum_status.ok()) { return checksum_status; } } - // The compression marker matched the blob index above, but SimpleGen2Blob - // payloads are currently written uncompressed only (see - // WriteSimpleGen2BlobRecord), so a record that actually carries a compression - // type is not readable here. This guards a corrupt or forward-incompatible - // record whose (compressed) marker happened to agree with a (compressed) - // index. - if (compression != kNoCompression) { - return Status::Corruption("Blob record compression " + - std::to_string(static_cast(compression)) + - " is not supported in " + file->file_name() + - " at offset " + std::to_string(record_offset)); - } return Status::OK(); } @@ -204,91 +213,172 @@ struct Gen2ReadSlot { size_t len = 0; char* buf = nullptr; Status* status = nullptr; + bool* copied = nullptr; }; -// Issues a single MultiRead for the records described by `slots` and copies -// each result into its buf, setting *slots[i].status. Shared by the -// whole-record and range batch readers; per-record format checks (compression -// marker / checksum) are done by the whole-record caller afterwards from the -// (now populated) buf. +// Reads the ranges described by `slots` and copies each result into its buf, +// setting *slots[i].status. Shared by the whole-record and range batch readers; +// the whole-record caller checks each record's format afterwards from the +// filled buffers. +// 1. Sort the slots by (offset, length). +// 2. Read an exact duplicate range once. The duplicate slot gets a copy of the +// bytes and the status, and *copied is set. +// 3. Put each range into the first batch whose last range ends at or before its +// offset, or into a new batch. Partly overlapping ranges thus go to +// different batches, because FileSystem::MultiRead requires non-overlapping +// requests. +// 4. Issue each batch as one MultiRead, or as one plain Read when the batch has +// a single range. void MultiReadGen2(const ReadOptions& read_options, RandomAccessFileReader* file, std::vector& slots) { assert(file != nullptr); assert(!slots.empty()); - // RandomAccessFileReader::MultiRead requires the requests sorted ascending by - // file offset -- it asserts this, and in direct-I/O mode it aligns and merges - // only consecutive requests, so unsorted input reads the wrong bytes. The - // callers group records by SST but not by offset, so sort here. Each slot - // carries its own output buf/status pointer, so sorting does not disturb - // result routing. + // Sort by (offset, length). Each batch is then in ascending offset order, as + // RandomAccessFileReader::MultiRead requires, and exact duplicates are next + // to each other. std::sort(slots.begin(), slots.end(), [](const Gen2ReadSlot& a, const Gen2ReadSlot& b) { - return a.offset < b.offset; + return std::make_pair(a.offset, a.len) < + std::make_pair(b.offset, b.len); }); - const size_t num = slots.size(); - std::vector read_reqs; - read_reqs.reserve(num); - for (size_t i = 0; i < num; ++i) { - FSReadRequest read_req{}; - read_req.offset = slots[i].offset; - read_req.len = slots[i].len; - read_reqs.emplace_back(std::move(read_req)); - } - - AlignedBuffer direct_io_buffer; - const bool direct_io = file->use_direct_io(); - if (direct_io) { - for (size_t i = 0; i < num; ++i) { - read_reqs[i].scratch = nullptr; + struct PhysicalRead { + uint64_t offset; + size_t len; + size_t slot_index; + }; + std::vector physical_reads; + physical_reads.reserve(slots.size()); + // Exact duplicates, as pairs of (duplicate slot, slot whose read it copies). + std::vector> duplicates; + for (size_t i = 0; i < slots.size(); ++i) { + if (slots[i].len == 0) { + *slots[i].status = Status::OK(); + continue; } - } else { - for (size_t i = 0; i < num; ++i) { - read_reqs[i].scratch = slots[i].buf; + if (slots[i].offset > std::numeric_limits::max() - slots[i].len) { + *slots[i].status = Status::Corruption("Blob read range overflow"); + continue; } + if (!physical_reads.empty() && + physical_reads.back().offset == slots[i].offset && + physical_reads.back().len == slots[i].len) { + duplicates.emplace_back(i, physical_reads.back().slot_index); + continue; + } + physical_reads.push_back(PhysicalRead{slots[i].offset, slots[i].len, i}); } - - IOOptions opts; - IODebugContext dbg; - Status s = file->PrepareIOOptions(read_options, opts, &dbg); - if (s.ok()) { - AlignedBufferAllocationContext direct_io_context{&direct_io_buffer}; - s = file->MultiRead(opts, read_reqs.data(), read_reqs.size(), - &direct_io_context, &dbg); + if (physical_reads.empty()) { + return; } - if (!s.ok()) { - // Batch read failed; s is written to every slot's status below, so these - // per-request FS statuses carry nothing extra. - for (auto& read_req : read_reqs) { - read_req.status.PermitUncheckedError(); + const auto copy_duplicates = [&]() { + for (const auto& [duplicate, source] : duplicates) { + if (slots[duplicate].copied != nullptr) { + *slots[duplicate].copied = true; + } + *slots[duplicate].status = *slots[source].status; + if (slots[duplicate].status->ok()) { + memcpy(slots[duplicate].buf, slots[source].buf, slots[duplicate].len); + } + } + }; + + struct ReadBatch { + uint64_t end_offset; + std::vector read_indexes; + }; + std::vector batches; + for (size_t i = 0; i < physical_reads.size(); ++i) { + const PhysicalRead& read = physical_reads[i]; + size_t batch_index = batches.size(); + for (size_t j = 0; j < batches.size(); ++j) { + if (batches[j].end_offset <= read.offset) { + batch_index = j; + break; + } } - for (size_t i = 0; i < num; ++i) { - *slots[i].status = s; + const uint64_t end_offset = read.offset + read.len; + if (batch_index == batches.size()) { + batches.push_back(ReadBatch{end_offset, {i}}); + } else { + batches[batch_index].end_offset = end_offset; + batches[batch_index].read_indexes.push_back(i); } - return; } - for (size_t i = 0; i < num; ++i) { - FSReadRequest& read_req = read_reqs[i]; - if (read_req.status.ok() && read_req.result.size() != read_req.len) { - read_req.status = IOStatus::Corruption( - "Incomplete blob record read from " + file->file_name() + - " at offset " + std::to_string(read_req.offset) + ": expected " + - std::to_string(read_req.len) + " bytes, got " + - std::to_string(read_req.result.size())); + const bool direct_io = file->use_direct_io(); + for (size_t batch_index = 0; batch_index < batches.size(); ++batch_index) { + const ReadBatch& batch = batches[batch_index]; + IOOptions opts; + IODebugContext dbg; + Status s = file->PrepareIOOptions(read_options, opts, &dbg); + if (!s.ok()) { + for (size_t unissued = batch_index; unissued < batches.size(); + ++unissued) { + for (size_t read_index : batches[unissued].read_indexes) { + const PhysicalRead& read = physical_reads[read_index]; + *slots[read.slot_index].status = s; + } + } + copy_duplicates(); + return; + } + + std::vector read_reqs; + read_reqs.reserve(batch.read_indexes.size()); + for (size_t read_index : batch.read_indexes) { + const PhysicalRead& read = physical_reads[read_index]; + FSReadRequest read_req{}; + read_req.offset = read.offset; + read_req.len = read.len; + read_req.scratch = direct_io ? nullptr : slots[read.slot_index].buf; + read_reqs.emplace_back(std::move(read_req)); } - *slots[i].status = read_req.status; - if (!slots[i].status->ok()) { + + AlignedBuffer direct_io_buffer; + AlignedBufferAllocationContext direct_io_context{&direct_io_buffer}; + TEST_SYNC_POINT_CALLBACK("MultiReadGen2:Requests", &read_reqs); + if (read_reqs.size() == 1) { + // A batch with one range (for example, a Get of one relocated value) uses + // a plain positional read, without the MultiRead submit and wait. + TEST_SYNC_POINT("MultiReadGen2:SingleRead"); + file->Read(opts, &read_reqs[0], &direct_io_context, &dbg); + s = Status::OK(); + } else { + s = file->MultiRead(opts, read_reqs.data(), read_reqs.size(), + &direct_io_context, &dbg); + } + if (!s.ok()) { + for (FSReadRequest& read_req : read_reqs) { + read_req.status.PermitUncheckedError(); + } + for (size_t read_index : batch.read_indexes) { + const PhysicalRead& read = physical_reads[read_index]; + *slots[read.slot_index].status = s; + } continue; } - // With mmap/direct reads the data lands outside the caller's buf; copy it - // in so the caller can rely on buf owning the bytes. - if (read_req.result.data() != slots[i].buf) { - memcpy(slots[i].buf, read_req.result.data(), read_req.len); + + for (size_t i = 0; i < read_reqs.size(); ++i) { + FSReadRequest& read_req = read_reqs[i]; + if (read_req.status.ok() && read_req.result.size() != read_req.len) { + read_req.status = IOStatus::Corruption( + "Incomplete blob record read from " + file->file_name() + + " at offset " + std::to_string(read_req.offset) + ": expected " + + std::to_string(read_req.len) + " bytes, got " + + std::to_string(read_req.result.size())); + } + const PhysicalRead& read = physical_reads[batch.read_indexes[i]]; + Gen2ReadSlot& slot = slots[read.slot_index]; + *slot.status = read_req.status; + if (slot.status->ok() && read_req.result.data() != slot.buf) { + memcpy(slot.buf, read_req.result.data(), read_req.len); + } } } + copy_duplicates(); } } // namespace @@ -309,8 +399,10 @@ void ReadAndVerifySimpleGen2BlobRecords(const ReadOptions& read_options, assert(reqs[i].status != nullptr); assert(reqs[i].record_size == reqs[i].payload_size + kSimpleGen2BlobTrailerSize); - slots.push_back({reqs[i].record_offset, reqs[i].record_size, reqs[i].buf, - reqs[i].status}); + slots.push_back( + {reqs[i].record_offset, reqs[i].record_size, reqs[i].buf, + reqs[i].status, + reqs[i].extras != nullptr ? &reqs[i].extras->copied : nullptr}); } MultiReadGen2(read_options, file, slots); @@ -335,21 +427,13 @@ void ReadAndVerifySimpleGen2BlobRecords(const ReadOptions& read_options, if (read_options.verify_checksums) { *reqs[i].status = VerifySimpleGen2BlobChecksum( checksum_type, base_context_checksum, record, payload_size, - reqs[i].record_offset, file->file_name()); + reqs[i].record_offset, file->file_name(), + reqs[i].extras != nullptr ? &reqs[i].extras->payload_crc32c + : nullptr); if (!reqs[i].status->ok()) { continue; } } - // See ReadAndVerifySimpleGen2BlobRecord: the marker matched the index, but - // only uncompressed SimpleGen2Blob payloads are currently supported. - if (compression != kNoCompression) { - *reqs[i].status = Status::Corruption( - "Blob record compression " + - std::to_string(static_cast(compression)) + - " is not supported in " + file->file_name() + " at offset " + - std::to_string(reqs[i].record_offset)); - continue; - } } } @@ -386,8 +470,10 @@ void ReadSimpleGen2BlobRanges(const ReadOptions& read_options, " at offset " + std::to_string(reqs[i].record_offset)); continue; } - slots.push_back({reqs[i].record_offset + reqs[i].range_offset, - reqs[i].range_length, reqs[i].buf, reqs[i].status}); + slots.push_back( + {reqs[i].record_offset + reqs[i].range_offset, reqs[i].range_length, + reqs[i].buf, reqs[i].status, + reqs[i].extras != nullptr ? &reqs[i].extras->copied : nullptr}); } if (slots.empty()) { @@ -399,6 +485,164 @@ void ReadSimpleGen2BlobRanges(const ReadOptions& read_options, MultiReadGen2(read_options, file, slots); } +void ReadCoalescedRanges(const ReadOptions& read_options, + RandomAccessFileReader* file, uint64_t coalesce_bytes, + uint64_t max_read_bytes, uint64_t max_total_bytes, + size_t num_ranges, const uint64_t* offsets, + const size_t* lengths, Slice* results, + Status* statuses, CoalescedReadBuffers* buffers, + size_t* num_reads, uint64_t* read_bytes) { + assert(file != nullptr); + assert(buffers != nullptr); + if (num_reads != nullptr) { + *num_reads = 0; + } + if (read_bytes != nullptr) { + *read_bytes = 0; + } + // One read covers the valid ranges after the previous read, up to `last`. + struct Read { + uint64_t offset = 0; + uint64_t end = 0; + size_t last = 0; + }; + std::vector reads; + std::vector valid; + valid.reserve(num_ranges); + uint64_t previous_end = 0; + for (size_t i = 0; i < num_ranges; ++i) { + statuses[i] = Status::OK(); + results[i] = Slice(); + if (lengths[i] > std::numeric_limits::max() - offsets[i] || + (!valid.empty() && offsets[i] < previous_end)) { + statuses[i] = Status::Corruption("Overlapping or invalid blob GC read"); + continue; + } + previous_end = offsets[i] + lengths[i]; + valid.push_back(i); + } + // Merging does not depend on max_total_bytes, so a larger coalesce_bytes + // never makes more reads for the same ranges. + for (size_t i : valid) { + const uint64_t end = offsets[i] + lengths[i]; + if (!reads.empty() && coalesce_bytes > 0) { + Read& read = reads.back(); + if (offsets[i] - read.end <= coalesce_bytes && + end - read.offset <= max_read_bytes) { + read.end = end; + read.last = i; + continue; + } + } + reads.push_back(Read{offsets[i], end, i}); + } + if (reads.empty()) { + return; + } + // Issue the reads that fit in max_total_bytes (at least one). The ranges of + // the later reads are deferred: Incomplete, for the caller to read again. A + // direct read also buffers the alignment around its range. + const uint64_t alignment = + file->use_direct_io() ? file->GetRequiredBufferAlignment() : 1; + const auto buffer_bytes = [alignment](const Read& read) { + return Roundup(read.end, alignment) - + TruncateToPageBoundary(alignment, read.offset); + }; + uint64_t total_bytes = reads.front().end - reads.front().offset; + uint64_t total_buffer_bytes = buffer_bytes(reads.front()); + size_t issued = 1; + while (issued < reads.size() && + buffer_bytes(reads[issued]) <= + max_total_bytes - std::min(total_buffer_bytes, max_total_bytes)) { + total_bytes += reads[issued].end - reads[issued].offset; + total_buffer_bytes += buffer_bytes(reads[issued]); + ++issued; + } + for (size_t v = 0; v < valid.size(); ++v) { + if (valid[v] > reads[issued - 1].last) { + statuses[valid[v]] = Status::Incomplete("Blob GC read is deferred"); + } + } + reads.resize(issued); + if (total_bytes > std::numeric_limits::max()) { + for (size_t i : valid) { + statuses[i] = Status::MemoryLimit("Blob GC read batch is too large"); + } + return; + } + + std::vector requests(reads.size()); + if (!file->use_direct_io()) { + buffers->buffer.reset(new char[static_cast(total_bytes)]); + } + size_t position = 0; + for (size_t r = 0; r < reads.size(); ++r) { + requests[r].offset = reads[r].offset; + requests[r].len = static_cast(reads[r].end - reads[r].offset); + requests[r].scratch = + file->use_direct_io() ? nullptr : buffers->buffer.get() + position; + position += requests[r].len; + } + if (num_reads != nullptr) { + *num_reads = requests.size(); + } + if (read_bytes != nullptr) { + *read_bytes = total_bytes; + } + TEST_SYNC_POINT_CALLBACK("ReadCoalescedRanges:Reads", &requests); + IOOptions io_options; + IODebugContext debug_context; + Status status = + file->PrepareIOOptions(read_options, io_options, &debug_context); + if (status.ok()) { + AlignedBufferAllocationContext direct_io_context{ + &buffers->direct_io_buffer}; + status = file->MultiRead(io_options, requests.data(), requests.size(), + &direct_io_context, &debug_context); + } + size_t next_valid = 0; + for (size_t r = 0; r < reads.size(); ++r) { + FSReadRequest& request = requests[r]; + Status read_status = status.ok() ? Status(request.status) : status; + request.status.PermitUncheckedError(); + if (read_status.ok() && request.result.size() != request.len) { + read_status = Status::Corruption("Incomplete blob GC read from " + + file->file_name()); + } + for (; next_valid < valid.size() && valid[next_valid] <= reads[r].last; + ++next_valid) { + const size_t i = valid[next_valid]; + if (!read_status.ok()) { + statuses[i] = read_status; + continue; + } + results[i] = Slice(request.result.data() + (offsets[i] - reads[r].offset), + lengths[i]); + } + } +} + +Status VerifySimpleGen2BlobRecord(ChecksumType checksum_type, + uint32_t base_context_checksum, + const Slice& record, size_t payload_size, + uint64_t record_offset, + CompressionType expected_compression, + const std::string& file_name) { + if (record.size() != payload_size + kSimpleGen2BlobTrailerSize) { + return Status::Corruption("Invalid blob record size in " + file_name); + } + if (static_cast(record[payload_size]) != + expected_compression) { + return Status::Corruption("Blob record compression mismatch in " + + file_name + " at offset " + + std::to_string(record_offset)); + } + return VerifySimpleGen2BlobChecksum(checksum_type, base_context_checksum, + record.data(), payload_size, + record_offset, file_name, + /*payload_crc32c=*/nullptr); +} + IOStatus WriteSimpleGen2BlobRecord(WritableFileWriter* file, const WriteOptions& write_options, ChecksumType checksum_type, @@ -406,9 +650,6 @@ IOStatus WriteSimpleGen2BlobRecord(WritableFileWriter* file, uint64_t record_offset, const Slice& payload, CompressionType compression) { assert(file != nullptr); - // Placeholder for future embedded blob compression support; only - // uncompressed payloads are currently written. - assert(compression == kNoCompression); std::array trailer; trailer[0] = lossless_cast(compression); @@ -431,4 +672,114 @@ IOStatus WriteSimpleGen2BlobRecord(WritableFileWriter* file, return file->Append(Slice(trailer.data(), trailer.size()), opts); } +Status StreamAndVerifyBlobStoredPayload( + const ReadOptions& read_options, RandomAccessFileReader* file, + uint64_t record_offset, uint64_t payload_size, CompressionType compression, + uint32_t stored_checksum, const BlobChunkConsumer& consumer) { + if (file == nullptr || !consumer || + payload_size > std::numeric_limits::max() - record_offset) { + return Status::InvalidArgument("Invalid stored blob stream"); + } + // The chunk buffer (kBlobRelocationStreamChunkSize, 256 KB) is too large for + // the stack, so it is on the heap. Indirect blob GC charges it in its I/O + // workspace (see DBImpl::IndirectBlobGCJob). + std::unique_ptr scratch(new char[kBlobRelocationStreamChunkSize]); + uint64_t position = 0; + uint32_t checksum = 0; + while (position < payload_size) { + const size_t length = lossless_cast(std::min( + kBlobRelocationStreamChunkSize, payload_size - position)); + IOOptions options; + IODebugContext debug_context; + Status status = + file->PrepareIOOptions(read_options, options, &debug_context); + if (!status.ok()) { + return status; + } + Slice chunk; + status = file->Read(record_offset + position, length, options, &chunk, + scratch.get(), &debug_context); + if (!status.ok()) { + return status; + } + if (chunk.size() != length) { + return Status::Corruption("Incomplete stored blob stream"); + } + TEST_SYNC_POINT_CALLBACK("StreamAndVerifyBlobStoredPayload:Chunk", &chunk); + if (chunk.size() != length) { + return Status::Corruption("Changed stored blob stream length"); + } + checksum = crc32c::Extend(checksum, chunk.data(), chunk.size()); + status = consumer(chunk); + if (!status.ok()) { + return status; + } + position += length; + } + if (BlobIndex::ExtendChecksumWithCompression(checksum, compression) != + stored_checksum) { + return Status::Corruption("Stored blob stream checksum mismatch"); + } + return Status::OK(); +} + +Status WriteSimpleGen2BlobRecordStream( + WritableFileWriter* file, const WriteOptions& write_options, + ChecksumType checksum_type, uint32_t base_context_checksum, + uint64_t record_offset, uint64_t payload_size, CompressionType compression, + uint32_t stored_checksum, const VerifiedBlobStoredSource& source) { + if (file == nullptr || !source || + (checksum_type != kCRC32c && checksum_type != kNoChecksum) || + payload_size > kMaxSimpleGen2BlobPayloadSize || + record_offset > std::numeric_limits::max() - payload_size - + kSimpleGen2BlobTrailerSize) { + return Status::InvalidArgument("Invalid streaming blob output"); + } + uint64_t delivered = 0; + Status consumer_status; + Status status = source([&](const Slice& chunk) { + if (!consumer_status.ok()) { + return consumer_status; + } + if (chunk.size() > payload_size - delivered) { + consumer_status = Status::Corruption("Stored blob stream is too long"); + return consumer_status; + } + IOOptions options; + consumer_status = + WritableFileWriter::PrepareIOOptions(write_options, options); + if (consumer_status.ok()) { + consumer_status = file->Append(chunk, options); + } + if (consumer_status.ok()) { + delivered += chunk.size(); + } + return consumer_status; + }); + // If the write failed, return its error even when the source returned OK, so + // a partial output is never published. + if (!consumer_status.ok()) { + status.PermitUncheckedError(); + return consumer_status; + } + if (!status.ok()) { + return status; + } + if (delivered != payload_size) { + return Status::Corruption("Stored blob stream is too short"); + } + std::array trailer; + trailer[0] = lossless_cast(compression); + const uint32_t checksum = + checksum_type == kCRC32c ? crc32c::Mask(stored_checksum) : 0; + EncodeFixed32(trailer.data() + 1, + checksum + ChecksumModifierForContext(base_context_checksum, + record_offset)); + IOOptions options; + status = WritableFileWriter::PrepareIOOptions(write_options, options); + return status.ok() ? Status(file->Append( + Slice(trailer.data(), trailer.size()), options)) + : status; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_gen2_format.h b/db/blob/blob_gen2_format.h index 0ca267f03c38..a5f8788d7ec7 100644 --- a/db/blob/blob_gen2_format.h +++ b/db/blob/blob_gen2_format.h @@ -7,12 +7,18 @@ #include #include +#include +#include +#include +#include +#include #include "cache/cache_key.h" #include "rocksdb/compression_type.h" #include "rocksdb/io_status.h" #include "rocksdb/rocksdb_namespace.h" #include "rocksdb/status.h" +#include "util/aligned_buffer.h" namespace ROCKSDB_NAMESPACE { @@ -33,7 +39,10 @@ enum ChecksumType : char; // context-modified by the record's absolute file offset (see // ChecksumModifierForContext / ComputeBuiltinChecksumWithLastByte in // table/format.h) so that an identical payload at a different offset gets a -// different stored checksum. Payloads are currently uncompressed only. +// different stored checksum. The payload holds the stored bytes as given by the +// writer: raw value bytes, or value bytes compressed with the type in the +// compression marker. The read functions in this file return the stored bytes. +// The caller decompresses them when the marker is not kNoCompression. // // This format is deliberately decoupled from any particular file type: the same // record shape is read out of SST files (embedded blobs) and is intended to be @@ -42,6 +51,33 @@ enum ChecksumType : char; // type + base context checksum), both passed in by the caller. inline constexpr size_t kSimpleGen2BlobTrailerSize = 5; +// Chunk size for streaming one stored value: 256 KB. The value is a choice, not +// a measurement. Indirect blob GC streams only a value larger than one read +// batch (kIndirectBlobGCReadBatchBytes, 1 MB), so a streamed value takes a few +// chunk reads instead of many small ones. The cost is memory: each streamed +// value uses one chunk buffer, and indirect blob GC charges two chunks in its +// I/O workspace. Memory use still does not depend on the value size. +inline constexpr size_t kBlobRelocationStreamChunkSize = 256 * 1024; + +// Receives each chunk of a streamed value. The Slice is valid only during the +// call. The consumer's output is valid only if the stream returns OK. +using BlobChunkConsumer = std::function; + +// A source streams one stored payload to a consumer in chunks. When it returns +// OK, it has delivered exactly the declared payload and checked its stored CRC +// (payload plus compression byte). When the consumer returns an error, the +// source must stop at once and keep no chunk Slice. +using VerifiedBlobStoredSource = + std::function; +// Largest payload size a SimpleGen2Blob record may have. It is UINT32_MAX, the +// 32-bit value size limit of the table format, or less when size_t cannot hold +// the payload plus the trailer. Writers and readers reject larger sizes. +inline constexpr uint64_t kMaxSimpleGen2BlobPayloadSize = + std::numeric_limits::max() - kSimpleGen2BlobTrailerSize < + std::numeric_limits::max() + ? std::numeric_limits::max() - kSimpleGen2BlobTrailerSize + : std::numeric_limits::max(); + // Cache key for the SimpleGen2Blob record at byte `record_offset` within a file // whose base cache key is `base_cache_key`. Delegates to the shared // OffsetableCacheKey::WithOffsetForMinSizeRecord scheme -- the exact same @@ -63,10 +99,10 @@ inline CacheKey GetSimpleGen2BlobCacheKey( // `checksum_type` and `base_context_checksum` are the file's checksum context // (e.g. from the SST footer). `expected_compression` is the compression type // the caller expects the record to carry (from the blob index); it must match -// the record's marker, and currently must be kNoCompression. +// the record's marker. // -// On success, buf[0, payload_size) holds the verified, uncompressed payload -// (the trailer remains at the tail of `buf` and can be ignored). +// On success, buf[0, payload_size) holds the verified stored payload. The +// caller decompresses it when the marker is not kNoCompression. Status ReadAndVerifySimpleGen2BlobRecord( const ReadOptions& read_options, RandomAccessFileReader* file, uint64_t record_offset, size_t payload_size, size_t record_size, @@ -91,6 +127,21 @@ Status ReadSimpleGen2BlobRange(const ReadOptions& read_options, uint64_t range_offset, size_t range_length, CompressionType expected_compression, char* buf); +// Optional outputs of one batched SimpleGen2 read (a record or a range +// request). The caller owns one per request and points the request's `extras` +// at it; the reader only writes it. +struct SimpleGen2ReadExtras { + // Record reads only: set to the CRC32C of the payload when the record + // checksum was checked with kCRC32c and passed. It is computed in the same + // pass as the check, so the bytes are not read a second time. For an + // uncompressed payload this is the CRC32C of the value. Left unset otherwise. + std::optional payload_crc32c; + // Set to true when this request is an exact duplicate of another request in + // the batch and got a copy of its bytes instead of its own read, so callers + // count read statistics once. + bool copied = false; +}; + // One whole-record request in a batched SimpleGen2Blob read (see // ReadAndVerifySimpleGen2BlobRecords). Mirrors the scalar // ReadAndVerifySimpleGen2BlobRecord inputs, with the read bytes going into @@ -103,6 +154,8 @@ struct SimpleGen2RecordReadRequest { CompressionType expected_compression = kNoCompression; char* buf = nullptr; Status* status = nullptr; + // Optional outputs; may be null. + SimpleGen2ReadExtras* extras = nullptr; }; // One sub-range request in a batched SimpleGen2Blob range read (see @@ -115,13 +168,21 @@ struct SimpleGen2RangeReadRequest { CompressionType expected_compression = kNoCompression; char* buf = nullptr; Status* status = nullptr; + // Optional outputs (only `copied` is used); may be null. + SimpleGen2ReadExtras* extras = nullptr; }; // Batch counterpart of ReadAndVerifySimpleGen2BlobRecord: reads `num_records` -// whole records from `file` in a single MultiRead, then per record verifies the -// compression marker and (when read_options.verify_checksums) the -// context-modified checksum, copying the record into reqs[i].buf. Per-record -// outcome is written to *reqs[i].status; on success reqs[i].buf[0, +// whole records from `file` into reqs[i].buf, then checks each record's +// compression marker and, when read_options.verify_checksums is set, its +// context-modified checksum. The reads are grouped as follows: +// - An exact duplicate request (same offset and size) is not read again. It +// gets a copy of the other request's bytes and sets extras->copied. +// - Overlapping records that are not exact duplicates go to separate groups, +// because MultiRead requires non-overlapping requests. +// - Each group is one MultiRead call, or one plain Read when it holds a single +// record. +// Per-record outcome is written to *reqs[i].status; on success reqs[i].buf[0, // payload_size) holds the verified payload (the trailer sits unused at the // tail). Requests may be given in any order; they are sorted internally, as // RandomAccessFileReader::MultiRead requires ascending offsets. @@ -133,15 +194,54 @@ void ReadAndVerifySimpleGen2BlobRecords(const ReadOptions& read_options, SimpleGen2RecordReadRequest* reqs); // Batch counterpart of ReadSimpleGen2BlobRange: reads the requested sub-range -// of `num_records` *uncompressed* payloads from `file` in a single MultiRead -// (no trailer read, no checksum). Per-record outcome is in *reqs[i].status; on -// success reqs[i].buf[0, range_length) holds the requested bytes. Requests may -// be given in any order; they are sorted internally, as +// of `num_records` *uncompressed* payloads from `file`, with no trailer read +// and no checksum check. The reads are grouped the same way as in +// ReadAndVerifySimpleGen2BlobRecords. Per-record outcome is in *reqs[i].status; +// on success reqs[i].buf[0, range_length) holds the requested bytes. Requests +// may be given in any order; they are sorted internally, as // RandomAccessFileReader::MultiRead requires ascending offsets. void ReadSimpleGen2BlobRanges(const ReadOptions& read_options, RandomAccessFileReader* file, size_t num_records, SimpleGen2RangeReadRequest* reqs); +// Holds the bytes that ReadCoalescedRanges() results point into. +struct CoalescedReadBuffers { + std::unique_ptr buffer; + AlignedBuffer direct_io_buffer; +}; + +// Reads `num_ranges` byte ranges of `file` with one MultiRead. The ranges are +// sorted by offset. +// - Merging: with `coalesce_bytes` > 0, a range whose gap (the bytes between +// it and the end of the previous read) is at most `coalesce_bytes` joins +// that read, while the read stays at most `max_read_bytes`. +// - Buffer limit: reads are issued in offset order while their buffer bytes +// (direct-I/O alignment included) stay at most `max_total_bytes`. The first +// read is always issued. The ranges of later reads get Incomplete, and the +// caller reads them in a later call. +// - Errors: a range that overlaps an earlier one fails with Corruption. If +// the bytes of the issued reads do not fit in size_t, every range that did +// not fail with Corruption gets MemoryLimit. +// On success results[i] points into `buffers`. statuses[i] is the outcome of +// range i. `num_reads` and `read_bytes`, when set, receive the number of reads +// issued and their bytes (merged gaps included). +void ReadCoalescedRanges(const ReadOptions& read_options, + RandomAccessFileReader* file, uint64_t coalesce_bytes, + uint64_t max_read_bytes, uint64_t max_total_bytes, + size_t num_ranges, const uint64_t* offsets, + const size_t* lengths, Slice* results, + Status* statuses, CoalescedReadBuffers* buffers, + size_t* num_reads, uint64_t* read_bytes); + +// Checks one whole SimpleGen2Blob record already read into memory: its size, +// its compression marker and its context-modified record checksum. +Status VerifySimpleGen2BlobRecord(ChecksumType checksum_type, + uint32_t base_context_checksum, + const Slice& record, size_t payload_size, + uint64_t record_offset, + CompressionType expected_compression, + const std::string& file_name); + // Writes a SimpleGen2Blob record for `payload` at the current end of `file`, // which the caller asserts is byte offset `record_offset`. Appends the payload // bytes followed by the 5-byte trailer (compression marker + context-modified @@ -149,8 +249,9 @@ void ReadSimpleGen2BlobRanges(const ReadOptions& read_options, // record format lives in one module. // // `checksum_type` and `base_context_checksum` are the file's checksum context -// (e.g. from the SST footer). `compression` is the payload's compression type, -// which currently must be kNoCompression. +// (e.g. from the SST footer). `payload` is already in stored form, and +// `compression` is the type it was compressed with (kNoCompression for raw +// bytes). This function writes it as is and never compresses it. IOStatus WriteSimpleGen2BlobRecord(WritableFileWriter* file, const WriteOptions& write_options, ChecksumType checksum_type, @@ -158,4 +259,27 @@ IOStatus WriteSimpleGen2BlobRecord(WritableFileWriter* file, uint64_t record_offset, const Slice& payload, CompressionType compression); +// Reads one stored payload in chunks of kBlobRelocationStreamChunkSize (256 +// KB), so memory use does not depend on the value size, and checks the +// stored CRC of the whole payload. The caller checks the file bounds. For a +// relocation file it also checks the record trailer first. The consumer can +// receive bytes before the check ends, so its output must not be published +// unless this function returns OK. +Status StreamAndVerifyBlobStoredPayload( + const ReadOptions& read_options, RandomAccessFileReader* file, + uint64_t record_offset, uint64_t payload_size, CompressionType compression, + uint32_t stored_checksum, const BlobChunkConsumer& consumer); + +// Writes a record from a verified source without holding the whole payload in +// memory. The source has already computed the CRC32C of the stored bytes, so +// the writer only masks it and adds the destination file's checksum context. +// Only kCRC32c and kNoChecksum are supported; other types return +// InvalidArgument. If the source fails, or delivers too few or too many bytes, +// the function returns an error and the partial file must not be published. +Status WriteSimpleGen2BlobRecordStream( + WritableFileWriter* file, const WriteOptions& write_options, + ChecksumType checksum_type, uint32_t base_context_checksum, + uint64_t record_offset, uint64_t payload_size, CompressionType compression, + uint32_t stored_checksum, const VerifiedBlobStoredSource& source); + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_index.h b/db/blob/blob_index.h index bcae4f6712f2..c8de8a40a7d6 100644 --- a/db/blob/blob_index.h +++ b/db/blob/blob_index.h @@ -11,13 +11,38 @@ #include "rocksdb/compression_type.h" #include "util/coding.h" #include "util/compression.h" +#include "util/crc32c.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { +// The fields of an indirect BlobIndex that describe its value bytes: the stored +// size, the compression type, the logical CRC (stable_checksum) and the stored +// CRC (stored_checksum). They never change, even when indirect blob GC moves +// the value. The stable blob cache keeps the fingerprint with each cached value +// (see StableBlobContents), so that a value saved to a secondary cache can be +// checked when it comes back. A cache hit does not compare the cached +// fingerprint with the fingerprint of the BlobIndex (see +// BlobSource::GetIndirectBlobFromCache()). +struct StableBlobFingerprint { + uint64_t stored_value_size = 0; + CompressionType compression = kNoCompression; + uint32_t stable_checksum = 0; + uint32_t stored_checksum = 0; + + bool operator==(const StableBlobFingerprint& other) const { + return stored_value_size == other.stored_value_size && + compression == other.compression && + stable_checksum == other.stable_checksum && + stored_checksum == other.stored_checksum; + } +}; + // BlobIndex is a pointer to the blob and metadata of the blob. The index is // stored in base DB as ValueType::kTypeBlobIndex. -// There are three types of blob index: +// There are five types of blob index. The fields of the two indirect types +// (generation, origin, original offset and the two CRCs) are explained after +// the diagrams. // // kInlinedTTL: // +------+------------+---------------+ @@ -40,15 +65,151 @@ namespace ROCKSDB_NAMESPACE { // | char | varint64 | varint64 | varint64 | varint64 | char | // +------+------------+-------------+----------+----------+-------------+ // +// kIndirectBlob (generation == origin, not stored): +// +------+--------+-----------------+----------+ +// | type | origin | original offset | size | +// +------+--------+-----------------+----------+ +// | char |varint64| varint64 | varint64 | +// +------+--------+-----------------+----------+ +// kIndirectFlattenedBlob (generation > origin): +// +------+------------+--------+-----------------+----------+ +// | type | generation | origin | original offset | size | +// +------+------------+--------+-----------------+----------+ +// | char | varint64 |varint64| varint64 | varint64 | +// +------+------------+--------+-----------------+----------+ +// both, after the size: +// +-------------+-------------+-----------------+ +// | compression | logical CRC | stored CRC (*) | +// +-------------+-------------+-----------------+ +// | char | fixed32 | fixed32 | +// +-------------+-------------+-----------------+ +// +// Background: indirect blob GC must move live values out of a blob file that +// is mostly garbage. GC copies them into a relocation file (see "Relocation +// file" below). A legacy kBlob or kBlobTTL index finds the value only by its +// physical file number and offset, so moving a value would mean rewriting +// every SST that points to it. The two indirect types (kIndirectBlob = 3 and +// kIndirectFlattenedBlob = 4) let an SST keep its index when the value moves. +// An index of one of these types is an "indirect BlobIndex". When +// enable_blob_indirection is on, the blob file builder writes kIndirectBlob +// indexes. +// +// Relocation file: a file into which GC copies live values. A reader finds a +// value in it by its stable identity (origin, original offset; see "Stable +// identity" below). +// +// Stable identity: (origin, original offset). +// - origin: the blob file where the value was first written. +// - original offset: the value's offset in the origin file. +// The stable identity names one immutable value and never changes, even +// when the value moves to another file. A relocation file stores the value +// under this key. +// +// Generation number (or generation): the file number that a reference uses to +// find its value. Type 4 stores it. Type 3 does not store it, because it equals +// the origin. A new value has generation number == origin. When GC moves the +// value, the reference keeps its generation number, and a route (see Route +// below) sends the read to the relocation file. The generation number is only +// a number: the file with that number may be a live blob file, a live +// relocation file, or an old file that has a route. +// - Live file: a blob file or relocation file that GC has not removed. +// - Old file: a file number whose file GC has removed. The number stays +// only as the key of a route, because SSTs may still store it as their +// generation number. +// - Route: a mapping from an old file number to the relocation file that +// now holds its values. Routes never chain, so a read follows at most one +// route. The read then finds the value in that file by its stable +// identity. +// - An index with generation number == origin is an "unflattened" +// reference. +// - Compaction can later rewrite the generation number to the relocation +// file that holds the value ("flatten"), so the reference does not need +// the route any more. The type becomes kIndirectFlattenedBlob; the stable +// identity, size, compression and checksums do not change. Only flatten +// makes the generation differ from the origin. +// +// Format version: the type byte also gives the format version. Types 3 and 4 +// are version 1 (kIndirectBlobFormatVersion), and a later format takes new type +// values. +// +// Why two types: type 3 has no generation field, because its generation equals +// its origin. This is true for every reference until compaction flattens it, so +// most references do not store the same file number twice. Type 4 stores the +// generation, which must be larger than the origin (a relocation file is newer +// than every origin it holds). The decoder rejects type 4 with generation <= +// origin, so each reference has one encoding. +// +// Why generation comes before origin: +// - A read uses the fields in this order. First it uses the generation to +// find the file that holds the value. Then it finds the value in that file +// by (origin, original offset). +// - The per-SST reference metadata sorts references by (encoding class, +// generation, origin, original offset) (see blob_reference_metadata.h). +// The index uses the same order. +// - Origin sits in the middle, so both pairs stay adjacent: (generation, +// origin) follows the encoding class in the group key of the reference +// metadata, and (origin, original offset) is the stable identity. +// - It matches the legacy kBlob layout, where the file the SST points to +// comes first. +// +// Checksums: both are raw CRC32C values (crc32c::Value, not crc32c::Mask, +// unlike the blob log record CRCs), because the stored CRC is derived with +// crc32c::Extend. They exclude the user key and do not change when the value is +// relocated. +// - logical CRC (stable_checksum()): covers the uncompressed value bytes. +// - stored CRC (stored_checksum()): covers the exact stored payload bytes, +// followed by the compression byte. GC uses it to check and copy compressed +// bytes without decompressing them. +// (*) For kNoCompression the stored CRC is not written. It equals the +// logical CRC extended with the compression byte, so the decoder computes +// it. +// +// See the relocation overview in db/blob/blob_relocation_manifest.h. +// // There isn't a kInlined (without TTL) type since we can store it as a plain // value (i.e. ValueType::kTypeValue). class BlobIndex { public: + // The only indirect format, given by the type byte (kIndirectBlob and + // kIndirectFlattenedBlob). Any later change to its fields, including the + // checksums, uses new type values and a new value here. + static constexpr uint32_t kIndirectBlobFormatVersion = 1; + + // The largest valid file number, 2^62 - 1. It equals kFileNumberMask in + // db/version_edit.h, which this header does not include (blob_source.cc + // checks that the two are equal). DecodeFrom() rejects an indirect index + // with a larger file number, so the stable cache key (see + // GetStableBlobCacheKey() in db/blob/blob_source.cc) can use the top two + // bits of a file number for other data. + static constexpr uint64_t kMaxFileNumber = 0x3FFFFFFFFFFFFFFFULL; + + // True for a built-in or custom compression type that a blob may be stored + // with. + static bool IsValidCompression(CompressionType compression) { + return compression <= kLastBuiltinCompression || + (compression >= kFirstCustomCompression && + compression <= kLastCustomCompression); + } + + static uint32_t ExtendChecksumWithCompression(uint32_t payload_checksum, + CompressionType compression) { + const char marker = static_cast(compression); + return crc32c::Extend(payload_checksum, &marker, 1); + } + + static uint32_t ComputeStoredChecksum(const Slice& payload, + CompressionType compression) { + return ExtendChecksumWithCompression( + crc32c::Value(payload.data(), payload.size()), compression); + } + enum class Type : unsigned char { kInlinedTTL = 0, kBlob = 1, kBlobTTL = 2, - kUnknown = 3, + kIndirectBlob = 3, + kIndirectFlattenedBlob = 4, + kUnknown = 5, }; BlobIndex() : type_(Type::kUnknown) {} @@ -62,6 +223,11 @@ class BlobIndex { bool IsInlined() const { return type_ == Type::kInlinedTTL; } + bool IsIndirect() const { + return type_ == Type::kIndirectBlob || + type_ == Type::kIndirectFlattenedBlob; + } + // True for a blob record stored in the same physical file as the table entry. // Only embedded-blob SST reader/writer paths should interpret file number // zero this way; generic blob metadata treats it as invalid. @@ -84,6 +250,10 @@ class BlobIndex { return value_; } + // For an indirect index, file_number() is the generation number and + // offset() is the original offset in the origin file. The two can name + // different files. Use generation_file_number(), origin_file_number() and + // original_value_offset() for indirect indexes. uint64_t file_number() const { assert(!IsInlined()); return file_number_; @@ -104,16 +274,84 @@ class BlobIndex { return compression_; } + uint64_t generation_file_number() const { + assert(IsIndirect()); + return file_number_; + } + + uint64_t origin_file_number() const { + assert(IsIndirect()); + return origin_file_number_; + } + + uint64_t original_value_offset() const { + assert(IsIndirect()); + return offset_; + } + + uint32_t stable_checksum() const { + assert(IsIndirect()); + return stable_checksum_; + } + + // For kNoCompression the encoding has no stored CRC: it is the logical CRC + // extended with the compression byte (ExtendChecksumWithCompression()). It + // is computed here, when a caller asks, so that a decode does not pay for + // it. Most decodes (for example in compaction) never read it. + uint32_t stored_checksum() const { + assert(IsIndirect()); + return compression_ == kNoCompression + ? ExtendChecksumWithCompression(stable_checksum_, compression_) + : stored_checksum_; + } + + StableBlobFingerprint fingerprint() const { + assert(IsIndirect()); + return StableBlobFingerprint{size_, compression_, stable_checksum_, + stored_checksum()}; + } + + // Checks the decoded fields. The type must be known, and then an inlined + // index always passes. Other types need a nonzero file number (except a + // same-file index) and a valid compression type, and an indirect index also + // needs a nonzero origin. DecodeFrom() does not run all of these checks for + // the legacy types, so existing callers that only decode keep their behavior. + // BlobReference::FromBlobIndex() (used by the reference collector) and + // WideColumnSerialization::ForEachBlobIndex() call Validate(). So a malformed + // index fails the metadata build. It is not skipped and not written into the + // metadata. + Status Validate() const { + const char* kErrorMessage = "Invalid blob index"; + if (type_ >= Type::kUnknown) { + return Status::Corruption(kErrorMessage, "Unknown blob index type"); + } + if (IsInlined()) { + return Status::OK(); + } + if (file_number_ == 0 && !IsSameFile()) { + return Status::Corruption(kErrorMessage, "Invalid blob file number"); + } + if (!IsValidCompression(compression_)) { + return Status::Corruption(kErrorMessage, "Invalid blob compression type"); + } + if (IsIndirect() && origin_file_number_ == 0) { + return Status::Corruption(kErrorMessage, + "Invalid indirect blob identity"); + } + return Status::OK(); + } + Status DecodeFrom(Slice slice) { const char* kErrorMessage = "Error while decoding blob index"; if (slice.empty()) { return Status::Corruption(kErrorMessage, "Empty blob index"); } - type_ = static_cast(*slice.data()); + Reset(); + type_ = static_cast(static_cast(slice[0])); if (type_ >= Type::kUnknown) { - return Status::Corruption(kErrorMessage, - "Unknown blob index type: " + - std::to_string(static_cast(type_))); + return Status::Corruption( + kErrorMessage, "Unknown blob index type: " + + std::to_string(static_cast(type_))); } slice = Slice(slice.data() + 1, slice.size() - 1); if (HasTTL()) { @@ -123,13 +361,65 @@ class BlobIndex { } if (IsInlined()) { value_ = slice; + } else if (IsIndirect()) { + const bool flattened = type_ == Type::kIndirectFlattenedBlob; + if ((flattened && !GetCanonicalVarint64(&slice, &file_number_)) || + !GetCanonicalVarint64(&slice, &origin_file_number_) || + !GetCanonicalVarint64(&slice, &offset_) || + !GetCanonicalVarint64(&slice, &size_)) { + return Status::Corruption(kErrorMessage, + "Corrupted indirect blob identity"); + } + if (!flattened) { + file_number_ = origin_file_number_; + } + if (file_number_ == 0 || origin_file_number_ == 0) { + return Status::Corruption(kErrorMessage, + "Invalid indirect blob file number"); + } + if (flattened && file_number_ <= origin_file_number_) { + return Status::Corruption(kErrorMessage, + "Flattened blob generation is not above its " + "origin"); + } + // The generation is at least the origin, so this also bounds the origin. + if (file_number_ > kMaxFileNumber) { + return Status::Corruption(kErrorMessage, + "Indirect blob file number is too large"); + } + if (slice.empty()) { + return Status::Corruption(kErrorMessage, + "Missing indirect blob compression"); + } + compression_ = + static_cast(static_cast(slice[0])); + slice.remove_prefix(1); + if (!IsValidCompression(compression_)) { + return Status::Corruption(kErrorMessage, + "Invalid blob compression type"); + } + if (!GetFixed32(&slice, &stable_checksum_)) { + return Status::Corruption(kErrorMessage, "Corrupted stable checksum"); + } + // For kNoCompression, stored_checksum() derives the stored CRC. + if (compression_ != kNoCompression && + !GetFixed32(&slice, &stored_checksum_)) { + return Status::Corruption(kErrorMessage, "Corrupted stored checksum"); + } + if (!slice.empty()) { + return Status::Corruption(kErrorMessage, + "Trailing indirect blob bytes"); + } } else { - if (GetVarint64(&slice, &file_number_) && GetVarint64(&slice, &offset_) && - GetVarint64(&slice, &size_) && slice.size() == 1) { - compression_ = static_cast(*slice.data()); - } else { + if (!GetVarint64(&slice, &file_number_) || + !GetVarint64(&slice, &offset_) || !GetVarint64(&slice, &size_)) { return Status::Corruption(kErrorMessage, "Corrupted blob offset"); } + if (slice.size() != 1) { + return Status::Corruption(kErrorMessage, "Corrupted blob compression"); + } + compression_ = + static_cast(static_cast(slice[0])); } return Status::OK(); } @@ -140,14 +430,22 @@ class BlobIndex { if (IsInlined()) { oss << "[inlined blob] value:" << value_.ToString(output_hex); } else { - oss << "[blob ref] file:"; + oss << (IsIndirect() ? "[indirect blob ref] generation:" + : "[blob ref] file:"); if (IsSameFile()) { oss << "same"; } else { oss << file_number_; } - oss << " offset:" << offset_ << " size:" << size_ - << " compression: " << CompressionTypeToString(compression_); + if (IsIndirect()) { + oss << " origin:" << origin_file_number_; + } + oss << " offset:" << offset_ << " size:" << size_; + if (IsIndirect()) { + oss << " logical_crc:" << stable_checksum_ + << " stored_crc:" << stored_checksum(); + } + oss << " compression: " << CompressionTypeToString(compression_); } if (HasTTL()) { @@ -164,6 +462,14 @@ class BlobIndex { } else if (HasTTL()) { EncodeBlobTTL(dst, expiration_, file_number_, offset_, size_, compression_); + } else if (IsIndirect()) { + EncodeIndirectBlob(dst, file_number_, + {.origin_file_number = origin_file_number_, + .original_value_offset = offset_, + .stored_value_size = size_, + .compression = compression_, + .stable_checksum = stable_checksum_, + .stored_checksum = stored_checksum()}); } else { EncodeBlob(dst, file_number_, offset_, size_, compression_); } @@ -206,13 +512,81 @@ class BlobIndex { dst->push_back(static_cast(compression)); } + // The fields of an indirect BlobIndex except the generation. Callers set + // them by name, so values of the same type (the file number, offset and size; + // the two CRCs) cannot be swapped by mistake. The generation is a separate + // argument of EncodeIndirectBlob(), so the common unflattened path + // (EncodeUnflattenedIndirectBlob()) cannot leave it unset. + struct IndirectBlobFields { + uint64_t origin_file_number = 0; + uint64_t original_value_offset = 0; + uint64_t stored_value_size = 0; + CompressionType compression = kNoCompression; + uint32_t stable_checksum = 0; + uint32_t stored_checksum = 0; + }; + + static void EncodeIndirectBlob(std::string* dst, + uint64_t generation_file_number, + const IndirectBlobFields& fields) { + const uint64_t origin_file_number = fields.origin_file_number; + const uint64_t original_value_offset = fields.original_value_offset; + const uint64_t stored_value_size = fields.stored_value_size; + const CompressionType compression = fields.compression; + const uint32_t stable_checksum = fields.stable_checksum; + const uint32_t stored_checksum = fields.stored_checksum; + assert(dst != nullptr); + assert(origin_file_number != 0); + assert(generation_file_number >= origin_file_number); + dst->clear(); + dst->reserve(kMaxVarint64Length * 5 + 2 * sizeof(stable_checksum) + 1); + if (generation_file_number == origin_file_number) { + dst->push_back(static_cast(Type::kIndirectBlob)); + } else { + dst->push_back(static_cast(Type::kIndirectFlattenedBlob)); + PutVarint64(dst, generation_file_number); + } + PutVarint64(dst, origin_file_number); + PutVarint64(dst, original_value_offset); + PutVarint64(dst, stored_value_size); + dst->push_back(static_cast(compression)); + PutFixed32(dst, stable_checksum); + if (compression != kNoCompression) { + PutFixed32(dst, stored_checksum); + } + } + + // Encodes an unflattened indirect BlobIndex: its generation is its origin. + static void EncodeUnflattenedIndirectBlob(std::string* dst, + const IndirectBlobFields& fields) { + EncodeIndirectBlob(dst, fields.origin_file_number, fields); + } + private: + void Reset() { + type_ = Type::kUnknown; + expiration_ = 0; + value_.clear(); + file_number_ = 0; + origin_file_number_ = 0; + offset_ = 0; + size_ = 0; + stable_checksum_ = 0; + stored_checksum_ = 0; + compression_ = kNoCompression; + } + Type type_ = Type::kUnknown; uint64_t expiration_ = 0; Slice value_; uint64_t file_number_ = 0; + uint64_t origin_file_number_ = 0; uint64_t offset_ = 0; uint64_t size_ = 0; + uint32_t stable_checksum_ = 0; + // The encoded stored CRC. It stays 0 for kNoCompression; see + // stored_checksum(). + uint32_t stored_checksum_ = 0; CompressionType compression_ = kNoCompression; }; diff --git a/db/blob/blob_log_format.cc b/db/blob/blob_log_format.cc index 8e26281e3745..294b3adfb82d 100644 --- a/db/blob/blob_log_format.cc +++ b/db/blob/blob_log_format.cc @@ -94,6 +94,11 @@ Status BlobLogFooter::DecodeFrom(Slice src) { } void BlobLogRecord::EncodeHeaderTo(std::string* dst) { + EncodeHeaderTo(dst, /*value_crc32c=*/nullptr); +} + +void BlobLogRecord::EncodeHeaderTo(std::string* dst, + const uint32_t* value_crc32c) { assert(dst != nullptr); dst->clear(); dst->reserve(BlobLogRecord::kHeaderSize + key.size() + value.size()); @@ -103,8 +108,13 @@ void BlobLogRecord::EncodeHeaderTo(std::string* dst) { header_crc = crc32c::Value(dst->c_str(), dst->size()); header_crc = crc32c::Mask(header_crc); PutFixed32(dst, header_crc); - blob_crc = crc32c::Value(key.data(), key.size()); - blob_crc = crc32c::Extend(blob_crc, value.data(), value.size()); + if (value_crc32c != nullptr) { + blob_crc = crc32c::Crc32cCombine(crc32c::Value(key.data(), key.size()), + *value_crc32c, value.size()); + } else { + blob_crc = crc32c::Value(key.data(), key.size()); + blob_crc = crc32c::Extend(blob_crc, value.data(), value.size()); + } blob_crc = crc32c::Mask(blob_crc); PutFixed32(dst, blob_crc); } diff --git a/db/blob/blob_log_format.h b/db/blob/blob_log_format.h index 837c83364825..17cb038ff83c 100644 --- a/db/blob/blob_log_format.h +++ b/db/blob/blob_log_format.h @@ -139,8 +139,16 @@ struct BlobLogRecord { uint64_t record_size() const { return kHeaderSize + key_size + value_size; } + // The base signature: EncodeHeaderTo(dst, /*value_crc32c=*/nullptr). void EncodeHeaderTo(std::string* dst); + // Encodes the record header into `dst`. The header ends with the record CRC + // (blob_crc), which covers the key and the value. `value_crc32c` is optional: + // when it is not null, it must be the CRC32C of `value`, already computed by + // the caller. The record CRC then combines it with the key CRC and does not + // read the value again. + void EncodeHeaderTo(std::string* dst, const uint32_t* value_crc32c); + Status DecodeHeaderFrom(Slice src); Status CheckBlobCRC() const; diff --git a/db/blob/blob_log_writer.cc b/db/blob/blob_log_writer.cc index 0fa12ba3a447..df2a35efbfd2 100644 --- a/db/blob/blob_log_writer.cc +++ b/db/blob/blob_log_writer.cc @@ -164,13 +164,34 @@ Status BlobLogWriter::AddRecord(const WriteOptions& write_options, return s; } +Status BlobLogWriter::AddRecordWithValueCrc(const WriteOptions& write_options, + const Slice& key, const Slice& val, + uint32_t value_crc32c, + uint64_t* key_offset, + uint64_t* blob_offset) { + assert(block_offset_ != 0); + assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord); + + std::string buf; + ConstructBlobHeader(&buf, key, val, 0, &value_crc32c); + + return EmitPhysicalRecord(write_options, buf, key, val, key_offset, + blob_offset); +} + void BlobLogWriter::ConstructBlobHeader(std::string* buf, const Slice& key, const Slice& val, uint64_t expiration) { + ConstructBlobHeader(buf, key, val, expiration, /*value_crc32c=*/nullptr); +} + +void BlobLogWriter::ConstructBlobHeader(std::string* buf, const Slice& key, + const Slice& val, uint64_t expiration, + const uint32_t* value_crc32c) { BlobLogRecord record; record.key = key; record.value = val; record.expiration = expiration; - record.EncodeHeaderTo(buf); + record.EncodeHeaderTo(buf, value_crc32c); } Status BlobLogWriter::EmitPhysicalRecord(const WriteOptions& write_options, diff --git a/db/blob/blob_log_writer.h b/db/blob/blob_log_writer.h index 5206c07cc229..19c36699e9d3 100644 --- a/db/blob/blob_log_writer.h +++ b/db/blob/blob_log_writer.h @@ -42,8 +42,14 @@ class BlobLogWriter { ~BlobLogWriter(); + // The base signature: ConstructBlobHeader(..., /*value_crc32c=*/nullptr). static void ConstructBlobHeader(std::string* buf, const Slice& key, const Slice& val, uint64_t expiration); + // `value_crc32c`, if not null, is the CRC32C of `val` (see + // BlobLogRecord::EncodeHeaderTo()). + static void ConstructBlobHeader(std::string* buf, const Slice& key, + const Slice& val, uint64_t expiration, + const uint32_t* value_crc32c); Status AddRecord(const WriteOptions& write_options, const Slice& key, const Slice& val, uint64_t* key_offset, @@ -53,6 +59,14 @@ class BlobLogWriter { const Slice& val, uint64_t expiration, uint64_t* key_offset, uint64_t* blob_offset); + // Same as AddRecord() without expiration, but `value_crc32c` is the CRC32C + // of `val`, already computed by the caller, so `val` is not read again for + // the record CRC. + Status AddRecordWithValueCrc(const WriteOptions& write_options, + const Slice& key, const Slice& val, + uint32_t value_crc32c, uint64_t* key_offset, + uint64_t* blob_offset); + Status EmitPhysicalRecord(const WriteOptions& write_options, const std::string& headerbuf, const Slice& key, const Slice& val, uint64_t* key_offset, diff --git a/db/blob/blob_read_request.h b/db/blob/blob_read_request.h index 95c839e529ef..e62915cd2c9a 100644 --- a/db/blob/blob_read_request.h +++ b/db/blob/blob_read_request.h @@ -6,7 +6,11 @@ #pragma once #include +#include +#include +#include +#include "db/blob/blob_constants.h" #include "rocksdb/compression_type.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" @@ -14,6 +18,10 @@ namespace ROCKSDB_NAMESPACE { +class BlobContents; +class BlobIndex; +struct SameFileBlobStandaloneRead; + // Non-owning metadata needed when opening a blob file for reading. The // checksum slices must remain valid while this object is in use. A file without // finalized checksum metadata, such as an active direct-write blob file, uses @@ -24,6 +32,21 @@ struct BlobFileOpenInfo { Slice file_checksum_func_name; }; +// The physical file that an indirect BlobIndex resolves to: a normal blob +// file, or a relocation file with its expected identity. Non-owning, like +// `open_info`. +struct BlobPhysicalFile { + BlobFileOpenInfo open_info; + uint64_t file_size = 0; + bool is_relocation = false; + // Relocation files only: the expected table tail size (index, properties, + // metaindex and footer; see db/blob/blob_relocation_file.h) and the expected + // directory digest. The code that installs the file records both values. + // BlobRelocationFileReader::Open checks that the file matches them. + uint64_t tail_size = 0; + uint32_t directory_digest = 0; +}; + // A read Blob request structure for use in BlobSource::MultiGetBlob and // BlobFileReader::MultiGetBlob. struct BlobReadRequest { @@ -47,6 +70,11 @@ struct BlobReadRequest { // Status of read Status* status = nullptr; + // Optional: when the record checksum is checked and passes, receives the + // CRC32C of the stored value bytes, computed in the same pass. For an + // uncompressed value, this is the logical CRC of the value. + std::optional* value_crc32c = nullptr; + BlobReadRequest(const Slice& _user_key, uint64_t _offset, size_t _len, CompressionType _compression, PinnableSlice* _result, Status* _status) @@ -113,4 +141,63 @@ using BlobFileRangeReadRequests = std::tuple>; +// One lookup of a value by its stable identity (origin, original offset) in a +// relocation file that the caller has already selected. The BlobIndex gives the +// origin, original offset, stored size and compression. The relocation reader +// does not check the logical CRC. BlobSource checks it after it has built the +// value, when read_options.verify_checksums is set or the value goes into the +// stable blob cache. +struct RelocationBlobReadRequest { + // Indirect BlobIndex to look up; not owned. + const BlobIndex* blob_index = nullptr; + // Requested range; kWholeBlobLength selects the whole value. A range of a + // compressed value is not supported and gets Corruption. + uint64_t range_offset = 0; + size_t range_length = kWholeBlobLength; + // Per-request status; not owned. + Status* status = nullptr; + // Receives the value (or range) as an owned buffer in `output->contents`, and + // the read outputs (see SameFileBlobStandaloneRead); not owned. The reader + // sets `output->allocator` and passes `output` down to the table reader as + // is. + SameFileBlobStandaloneRead* output = nullptr; +}; + +// One read of an indirect BlobIndex (kIndirectBlob or kIndirectFlattenedBlob). +// The caller has already resolved all requests in the batch to the same +// physical file. BlobSource looks up the stable blob cache and sends the reads +// to the normal blob file reader or the relocation file reader of that file. +// It checks the logical CRC when read_options.verify_checksums is set or the +// value goes into the stable blob cache. +struct IndirectBlobReadRequest { + // User key of the record in a normal blob file (used for its record + // header); not used for relocation files. + Slice user_key; + // Indirect BlobIndex to read; not owned. + const BlobIndex* blob_index = nullptr; + // Requested range; kWholeBlobLength selects the whole value. + uint64_t range_offset = 0; + size_t range_length = kWholeBlobLength; + // Output value and per-request status; not owned. + PinnableSlice* result = nullptr; + Status* status = nullptr; + // True when the caller already looked up the stable blob cache for this + // value and missed (see BlobSource::GetIndirectBlobFromCache()), so + // BlobSource does not look it up again. + bool cache_checked = false; +}; + +// The indirect reads of one batch that go to one physical file. It is a named +// type, like BlobFileReadRequests, and not a struct local to +// Version::MultiGetBlobLazy(): gcc 16 merged the vector code of that local +// struct with other vector code in a unity build and then reported a false +// -Wfree-nonheap-object error. +struct BlobFileIndirectReadRequests { + BlobPhysicalFile file; + // BlobSource takes a pointer and a count, so the requests must be + // contiguous. autovector splits its inline and overflow storage, so + // &requests[0] does not point to a contiguous array. + std::vector requests; +}; + } // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_reference_census.cc b/db/blob/blob_reference_census.cc new file mode 100644 index 000000000000..f9651f1b74cc --- /dev/null +++ b/db/blob/blob_reference_census.cc @@ -0,0 +1,2499 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_reference_census.h" + +#include +#include +#include +#include + +#include "db/blob/blob_gen2_format.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/math.h" + +namespace ROCKSDB_NAMESPACE { + +int BlobGcCandidateHeap::CompareRatios(uint64_t lhs_garbage, uint64_t lhs_total, + uint64_t rhs_garbage, + uint64_t rhs_total) { + assert(lhs_total != 0 && rhs_total != 0); + if ((lhs_garbage | lhs_total | rhs_garbage | rhs_total) <= + std::numeric_limits::max()) { + const uint64_t lhs = lhs_garbage * rhs_total; + const uint64_t rhs = rhs_garbage * lhs_total; + return lhs == rhs ? 0 : (lhs < rhs ? -1 : 1); + } + // Large values: compare the two fractions exactly as continued fractions. + // Each round compares the integer parts, then takes the reciprocals of the + // remainders, which reverses the order. Like Euclid's algorithm, this ends in + // O(log(max denominator)) rounds. It needs no floating point, which can + // round. It needs no products, which can overflow. It needs no 128-bit + // integer, which is not portable. + bool reversed = false; + for (;;) { + const uint64_t lhs_integer = lhs_garbage / lhs_total; + const uint64_t rhs_integer = rhs_garbage / rhs_total; + if (lhs_integer != rhs_integer) { + const int result = lhs_integer < rhs_integer ? -1 : 1; + return reversed ? -result : result; + } + lhs_garbage %= lhs_total; + rhs_garbage %= rhs_total; + if (lhs_garbage == 0 || rhs_garbage == 0) { + const int result = + lhs_garbage == rhs_garbage ? 0 : (lhs_garbage == 0 ? -1 : 1); + return reversed ? -result : result; + } + std::swap(lhs_garbage, lhs_total); + std::swap(rhs_garbage, rhs_total); + reversed = !reversed; + } +} + +bool BlobGcCandidateHeap::Before(const Candidate* lhs, const Candidate* rhs) { + const Candidate& left = *lhs; + const Candidate& right = *rhs; + const int order = CompareRatios( + left.total_bytes - std::min(left.live_bytes, left.total_bytes), + left.total_bytes, + right.total_bytes - std::min(right.live_bytes, right.total_bytes), + right.total_bytes); + return order != 0 ? order > 0 : left.file_number < right.file_number; +} + +void BlobGcCandidateHeap::Swap(size_t lhs, size_t rhs) { + std::swap(heap_[lhs], heap_[rhs]); + heap_[lhs]->position = lhs; + heap_[rhs]->position = rhs; +} + +void BlobGcCandidateHeap::Fix(size_t position) { + while (position != 0) { + const size_t parent = (position - 1) / 2; + if (!Before(heap_[position], heap_[parent])) { + break; + } + Swap(position, parent); + position = parent; + } + for (;;) { + const size_t left = position * 2 + 1; + if (left >= heap_.size()) { + break; + } + size_t child = left; + if (left + 1 < heap_.size() && Before(heap_[left + 1], heap_[left])) { + child = left + 1; + } + if (!Before(heap_[child], heap_[position])) { + break; + } + Swap(position, child); + position = child; + } +} + +void BlobGcCandidateHeap::Activate(uint64_t file_number, Candidate* candidate) { + if (candidate->retry_micros != 0) { + retries_.erase({candidate->retry_micros, file_number}); + } + candidate->retry_micros = 0; + candidate->suppressed = false; + // These files stay out of the heap: + // - A file pinned by the lowest coarse frontier of the Version. The coarse + // frontier of an SST is the oldest indirect-origin file number a coarse SST + // may reference (see "Coarse SST" in db/blob/blob_reference_metadata.h). + // - A lost relocation file while references to it remain (live bytes > 0). + // It has no readable values, so indirect blob GC cannot copy from it. When + // no references remain, it enters the heap so that the empty-file cleanup + // can pick it (see SelectEmpty()). + if (candidate->coarse_pinned || + (candidate->is_lost && candidate->live_bytes != 0)) { + Remove(candidate); + return; + } + if (candidate->position == kNotInHeap) { + candidate->position = heap_.size(); + heap_.push_back(candidate); + } + Fix(candidate->position); +} + +void BlobGcCandidateHeap::Remove(Candidate* candidate) { + if (candidate->position == kNotInHeap) { + return; + } + const size_t position = candidate->position; + Swap(position, heap_.size() - 1); + heap_.pop_back(); + candidate->position = kNotInHeap; + if (position < heap_.size()) { + Fix(position); + } +} + +uint64_t BlobGcCandidateHeap::RelocationGarbage(const Candidate& candidate) { + return candidate.is_indirect_origin + ? 0 + : candidate.total_bytes - + std::min(candidate.live_bytes, candidate.total_bytes); +} + +void BlobGcCandidateHeap::AdjustRelocationGarbage(uint64_t before, + uint64_t after) { + if (relocation_garbage_overflow_) { + return; + } + if (before > relocation_garbage_bytes_ || + after > std::numeric_limits::max() - + (relocation_garbage_bytes_ - before)) { + // The sum would overflow or go below zero. Stop tracking it and report 0 + // garbage bytes until the census is rebuilt. Reporting less garbage is + // safe: the space estimate then counts these bytes as live. + relocation_garbage_overflow_ = true; + relocation_garbage_bytes_ = 0; + return; + } + relocation_garbage_bytes_ = relocation_garbage_bytes_ - before + after; +} + +void BlobGcCandidateHeap::AccountTotals(const Candidate& candidate, bool add) { + const uint64_t garbage = + candidate.total_bytes - + std::min(candidate.live_bytes, candidate.total_bytes); + const auto apply = [add](uint64_t* total, uint64_t value) { + *total = add ? *total + value : *total - std::min(*total, value); + }; + if (candidate.suppressed) { + apply(&totals_.suppressed_garbage_bytes, garbage); + } else if (candidate.coarse_pinned) { + apply(&totals_.coarse_pinned_files, 1); + apply(&totals_.coarse_pinned_bytes, candidate.total_bytes); + apply(&totals_.coarse_pinned_garbage_bytes, garbage); + } else { + apply(&totals_.collectable_garbage_bytes, garbage); + } +} + +void BlobGcCandidateHeap::SetPhysicalFile( + const BlobGcPhysicalFileSummary& file) { + assert(file.file_number != 0); + if (file.total_record_bytes == 0) { + DeletePhysicalFile(file.file_number); + return; + } + Candidate& candidate = candidates_[file.file_number]; + candidate.file_number = file.file_number; + if (candidate.total_bytes == file.total_record_bytes && + candidate.is_indirect_origin == file.is_indirect_origin && + candidate.is_lost == file.is_lost) { + return; + } + const uint64_t previous_garbage = RelocationGarbage(candidate); + AccountTotals(candidate, /*add=*/false); + candidate.total_bytes = file.total_record_bytes; + candidate.is_indirect_origin = file.is_indirect_origin; + candidate.is_lost = file.is_lost; + AdjustRelocationGarbage(previous_garbage, RelocationGarbage(candidate)); + candidate.coarse_pinned = + file.is_indirect_origin && coarse_indirect_origin_frontier_ != 0 && + file.file_number >= coarse_indirect_origin_frontier_; + candidate.failures = 0; + candidate.revision = next_revision_++; + if (next_revision_ == 0) { + ++next_revision_; + } + Activate(file.file_number, &candidate); + AccountTotals(candidate, /*add=*/true); +} + +void BlobGcCandidateHeap::SetCoarseIndirectOriginFrontier(uint64_t frontier) { + if (frontier == coarse_indirect_origin_frontier_) { + return; + } + const uint64_t old_boundary = coarse_indirect_origin_frontier_ == 0 + ? std::numeric_limits::max() + : coarse_indirect_origin_frontier_; + const uint64_t new_boundary = + frontier == 0 ? std::numeric_limits::max() : frontier; + coarse_indirect_origin_frontier_ = frontier; + // Visit only the file numbers between the old and the new frontier, whose pin + // state can change. This runs in the background worker, not on each selection + // or install. + for (std::map::iterator entry = + candidates_.lower_bound(std::min(old_boundary, new_boundary)); + entry != candidates_.end() && + entry->first < std::max(old_boundary, new_boundary); + ++entry) { + Candidate& candidate = entry->second; + if (!candidate.is_indirect_origin) { + continue; + } + AccountTotals(candidate, /*add=*/false); + candidate.coarse_pinned = frontier != 0 && entry->first >= frontier; + candidate.revision = next_revision_++; + if (next_revision_ == 0) { + ++next_revision_; + } + Activate(entry->first, &candidate); + AccountTotals(candidate, /*add=*/true); + } + // Indirect blob GC skips (suppresses) a relocation file while a coarse SST + // pins one of the old file numbers routed to it. A routed file number is + // older than its relocation file, so a higher frontier can unpin only the + // routed file numbers of relocation files above old_boundary (the old + // frontier, or UINT64_MAX when there was none). The heap does not store why a + // file was suppressed, so it retries every suppressed relocation file above + // old_boundary. The frontier is expected to change rarely (assumed, not + // measured), so this loop is cheap. + if (new_boundary > old_boundary) { + for (std::map::iterator entry = + candidates_.upper_bound(old_boundary); + entry != candidates_.end(); ++entry) { + Candidate& candidate = entry->second; + if (candidate.is_indirect_origin || !candidate.suppressed) { + continue; + } + AccountTotals(candidate, /*add=*/false); + candidate.revision = next_revision_++; + if (next_revision_ == 0) { + ++next_revision_; + } + Activate(entry->first, &candidate); + AccountTotals(candidate, /*add=*/true); + } + } +} + +Status BlobGcCandidateHeap::AdjustLiveBytes(uint64_t file_number, + uint64_t bytes, bool add) { + const std::map::iterator entry = + candidates_.find(file_number); + if (entry == candidates_.end() || bytes == 0) { + return Status::OK(); + } + Candidate& candidate = entry->second; + if ((add && + bytes > std::numeric_limits::max() - candidate.live_bytes) || + (!add && bytes > candidate.live_bytes)) { + return Status::Corruption("Blob GC candidate live-byte overflow"); + } + const uint64_t previous_garbage = RelocationGarbage(candidate); + AccountTotals(candidate, /*add=*/false); + if (add) { + candidate.live_bytes += bytes; + } else { + candidate.live_bytes -= bytes; + } + AdjustRelocationGarbage(previous_garbage, RelocationGarbage(candidate)); + candidate.failures = 0; + candidate.revision = next_revision_++; + if (next_revision_ == 0) { + ++next_revision_; + } + Activate(file_number, &candidate); + AccountTotals(candidate, /*add=*/true); + return Status::OK(); +} + +void BlobGcCandidateHeap::DeletePhysicalFile(uint64_t file_number) { + const std::map::iterator entry = + candidates_.find(file_number); + if (entry == candidates_.end()) { + return; + } + Candidate& candidate = entry->second; + if (candidate.retry_micros != 0) { + retries_.erase({candidate.retry_micros, file_number}); + } + Remove(&candidate); + AccountTotals(candidate, /*add=*/false); + AdjustRelocationGarbage(RelocationGarbage(candidate), 0); + candidates_.erase(entry); +} + +bool BlobGcCandidateHeap::IsIndirectOrigin(uint64_t file_number) const { + const std::map::const_iterator entry = + candidates_.find(file_number); + return entry != candidates_.end() && entry->second.is_indirect_origin; +} + +uint64_t BlobGcCandidateHeap::Revision(uint64_t file_number) const { + const std::map::const_iterator entry = + candidates_.find(file_number); + return entry != candidates_.end() ? entry->second.revision : 0; +} + +BlobGcCandidateHeap::Selection BlobGcCandidateHeap::Select() const { + Selection selected{}; + if (heap_.empty()) { + return selected; + } + // Pick the best kMaxSelectedFiles (8) files without sorting or scanning every + // file. `frontier` holds the heap positions that can be the next best file: + // start with the root, and after each pick add the children of the picked + // position. `frontier` is a fixed array, so selection does not allocate. Stop + // at the first file whose live bytes are more than half of its total bytes + // (garbage ratio below 50%). For such a file, indirect blob GC would copy + // more live bytes than the garbage bytes it removes. + std::array frontier{}; + size_t frontier_size = 1; + for (size_t output = 0; output < selected.size() && frontier_size != 0; + ++output) { + size_t best = 0; + for (size_t i = 1; i < frontier_size; ++i) { + if (Before(heap_[frontier[i]], heap_[frontier[best]])) { + best = i; + } + } + const size_t position = frontier[best]; + frontier[best] = frontier[--frontier_size]; + const Candidate& candidate = *heap_[position]; + if (candidate.live_bytes > candidate.total_bytes / 2) { + break; + } + selected[output] = candidate.file_number; + for (size_t child = position * 2 + 1; + child < heap_.size() && child <= position * 2 + 2; ++child) { + assert(frontier_size < frontier.size()); + frontier[frontier_size++] = child; + } + } + return selected; +} + +void BlobGcCandidateHeap::Suppress(uint64_t file_number, + uint64_t retry_micros) { + const std::map::iterator entry = + candidates_.find(file_number); + if (entry == candidates_.end()) { + return; + } + Candidate& candidate = entry->second; + if (candidate.retry_micros != 0) { + retries_.erase({candidate.retry_micros, file_number}); + } + Remove(&candidate); + AccountTotals(candidate, /*add=*/false); + candidate.suppressed = true; + candidate.retry_micros = retry_micros; + if (retry_micros != 0) { + retries_.emplace(retry_micros, file_number); + } + AccountTotals(candidate, /*add=*/true); +} + +BlobGcCandidateHeap::Selection BlobGcCandidateHeap::SelectEmpty() const { + Selection result = Select(); + // Keep only the leading files with 0 live bytes. They rank first, because + // their garbage ratio is 100%. This checks only the selected files, not every + // physical file. Live bytes can be too high (see the class comment), so they + // can delay a cleanup but never prove that a file is empty. Indirect blob GC + // checks the exact holders before it acts. + bool found_live = false; + for (uint64_t& file_number : result) { + if (file_number != 0 && !found_live) { + found_live = candidates_.at(file_number).live_bytes != 0; + } + if (found_live) { + file_number = 0; + } + } + return result; +} + +uint64_t BlobGcCandidateHeap::RetryDue(uint64_t now_micros) { + // At most kMaxRetriesPerSlice (32, an estimate) files per call, to bound the + // work of one call. Files still due wait for the next call. + constexpr size_t kMaxRetriesPerSlice = 32; + size_t count = 0; + while (!retries_.empty() && retries_.begin()->first <= now_micros && + count++ < kMaxRetriesPerSlice) { + const uint64_t file_number = retries_.begin()->second; + const std::map::iterator entry = + candidates_.find(file_number); + assert(entry != candidates_.end()); + AccountTotals(entry->second, /*add=*/false); + Activate(file_number, &entry->second); + AccountTotals(entry->second, /*add=*/true); + } + return retries_.empty() ? 0 : retries_.begin()->first; +} + +void BlobGcCandidateHeap::SuppressTransient(uint64_t file_number, + uint64_t now_micros) { + const std::map::iterator entry = + candidates_.find(file_number); + if (entry == candidates_.end()) { + return; + } + Candidate& candidate = entry->second; + // Retry delay: 1 s after the first failure, doubled after each failure, at + // most 32 s (from the sixth failure on). These are fixed choices, not + // measured. + constexpr uint64_t kFirstFailureRetryMicros = 1000 * 1000; + constexpr uint8_t kMaxCountedFailures = 6; + candidate.failures = + std::min(candidate.failures + 1, kMaxCountedFailures); + const uint64_t delay = kFirstFailureRetryMicros << (candidate.failures - 1); + Suppress(file_number, + now_micros + std::min(delay, std::numeric_limits::max() - + now_micros)); +} + +size_t BlobGcCandidateHeap::MemoryUsage(size_t additional_candidates) const { + // Bytes per candidate (estimate): a map node with its tree pointers, up to + // two heap pointers (vector growth) and one retry set node. + constexpr size_t kCandidateBytes = + sizeof(std::map::value_type) + 4 * sizeof(void*) + + 2 * sizeof(Candidate*) + sizeof(std::pair) + + 4 * sizeof(void*); + const size_t max_count = std::numeric_limits::max() / kCandidateBytes; + if (candidates_.size() > max_count || + additional_candidates > max_count - candidates_.size()) { + return std::numeric_limits::max(); + } + return (candidates_.size() + additional_candidates) * kCandidateBytes; +} + +Status BlobCensusSlotTable::Assign(uint64_t file_number, uint32_t* slot) { + assert(slot != nullptr); + if (file_number == 0 || slots_.count(file_number) != 0) { + return Status::Corruption("Invalid or duplicate census SST slot"); + } + uint32_t result = static_cast(files_.size()); + for (size_t word = first_free_word_; word < free_words_.size(); ++word) { + if (free_words_[word] != 0) { + const uint32_t bit = + static_cast(CountTrailingZeroBits(free_words_[word])); + result = static_cast(word * 64 + bit); + free_words_[word] &= ~(uint64_t{1} << bit); + first_free_word_ = word; + break; + } + first_free_word_ = word + 1; + } + if (result == files_.size()) { + if (files_.size() == kNoSlot) { + return Status::MemoryLimit("Too many census SST slots"); + } + files_.push_back(file_number); + free_words_.resize((files_.size() + 63) / 64); + } else { + files_[result] = file_number; + } + slots_.emplace(file_number, result); + *slot = result; + return Status::OK(); +} + +Status BlobCensusSlotTable::Release(uint64_t file_number, uint32_t* slot) { + assert(slot != nullptr); + const auto found = slots_.find(file_number); + if (found == slots_.end()) { + return Status::Corruption("Missing census SST slot"); + } + *slot = found->second; + slots_.erase(found); + files_[*slot] = 0; + free_words_[*slot / 64] |= uint64_t{1} << (*slot % 64); + first_free_word_ = std::min(first_free_word_, *slot / 64); + // Keep U (the slot count) tight: drop the free slots at the end. + while (!files_.empty() && files_.back() == 0) { + const size_t last = files_.size() - 1; + free_words_[last / 64] &= ~(uint64_t{1} << (last % 64)); + files_.pop_back(); + } + free_words_.resize((files_.size() + 63) / 64); + first_free_word_ = std::min(first_free_word_, free_words_.size()); + return Status::OK(); +} + +uint32_t BlobCensusSlotTable::Find(uint64_t file_number) const { + const auto found = slots_.find(file_number); + return found == slots_.end() ? kNoSlot : found->second; +} + +size_t BlobCensusSlotTable::MemoryUsage() const { + // An estimate for both hash map types behind UnorderedMap: per entry the key, + // the value and three pointers (links, metadata and allocator rounding), plus + // one pointer per bucket. + return files_.capacity() * sizeof(uint64_t) + + free_words_.capacity() * sizeof(uint64_t) + + slots_.bucket_count() * sizeof(void*) + + slots_.size() * + (sizeof(uint64_t) + sizeof(uint32_t) + 3 * sizeof(void*)); +} + +bool BlobCensusSlotSet::Contains(uint32_t slot) const { + if (bitmap_) { + return slot / 64 < words_.size() && + (words_[slot / 64] >> (slot % 64) & 1) != 0; + } + Slice input(list_); + uint32_t value = 0; + for (uint32_t i = 0; i < count_; ++i) { + uint32_t delta = 0; + if (!GetVarint32(&input, &delta)) { + assert(false); + return false; + } + value = i == 0 ? delta : value + delta; + if (value >= slot) { + return value == slot; + } + } + return false; +} + +void BlobCensusSlotSet::AppendTo(std::vector* slots) const { + assert(slots != nullptr); + slots->reserve(slots->size() + count_); + if (bitmap_) { + for (size_t word = 0; word < words_.size(); ++word) { + uint64_t bits = words_[word]; + while (bits != 0) { + const int bit = CountTrailingZeroBits(bits); + slots->push_back(static_cast(word * 64 + bit)); + bits &= bits - 1; + } + } + return; + } + Slice input(list_); + uint32_t value = 0; + for (uint32_t i = 0; i < count_; ++i) { + uint32_t delta = 0; + const bool ok = GetVarint32(&input, &delta); + assert(ok); + (void)ok; + value = i == 0 ? delta : value + delta; + slots->push_back(value); + } +} + +Status BlobCensusSlotSet::Update(const std::vector& removed, + const std::vector& added, + uint32_t slot_count, + std::vector* scratch) { + assert(scratch != nullptr); + if (bitmap_) { + return UpdateBitmap(removed, added, slot_count, scratch); + } + std::vector current; + AppendTo(¤t); + scratch->clear(); + scratch->reserve(current.size() + added.size()); + size_t r = 0; + size_t a = 0; + for (size_t c = 0; c <= current.size(); ++c) { + const uint32_t value = + c < current.size() ? current[c] : std::numeric_limits::max(); + // Copy the additions that are below the next kept slot. + while (a < added.size() && added[a] < value) { + if (!scratch->empty() && scratch->back() >= added[a]) { + return Status::Corruption("Unsorted census slot addition"); + } + scratch->push_back(added[a++]); + } + if (c == current.size()) { + break; + } + if (r < removed.size() && removed[r] < value) { + return Status::Corruption("Census slot removal does not match"); + } + if (r < removed.size() && removed[r] == value) { + ++r; + // A slot freed and reused in the same change is added back here. + if (a < added.size() && added[a] == value) { + scratch->push_back(added[a++]); + } + continue; + } + if (a < added.size() && added[a] == value) { + return Status::Corruption("Census slot added twice"); + } + scratch->push_back(value); + } + if (r != removed.size() || a != added.size()) { + return Status::Corruption("Census slot removal does not match"); + } + Encode(*scratch, slot_count); + return Status::OK(); +} + +Status BlobCensusSlotSet::UpdateBitmap(const std::vector& removed, + const std::vector& added, + uint32_t slot_count, + std::vector* scratch) { + assert(bitmap_); + const auto is_set = [this](uint32_t slot) { + return slot / 64 < words_.size() && + (words_[slot / 64] >> (slot % 64) & 1) != 0; + }; + // Check everything first, so a bad change leaves the set as it was. + for (size_t i = 0; i < removed.size(); ++i) { + if ((i > 0 && removed[i - 1] >= removed[i]) || !is_set(removed[i])) { + return Status::Corruption("Census slot removal does not match"); + } + } + for (size_t i = 0; i < added.size(); ++i) { + if (i > 0 && added[i - 1] >= added[i]) { + return Status::Corruption("Unsorted census slot addition"); + } + // A slot freed and reused in the same change is added back. + if (is_set(added[i]) && + !std::binary_search(removed.begin(), removed.end(), added[i])) { + return Status::Corruption("Census slot added twice"); + } + } + // Set and clear bits in place; no decode and re-encode of the set. + for (uint32_t slot : removed) { + words_[slot / 64] &= ~(uint64_t{1} << (slot % 64)); + } + if (!added.empty() && added.back() / 64 >= words_.size()) { + words_.resize(added.back() / 64 + 1, 0); + } + for (uint32_t slot : added) { + words_[slot / 64] |= uint64_t{1} << (slot % 64); + } + count_ = count_ - static_cast(removed.size()) + + static_cast(added.size()); + // Drop the zero words after the highest set bit. + while (!words_.empty() && words_.back() == 0) { + words_.pop_back(); + } + // Encode again only when the set may cross the switch point: a list takes at + // least one byte per slot, so with count_ >= U/16 it stays a bitmap. + const size_t bitmap_bytes = (size_t{slot_count} + 7) / 8; + if (count_ == 0 || count_ < bitmap_bytes / 2) { + scratch->clear(); + AppendTo(scratch); + Encode(*scratch, slot_count); + } else if (words_.capacity() > 2 * words_.size() + 8) { + words_.shrink_to_fit(); + } + return Status::OK(); +} + +void BlobCensusSlotSet::Assign(const std::vector& slots, + uint32_t slot_count) { + assert(std::adjacent_find(slots.begin(), slots.end(), + std::greater_equal()) == slots.end()); + Encode(slots, slot_count); +} + +void BlobCensusSlotSet::Encode(const std::vector& slots, + uint32_t slot_count) { + size_t list_bytes = 0; + for (size_t i = 0; i < slots.size(); ++i) { + list_bytes += VarintLength(i == 0 ? slots[i] : slots[i] - slots[i - 1]); + } + const size_t bitmap_bytes = (size_t{slot_count} + 7) / 8; + const bool use_bitmap = + bitmap_ ? list_bytes >= bitmap_bytes / 2 : list_bytes > bitmap_bytes; + count_ = static_cast(slots.size()); + bitmap_ = use_bitmap && !slots.empty(); + if (bitmap_) { + list_.clear(); + words_.assign(slots.back() / 64 + 1, 0); + for (uint32_t slot : slots) { + words_[slot / 64] |= uint64_t{1} << (slot % 64); + } + } else { + words_.clear(); + // Exact size, so no capacity slack is charged. + std::string list; + list.reserve(list_bytes); + for (size_t i = 0; i < slots.size(); ++i) { + PutVarint32(&list, i == 0 ? slots[i] : slots[i] - slots[i - 1]); + } + list_.swap(list); + list_.shrink_to_fit(); + } + if (words_.capacity() > words_.size()) { + words_.shrink_to_fit(); + } +} + +Status BlobReferenceCensusBuildContext::ReserveInitialFiles(size_t file_count) { + // The context remembers only removed SSTs of the pinned starting Version, so + // their number cannot exceed that Version's file count. The change log limits + // the bytes of deltas returned but not yet applied. Here we charge only their + // list nodes: at most kMaxDeltasPerRead. + constexpr size_t kDeletedFileBytes = sizeof(uint64_t) + 4 * sizeof(void*); + constexpr size_t kPendingNodesBytes = + BlobReferenceCensus::kMaxDeltasPerRead * + (sizeof(std::shared_ptr) + + 3 * sizeof(void*)); + constexpr size_t kBaseBytes = + sizeof(BlobReferenceCensusBuildContext) + kPendingNodesBytes; + if (file_count > + (std::numeric_limits::max() - kBaseBytes) / kDeletedFileBytes) { + return Status::MemoryLimit("Blob census initial file count overflow"); + } + return reservation_manager != nullptr + ? reservation_manager->UpdateCacheReservation( + kBaseBytes + file_count * kDeletedFileBytes) + : Status::OK(); +} + +size_t BlobReferenceCensus::WorkerState::MemoryUsage( + size_t additional_slots, size_t additional_candidates) const { + // A node of the `generations` map holds the WorkerGeneration record. Its + // holder set and pending vectors are counted by capacity in slot_set_bytes. + // The worker keeps no SST summaries: each delta carries the groups of its + // removed SSTs. + constexpr size_t kGenerationBytes = + sizeof(std::map::value_type) + + 4 * sizeof(void*); + size_t bytes = sizeof(*this); + const auto add = [&bytes](size_t count, size_t item_size) { + if (count == 0 || item_size == 0) { + return; + } + if (count > (std::numeric_limits::max() - bytes) / item_size) { + bytes = std::numeric_limits::max(); + } else { + bytes += count * item_size; + } + }; + add(generations.size(), kGenerationBytes); + add(1, slot_set_bytes); + add(1, slots.MemoryUsage()); + add(scratch.capacity(), sizeof(uint32_t)); + // Room for the slot changes of the next operation: one slot per group in a + // pending vector or the merge scratch, plus a new slot-table entry. + add(additional_slots, 2 * sizeof(uint32_t) + sizeof(uint64_t)); + // Candidate map nodes, heap pointer capacity and retry set nodes (see + // BlobGcCandidateHeap::MemoryUsage()). Only this worker changes the heap. + add(1, candidates.MemoryUsage(additional_candidates)); + return bytes; +} + +Status BlobReferenceCensus::WorkerState::Reserve(size_t additional_slots) { + return Reserve(additional_slots, /*additional_candidates=*/0); +} + +Status BlobReferenceCensus::WorkerState::Reserve(size_t additional_slots, + size_t additional_candidates) { + const size_t bytes = MemoryUsage(additional_slots, additional_candidates); + if (bytes == std::numeric_limits::max()) { + return Status::MemoryLimit("Blob census index size overflow"); + } + reserved_bytes.store(bytes, std::memory_order_relaxed); + return reservation_manager != nullptr + ? reservation_manager->UpdateCacheReservation(bytes) + : Status::OK(); +} + +size_t BlobReferenceCensus::WorkerState::GenerationBytes( + const WorkerGeneration& entry) { + return entry.holders.MemoryUsage() + + (entry.pending_added.capacity() + entry.pending_removed.capacity()) * + sizeof(uint32_t); +} + +Status BlobReferenceCensus::WorkerState::ApplyFiles( + const std::vector>& removed, + const std::vector>& added) { + // First collect the slot changes per generation number, so that each touched + // generation number gets one merge pass. Removals free their slots before + // additions take the lowest free slots. + struct GenerationChange { + std::vector removed; + std::vector added; + uint64_t removed_count = 0; + uint64_t removed_stored = 0; + uint64_t removed_record = 0; + uint64_t added_count = 0; + uint64_t added_stored = 0; + uint64_t added_record = 0; + }; + std::map changes; + const auto has_census_group = [](const BlobReferenceSstSummary& file) { + return std::any_of(file.groups.begin(), file.groups.end(), IsCensusGroup); + }; + const auto add_totals = [](uint64_t* total, uint64_t value) { + if (value > std::numeric_limits::max() - *total) { + return false; + } + *total += value; + return true; + }; + // Adds the census groups of `file`, whose holder slot is `slot`, to + // `changes`: as a removal, or with `add` as an addition. Only an added group + // is checked for a zero generation number or origin; a removed group was + // checked when it was added. + const auto collect_groups = [&changes, &add_totals]( + const BlobReferenceSstSummary& file, + uint32_t slot, bool add) -> Status { + for (const BlobReferenceGroup& group : file.groups) { + if (!IsCensusGroup(group)) { + continue; + } + if (add && (group.generation_file_number == 0 || + group.origin_file_number == 0)) { + return Status::Corruption("Invalid census generation group"); + } + GenerationChange& change = changes[group.generation_file_number]; + std::vector& change_slots = add ? change.added : change.removed; + // Several groups of one generation number in this SST add only one holder + // slot. + if (change_slots.empty() || change_slots.back() != slot) { + change_slots.push_back(slot); + } + if (!add_totals(add ? &change.added_count : &change.removed_count, + group.reference_count) || + !add_totals(add ? &change.added_stored : &change.removed_stored, + group.stored_value_bytes) || + !add_totals(add ? &change.added_record : &change.removed_record, + group.record_bytes)) { + return Status::Corruption("Blob census group accounting overflow"); + } + } + return Status::OK(); + }; + for (const std::shared_ptr& file : removed) { + if (file == nullptr || file->file_number == 0) { + return Status::Corruption("Invalid census SST removal"); + } + if (!has_census_group(*file)) { + continue; + } + uint32_t slot = BlobCensusSlotTable::kNoSlot; + if (slots.Find(file->file_number) == BlobCensusSlotTable::kNoSlot) { + if (finalized) { + return Status::Corruption("Removed SST has no census slot"); + } + // The initial scan has not reached this starting SST yet. + continue; + } + Status status = slots.Release(file->file_number, &slot); + if (!status.ok()) { + return status; + } + status = collect_groups(*file, slot, /*add=*/false); + if (!status.ok()) { + return status; + } + } + for (const std::shared_ptr& file : added) { + if (file == nullptr || file->file_number == 0) { + return Status::Corruption("Invalid or duplicate census SST summary"); + } + if (!has_census_group(*file)) { + continue; + } + uint32_t slot = 0; + Status status = slots.Assign(file->file_number, &slot); + if (!status.ok()) { + return status; + } + status = collect_groups(*file, slot, /*add=*/true); + if (!status.ok()) { + return status; + } + } + for (auto& [generation, change] : changes) { + auto found = generations.find(generation); + if (found == generations.end()) { + if (!change.removed.empty()) { + return Status::Corruption("Missing census SST generation accounting"); + } + found = generations.emplace(generation, WorkerGeneration()).first; + found->second.physical_file_number = ResolvePhysical(generation); + slot_set_bytes += GenerationBytes(found->second); + } + WorkerGeneration& entry = found->second; + if (entry.reference_count < change.removed_count || + entry.stored_value_bytes < change.removed_stored || + entry.record_bytes < change.removed_record) { + return Status::Corruption("Missing census SST generation accounting"); + } + Status status = AdjustGenerationBytes(entry, /*add=*/false); + if (!status.ok()) { + return status; + } + entry.reference_count -= change.removed_count; + entry.stored_value_bytes -= change.removed_stored; + entry.record_bytes -= change.removed_record; + if (!add_totals(&entry.reference_count, change.added_count) || + !add_totals(&entry.stored_value_bytes, change.added_stored) || + !add_totals(&entry.record_bytes, change.added_record)) { + return Status::Corruption("Blob census group accounting overflow"); + } + const size_t bytes_before = GenerationBytes(entry); + if (finalized) { + std::sort(change.removed.begin(), change.removed.end()); + std::sort(change.added.begin(), change.added.end()); + status = entry.holders.Update(change.removed, change.added, + slots.slot_count(), &scratch); + if (!status.ok()) { + return status; + } + } else { + entry.pending_removed.insert(entry.pending_removed.end(), + change.removed.begin(), + change.removed.end()); + entry.pending_added.insert(entry.pending_added.end(), + change.added.begin(), change.added.end()); + } + slot_set_bytes = slot_set_bytes - bytes_before + GenerationBytes(entry); + status = AdjustGenerationBytes(entry, /*add=*/true); + if (!status.ok()) { + return status; + } + if (finalized && entry.holders.empty()) { + if (entry.reference_count != 0 || entry.stored_value_bytes != 0 || + entry.record_bytes != 0) { + return Status::Corruption("Census totals remain without holders"); + } + slot_set_bytes -= GenerationBytes(entry); + generations.erase(found); + } + } + return Status::OK(); +} + +Status BlobReferenceCensus::WorkerState::Finalize() { + if (finalized) { + return Status::OK(); + } + // Build each holder set once: collect, sort, encode. During the build a slot + // can be added, removed, and then reused by another SST that holds the same + // generation number. So a slot is in the set if its additions are one more + // than its removals. + std::vector slots_held; + for (auto found = generations.begin(); found != generations.end();) { + WorkerGeneration& record = found->second; + std::sort(record.pending_added.begin(), record.pending_added.end()); + std::sort(record.pending_removed.begin(), record.pending_removed.end()); + slots_held.clear(); + size_t r = 0; + for (size_t a = 0; a < record.pending_added.size();) { + const uint32_t slot = record.pending_added[a]; + size_t adds = 0; + while (a < record.pending_added.size() && + record.pending_added[a] == slot) { + ++adds; + ++a; + } + while (r < record.pending_removed.size() && + record.pending_removed[r] < slot) { + return Status::Corruption("Census slot removal does not match"); + } + size_t removes = 0; + while (r < record.pending_removed.size() && + record.pending_removed[r] == slot) { + ++removes; + ++r; + } + if (adds < removes || adds - removes > 1) { + return Status::Corruption("Census slot removal does not match"); + } + if (adds - removes == 1) { + slots_held.push_back(slot); + } + } + if (r != record.pending_removed.size()) { + return Status::Corruption("Census slot removal does not match"); + } + slot_set_bytes -= GenerationBytes(record); + std::vector().swap(record.pending_added); + std::vector().swap(record.pending_removed); + record.holders.Assign(slots_held, slots.slot_count()); + slot_set_bytes += GenerationBytes(record); + if (record.holders.empty()) { + if (record.reference_count != 0 || record.stored_value_bytes != 0 || + record.record_bytes != 0) { + return Status::Corruption("Census totals remain without holders"); + } + slot_set_bytes -= GenerationBytes(record); + found = generations.erase(found); + } else { + ++found; + } + } + finalized = true; + return Status::OK(); +} + +uint64_t BlobReferenceCensus::WorkerState::ResolvePhysical( + uint64_t generation) const { + const uint64_t target = forward_state != nullptr + ? forward_state->FindRouteTarget(generation) + : kInvalidBlobFileNumber; + return target != kInvalidBlobFileNumber ? target : generation; +} + +// Adds (`add` true) or subtracts the live bytes of one generation number on its +// physical file in the candidate heap. ApplyFiles() calls it when holders +// change, and ApplyWorkerDelta() calls it when a route moves the generation +// number to a new physical file. Live bytes must use the same unit as the total +// bytes of the physical file. An indirect-origin blob file counts whole blob +// records (header, key and value), so it uses the record bytes of the +// generation number. A relocation file stores each value with a trailer of +// kSimpleGen2BlobTrailerSize (5 bytes, see db/blob/blob_gen2_format.h), so its +// live bytes are the stored value bytes plus 5 bytes per reference. +Status BlobReferenceCensus::WorkerState::AdjustGenerationBytes( + const WorkerGeneration& entry, bool add) { + uint64_t bytes = entry.record_bytes; + if (!candidates.IsIndirectOrigin(entry.physical_file_number)) { + if (entry.reference_count > + (std::numeric_limits::max() - entry.stored_value_bytes) / + kSimpleGen2BlobTrailerSize) { + return Status::Corruption("Blob GC candidate generation-byte overflow"); + } + bytes = entry.stored_value_bytes + + entry.reference_count * kSimpleGen2BlobTrailerSize; + } + return candidates.AdjustLiveBytes(entry.physical_file_number, bytes, add); +} + +std::shared_ptr +BlobReferenceCensus::LoadWorkerState() const { + std::lock_guard lock(worker_state_mutex_); + return worker_state_; +} + +std::shared_ptr +BlobReferenceCensus::ExchangeWorkerState(std::shared_ptr next) { + std::lock_guard lock(worker_state_mutex_); + worker_state_.swap(next); + // The caller frees the old WorkerState after this lock is released. + return next; +} + +Status BlobReferenceCensus::ResetWorkerState( + uint64_t token, uint64_t version_number, + std::shared_ptr forward_state, + std::shared_ptr block_cache) { + if (token == 0 || version_number == 0) { + return Status::InvalidArgument("Invalid census worker initialization"); + } + std::shared_ptr next = std::make_shared(); + next->token = token; + next->version_number = version_number; + next->forward_state = std::move(forward_state); + next->block_cache = std::move(block_cache); + if (next->block_cache != nullptr) { + next->reservation_manager = std::make_shared< + CacheReservationManagerImpl>( + next->block_cache); + } + Status status = next->Reserve(0); + if (!status.ok()) { + return status; + } + if (next->forward_state != nullptr) { + status = next->Reserve(0, next->forward_state->files().size()); + if (!status.ok()) { + return status; + } + for (const std::shared_ptr& file : + next->forward_state->files()) { + next->candidates.SetPhysicalFile( + {file->GetFileNumber(), file->GetTotalRecordBytes(), + /*is_indirect_origin=*/false, file->IsMarkedLost()}); + } + status = next->Reserve(0); + if (!status.ok()) { + return status; + } + } + { + std::lock_guard worker_lock(worker_mutex_); + std::lock_guard log_lock(mutex_); + if (canceled_ || token != build_cursor_.token) { + return Status::Aborted("Census worker initialization was canceled"); + } + worker_ready_version_.store(0, std::memory_order_release); + next = ExchangeWorkerState(std::move(next)); + worker_active_token_.store(token, std::memory_order_release); + } + // Replaced maps and reservations are destroyed outside both mutexes. + return Status::OK(); +} + +bool BlobReferenceCensus::IsActiveWorker(const WorkerState* worker, + uint64_t token) const { + return worker != nullptr && worker->token == token && + token == worker_active_token_.load(std::memory_order_acquire); +} + +Status BlobReferenceCensus::AddStartingFile( + uint64_t token, std::shared_ptr file) { + std::shared_ptr worker = LoadWorkerState(); + TEST_SYNC_POINT("BlobReferenceCensus::AddStartingFile:CapturedWorker"); + TEST_SYNC_POINT("BlobReferenceCensus::AddStartingFile:BeforeWorkerLock"); + std::lock_guard lock(worker_mutex_); + if (!IsActiveWorker(worker.get(), token)) { + return Status::Aborted("Census worker was replaced"); + } + // The token checks above stop a run of a canceled build from adding its SST + // to the worker of a newer build. Starting SSTs are added before the census + // is first marked ready, so such an add would make the new census wrong when + // it becomes ready. + if (file == nullptr) { + return Status::Corruption("Invalid or duplicate census SST summary"); + } + Status status = worker->Reserve(file->groups.size()); + if (!status.ok()) { + return status; + } + status = worker->ApplyFiles({}, {std::move(file)}); + if (!status.ok()) { + return status; + } + return worker->Reserve(0); +} + +Status BlobReferenceCensus::FinalizeWorkerBuild(uint64_t token) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (!IsActiveWorker(worker.get(), token)) { + return Status::Aborted("Census worker was replaced"); + } + Status status = worker->Finalize(); + if (!status.ok()) { + return status; + } + return worker->Reserve(0); +} + +Status BlobReferenceCensus::SetWorkerPhysicalFiles( + uint64_t token, const std::vector& files) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (!IsActiveWorker(worker.get(), token)) { + return Status::Aborted("Census worker was replaced"); + } + Status status = worker->Reserve(0, files.size()); + if (!status.ok()) { + return status; + } + for (const BlobGcPhysicalFileSummary& file : files) { + if (file.file_number == 0) { + return Status::Corruption("Invalid blob GC physical file number"); + } + worker->candidates.SetPhysicalFile(file); + } + return worker->Reserve(0); +} + +BlobGcCandidateHeap::Selection BlobReferenceCensus::SelectWorkerCandidates( + uint64_t token) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (!IsActiveWorker(worker.get(), token)) { + return {}; + } + // A failure report or the end of a metadata-only commit that comes before + // this selection is published sets the flag again. Then the next job run + // publishes a fresh selection, even without a Version change. + candidate_refresh_requested_.store(false, std::memory_order_release); + return worker->candidates.Select(); +} + +uint64_t BlobReferenceCensus::RetryWorkerCandidates(uint64_t token, + uint64_t now_micros) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + // Checks only the worker's token, not IsActiveWorker(): it reads or changes + // only the candidate heap of this worker object (RetryDue()), and a replaced + // worker is dropped with its heap. The caller publishes the result only when + // TryPublishWorker() accepts the same token. + return worker != nullptr && worker->token == token + ? worker->candidates.RetryDue(now_micros) + : 0; +} + +BlobGcCandidateHeap::Selection BlobReferenceCensus::SelectWorkerEmptyCandidates( + uint64_t token) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + // Checks only the worker's token, not IsActiveWorker(): it reads or changes + // only the candidate heap of this worker object (SelectEmpty()), and a + // replaced worker is dropped with its heap. The caller publishes the result + // only when TryPublishWorker() accepts the same token. + return worker != nullptr && worker->token == token + ? worker->candidates.SelectEmpty() + : BlobGcCandidateHeap::Selection{}; +} + +bool BlobReferenceCensus::NeedsWorkerRun(uint64_t version_number) const { + std::lock_guard lock(mutex_); + return !build_waiting_for_install_ && + (version_number == 0 || + worker_ready_version_.load(std::memory_order_acquire) != + version_number || + candidate_refresh_requested_.load(std::memory_order_acquire)); +} + +void BlobReferenceCensus::SetBuildWakeup(BuildWakeup wakeup, void* db, + void* cfd) { + build_wakeup_ = wakeup; + build_wakeup_db_ = db; + build_wakeup_cfd_ = cfd; +} + +BlobGcCandidateHeap::Totals BlobReferenceCensus::WorkerCandidateTotals( + uint64_t token) const { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + // Same token rule as WorkerRelocationGarbageBytes(). + return worker != nullptr && worker->token == token + ? worker->candidates.GetTotals() + : BlobGcCandidateHeap::Totals{}; +} + +uint64_t BlobReferenceCensus::WorkerRelocationGarbageBytes( + uint64_t token) const { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + // Checks only the worker's token, not IsActiveWorker(): it only reads the + // candidate heap of this worker object. The caller publishes the result only + // when TryPublishWorker() accepts the same token. + return worker != nullptr && worker->token == token + ? worker->candidates.RelocationGarbageBytes() + : 0; +} + +Status BlobReferenceCensus::CaptureCandidateStamp( + uint64_t version_number, const std::vector& files, + BlobGcCandidateStamp* stamp) const { + if (stamp == nullptr || files.size() > stamp->files.size()) { + return Status::InvalidArgument("Invalid GC candidate stamp"); + } + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (worker == nullptr || worker->version_number != version_number) { + return Status::TryAgain("GC candidate worker epoch changed"); + } + *stamp = {}; + stamp->worker_token = worker->token; + for (size_t index = 0; index < files.size(); ++index) { + stamp->files[index] = files[index]; + stamp->revisions[index] = worker->candidates.Revision(files[index]); + } + return Status::OK(); +} + +void BlobReferenceCensus::SuppressWorkerCandidates( + const BlobGcCandidateStamp& stamp, const std::vector& files, + const std::set& transient_files, uint64_t now_micros) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (!IsActiveWorker(worker.get(), stamp.worker_token)) { + return; + } + if (!files.empty()) { + // The caller may have removed these files from the candidate list in the + // Version, even when the report is stale and ignored below. Ask the worker + // to publish the list again from its current heap. + candidate_refresh_requested_.store(true, std::memory_order_release); + } + for (uint64_t file_number : files) { + for (size_t index = 0; index < stamp.files.size(); ++index) { + if (stamp.files[index] != file_number || stamp.revisions[index] == 0 || + worker->candidates.Revision(file_number) != stamp.revisions[index]) { + continue; + } + if (transient_files.count(file_number) != 0) { + worker->candidates.SuppressTransient(file_number, now_micros); + } else { + worker->candidates.Suppress(file_number, /*retry_micros=*/0); + } + candidate_refresh_requested_.store(true, std::memory_order_release); + break; + } + } +} + +Status BlobReferenceCensus::SetWorkerCoarseIndirectOriginFrontier( + uint64_t token, uint64_t frontier) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (!IsActiveWorker(worker.get(), token)) { + return Status::Aborted("Census worker was replaced"); + } + worker->candidates.SetCoarseIndirectOriginFrontier(frontier); + return Status::OK(); +} + +Status BlobReferenceCensus::ApplyWorkerDelta( + uint64_t token, const BlobReferenceCensusDelta& delta) { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + if (worker == nullptr || worker->token != token) { + return Status::Aborted("Census worker was replaced"); + } + { + std::lock_guard log_lock(mutex_); + if (token != build_cursor_.token || + token != worker_active_token_.load(std::memory_order_acquire)) { + return Status::Aborted("Census worker was replaced"); + } + if (snapshot_pending_) { + return Status::TryAgain("Blob GC is copying its selected memberships"); + } + worker_ready_version_.store(0, std::memory_order_release); + } + if (worker->version_number != delta.base_version_number || + delta.version_number <= delta.base_version_number) { + return Status::Corruption("Noncontiguous census worker delta"); + } + if (delta.rebuild) { + // A rebuild delta never reaches the worker: Install() invalidates the + // census instead of linking the delta into the change log. + return Status::Corruption("Census rebuild delta reached the worker"); + } + size_t additional_slots = 0; + for (const std::shared_ptr& file : + delta.added_files) { + if (file == nullptr || + file->groups.size() > + std::numeric_limits::max() - additional_slots) { + return Status::Corruption("Invalid census worker delta summary"); + } + additional_slots += file->groups.size(); + } + for (const std::shared_ptr& file : + delta.removed_summaries) { + if (file == nullptr || + file->groups.size() > + std::numeric_limits::max() - additional_slots) { + return Status::Corruption("Invalid census worker delta summary"); + } + additional_slots += file->groups.size(); + } + Status status = + worker->Reserve(additional_slots, delta.physical_files.size()); + if (!status.ok()) { + return status; + } + // Add new physical files before applying the SST changes. Remove the gone + // files (total 0) only after the changed routes moved their generation + // numbers to the new targets. + for (const BlobGcPhysicalFileSummary& file : delta.physical_files) { + if (file.file_number == 0) { + return Status::Corruption("Invalid blob GC physical file delta"); + } + if (file.total_record_bytes != 0) { + worker->candidates.SetPhysicalFile(file); + } + } + worker->candidates.SetCoarseIndirectOriginFrontier( + delta.coarse_indirect_origin_frontier); + // Apply the removed and added SSTs as one change. Nothing is published in + // between, even when a compaction has no output SST. + status = worker->ApplyFiles(delta.removed_summaries, delta.added_files); + if (!status.ok()) { + return status; + } + worker->forward_state = delta.forward_state; + for (uint64_t generation : delta.changed_routes) { + const std::map::iterator found = + worker->generations.find(generation); + if (found == worker->generations.end()) { + continue; + } + const uint64_t target = worker->ResolvePhysical(generation); + if (target == found->second.physical_file_number) { + continue; + } + status = worker->AdjustGenerationBytes(found->second, /*add=*/false); + if (!status.ok()) { + return status; + } + found->second.physical_file_number = target; + status = worker->AdjustGenerationBytes(found->second, /*add=*/true); + if (!status.ok()) { + return status; + } + } + for (const BlobGcPhysicalFileSummary& file : delta.physical_files) { + if (file.total_record_bytes == 0) { + worker->candidates.DeletePhysicalFile(file.file_number); + } + } + worker->version_number = delta.version_number; + return worker->Reserve(0); +} + +uint64_t BlobReferenceCensus::WorkerVersionNumber() const { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard lock(worker_mutex_); + return worker != nullptr ? worker->version_number : 0; +} + +BlobReferenceCensus::PublishResult BlobReferenceCensus::TryPublishWorker( + uint64_t token, uint64_t applied_version_number, + uint64_t current_version_number) { + std::lock_guard lock(mutex_); + if (token == 0 || token != build_cursor_.token || canceled_ || + build_cursor_.overflowed) { + return PublishResult::kStale; + } + // Indirect blob GC found that the census does not match the SSTs + // (ReportMismatch()), or a rebuild was requested. The census must not become + // ready again before the census job fails this build and rebuilds it. Without + // this check, the apply after an install could publish the mismatching census + // first. + if (mismatch_reported_ || rebuild_requested_) { + return PublishResult::kStale; + } + if (HasPending(build_cursor_) || build_cursor_.outstanding_bytes != 0) { + return PublishResult::kQueueNotEmpty; + } + if (active_install_reservation_.load(std::memory_order_acquire) != 0) { + // Return the LOW pool thread instead of scheduling the unready census again + // and again. Install() or CancelInstallPreparation() wakes the census job + // of this column family. + build_waiting_for_install_ = true; + return PublishResult::kInstallPending; + } + if (applied_version_number != current_version_number) { + return PublishResult::kStale; + } + worker_ready_version_.store(current_version_number, + std::memory_order_release); + overflow_restart_used_ = false; + // The restart limit ends here (see UseOverflowRestart()). Later deltas have + // the normal limit again. + build_cursor_.restart_build = false; + return PublishResult::kPublished; +} + +std::shared_ptr +BlobReferenceCensus::GetBuildContext() const { + std::lock_guard lock(mutex_); + return build_context_; +} + +bool BlobReferenceCensus::TrySetBuildContext( + std::shared_ptr context) { + { + std::lock_guard lock(mutex_); + if (context == nullptr || context->token == 0 || + context->token != build_cursor_.token || canceled_ || + build_context_ != nullptr) { + return false; + } + build_context_ = std::move(context); + } + return true; +} + +Status BlobReferenceCensus::SnapshotSelected( + uint64_t token, uint64_t version_number, + const std::set& generation_numbers, + const std::shared_ptr& gc_block_cache, + std::shared_ptr* state) { + if (state == nullptr) { + return Status::InvalidArgument("Null selected census snapshot"); + } + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard worker_lock(worker_mutex_); + { + std::lock_guard log_lock(mutex_); + if (token == 0 || token != gc_job_token_ || canceled_ || + !snapshot_pending_) { + return Status::Aborted("Selected census snapshot is no longer active"); + } + if (worker == nullptr || + worker->token != worker_active_token_.load(std::memory_order_acquire) || + worker->version_number != version_number || + worker_ready_version_.load(std::memory_order_acquire) != + version_number) { + snapshot_pending_ = false; + return Status::TryAgain("Selected census snapshot epoch changed"); + } + } + Status status = CopyWorkerSnapshot(*worker, version_number, + generation_numbers, gc_block_cache, state); + { + std::lock_guard log_lock(mutex_); + snapshot_pending_ = false; + } + return status; +} + +Status BlobReferenceCensus::ReadSelectedSnapshot( + uint64_t version_number, const std::set& generation_numbers, + const std::shared_ptr& gc_block_cache, + std::shared_ptr* state) const { + if (state == nullptr) { + return Status::InvalidArgument("Null selected census snapshot"); + } + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard worker_lock(worker_mutex_); + if (worker == nullptr || + worker->token != worker_active_token_.load(std::memory_order_acquire) || + worker->version_number != version_number || + worker_ready_version_.load(std::memory_order_acquire) != version_number) { + return Status::TryAgain("Selected census snapshot epoch changed"); + } + std::shared_ptr copy; + Status status = CopyWorkerSnapshot(*worker, version_number, + generation_numbers, gc_block_cache, ©); + if (status.ok()) { + *state = std::move(copy); + } + return status; +} + +std::shared_ptr +BlobReferenceCensus::TEST_GetReadyState(uint64_t version_number) const { + std::shared_ptr worker = LoadWorkerState(); + std::lock_guard worker_lock(worker_mutex_); + if (worker == nullptr || worker->version_number != version_number || + worker_ready_version_.load(std::memory_order_acquire) != version_number) { + return nullptr; + } + std::set generation_numbers; + for (const auto& entry : worker->generations) { + generation_numbers.insert(entry.first); + } + std::shared_ptr result; + const Status status = + CopyWorkerSnapshot(*worker, version_number, generation_numbers, + /*gc_block_cache=*/nullptr, &result); + assert(status.ok()); + status.PermitUncheckedError(); + return result; +} + +Status BlobReferenceCensus::CopyWorkerSnapshot( + const WorkerState& worker, uint64_t version_number, + const std::set& generation_numbers, + const std::shared_ptr& gc_block_cache, + std::shared_ptr* state) const { + if (!worker.finalized) { + return Status::TryAgain("Census build has not finished"); + } + // Count only the copied data, not all DB files: one map node per selected + // generation number and one SST number per holder. + constexpr size_t kSelectedGenerationBytes = + sizeof(uint64_t) + 4 * sizeof(void*) + + sizeof(BlobReferenceCensusState::SstFilesByGeneration::value_type) + + 4 * sizeof(void*); + size_t bytes = sizeof(BlobReferenceCensusState); + if (generation_numbers.size() > + (std::numeric_limits::max() - bytes) / kSelectedGenerationBytes) { + return Status::MemoryLimit("Selected blob census size overflow"); + } + bytes += generation_numbers.size() * kSelectedGenerationBytes; + for (uint64_t generation : generation_numbers) { + const auto found = worker.generations.find(generation); + if (found == worker.generations.end()) { + continue; + } + if (found->second.holders.count() > + (std::numeric_limits::max() - bytes) / sizeof(uint64_t)) { + return Status::MemoryLimit("Selected blob census size overflow"); + } + bytes += found->second.holders.count() * sizeof(uint64_t); + } + std::shared_ptr reservation_manager; + if (gc_block_cache != nullptr) { + reservation_manager = std::make_shared>(gc_block_cache); + const Status status = reservation_manager->UpdateCacheReservation(bytes); + if (!status.ok()) { + return status; + } + } + std::shared_ptr result = + std::make_shared(version_number); + result->reservation_manager_ = std::move(reservation_manager); + result->snapshot_memory_bound_ = bytes; + result->selected_generations_ = + std::make_shared>(generation_numbers); + std::vector slots; + for (uint64_t generation : generation_numbers) { + const auto found = worker.generations.find(generation); + if (found == worker.generations.end()) { + continue; + } + BlobReferenceCensusState::GenerationEntry& entry = + result->generations_[generation]; + slots.clear(); + found->second.holders.AppendTo(&slots); + entry.sst_files.reserve(slots.size()); + for (uint32_t slot : slots) { + const uint64_t file_number = worker.slots.FileNumber(slot); + if (file_number == 0) { + return Status::Corruption("Census holder slot has no SST"); + } + entry.sst_files.push_back(file_number); + } + std::sort(entry.sst_files.begin(), entry.sst_files.end()); + entry.totals.reference_count = found->second.reference_count; + entry.totals.stored_value_bytes = found->second.stored_value_bytes; + entry.totals.record_bytes = found->second.record_bytes; + } + // The estimate above is an upper bound; charge the real size. + result->snapshot_memory_bound_ = SnapshotMemoryBound(*result); + if (result->reservation_manager_ != nullptr) { + Status status = result->reservation_manager_->UpdateCacheReservation( + result->snapshot_memory_bound_); + if (!status.ok()) { + return status; + } + } + *state = std::move(result); + return Status::OK(); +} + +size_t BlobReferenceCensus::SnapshotMemoryBound( + const BlobReferenceCensusState& state) { + // The real size: the state, the selected generation set, one map node per + // generation number, and 8 bytes per holder SST (vector capacity). A tree + // node costs its value plus about 4 pointers (an estimate of the node + // header). + size_t bytes = sizeof(state); + const auto add = [&bytes](size_t count, size_t item_bytes) { + if (count > (std::numeric_limits::max() - bytes) / item_bytes) { + bytes = std::numeric_limits::max(); + } else { + bytes += count * item_bytes; + } + }; + if (state.selected_generations_ != nullptr) { + add(state.selected_generations_->size(), + sizeof(uint64_t) + 4 * sizeof(void*)); + } + add(state.generations_.size(), + sizeof(BlobReferenceCensusState::SstFilesByGeneration::value_type) + + 4 * sizeof(void*)); + for (const BlobReferenceCensusState::SstFilesByGeneration::value_type& + generation : state.generations_) { + add(generation.second.sst_files.capacity(), sizeof(uint64_t)); + } + return bytes; +} + +const std::vector* BlobReferenceCensusState::FindSstFiles( + uint64_t generation) const { + const auto iterator = generations_.find(generation); + return iterator == generations_.end() ? nullptr : &iterator->second.sst_files; +} + +const BlobCensusGenerationTotals* BlobReferenceCensusState::FindTotals( + uint64_t generation) const { + const auto iterator = generations_.find(generation); + return iterator == generations_.end() ? nullptr : &iterator->second.totals; +} + +std::vector BlobReferenceCensusState::SstFileNumbers() const { + std::set files; + for (const auto& generation : generations_) { + files.insert(generation.second.sst_files.begin(), + generation.second.sst_files.end()); + } + return std::vector(files.begin(), files.end()); +} + +bool BlobReferenceCensusState::SameMemberships( + const BlobReferenceCensusState& other, std::string* difference) const { + assert(difference != nullptr); + if (generations_.size() != other.generations_.size()) { + *difference = "generation count " + std::to_string(generations_.size()) + + " vs " + std::to_string(other.generations_.size()); + return false; + } + for (const auto& [generation, entry] : generations_) { + const std::vector* const files = other.FindSstFiles(generation); + const BlobCensusGenerationTotals* const totals = + other.FindTotals(generation); + if (files == nullptr || *files != entry.sst_files || + totals->reference_count != entry.totals.reference_count || + totals->stored_value_bytes != entry.totals.stored_value_bytes || + totals->record_bytes != entry.totals.record_bytes) { + *difference = "generation " + std::to_string(generation); + return false; + } + } + return true; +} + +size_t BlobReferenceCensusState::GetSstCount(uint64_t generation) const { + const auto* files = FindSstFiles(generation); + return files == nullptr ? 0 : files->size(); +} + +size_t BlobReferenceCensusDelta::QueueWeight() const { + size_t bytes = sizeof(*this) + sizeof(BlobReferenceChangeLogEntry); + const auto add_bytes = [&bytes](size_t count, size_t element_size) { + if (count == 0 || element_size == 0) { + return; + } + if (count > (std::numeric_limits::max() - bytes) / element_size) { + bytes = std::numeric_limits::max(); + } else { + bytes += count * element_size; + } + }; + add_bytes(removed_files.capacity(), sizeof(uint64_t)); + add_bytes(removed_summaries.capacity(), + sizeof(std::shared_ptr)); + add_bytes(added_files.capacity(), + sizeof(std::shared_ptr)); + add_bytes(changed_routes.capacity(), sizeof(uint64_t)); + add_bytes(changed_physical_files.capacity(), sizeof(uint64_t)); + add_bytes(physical_files.capacity(), sizeof(BlobGcPhysicalFileSummary)); + add_bytes(1, retained_forward_state_bytes); + for (const auto* files : {&added_files, &removed_summaries}) { + for (const std::shared_ptr& file : *files) { + if (file != nullptr) { + add_bytes(1, file->MemoryUsage()); + } + } + } + return bytes; +} + +size_t BlobReferenceCensusDelta::ChargeBytes() const { + size_t bytes = QueueWeight(); + if (bytes == std::numeric_limits::max()) { + return bytes; + } + for (const auto* files : {&added_files, &removed_summaries}) { + for (const std::shared_ptr& file : *files) { + // A summary with its own cache charge is charged by the code that created + // it (see the comment at ChargeBytes() in the header). + if (file != nullptr && file->reservation_handle != nullptr) { + bytes -= std::min(bytes, file->MemoryUsage()); + } + } + } + return bytes; +} + +BlobReferenceChangeLogEntry::~BlobReferenceChangeLogEntry() { + if (log_bytes != nullptr) { + log_bytes->fetch_sub(retained_bytes, std::memory_order_relaxed); + } + // An acknowledged prefix can hold many small entries. Free the entries that + // only this chain owns in a loop, so that shared_ptr destruction does not + // recurse once per entry. + std::shared_ptr pending = std::move(next); + while (pending != nullptr && pending.use_count() == 1) { + std::shared_ptr following = + std::move(pending->next); + pending.reset(); + pending = std::move(following); + } +} + +bool BlobReferenceCensus::IsCensusGroup(const BlobReferenceGroup& group) { + return group.encoding_class == + BlobReferenceEncodingClass::kIndirectUnflattened || + group.encoding_class == BlobReferenceEncodingClass::kIndirectFlattened; +} + +Status BlobReferenceCensus::SumGroupsByGeneration( + const BlobReferenceSstSummary& file, + const std::set* selected_names, + std::map* totals) { + // An SST can hold several groups of one generation number (one per origin, or + // one per encoding class). The census counts the SST once per generation + // number. + for (const BlobReferenceGroup& group : file.groups) { + if (!IsCensusGroup(group) || + (selected_names != nullptr && + selected_names->count(group.generation_file_number) == 0)) { + continue; + } + BlobCensusGenerationTotals& sum = (*totals)[group.generation_file_number]; + if (group.reference_count > + std::numeric_limits::max() - sum.reference_count || + group.stored_value_bytes > + std::numeric_limits::max() - sum.stored_value_bytes || + group.record_bytes > + std::numeric_limits::max() - sum.record_bytes) { + return Status::Corruption("Blob census group accounting overflow"); + } + sum.reference_count += group.reference_count; + sum.stored_value_bytes += group.stored_value_bytes; + sum.record_bytes += group.record_bytes; + } + return Status::OK(); +} + +Status BlobReferenceCensus::AddFile(const BlobReferenceSstSummary& file, + BlobReferenceCensusState* state) { + if (state == nullptr || file.file_number == 0) { + return Status::Corruption("Invalid blob reference SST summary"); + } + std::map sums; + Status status = + SumGroupsByGeneration(file, state->selected_generations_.get(), &sums); + if (!status.ok()) { + return status; + } + for (const auto& [generation, sum] : sums) { + BlobReferenceCensusState::GenerationEntry& entry = + state->generations_[generation]; + const auto position = std::lower_bound( + entry.sst_files.begin(), entry.sst_files.end(), file.file_number); + if (position != entry.sst_files.end() && *position == file.file_number) { + return Status::Corruption("Duplicate SST generation membership"); + } + BlobCensusGenerationTotals& totals = entry.totals; + if (sum.reference_count > + std::numeric_limits::max() - totals.reference_count || + sum.stored_value_bytes > + std::numeric_limits::max() - totals.stored_value_bytes || + sum.record_bytes > + std::numeric_limits::max() - totals.record_bytes) { + return Status::Corruption("Blob census group accounting overflow"); + } + entry.sst_files.insert(position, file.file_number); + totals.reference_count += sum.reference_count; + totals.stored_value_bytes += sum.stored_value_bytes; + totals.record_bytes += sum.record_bytes; + } + return Status::OK(); +} + +Status BlobReferenceCensus::BuildState( + uint64_t version_number, + const std::vector>& files, + std::shared_ptr* state) { + if (state == nullptr) { + return Status::InvalidArgument("Null blob reference census output"); + } + auto result = std::make_shared(version_number); + for (const auto& file : files) { + if (file == nullptr) { + return Status::Corruption("Invalid blob reference SST summary"); + } + Status status = AddFile(*file, result.get()); + if (!status.ok()) { + return status; + } + } + *state = std::move(result); + return Status::OK(); +} + +Status BlobReferenceCensus::PrepareInstall( + std::shared_ptr delta, + uint64_t reservation_id, + BlobReferenceCensusPreparedInstall* prepared) const { + return PrepareInstallWithCache(std::move(delta), reservation_id, + /*block_cache=*/nullptr, prepared); +} + +Status BlobReferenceCensus::PrepareInstallWithCache( + std::shared_ptr delta, + uint64_t reservation_id, std::shared_ptr block_cache, + BlobReferenceCensusPreparedInstall* prepared) const { + if (delta == nullptr || prepared == nullptr) { + return Status::InvalidArgument("Null blob reference census install"); + } + if (!std::is_sorted(delta->removed_files.begin(), + delta->removed_files.end()) || + std::adjacent_find(delta->removed_files.begin(), + delta->removed_files.end()) != + delta->removed_files.end()) { + return Status::Corruption( + "Blob reference census deletions are not sorted and unique"); + } + if (delta->rebuild) { + // The census is rebuilt on the new Version. The added summaries are already + // charged by their builders, and they become seeds of that build. + prepared->delta = std::move(delta); + prepared->log_entry.reset(); + prepared->invalidate_census = true; + prepared->reservation_id = reservation_id; + prepared->delta_queue_weight = 0; + return Status::OK(); + } + std::unique_ptr charge; + if (block_cache != nullptr) { + std::shared_ptr manager; + { + std::lock_guard lock(mutex_); + if (log_block_cache_ == block_cache) { + manager = log_reservation_manager_; + } + } + if (manager == nullptr) { + manager = std::make_shared( + std::make_shared>(block_cache)); + // The MANIFEST queue runs one preparation at a time for this column + // family. Construct the cache objects without a lock, and swap only the + // handles under the log mutex. + std::shared_ptr previous_manager; + std::shared_ptr previous_cache; + { + std::lock_guard lock(mutex_); + previous_manager = std::move(log_reservation_manager_); + previous_cache = std::move(log_block_cache_); + log_reservation_manager_ = manager; + log_block_cache_ = std::move(block_cache); + } + } + // ChargeBytes() leaves out the summaries that have their own charge, so + // they are not charged twice. + const size_t bytes = delta->ChargeBytes(); + if (bytes == std::numeric_limits::max()) { + return Status::MemoryLimit("Blob change log retention size overflow"); + } + const Status status = manager->MakeCacheReservation(bytes, &charge); + if (!status.ok()) { + return status; + } + } + prepared->delta = std::move(delta); + prepared->log_entry = std::make_shared( + prepared->delta, std::move(charge)); + prepared->invalidate_census = false; + prepared->reservation_id = reservation_id; + prepared->delta_queue_weight = prepared->delta->QueueWeight(); + + // The table builder has already checked and prepared the SST metadata. The + // census worker applies the holder changes later through the build cursor; + // nothing is copied at the MANIFEST queue head. + return Status::OK(); +} + +Status BlobReferenceCensus::PrepareInvalidation( + uint64_t reservation_id, + BlobReferenceCensusPreparedInstall* prepared) const { + if (reservation_id == 0 || prepared == nullptr) { + return Status::InvalidArgument( + "Invalid blob reference census invalidation"); + } + prepared->delta.reset(); + prepared->log_entry.reset(); + prepared->invalidate_census = true; + prepared->reservation_id = reservation_id; + prepared->delta_queue_weight = 0; + return Status::OK(); +} + +uint64_t BlobReferenceCensus::BeginInstallPreparation() { + uint64_t reservation_id = next_reservation_id_++; + if (reservation_id == 0) { + reservation_id = next_reservation_id_++; + } + assert(active_install_reservation_.load(std::memory_order_relaxed) == 0); + active_install_reservation_.store(reservation_id, std::memory_order_release); + return reservation_id; +} + +void BlobReferenceCensus::CancelInstallPreparation(uint64_t reservation_id) { + if (reservation_id == 0) { + return; + } + bool wake_build = false; + { + std::lock_guard lock(mutex_); + uint64_t expected = reservation_id; + if (active_install_reservation_.compare_exchange_strong( + expected, 0, std::memory_order_acq_rel, + std::memory_order_acquire)) { + wake_build = std::exchange(build_waiting_for_install_, false); + } + } + if (wake_build && build_wakeup_ != nullptr) { + build_wakeup_(build_wakeup_db_, build_wakeup_cfd_); + } +} + +bool BlobReferenceCensus::HasPending(const ChangeLogCursor& cursor) { + return cursor.delivered != nullptr ? cursor.delivered->next != nullptr + : cursor.first != nullptr; +} + +void BlobReferenceCensus::DetachCursor(ChangeLogCursor* cursor, + BlobReferenceCensusGarbage* garbage, + size_t first_slot) { + if (cursor->first != nullptr) { + assert(garbage->log_entries[first_slot] == nullptr); + garbage->log_entries[first_slot] = std::move(cursor->first); + } + if (cursor->acknowledged != nullptr) { + assert(garbage->log_entries[first_slot + 1] == nullptr); + garbage->log_entries[first_slot + 1] = std::move(cursor->acknowledged); + } + if (cursor->delivered != nullptr) { + assert(garbage->log_entries[first_slot + 2] == nullptr); + garbage->log_entries[first_slot + 2] = std::move(cursor->delivered); + } + cursor->outstanding_bytes = 0; +} + +void BlobReferenceCensus::AppendChange( + const BlobReferenceCensusPreparedInstall& prepared, + BlobReferenceCensusGarbage* garbage) { + const size_t bytes = prepared.delta_queue_weight; + size_t limit = kMaxLogMemory; + // Tests lower the limit through this sync point. The restart limit keeps the + // same factor (kMaxRestartLogMemory / kMaxLogMemory) to the limit. + TEST_SYNC_POINT_CALLBACK("BlobReferenceCensus::AppendChange:Limit", &limit); + constexpr size_t kRestartFactor = kMaxRestartLogMemory / kMaxLogMemory; + const size_t restart_limit = + limit > std::numeric_limits::max() / kRestartFactor + ? std::numeric_limits::max() + : limit * kRestartFactor; + bool retain = false; + ChangeLogCursor& cursor = build_cursor_; + // The limit is checked before the append: the cursor overflows only when its + // outstanding weight has already reached the limit. So it holds at most the + // limit plus one delta. One large but valid delta (for example, a compaction + // that rewrites a whole level) then does not overflow a cursor that holds + // only small deltas, such as flushes. + if (cursor.token != 0 && !cursor.overflowed) { + if (cursor.outstanding_bytes >= + (cursor.restart_build ? restart_limit : limit)) { + cursor.overflowed = true; + DetachCursor(&cursor, garbage, + BlobReferenceCensusGarbage::kBuildCursorSlot); + } else { + retain = true; + cursor.outstanding_bytes += bytes; + if (cursor.delivered == nullptr && cursor.first == nullptr) { + cursor.first = prepared.log_entry; + } + } + } + if (!retain) { + garbage->log_entries[BlobReferenceCensusGarbage::kLogTailSlot] = + std::move(log_tail_); + return; + } + assert(prepared.log_entry != nullptr && prepared.log_entry->next == nullptr); + prepared.log_entry->log_bytes = log_bytes_; + log_bytes_->fetch_add(prepared.log_entry->retained_bytes, + std::memory_order_relaxed); + std::shared_ptr previous = std::move(log_tail_); + if (previous != nullptr) { + assert(previous->next == nullptr); + previous->next = prepared.log_entry; + // A detached reader can release the only other owner at the same time. Hand + // the old tail to the caller, so that it never frees a large prefix under + // the DB mutex. + garbage->log_entries[BlobReferenceCensusGarbage::kLogTailSlot] = + std::move(previous); + } + log_tail_ = prepared.log_entry; +} + +BlobReferenceCensusGarbage BlobReferenceCensus::Install( + BlobReferenceCensusPreparedInstall& prepared) { + std::unique_lock lock(mutex_); + const auto finish_install = [&]() { + const bool wake_build = std::exchange(build_waiting_for_install_, false); + lock.unlock(); + // The caller still holds the DB mutex. The callback takes the log mutex to + // check readiness, so it must run after this unlock. + if (wake_build && build_wakeup_ != nullptr) { + build_wakeup_(build_wakeup_db_, build_wakeup_cfd_); + } + }; + BlobReferenceCensusGarbage garbage; + // Guard: this prepared install cannot be trusted. This happens when the + // census was canceled (for example, the column family was dropped), when the + // install has no reservation or no prepared result, or when the active + // reservation is not the one of this install. The census drops all of its + // state (worker, build context, build cursor and log tail). If the census was + // not canceled, the next census job builds it again. Unlike the invalidate + // branch below, it adds no rebuild seeds and does not request a rebuild. A + // running build fails its token check and waits for the normal retry delay. + if ((!prepared.invalidate_census && prepared.delta == nullptr) || canceled_ || + prepared.reservation_id == 0 || + active_install_reservation_.load(std::memory_order_acquire) != + prepared.reservation_id) { + worker_ready_version_.store(0, std::memory_order_release); + garbage.build_context = std::move(build_context_); + DetachCursor(&build_cursor_, &garbage, + BlobReferenceCensusGarbage::kBuildCursorSlot); + garbage.worker_state = ExchangeWorkerState(std::shared_ptr()); + worker_active_token_.store(0, std::memory_order_release); + build_cursor_.token = 0; + garbage.log_entries[BlobReferenceCensusGarbage::kLogTailSlot] = + std::move(log_tail_); + active_install_reservation_.store(0, std::memory_order_release); + finish_install(); + return garbage; + } + active_install_reservation_.store(0, std::memory_order_release); + if (prepared.invalidate_census) { + worker_ready_version_.store(0, std::memory_order_release); + if (prepared.delta != nullptr && prepared.delta->rebuild) { + // The old seeds go to the garbage, which is released without the mutexes. + garbage.summaries = + std::exchange(rebuild_seeds_, prepared.delta->added_files); + rebuild_requested_ = true; + TEST_SYNC_POINT("BlobReferenceCensus::Install:Rebuild"); + } + // A later retry must pin a new Version and get a new token, not reuse this + // canceled context. An active worker run keeps its own pin until it + // returns. + garbage.build_context = std::move(build_context_); + garbage.worker_state = ExchangeWorkerState(std::shared_ptr()); + worker_active_token_.store(0, std::memory_order_release); + DetachCursor(&build_cursor_, &garbage, + BlobReferenceCensusGarbage::kBuildCursorSlot); + build_cursor_.token = 0; + garbage.log_entries[BlobReferenceCensusGarbage::kLogTailSlot] = + std::move(log_tail_); + finish_install(); + return garbage; + } + + AppendChange(prepared, &garbage); + finish_install(); + return garbage; +} + +uint64_t BlobReferenceCensus::StayingMemoryBytes() const { + const std::shared_ptr worker = LoadWorkerState(); + return (worker != nullptr + ? worker->reserved_bytes.load(std::memory_order_relaxed) + : 0) + + log_bytes_->load(std::memory_order_relaxed); +} + +uint64_t BlobReferenceCensus::BeginBuild() { + build_count_.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(mutex_); + assert(build_cursor_.token == 0); + canceled_ = false; + build_cursor_.token = next_build_token_++; + if (build_cursor_.token == 0) { + build_cursor_.token = next_build_token_++; + } + build_cursor_.overflowed = false; + build_cursor_.restart_build = overflow_restart_used_; + rebuild_requested_ = false; + return build_cursor_.token; +} + +bool BlobReferenceCensus::RebuildRequested() const { + std::lock_guard lock(mutex_); + return rebuild_requested_; +} + +bool BlobReferenceCensus::ReportMismatch(uint64_t file_number) { + std::lock_guard lock(mutex_); + if (file_number != 0 && unresolved_mismatch_file_ == file_number) { + gc_mismatch_suppress_count_.fetch_add(1, std::memory_order_relaxed); + return true; + } + unresolved_mismatch_file_ = file_number; + worker_ready_version_.store(0, std::memory_order_release); + rebuild_requested_ = true; + mismatch_reported_ = true; + return false; +} + +void BlobReferenceCensus::ClearMismatch(uint64_t file_number) { + std::lock_guard lock(mutex_); + if (unresolved_mismatch_file_ == file_number) { + unresolved_mismatch_file_ = 0; + } +} + +bool BlobReferenceCensus::TakeMismatchReport() { + std::lock_guard lock(mutex_); + return std::exchange(mismatch_reported_, false); +} + +std::vector> +BlobReferenceCensus::TakeRebuildSeeds() { + std::lock_guard lock(mutex_); + return std::exchange(rebuild_seeds_, {}); +} + +bool BlobReferenceCensus::UseOverflowRestart() { + std::lock_guard lock(mutex_); + if (overflow_restart_used_) { + return false; + } + overflow_restart_used_ = true; + overflow_restart_count_.fetch_add(1, std::memory_order_relaxed); + return true; +} + +Status BlobReferenceCensus::TakeBuildDeltas( + uint64_t token, BlobReferenceCensusDeltaQueue* deltas, bool* overflowed) { + if (deltas == nullptr || overflowed == nullptr || !deltas->empty()) { + return Status::InvalidArgument("Invalid blob change log output"); + } + std::array, kMaxDeltasPerRead> + entries; + size_t count = 0; + { + std::lock_guard lock(mutex_); + if (token == 0 || token != build_cursor_.token || canceled_) { + return Status::Aborted("Blob change log cursor is no longer active"); + } + *overflowed = build_cursor_.overflowed; + if (build_cursor_.overflowed) { + return Status::OK(); + } + std::shared_ptr entry = + build_cursor_.delivered != nullptr ? build_cursor_.delivered->next + : build_cursor_.first; + while (entry != nullptr && count < entries.size()) { + entries[count++] = entry; + entry = entry->next; + } + if (count != 0) { + build_cursor_.delivered = entries[count - 1]; + } + } + // Allocate the list nodes without the log mutex. The entry handles above keep + // the immutable deltas alive while they are copied. + for (size_t index = 0; index < count; ++index) { + assert(entries[index]->delta != nullptr); + deltas->emplace_back(entries[index]->delta); + } + return Status::OK(); +} + +Status BlobReferenceCensus::AcknowledgeBuildDeltas(uint64_t token, + size_t retained_bytes) { + std::shared_ptr old_acknowledged; + std::shared_ptr old_first; + std::shared_ptr drained_acknowledged; + std::shared_ptr drained_delivered; + std::shared_ptr drained_tail; + { + std::lock_guard lock(mutex_); + if (token == 0 || token != build_cursor_.token || canceled_ || + build_cursor_.overflowed) { + return Status::Aborted("Blob change log cursor is no longer active"); + } + if (retained_bytes > build_cursor_.outstanding_bytes) { + return Status::Corruption("Blob change log acknowledged excess bytes"); + } + if (retained_bytes == 0) { + return Status::OK(); + } + if (build_cursor_.delivered == nullptr) { + return Status::Corruption( + "Blob change log acknowledged undelivered bytes"); + } + std::shared_ptr entry = + build_cursor_.acknowledged != nullptr ? build_cursor_.acknowledged->next + : build_cursor_.first; + std::shared_ptr acknowledged; + size_t remaining = retained_bytes; + for (size_t count = 0; count < kMaxDeltasPerRead && entry != nullptr; + ++count) { + if (entry->retained_bytes > remaining) { + break; + } + remaining -= entry->retained_bytes; + acknowledged = entry; + if (remaining == 0 || entry == build_cursor_.delivered) { + break; + } + entry = entry->next; + } + if (remaining != 0 || acknowledged == nullptr) { + return Status::Corruption( + "Blob change log acknowledged undelivered bytes"); + } + old_acknowledged = std::move(build_cursor_.acknowledged); + old_first = std::move(build_cursor_.first); + build_cursor_.acknowledged = std::move(acknowledged); + build_cursor_.outstanding_bytes -= retained_bytes; + // Appends and acknowledgments both use QueueWeight(), so a cursor that has + // acknowledged every entry appended to it has no outstanding weight. + assert(build_cursor_.acknowledged->next != nullptr || + build_cursor_.outstanding_bytes == 0); + if (build_cursor_.outstanding_bytes == 0) { + drained_acknowledged = std::move(build_cursor_.acknowledged); + drained_delivered = std::move(build_cursor_.delivered); + drained_tail = std::move(log_tail_); + } + } + return Status::OK(); +} + +bool BlobReferenceCensus::BuildCursorOverflowed(uint64_t token) const { + std::lock_guard lock(mutex_); + return token != 0 && token == build_cursor_.token && build_cursor_.overflowed; +} + +BlobReferenceCensusGarbage BlobReferenceCensus::FailBuild(uint64_t token) { + std::lock_guard lock(mutex_); + BlobReferenceCensusGarbage garbage; + if (token != 0 && token == build_cursor_.token) { + worker_ready_version_.store(0, std::memory_order_release); + garbage.worker_state = ExchangeWorkerState(std::shared_ptr()); + worker_active_token_.store(0, std::memory_order_release); + garbage.build_context = std::move(build_context_); + DetachCursor(&build_cursor_, &garbage, + BlobReferenceCensusGarbage::kBuildCursorSlot); + build_cursor_.token = 0; + build_cursor_.overflowed = false; + garbage.log_entries[BlobReferenceCensusGarbage::kLogTailSlot] = + std::move(log_tail_); + } + return garbage; +} + +BlobReferenceCensusGarbage BlobReferenceCensus::Cancel() { + std::lock_guard lock(mutex_); + { + std::lock_guard commit_lock(metadata_commit_mutex_); + metadata_commit_canceled_ = true; + } + canceled_ = true; + build_waiting_for_install_ = false; + build_wakeup_ = nullptr; + build_wakeup_db_ = nullptr; + build_wakeup_cfd_ = nullptr; + snapshot_pending_ = false; + worker_ready_version_.store(0, std::memory_order_release); + BlobReferenceCensusGarbage garbage; + garbage.worker_state = ExchangeWorkerState(std::shared_ptr()); + worker_active_token_.store(0, std::memory_order_release); + garbage.build_context = std::move(build_context_); + DetachCursor(&build_cursor_, &garbage, + BlobReferenceCensusGarbage::kBuildCursorSlot); + build_cursor_.token = 0; + gc_job_token_ = 0; + garbage.log_entries[BlobReferenceCensusGarbage::kLogTailSlot] = + std::move(log_tail_); + // The seeds are charged summaries: hand them to the caller, so they are freed + // without the DB mutex. + garbage.summaries = std::exchange(rebuild_seeds_, {}); + active_install_reservation_.store(0, std::memory_order_release); + return garbage; +} + +bool BlobReferenceCensus::IsReady(uint64_t version_number) const { + std::lock_guard lock(mutex_); + return version_number != 0 && + worker_ready_version_.load(std::memory_order_acquire) == + version_number; +} + +bool BlobReferenceCensus::IsSnapshotPending() const { + std::lock_guard lock(mutex_); + return snapshot_pending_; +} + +Status BlobReferenceCensus::BeginGcJob(uint64_t version_number, + uint64_t* token) { + if (token == nullptr || version_number == 0) { + return Status::InvalidArgument("Invalid blob GC job start"); + } + { + // Return TryAgain early, without mutex_, when a metadata commit is in + // progress. The check is repeated under mutex_ below. + std::lock_guard commit_lock(metadata_commit_mutex_); + if (metadata_commit_pending_) { + return Status::TryAgain("A blob metadata commit is in progress"); + } + } + std::lock_guard lock(mutex_); + if (canceled_) { + return Status::Aborted("Blob reference census is canceled"); + } + { + // A metadata-only commit (see the top of blob_reference_census.h) in + // progress can remove a route that this indirect blob GC job would + // re-point, so start after it. The lock order is the same as in Cancel(). + std::lock_guard commit_lock(metadata_commit_mutex_); + if (metadata_commit_ambiguous_) { + return Status::Busy("Blob metadata commit result is unknown"); + } + if (metadata_commit_pending_) { + return Status::TryAgain("A blob metadata commit is in progress"); + } + } + const bool worker_ready = + worker_ready_version_.load(std::memory_order_acquire) == version_number; + if (!worker_ready) { + return Status::TryAgain("Blob reference census is not ready"); + } + if (gc_job_token_ != 0) { + return Status::Busy("A blob GC job is already registered"); + } + gc_job_token_ = next_gc_token_++; + if (gc_job_token_ == 0) { + gc_job_token_ = next_gc_token_++; + } + snapshot_pending_ = worker_ready; + *token = gc_job_token_; + return Status::OK(); +} + +bool BlobReferenceCensus::IsGcJobActive(uint64_t token) const { + std::lock_guard lock(mutex_); + return token != 0 && token == gc_job_token_ && !canceled_; +} + +bool BlobReferenceCensus::HasGcJob() const { + std::lock_guard lock(mutex_); + return gc_job_token_ != 0; +} + +void BlobReferenceCensus::EndGcJob(uint64_t token) { + std::lock_guard lock(mutex_); + if (token != 0 && token == gc_job_token_) { + gc_job_token_ = 0; + snapshot_pending_ = false; + } +} + +// Groundwork for indirect blob GC jobs (see "Groundwork term" in the top +// comment of blob_reference_census.h): the metadata-only commit. + +Status BlobReferenceCensus::BeginMetadataCommit() { + std::lock_guard lock(metadata_commit_mutex_); + if (metadata_commit_canceled_) { + return Status::ColumnFamilyDropped(); + } + if (metadata_commit_pending_ || metadata_commit_ambiguous_) { + return Status::TryAgain("A blob metadata commit is in progress"); + } + metadata_commit_pending_ = true; + return Status::OK(); +} + +void BlobReferenceCensus::EndMetadataCommit(bool result_known) { + { + std::lock_guard lock(metadata_commit_mutex_); + metadata_commit_pending_ = false; + metadata_commit_ambiguous_ |= !result_known; + candidate_refresh_requested_.store(true, std::memory_order_release); + } + if (result_known && build_wakeup_ != nullptr) { + build_wakeup_(build_wakeup_db_, build_wakeup_cfd_); + } +} + +void BlobReferenceCensus::ResolveMetadataCommitAmbiguity() { + { + std::lock_guard lock(metadata_commit_mutex_); + if (!metadata_commit_ambiguous_ || metadata_commit_pending_ || + metadata_commit_canceled_) { + return; + } + metadata_commit_ambiguous_ = false; + candidate_refresh_requested_.store(true, std::memory_order_release); + } + if (build_wakeup_ != nullptr) { + build_wakeup_(build_wakeup_db_, build_wakeup_cfd_); + } +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_reference_census.h b/db/blob/blob_reference_census.h new file mode 100644 index 000000000000..9d123deab45c --- /dev/null +++ b/db/blob/blob_reference_census.h @@ -0,0 +1,1492 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_reference_metadata.h" +#include "db/blob/blob_relocation_manifest.h" +#include "rocksdb/status.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +// Blob reference census +// +// Background: before indirect blob GC can move the live values out of a blob +// file or a relocation file and drop it, it must know which SSTs still +// reference that file and how many live bytes they keep in it. Ordinary reads +// and writes never use the census. +// +// A RocksDB Version (class Version: the live SST, blob and relocation files of +// one column family at one point in time) cannot answer this: +// - For each SST, FileMetaData keeps oldest_blob_file_number (the oldest blob +// file that the SST references) and, for an exact SST, the descriptor of its +// reference metadata. None of these fields lists the other files that the +// SST references. +// - BlobFileTable maps a generation number to a live file, or to a route (an +// old file number -> the relocation file that now holds its values). It says +// where the values of a generation number are now, not which SSTs reference +// it. +// So a Version has no mapping from a generation number to SSTs, and none can +// be derived from it. The full list of generation numbers that an SST +// references is stored only in that SST, in its exact reference metadata (see +// the top of db/blob/blob_reference_metadata.h). +// +// The census is that reverse index: for each generation number, the SSTs that +// reference it and the total bytes that those references keep. It is built by +// reading the reference metadata of every SST, lives only in memory, and is +// built again after each DB open. +// +// Terms: +// - generation number: the file number that a reference uses to find its +// value (see "Generation number" in db/blob/blob_index.h). +// kIndirectFlattenedBlob stores it. kIndirectBlob does not store it, +// because it equals the origin. The census is keyed by generation number. +// - exact SST: an SST with exact reference metadata (a nonempty +// FileMetaData::blob_reference_descriptor). The census counts only exact +// SSTs, and in them only the kIndirectUnflattened and kIndirectFlattened +// groups (IsCensusGroup()). kDirect groups are not counted. +// - coarse SST and coarse frontier: exact metadata costs collector memory +// while an SST is built, so an SST can skip it and record only a coarse +// frontier (see "Coarse SST" in db/blob/blob_reference_metadata.h). GC then +// gives up exact selection for the indirect-origin blob files at or above +// the lowest frontier of the Version +// (VersionStorageInfo::OldestCoarseIndirectOriginFileNumber()): such a file +// is coarse pinned, and the census keeps it out of the candidate heap. +// - SST summary (BlobReferenceSstSummary): the group totals of one exact SST, +// in memory. The table builder makes it for a new SST. A reader decodes it +// from the reference index of an existing SST. +// - holder: an exact SST that has at least one counted group with a given +// generation number. Several such groups in one SST make one holder. +// - slot: a small dense number for each exact SST (BlobCensusSlotTable). A +// new SST gets the lowest free slot. +// - physical file (in the census): an indirect-origin blob file or a +// relocation file. Such a file holds value bytes and can be a GC +// candidate. Normal blob files written with the option off are not +// tracked, because the census counts only indirect references. +// - lost file: a relocation file that the Version marks lost +// (BlobRelocationFileMetaData::IsMarkedLost()). Its values cannot be read. +// A lost file that still has live bytes stays out of the candidate heap. +// - forward state: the BlobRelocationState (routes and relocation files) of +// a Version (see db/blob/blob_relocation_manifest.h). It maps a generation +// number forward to its physical file. +// - delta (BlobReferenceCensusDelta): the census change of one Version +// install: the added and removed SSTs with their groups, and the changed +// routes and physical files. +// - change log: one ordered list of deltas. Install() appends one entry for +// each Version install while the build cursor is open. +// - cursor: the position of the census build in the change log (the build +// cursor). +// - ready: the census has applied every delta up to the current Version and +// is published for that Version. The GC API works only on a ready census. +// Groundwork term. The term below, and the GC job registration and selected +// copy under "Readers", serve GC jobs (intended users: the GC job that +// installs its relocation output, and route retirement). The census itself +// does not need them, so a reader who wants only the census can skip them. +// - metadata-only commit: a MANIFEST commit that changes relocation state +// (routes, relocation files) but adds no SST +// (BeginMetadataCommit() / EndMetadataCommit()). At most one runs per +// census at a time, and no GC job starts while one runs. +// +// Ownership: a column family that has the per-CF format fence +// (VersionStorageInfo::HasBlobIndirectionFormat()) owns one census +// (ColumnFamilyData::GetOrCreateBlobReferenceCensus()). A column family with +// enable_blob_indirection also gets the census object when the DB has the +// DB-global format fence (see ColumnFamilyData::blob_reference_census()), so +// that its installs can be reserved. +// +// Who updates the census: +// 1. Build. At DB open, and in every rebuild (see "Errors and rebuild"), one +// background job on the LOW pool (the census job) builds the census. It +// reads the reference metadata of every exact SST, a few SSTs per run +// and one column family at a time (see "Build"). +// 2. Normal updates. After the build, the flush or compaction thread that +// installed a new Version applies the delta of that install itself, +// right after the install (see "Apply after an install"). So while the +// DB runs normally, the census job has little or no work. +// 3. The rest. The census job applies the deltas that step 2 did not apply: +// installs on other threads (for example a user CompactFiles()), and +// column families that step 2 skipped because another thread held the +// worker. It also runs every retry and every requested refresh of the +// GC candidates (see "Scheduling"). +// In this file, "the worker" means the code that changes the census +// contents. Only one thread runs it at a time (worker_run_mutex()). Only the +// worker changes the generation numbers, slots and holder sets. GC can only +// mark candidates in the heap as suppressed (SuppressWorkerCandidates()). +// +// Structure of one census: +// ColumnFamilyData +// `-- BlobReferenceCensus (one per column family that needs it) +// |-- WorkerState: the census contents +// | |-- generations: generation number -> WorkerGeneration +// | | (see "Per generation number") +// | |-- BlobCensusSlotTable: SST file number <-> slot +// | |-- BlobGcCandidateHeap: physical files ranked by garbage ratio +// | `-- forward state of the applied Version (the routes) +// |-- BlobReferenceCensusBuildContext: the state of one build +// `-- change log: BlobReferenceChangeLogEntry -> ... -> log tail +// | (each entry holds one shared, immutable delta) +// `-- build cursor: the worker reads the log through it +// +// - ResetWorkerState() creates the WorkerState when a build starts. A new +// build, an invalidating install, FailBuild() or Cancel() swaps it out. +// - VersionSet::PrepareBlobReferenceCensusInstalls() creates one delta for +// each new Version of a column family that needs a census +// (BuildBlobReferenceCensusDelta()). A log entry stays alive while a +// cursor, an older entry or the log tail still points to it. +// - DBImpl::BuildBlobReferenceCensus() creates the build context. It stays +// attached after the build, so later runs keep the same build token (see +// Build). +// - A call under the DB mutex returns the state that it replaces in a +// BlobReferenceCensusGarbage. The caller frees it without the DB mutex. +// +// Data flow of one install: +// 1. The table builder makes the summary of each new exact SST +// (FileMetaData::blob_reference_summary). At job start, a compaction +// reads the summary of each exact input SST +// (CompactionJob::ReadInputBlobReferenceSummaries()). The VersionEdit +// carries both. +// 2. Under the DB mutex, VersionSet::ReserveBlobReferenceCensusInstalls() +// calls BeginInstallPreparation() (O(1)). This file calls this an install +// reservation. Until step 4 of this list ends it, the worker does not +// publish. The comment at BlobReferenceCensusInstallBatch in +// db/version_set.h gives the same path in more detail, in its own four +// steps (reserve, prepare, install, free). +// 3. Without the DB mutex, PrepareBlobReferenceCensusInstalls() builds the +// delta. PrepareInstallWithCache() allocates the log entry and charges its +// memory. +// 4. After the MANIFEST write, under the DB mutex, Install() links the entry +// at the log tail without allocating, when a cursor is open. +// 5. The SuperVersion install queues the census job of the column family. +// 6. The worker runs without the DB mutex: in the flush or compaction thread +// right after the install, or in the census job. It takes up to +// kMaxDeltasPerRead (64, an estimate) deltas at a time, applies each one +// and acknowledges it. +// 7. When the build token is valid, the build cursor is empty, no install is +// being prepared, and the census has reached the current Version, +// TryPublishWorker() marks it ready. DBImpl then stores the top GC +// candidates, the empty-file candidates and the relocation garbage bytes +// in that Version. +// +// Scheduling: DBImpl runs the census job (DBImpl::ScheduleBlobReferenceCensus() +// and BGWorkBlobReferenceCensus()). A column family is in one of four states: +// - In the run queue: it has work (NeedsWorkerRun(): the census is not ready +// for the current Version, or a candidate refresh was requested). Open, a +// SuperVersion install and a census wakeup add it. +// - Running: the job takes the column family at the head of the run queue +// and runs one slice for it. If it still has work, it goes to the back of +// the run queue. +// - In the retry list: after a failed build (1 s, doubled after each failure, +// at most 32 s), while a GC job copies its snapshot (1 ms), or while a +// suppressed GC candidate waits for its retry time (candidate_retry_micros +// in BlobReferenceCensusBuildContext). The periodic task +// (PeriodicTaskType::kBlobReferenceCensus, once per second) moves it back +// to the run queue when its time has come, or earlier when it has no +// failures and has new work. +// - Waiting for an install: when a publish finds an active install +// reservation (TryPublishWorker() returns kInstallPending), +// NeedsWorkerRun() is false, so the column family is not queued. Install() +// or CancelInstallPreparation() wakes it (SetBuildWakeup()). +// +// Apply after an install: the census job shares the LOW pool with compactions +// and has no thread of its own. With few background threads, a delta could wait +// in the change log for a long time, and the census would stay unready and hold +// the delta's memory. So the flush and compaction threads also run the worker: +// 1. At the end of each background flush or compaction, the thread releases +// the DB mutex (DBImpl::ApplyBlobReferenceCensusAfterInstall()). +// 2. It visits only the column families with a census that this job +// flushed or compacted (also when the job failed), so a DB without blob +// indirection has nothing to visit. A column family dropped before the +// job ends is not visited. It visits them in any scheduling state, also in +// the retry list, so a delta there does not wait for the periodic task. +// 3. For each column family, it tries worker_run_mutex(). If another thread +// holds it (for example the census job), the thread skips that column +// family and does not wait. +// 4. It skips a column family whose build has not started or has not finished +// its scan of the SSTs, because the census job does the scan. It also +// skips a build with a stored failure (BlobReferenceCensusBuildContext:: +// failure, below). +// 5. Otherwise it applies the queued deltas up to the end of the change log. +// If it applied at least one delta, it then tries to publish the census +// (DBImpl::ApplyBlobReferenceCensusDeltas()). +// This work never fails the flush or the compaction. When it cannot go on, it +// leaves the rest to the census job. The SuperVersion install has queued that +// job, or, for a column family in the retry list, the periodic task runs it: +// - TryAgain (a registered GC job has not yet copied its snapshot) changes +// nothing. The delta stays at the front of the queue, and the census job +// applies it later. +// - An overflow of the build cursor: the thread stops and stores no failure. +// The census job restarts or fails the build (see "Errors and rebuild"). +// If a delta failed at the same time, the thread drops the build's queued +// deltas, so that delta is never applied again. +// - Any other failure may have changed the worker part way, so the delta must +// never be applied again. The thread stores the failure in +// BlobReferenceCensusBuildContext::failure and logs it once. Later +// applies after an install skip the build (step 4), and the next census +// job run fails the build with it. +// - A census job run that fails stores its failure the same way before it +// releases worker_run_mutex(), so an apply after an install that runs +// between the failure and FailBuild() applies nothing and publishes +// nothing. +// - A publish that fails or must wait: the census job publishes later. +// The census job still runs every build and every retry. Installs on other +// threads, for example a user CompactFiles(), do not run this step. The census +// job, or the next flush or compaction of the same column family, applies +// their deltas. +// +// Build: DB open and every rebuild run the same build. +// 1. Under the DB mutex, BeginBuild() opens the build cursor and returns a +// new token (a build id). The caller pins the current Version in the same +// mutex hold, so the cursor sees every later install. +// 2. Each job run is one slice. It visits at most 16 SSTs of the pinned +// Version (kMaxFilesPerSlice in DBImpl::BuildBlobReferenceCensus(), an +// estimate) and reads the reference index of each exact one. Between +// reads, it applies the deltas installed after the pin (catch-up). It +// skips an SST of the pinned Version that such a delta removed. It uses a +// summary that is already in memory (a seed from a rebuild install) +// instead of a read. While a registered GC job has not yet copied its +// snapshot (TryAgain), the slice ends early and the next run visits the +// same SST again. A summary that the slice already read is kept as a seed, +// so that SST is not read twice. +// 3. Until the scan ends, the slot changes of each generation number wait in +// the pending lists of its WorkerGeneration. FinalizeWorkerBuild() then +// builds each holder set once. +// 4. The census publishes as in step 7 above. The build cursor stays open, +// so later deltas keep the census current. +// +// An install that invalidates the census, FailBuild() and Cancel() clear the +// build token. A run of an old build then fails its token check, so it cannot +// change the worker state of a newer build. +// +// Example of one build. Token 7 and Versions 100 to 103 are example values. +// 1. Under the DB mutex, BeginBuild() returns token 7 and pins Version 100. +// 2. Run 1 visits the first 16 SSTs of Version 100 and reads the exact +// ones. Between reads, it applies the deltas up to Version 102. +// 3. Run 2 visits the next 16 SSTs and applies the delta to Version 103. +// 4. The last run visits the rest, applies the deltas that are still +// queued, and calls FinalizeWorkerBuild(). +// 5. The census publishes as ready for Version 103, the current Version. +// 6. From now on, the flush or compaction thread that installs a new +// Version applies its delta and publishes again (see "Apply after an +// install"). +// +// Per generation number: `generations` maps each generation number to one +// WorkerGeneration. It holds the holder set (BlobCensusSlotSet), the totals +// (reference count, stored value bytes, record bytes), and the physical file +// that holds the values now: the generation number itself, or its route +// target. The worker adds the live bytes of each generation number to its +// physical file in the candidate heap (WorkerState::AdjustGenerationBytes()), +// which keeps the total bytes and the live bytes of each physical file. A +// holder set is a varint list or a bitmap of slots (see BlobCensusSlotSet for +// the switch rule). +// +// Example: two generation numbers, with 1,000 slots in use. All numbers are +// example values. +// BlobCensusSlotTable: slot 5 = SST 500, slot 17 = SST 501, ... +// generation 10: +// - holders: list {5, 17}, which means SSTs 500 and 501 (2 bytes) +// - totals: the reference count and bytes of both SSTs +// - physical file: 10 (no route; an indirect-origin blob file) +// generation 300: +// - holders: a bitmap with 600 of 1,000 slots set. The highest slot is +// 999, so it takes 16 words of 64 bits (128 bytes). +// - totals: the reference count and bytes of the 600 SSTs +// - physical file: 900 (route 300 -> relocation file 900) +// BlobGcCandidateHeap: the live bytes of file 10 include generation 10, and +// the live bytes of file 900 include generation 300. +// +// Readers: a GC job registers itself at a ready Version (BeginGcJob()), and +// SnapshotSelected() copies only the generation numbers that it selected. From +// BeginGcJob() until SnapshotSelected() ends or the job ends (EndGcJob()), the +// worker does not apply deltas, so the copy shows exactly the census at the +// job's input Version. The job does not read later deltas. At most one GC job +// is registered per census. +// +// Threads and locks: +// - Under the DB mutex, the census does only O(1) or bounded work, for +// example BeginInstallPreparation(), Install(), BeginBuild(), FailBuild(), +// Cancel(), the GC job calls, the metadata-only commit calls, and the +// TryPublishWorker() call of DBImpl. These calls never allocate or free +// state as large as the census. +// - worker_run_mutex_ lets only one thread run the worker: the census job or +// an installing thread (see "Apply after an install"). The worker takes the +// DB mutex to publish while it holds worker_run_mutex_. So no code may take +// worker_run_mutex_ while it holds the DB mutex. It is the first lock in +// the lock order, before worker_mutex_. +// - worker_mutex_ guards the WorkerState contents. The worker holds it to +// apply a delta. GC holds it to copy generation numbers, to capture the +// candidate revisions and to suppress candidates. No code takes it while +// it holds the DB mutex. +// - mutex_ (the log mutex) guards the change log, the build cursor, the GC +// job registration, the build context pointer and some small flags. +// - metadata_commit_mutex_ guards the metadata-only commit state: a commit +// in progress, a commit with an unknown result, and the canceled flag. +// - A caller may hold the DB mutex when it takes mutex_ or +// metadata_commit_mutex_, so both are held only for short, bounded work. +// - The lock order is worker_mutex_, then mutex_, then +// metadata_commit_mutex_. +// - worker_state_ is read and replaced under worker_state_mutex_ +// (LoadWorkerState() and ExchangeWorkerState()), so a call under the DB +// mutex can drop the WorkerState without worker_mutex_. A worker call that +// still runs keeps its own handle and then fails its token check. +// worker_state_mutex_ is a leaf lock: no other lock is taken while it is +// held, and each hold only copies or swaps one shared_ptr (a few atomic +// operations). The old WorkerState is freed after the lock is released. A +// mutex is used because std::atomic_load() and std::atomic_exchange() on a +// shared_ptr are deprecated in C++20, and some standard libraries do not +// fully support std::atomic. +// +// Why this design: +// - Background worker: building or copying a reverse index as large as the +// DB at the head of the MANIFEST queue would delay every Version install. +// So an install only links one prepared entry. +// - A delta at every install: a delta touches only the changed SSTs, and a +// recount reads every SST. A full compaction also installs a delta. When +// a compaction rewrites almost every SST, its delta can cost more CPU than +// a rebuild. One install path is simpler. +// - Summaries only on edits: a Version would keep each summary for the life +// of its SST, but the census needs it only to add and to remove the SST. So +// VersionBuilder drops it, and a compaction reads its input summaries again +// (reference index only, fill_cache off). Keeping the summaries would use +// much more memory, in the census and in every Version, than the reverse +// index itself. +// - One shared log entry per install: Install() runs under the DB mutex, so +// it must not allocate. It links the one prepared entry. +// - Bounded change log: a slow reader must not hold unbounded memory. After +// the scan of a build ends, an installing thread usually applies each delta +// right after its install, so the change log stays short. During the scan +// of a build, installing threads skip the column family. So the deltas wait +// in the log between build slices, and a build of a very large DB can take +// minutes (an estimate). One limit, kMaxLogMemory (256 MB), covers all +// memory that the build cursor keeps in the change log, including the SST +// summaries that have their own cache charge. The value is an estimate, not +// a measurement: only a long build under heavy write load should reach it. +// When the cursor already holds the limit, it overflows at its next delta, +// and the build restarts at once. The restarted build, and every later +// build until one publishes, has a larger limit, kMaxRestartLogMemory (1 +// GB, 4 x kMaxLogMemory, an estimate, not tuned). The factor 4 leaves room +// for one slow full rebuild under write load. If a build overflows the +// larger limit too, it fails and retries after the normal backoff (see +// "Errors and rebuild"). So overflows never restart builds in a loop +// without a delay, and the log memory stays bounded. GC waits while the +// census is not ready, so no value is lost. +// - Slices: one LOW pool task never runs long, so compactions and other +// column families can run between slices. +// - Key per generation number, not per (generation number, origin): GC asks +// which SSTs use a file. A key per origin would put one SST into several +// holder sets. +// - Slots and list/bitmap sets: an SST file number takes 8 bytes. A slot is +// a small dense number. A list costs 1 byte per holder for gaps below 128 +// and 2 bytes for gaps below 16,384 (varint sizes). A bitmap costs 1 bit +// per slot. Each set takes about the smaller of the two sizes. +// - Pending lists during the build: encoding a list again costs time in +// proportion to its size, so the build encodes each set only once. +// - Routes in the delta: the worker maps each generation number through the +// routes of the same Version as the SST change, also when it applies the +// delta much later. +// - Install reservation: the worker must not publish a Version that an +// install is about to replace. +// +// Census memory charge: the census charges its memory to the block cache under +// CacheEntryRole::kBlobReferenceCensus, unless that role is disabled in +// cache_usage_options, so the memory counts against the block cache capacity. +// The charged memory is the worker state, the build state, the summaries that +// a build or a compaction reads, and the change log entries. An indirect blob +// GC copy of the selected generation numbers (BlobReferenceCensusState) belongs +// to the GC job. It is charged to CacheEntryRole::kIndirectBlobGcWorkingMemory +// in the GC job's block cache instead (see SnapshotSelected()). +// +// Errors and rebuild: a change that the census cannot apply as a delta makes +// it unready, and a new build starts. The GC API waits until the build +// publishes. DB open also runs a build, because the census is kept only in +// memory. The error cases: +// - A delta with `rebuild` set: an exact SST was removed without its summary +// (only CompactionJob passes the summaries of the SSTs it deletes, so for +// example DeleteFile, DeleteFilesInRange, or a compaction that could not +// read an input), or a new exact SST came without its summary. The +// summaries that the install adds become seeds. The build starts at once. +// - PrepareInstallWithCache() failed: PrepareBlobReferenceCensusInstalls() +// prepares an invalidation instead (PrepareInvalidation()). The install +// still commits, and Install() makes the census unready. A running +// build fails its token check and waits for the normal retry delay below. +// - The build cursor overflowed: the build restarts at once with the larger +// limit kMaxRestartLogMemory (see "Bounded change log"). A second overflow +// before a publish fails the build like any other error below. The census +// job then logs a WARN line and counts it in the ticker +// BLOB_DB_CENSUS_RESTART_OVERFLOW_COUNT. +// - The apply after an install failed with an error other than TryAgain: the +// delta may be partly applied, so it is never applied again. The next +// census job run fails the build with that error +// (BlobReferenceCensusBuildContext::failure). +// - Any other build error (for example Corruption): FailBuild() drops the +// worker state. The next try waits 1 s, doubled after each failure, at +// most 32 s (PeriodicTaskType::kBlobReferenceCensus). +// - Not a census error: an edit that adds an SST to a fenced column family +// with no exact metadata, no coarse frontier and no legacy marker fails the +// commit with Corruption (BuildBlobReferenceCensusDelta()). +// +// Self-check: debug builds compare the census with a fresh BuildState() of the +// same Version after 1 in 16 publications of the census job +// (DBImpl::CheckBlobReferenceCensus()). A publish by the apply after an install +// is not checked. + +class Version; +struct BlobReferenceCensusDelta; + +// The size and state of one physical file, as a Version install passes it to +// the candidate heap. The same delta also carries the route changes of that +// install. A total of 0 means the file is not a physical file of the census: it +// was removed, or it is a normal blob file. The heap drops it if it has it. +// Indirect-origin blob files and relocation files of one column family share +// one heap. +struct BlobGcPhysicalFileSummary { + uint64_t file_number = 0; + uint64_t total_record_bytes = 0; + bool is_indirect_origin = false; + bool is_lost = false; +}; + +// Indirect blob GC candidates of one column family. The map holds every +// physical file. The heap holds the files that GC may pick, ranked by garbage +// ratio (garbage bytes / total bytes), highest first. Suppressed files, coarse +// pinned files and lost files that still have live bytes stay out of the heap +// (see Activate()). Only the census worker changes it, and never under the DB +// mutex. Each file also knows its position in the heap, so a change to one file +// fixes the heap in O(log n), where n is the number of files in the heap. +// +// Live bytes of a file are the sum, over all generation numbers that map to the +// file, of the bytes their references keep. A value that several SSTs reference +// is counted once per reference, so the sum can be too high, never too low. +// This is safe for ranking, but it does not prove that a value is dead. GC +// checks the exact references of the files it selects before it moves any +// value. +class BlobGcCandidateHeap { + public: + static constexpr size_t kMaxSelectedFiles = kBlobGcMaxSelectedFiles; + using Selection = std::array; + + void SetPhysicalFile(const BlobGcPhysicalFileSummary& file); + void SetCoarseIndirectOriginFrontier(uint64_t frontier); + Status AdjustLiveBytes(uint64_t file_number, uint64_t bytes, bool add); + void DeletePhysicalFile(uint64_t file_number); + bool IsIndirectOrigin(uint64_t file_number) const; + Selection Select() const; + Selection SelectEmpty() const; + uint64_t Revision(uint64_t file_number) const; + size_t size() const { return candidates_.size(); } + uint64_t RelocationGarbageBytes() const { return relocation_garbage_bytes_; } + + // Totals over the physical files, for monitoring. Each file counts in one + // group: suppressed (Suppress()), else pinned by the coarse frontier, else + // collectable. Garbage is total bytes minus live bytes. The heap keeps them + // up to date at each change, so reading them is O(1). + struct Totals { + uint64_t candidate_count = 0; + uint64_t collectable_garbage_bytes = 0; + uint64_t suppressed_garbage_bytes = 0; + uint64_t coarse_pinned_files = 0; + uint64_t coarse_pinned_bytes = 0; + uint64_t coarse_pinned_garbage_bytes = 0; + }; + Totals GetTotals() const { + Totals totals = totals_; + totals.candidate_count = heap_.size(); + return totals; + } + size_t MemoryUsage(size_t additional_candidates) const; + + // GC reports a file it could not use, and the heap takes the file out + // (suppresses it). The suppression stays across Version changes that do not + // touch the file. A change of the file's total bytes, live bytes, flags + // (is_indirect_origin, is_lost) or coarse pin ends it at once. A rise of the + // coarse frontier ends it for every suppressed relocation file above the old + // frontier (see SetCoarseIndirectOriginFrontier()). + // SuppressTransient() is for transient failures: the file comes back after a + // retry delay (1 s, doubled per failure, at most 32 s). Suppress(file, 0) is + // for a corrupt file: it comes back only after a change to the file or a + // census rebuild. RetryDue() brings back the files whose delay has passed and + // returns the next retry time (0 if none). + void Suppress(uint64_t file_number, uint64_t retry_micros); + void SuppressTransient(uint64_t file_number, uint64_t now_micros); + uint64_t RetryDue(uint64_t now_micros); + + static int CompareRatios(uint64_t lhs_garbage, uint64_t lhs_total, + uint64_t rhs_garbage, uint64_t rhs_total); + + private: + static constexpr size_t kNotInHeap = std::numeric_limits::max(); + struct Candidate { + uint64_t file_number = 0; + uint64_t total_bytes = 0; + uint64_t live_bytes = 0; + uint64_t retry_micros = 0; + uint64_t revision = 0; + size_t position = kNotInHeap; + bool is_indirect_origin = false; + bool is_lost = false; + bool suppressed = false; + bool coarse_pinned = false; + uint8_t failures = 0; + }; + static bool Before(const Candidate* lhs, const Candidate* rhs); + void Swap(size_t lhs, size_t rhs); + void Fix(size_t position); + void Activate(uint64_t file_number, Candidate* candidate); + void Remove(Candidate* candidate); + static uint64_t RelocationGarbage(const Candidate& candidate); + void AdjustRelocationGarbage(uint64_t before, uint64_t after); + // Adds (`add` true) or removes the contribution of `candidate` to totals_. + // Every change of a candidate calls it once before and once after. + void AccountTotals(const Candidate& candidate, bool add); + Totals totals_; + std::map candidates_; + // std::map node addresses are stable, so the heap can hold direct pointers. + // Each heap comparison is then O(1), without a map lookup at every level. + std::vector heap_; + // (retry time, file number) of each suppressed file with a retry time. A + // timer tick reads only the due entries, not every blob file. + std::set> retries_; + uint64_t next_revision_ = 1; + uint64_t coarse_indirect_origin_frontier_ = 0; + uint64_t relocation_garbage_bytes_ = 0; + bool relocation_garbage_overflow_ = false; +}; + +// Dense ids (slots) for the SSTs that hold at least one census group. Each new +// SST gets the lowest free slot, so slot numbers stay small. Small slot numbers +// keep the holder lists short and the holder bitmaps short. +class BlobCensusSlotTable { + public: + static constexpr uint32_t kNoSlot = std::numeric_limits::max(); + + // Gives `file_number` the lowest free slot. Corruption if it has one. + Status Assign(uint64_t file_number, uint32_t* slot); + + // Frees the slot of `file_number` and returns it. Corruption if it has none. + Status Release(uint64_t file_number, uint32_t* slot); + + // kNoSlot if `file_number` has no slot. + uint32_t Find(uint64_t file_number) const; + + // 0 if `slot` is free or out of range. + uint64_t FileNumber(uint32_t slot) const { + return slot < files_.size() ? files_[slot] : 0; + } + + // The number of slots, U: one past the highest slot in use. + uint32_t slot_count() const { return static_cast(files_.size()); } + size_t live_count() const { return slots_.size(); } + size_t MemoryUsage() const; + + private: + // Slot -> file number; 0 means free. + std::vector files_; + UnorderedMap slots_; + // Bit (i % 64) of word i / 64 is set when slot i (< files_.size()) is free. + std::vector free_words_; + // No free slot is below 64 * first_free_word_. + size_t first_free_word_ = 0; +}; + +// The holder set of one generation number: the slots of the SSTs that hold the +// generation number. +// +// Encoding: a small set is a list. The list stores the slots in ascending order +// as varints: the first slot, then the gap to each next slot. A large set is a +// bitmap with one bit per slot. Its words end at the word of the highest set +// bit. +// +// Switch: let U be the number of slots (BlobCensusSlotTable::slot_count()). A +// full bitmap takes U/8 bytes. A list longer than U/8 bytes becomes a bitmap. A +// bitmap becomes a list again only when the list would be shorter than U/16 +// bytes. The gap between the two limits keeps a set near one limit from +// switching back and forth on each change. +// +// Updates: a bitmap is updated in place. A list is encoded again at its exact +// size, so no spare capacity is charged. +class BlobCensusSlotSet { + public: + uint32_t count() const { return count_; } + bool empty() const { return count_ == 0; } + bool is_bitmap() const { return bitmap_; } + bool Contains(uint32_t slot) const; + + // Appends every slot in ascending order. + void AppendTo(std::vector* slots) const; + + // Replaces the set by (set - removed) + added in one pass. Both inputs are + // sorted and unique. Corruption if a removed slot is absent or an added slot + // is already present after the removals. `slot_count` is U. `scratch` is + // reused between calls. + Status Update(const std::vector& removed, + const std::vector& added, uint32_t slot_count, + std::vector* scratch); + + // Replaces the set by `slots` (sorted and unique). + void Assign(const std::vector& slots, uint32_t slot_count); + + // Heap bytes of the list or bitmap, by capacity. + size_t MemoryUsage() const { + return list_.capacity() + 8 * words_.capacity(); + } + + private: + void Encode(const std::vector& slots, uint32_t slot_count); + + // Update() of a bitmap: sets and clears bits in place, and encodes again only + // when the set may cross the list/bitmap switch point. + Status UpdateBitmap(const std::vector& removed, + const std::vector& added, uint32_t slot_count, + std::vector* scratch); + std::string list_; + std::vector words_; + uint32_t count_ = 0; + bool bitmap_ = false; +}; + +// The candidate revisions that an indirect blob GC job sees when it starts. +// Each candidate gets a new revision when its total bytes, live bytes, flags +// (is_indirect_origin, is_lost) or coarse pin change. A later failure report +// (SuppressWorkerCandidates()) is applied only to a file whose revision is +// unchanged. So an unrelated Version change does not cancel the report, and a +// report about an old state of the file is ignored. +struct BlobGcCandidateStamp { + uint64_t worker_token = 0; + BlobGcCandidateHeap::Selection files{}; + std::array revisions{}; +}; + +// The state of one census build (see "Build" at the top of this file) between +// runs of the background job: the pinned starting Version and the next SST to +// read. +// +// The deleter of `starting_version` (set by DBImpl) takes the DB mutex to unref +// the Version and then releases the column family pin. So the last handle must +// be dropped without the DB mutex held. Cancel() only detaches this context; +// the caller drops it later (see BlobReferenceCensusGarbage). +// +// Locks: the thread that runs the worker owns every field (it holds +// BlobReferenceCensus::worker_run_mutex()). There are two exceptions: +// - token is set before the context is shared and never changes, so any +// thread may read it. +// - candidate_retry_micros: the census job reads it under the DB mutex after +// the run, so it is written only under the DB mutex. +struct BlobReferenceCensusBuildContext { + // The census job reads the failure of the current context at the start of + // every run (the first catch-up in DBImpl::BuildBlobReferenceCensus()) and + // fails the build if it is set. A context that a new build replaced, that + // Cancel() detached, or that is dropped at shutdown is never read again. + // There, failure is stored state, not a dropped result, so its destructor + // needs no check. + ~BlobReferenceCensusBuildContext() { failure.PermitUncheckedError(); } + + std::shared_ptr reservation_manager; + // Charges the summary of the SST being read. Each summary is dropped after + // the worker applies its groups. + std::shared_ptr summary_reservation_manager; + std::shared_ptr starting_version; + uint64_t token = 0; + int level = 0; + size_t next_file = 0; + std::set deleted_starting_files; + // Summaries already in memory: the added SSTs of the rebuild install that + // started this build (for example, the outputs of a compaction that could not + // read an input summary). The scan uses them instead of reading the SST. + std::map> seeds; + std::list> pending_deltas; + bool scan_complete = false; + // The time (DB clock, in microseconds) when the next suppressed GC candidate + // may come back (BlobGcCandidateHeap::RetryDue()), or 0 if none waits. + uint64_t candidate_retry_micros = 0; + // A failure that may have changed the worker: of an apply after an install, + // or of a census job run (see "Apply after an install" in this file). While + // it is set, applies after an install skip this build, and the census job + // fails the build with it. + Status failure; + + Status ReserveInitialFiles(size_t file_count); +}; + +// Totals of one generation number over all SSTs that hold it. +struct BlobCensusGenerationTotals { + uint64_t reference_count = 0; + uint64_t stored_value_bytes = 0; + uint64_t record_bytes = 0; +}; + +// A copy of part of the census at one Version: for each generation number in +// the copy, the SSTs that hold it (sorted by file number) and its totals. +// +// SnapshotSelected() copies only the generation numbers that an indirect blob +// GC job selected. The job owns its copy alone. BuildState() copies every +// generation number of a list of SST summaries; debug builds use it to check +// the worker. +class BlobReferenceCensusState { + public: + explicit BlobReferenceCensusState(uint64_t version_number) + : version_number_(version_number) {} + + // A copy would share reservation_manager_, which is not thread safe. A job + // owns its copy alone. + BlobReferenceCensusState(const BlobReferenceCensusState&) = delete; + BlobReferenceCensusState& operator=(const BlobReferenceCensusState&) = delete; + + uint64_t version_number() const { return version_number_; } + size_t generation_count() const { return generations_.size(); } + const std::vector* FindSstFiles(uint64_t generation) const; + + // Null if no SST holds `generation`. + const BlobCensusGenerationTotals* FindTotals(uint64_t generation) const; + size_t GetSstCount(uint64_t generation) const; + + // Every SST that holds a selected generation number, sorted. + std::vector SstFileNumbers() const; + + // True if both have the same generation numbers, and for each one the same + // SSTs and totals. Otherwise describes the first difference in `*difference`. + bool SameMemberships(const BlobReferenceCensusState& other, + std::string* difference) const; + + private: + friend class BlobReferenceCensus; + + struct GenerationEntry { + std::vector sst_files; + BlobCensusGenerationTotals totals; + }; + using SstFilesByGeneration = std::map; + + uint64_t version_number_; + SstFilesByGeneration generations_; + // The generation numbers in this copy; null means every generation number. + std::shared_ptr> selected_generations_; + // The block cache charge of this copy + // (CacheEntryRole::kIndirectBlobGcWorkingMemory, because the indirect blob GC + // job owns the copy), or null when the copy is not charged. The charge is the + // real size of the copy (SnapshotMemoryBound()). + std::shared_ptr reservation_manager_; + size_t snapshot_memory_bound_ = 0; +}; + +// The census change of one Version install, from the Version +// `base_version_number` to the Version `version_number`. +struct BlobReferenceCensusDelta { + uint64_t base_version_number = 0; + uint64_t version_number = 0; + std::vector removed_files; + // The groups of every removed exact SST, sorted by file number. An SST + // without exact metadata needs no summary. + std::vector> removed_summaries; + // The summaries of the added exact SSTs, sorted by file number. + std::vector> added_files; + // True if the change cannot be applied as a delta, in one of these cases: + // - An exact SST was removed without its groups. + // - A new exact SST came without its prepared summary. + // Installing it invalidates the census. The next build runs on the new + // Version and uses `added_files` as seeds, so it does not read those SSTs. + bool rebuild = false; + // The relocation state (routes and relocation files) of the new Version, and + // the routes and physical files that this install changes. The worker maps + // each generation number to its physical file through the routes of the same + // Version as the SST change, also when it applies the delta much later. + std::shared_ptr forward_state; + std::vector changed_routes; + std::vector changed_physical_files; + std::vector physical_files; + size_t retained_forward_state_bytes = 0; + uint64_t coarse_indirect_origin_frontier = 0; + + // An upper bound of the bytes that this delta keeps in memory while it waits + // in the change log. It includes the delta's own vectors, every SST summary + // that it holds, and the parts of the relocation state that no other + // reservation charges (see BlobRelocationState::IncrementalRetentionBytes()). + // It leaves out the file table of the relocation state, which the file + // metadata reservation already charges. The build cursor counts this weight + // against its limit (kMaxLogMemory, or kMaxRestartLogMemory after an overflow + // restart), and the worker acknowledges a delta with the same weight + // (AcknowledgeBuildDeltas()). + size_t QueueWeight() const; + + // The part of QueueWeight() that the change log charges to the block cache + // (see PrepareInstallWithCache()). It leaves out each summary that has its + // own cache charge (reservation_handle set), so that memory is not charged + // twice. The code that creates such a summary charges it: the table builder + // for the summary of a new SST, or the compaction's input read + // (CompactionJob::ReadInputBlobReferenceSummaries()) for the summary of a + // removed SST. The charge stays with the summary object until its last owner + // (the delta, a FileMetaData or the census) drops it. + size_t ChargeBytes() const; +}; + +using BlobReferenceCensusDeltaQueue = + std::list>; + +// One node of the change log: a singly linked list of installed deltas that the +// build cursor reads. Only the tail's `next` changes; the inner links never +// change. +struct BlobReferenceChangeLogEntry { + explicit BlobReferenceChangeLogEntry( + std::shared_ptr value, + std::unique_ptr charge) + : delta(std::move(value)), + retained_bytes(delta->QueueWeight()), + reservation(std::move(charge)) {} + ~BlobReferenceChangeLogEntry(); + const std::shared_ptr delta; + std::shared_ptr next; + size_t retained_bytes; + std::unique_ptr reservation; + // The census counter of change log bytes (see + // BlobReferenceCensus::StayingMemoryBytes()), set when the census links this + // entry. The destructor subtracts retained_bytes from it. + std::shared_ptr> log_bytes; +}; + +struct BlobReferenceCensusPreparedInstall { + std::shared_ptr delta; + // One log entry, allocated before the install. Install() links it and does + // not allocate. + std::shared_ptr log_entry; + bool invalidate_census = false; + uint64_t reservation_id = 0; + size_t delta_queue_weight = 0; +}; + +// State that a census call under the DB mutex hands back to the caller. The +// caller must destroy it after it releases the DB mutex, because it can be as +// large as the census. +struct BlobReferenceCensusGarbage { + // The replaced WorkerState, with its type hidden. It is taken without + // worker_mutex_. A worker run that is still active keeps its own handle until + // it returns. + std::shared_ptr worker_state; + std::shared_ptr build_context; + std::vector> summaries; + // Change log handles. The build cursor has kSlotsPerCursor (3) slots, + // starting at kBuildCursorSlot (0): its first, acknowledged and delivered + // handles, in that order. kLogTailSlot (3) holds the log tail. Dropping the + // last handle of a log prefix frees it. + static constexpr size_t kSlotsPerCursor = 3; + static constexpr size_t kBuildCursorSlot = 0; + static constexpr size_t kLogTailSlot = kSlotsPerCursor; + std::array, kLogTailSlot + 1> + log_entries; +}; + +// The census of one column family (see the top of this file). It owns the +// worker state, the change log with its build cursor, the ready state and the +// metadata-only commit state. The log mutex (`mutex_`) is never held while +// taking the DB mutex, or while parsing, allocating, copying or destroying +// state as large as the census. +class BlobReferenceCensus { + public: + // The limit of QueueWeight() bytes that the build cursor may have queued and + // not acknowledged (256 MB, an estimate; "Bounded change log" at the top of + // this file gives the reason). QueueWeight() counts all memory that a delta + // keeps in the change log, also the summaries that have their own cache + // charge, so this one limit bounds the change log. A cursor whose queued + // weight has already reached the limit overflows at its next delta, and the + // build restarts. A cursor below the limit accepts a delta of any size, so it + // holds at most the limit plus one delta, and a large change still goes + // through. + static constexpr size_t kMaxLogMemory = 256 * 1024 * 1024; + // The limit of the build after an overflow restart, and of every later build + // until a build publishes (1 GB: 4 x kMaxLogMemory, an estimate, not tuned; + // see "Bounded change log" at the top of this file). + static constexpr size_t kMaxRestartLogMemory = 4 * kMaxLogMemory; + // The number of deltas that one TakeBuildDeltas() call returns at most (64, + // an estimate). A fixed array holds them, so the log mutex is held for a + // bounded time. + static constexpr size_t kMaxDeltasPerRead = 64; + + enum class PublishResult : uint8_t { + kPublished, + kQueueNotEmpty, + kInstallPending, + kStale, + }; + + BlobReferenceCensus() = default; + BlobReferenceCensus(const BlobReferenceCensus&) = delete; + BlobReferenceCensus& operator=(const BlobReferenceCensus&) = delete; + + // Metadata-only commit (see the top of this file). REQUIRES: DB mutex held. + // BeginMetadataCommit() marks that one such commit is in progress, from the + // Version preparation without the DB mutex until the MANIFEST sync. It + // returns TryAgain while another commit is in progress or while a commit + // result is unknown, and ColumnFamilyDropped after Cancel(). While a commit + // is in progress, BeginGcJob() returns TryAgain. + // EndMetadataCommit(false) means the commit result is unknown (for example, + // after a failed MANIFEST write). BeginGcJob() then returns Busy, and + // BeginMetadataCommit() returns TryAgain, until + // ResolveMetadataCommitAmbiguity(). Flushes and compactions never wait for a + // metadata-only commit. + Status BeginMetadataCommit(); + void EndMetadataCommit(bool result_known); + + // REQUIRES: DB mutex held. Call only after a new MANIFEST and CURRENT file + // were written and the Version was installed. Only this proves which state is + // durable. A later successful MANIFEST append alone is not enough. + void ResolveMetadataCommitAmbiguity(); + + // REQUIRES: DB mutex held. Marks that a Version install of this column family + // is being prepared (O(1)). While it is active, the worker does not publish + // (TryPublishWorker() returns kInstallPending), because an install that it + // has not seen is about to come. Install() or CancelInstallPreparation() ends + // it and wakes a waiting worker. + uint64_t BeginInstallPreparation(); + void CancelInstallPreparation(uint64_t reservation_id); + + // PrepareInstall() and PrepareInstallWithCache() prepare the log entry of + // `delta` without the DB mutex. PrepareInstallWithCache() also charges its + // memory to `block_cache` if it is not null. For a rebuild delta, both + // prepare an invalidation instead. PrepareInstall() (no cache charge) is for + // tests. + Status PrepareInstall(std::shared_ptr delta, + uint64_t reservation_id, + BlobReferenceCensusPreparedInstall* prepared) const; + Status PrepareInstallWithCache( + std::shared_ptr delta, + uint64_t reservation_id, std::shared_ptr block_cache, + BlobReferenceCensusPreparedInstall* prepared) const; + + // Prepares an invalidation for when PrepareInstall() fails. It does not + // allocate. The Version install still goes ahead, and Install() makes the + // census unready, so no old or partial census stays visible. + Status PrepareInvalidation( + uint64_t reservation_id, + BlobReferenceCensusPreparedInstall* prepared) const; + + // REQUIRES: DB mutex held. Links the prepared log entry without allocation, + // or invalidates the census. Returns the replaced state, which the caller + // destroys without the DB mutex. + BlobReferenceCensusGarbage Install( + BlobReferenceCensusPreparedInstall& prepared); + + // REQUIRES: DB mutex held. Opens the build cursor. The caller pins the + // current Version under the same DB mutex hold, so the cursor sees every + // later install. The returned token identifies this build attempt. + uint64_t BeginBuild(); + + // Returns the next deltas of the build cursor, to apply without the DB mutex + // and the log mutex. + Status TakeBuildDeltas(uint64_t token, BlobReferenceCensusDeltaQueue* deltas, + bool* overflowed); + + // Releases the queue budget of deltas that the builder has applied. A delta + // that was returned but not yet applied stays counted, so a slow initial scan + // cannot build up an unbounded private backlog. The caller passes `weight`, + // the sum of QueueWeight() of the deltas it applied, in order. The cursor + // limit (kMaxLogMemory, or kMaxRestartLogMemory after an overflow restart) is + // in bytes, so the acknowledgment is in bytes too. The census moves this + // cursor past the log entries whose weights add up to this sum. An entry is + // freed when no cursor, older entry or log tail still holds it. + Status AcknowledgeBuildDeltas(uint64_t token, size_t weight); + + // True if the build cursor of build `token` has overflowed. A failed + // AcknowledgeBuildDeltas() can mean an overflow. The census job then handles + // it like an overflow that TakeBuildDeltas() reports (see + // UseOverflowRestart()). + bool BuildCursorOverflowed(uint64_t token) const; + + // REQUIRES: DB mutex held. Ends the build `token` after an error: the worker + // state is dropped, the census becomes unready (GC pauses) and the build + // cursor closes. Returns the replaced state, which the caller destroys + // without the DB mutex. + BlobReferenceCensusGarbage FailBuild(uint64_t token); + + // Stops the census when its column family goes away: closes metadata-only + // commits, closes the build cursor, ends the GC job registration and the + // snapshot pause, and drops the worker state, the build state and the rebuild + // seeds. Returns the replaced state, which the caller destroys without the DB + // mutex. REQUIRES: DB mutex held (or exclusive use in tests). Because of the + // DB mutex, clearing the wakeup callback cannot race with EndMetadataCommit + // or ResolveMetadataCommitAmbiguity. + BlobReferenceCensusGarbage Cancel(); + + // One thread at a time runs the census worker: the census job (one build + // slice) or an installing thread that applies the queued deltas (see "Apply + // after an install" at the top of this file). The installing thread only + // tries the lock, so it never waits for a build slice. Take it before the DB + // mutex, never while holding it. + std::mutex& worker_run_mutex() { return worker_run_mutex_; } + + // True if the census is published for `version_number`. + bool IsReady(uint64_t version_number) const; + bool IsSnapshotPending() const; + + // REQUIRES: DB mutex held. Registers an indirect blob GC job at the ready + // Version `version_number` and returns its token. At most one GC job is + // registered per column family. The worker then does not apply deltas until + // SnapshotSelected() has copied the selected generation numbers, or the job + // ends. Returns TryAgain if the census is not ready at `version_number` or a + // metadata commit (BeginMetadataCommit()) is in progress. Returns Busy while + // a metadata commit result is unknown, or if a GC job is already registered. + // Returns Aborted after Cancel(). + Status BeginGcJob(uint64_t version_number, uint64_t* token); + + // True while the GC job `token` is registered and the census is not + // canceled. + bool IsGcJobActive(uint64_t token) const; + + // True while any GC job is registered (see BeginGcJob()). + bool HasGcJob() const; + + // REQUIRES: DB mutex held. Ends the GC job `token` (no-op for another + // token). O(1), no allocation. + void EndGcJob(uint64_t token); + + // Builds a copy of every generation number from a list of SST summaries. + static Status BuildState( + uint64_t version_number, + const std::vector>& files, + std::shared_ptr* state); + + // The summaries handed over by the last rebuild install, for the next build. + std::vector> + TakeRebuildSeeds(); + + // True from a rebuild install until the next BeginBuild(). A build slice that + // fails because of that install restarts at once, without backoff. + bool RebuildRequested() const; + + // Before indirect blob GC copies a blob file, it reads the references of that + // file from the SSTs and checks them against the census. GC calls + // ReportMismatch() when the SSTs or the totals differ from the census for + // blob file `file_number`. + // + // Census mismatch state: ReportMismatch(), ClearMismatch() and + // TakeMismatchReport() share the file of the last mismatch report ("last + // file", unresolved_mismatch_file_, 0 means none) and two flags, + // mismatch_reported_ and rebuild_requested_. TryPublishWorker() returns + // kStale while either flag is set, so a mismatching census cannot become + // ready again. BeginBuild() clears rebuild_requested_. In the table, f is the + // file number that GC passes. + // + // Call Condition Action Returns + // -------------------- ----------------- --------------------- ------- + // ReportMismatch(f) f == last file count a suppression true + // and f != 0 + // ReportMismatch(f) otherwise last file = f; census false + // not ready; set both + // flags + // ClearMismatch(f) f == last file last file = 0 - + // ClearMismatch(f) f != last file nothing - + // TakeMismatchReport() - clear the old + // mismatch_reported_ flag + // + // After a first report, the next build slice rebuilds the census from the + // current Version. A repeat report (true in the table) means that the rebuild + // did not clear the mismatch. GC then suppresses that file until the file + // changes or until the next census rebuild (see + // BlobGcCandidateHeap::Suppress()). The census remembers only the last file, + // so reports for two files that alternate both request a rebuild each time. + bool ReportMismatch(uint64_t file_number); + + // GC calls this for each checked input of a job when no input of that job + // mismatched the SSTs. When `file_number` is the file of the last mismatch + // report, the census forgets that report, so a later mismatch of the file + // requests a rebuild again. A match of another file keeps the report, so a + // match of one file does not hide a repeat mismatch of a different file. + void ClearMismatch(uint64_t file_number); + + // Returns how many times GC suppressed a candidate because a census rebuild + // did not clear its mismatch. + uint64_t gc_mismatch_suppress_count() const { + return gc_mismatch_suppress_count_.load(std::memory_order_relaxed); + } + + // Returns true if a ReportMismatch() requested a rebuild since the last call, + // and clears the flag. The census build slice calls it to fail the running + // build. + bool TakeMismatchReport(); + + // After a change log overflow, the build restarts at once, one time only. + // From then until a build publishes, each build has the larger limit + // kMaxRestartLogMemory. Returns false if that restart was already used + // without a publish, which means that the restarted build overflowed its + // limit too. The caller then fails the build, and it retries after the normal + // backoff (see "Bounded change log" at the top of this file). + bool UseOverflowRestart(); + uint64_t overflow_restart_count() const { + return overflow_restart_count_.load(std::memory_order_relaxed); + } + + // Counts one rebuild caused by a commit that added an exact SST without its + // prepared summary. The census is then rebuilt from disk instead of failing + // the commit. + void NoteMissingSummaryRebuild() { + missing_summary_rebuild_count_.fetch_add(1, std::memory_order_relaxed); + } + uint64_t missing_summary_rebuild_count() const { + return missing_summary_rebuild_count_.load(std::memory_order_relaxed); + } + + // Monitoring (the property rocksdb.indirect-blob-gc-stats). These read and + // write relaxed atomics only, so a property read takes no census lock. + // + // An estimate of the census memory that stays: the worker state (its last + // cache reservation size) plus the change log entries that the census holds. + // It leaves out the short-lived build context, GC copies and the summaries + // that a compaction reads. + uint64_t StayingMemoryBytes() const; + // Number of builds started (BeginBuild()). + uint64_t build_count() const { + return build_count_.load(std::memory_order_relaxed); + } + // The census is unready from an install, a build start or a build failure + // until a publish. NoteUnready() keeps the DB clock time of the first such + // event; NoteReady() clears it at a publish and returns it. A build start + // also remembers its time, so the publish that ends the build can log how + // long it took (TakeBuildStartMicros()). + void NoteUnready(uint64_t now_micros) { + uint64_t expected = 0; + unready_since_micros_.compare_exchange_strong(expected, now_micros, + std::memory_order_relaxed); + } + uint64_t NoteReady() { + return unready_since_micros_.exchange(0, std::memory_order_relaxed); + } + uint64_t UnreadySinceMicros() const { + return unready_since_micros_.load(std::memory_order_relaxed); + } + // The candidate totals of the last publish (see BlobGcCandidateHeap::Totals), + // and the relocation garbage bytes it published. + void SetPublishedTotals(const BlobGcCandidateHeap::Totals& totals, + uint64_t relocation_garbage_bytes) { + published_candidate_count_.store(totals.candidate_count, + std::memory_order_relaxed); + published_collectable_garbage_.store(totals.collectable_garbage_bytes, + std::memory_order_relaxed); + published_suppressed_garbage_.store(totals.suppressed_garbage_bytes, + std::memory_order_relaxed); + published_coarse_pinned_files_.store(totals.coarse_pinned_files, + std::memory_order_relaxed); + published_coarse_pinned_bytes_.store(totals.coarse_pinned_bytes, + std::memory_order_relaxed); + published_coarse_pinned_garbage_.store(totals.coarse_pinned_garbage_bytes, + std::memory_order_relaxed); + published_relocation_garbage_.store(relocation_garbage_bytes, + std::memory_order_relaxed); + } + BlobGcCandidateHeap::Totals PublishedTotals() const { + BlobGcCandidateHeap::Totals totals; + totals.candidate_count = + published_candidate_count_.load(std::memory_order_relaxed); + totals.collectable_garbage_bytes = + published_collectable_garbage_.load(std::memory_order_relaxed); + totals.suppressed_garbage_bytes = + published_suppressed_garbage_.load(std::memory_order_relaxed); + totals.coarse_pinned_files = + published_coarse_pinned_files_.load(std::memory_order_relaxed); + totals.coarse_pinned_bytes = + published_coarse_pinned_bytes_.load(std::memory_order_relaxed); + totals.coarse_pinned_garbage_bytes = + published_coarse_pinned_garbage_.load(std::memory_order_relaxed); + return totals; + } + uint64_t PublishedRelocationGarbageBytes() const { + return published_relocation_garbage_.load(std::memory_order_relaxed); + } + void NoteBuildStart(uint64_t now_micros) { + build_start_micros_.store(now_micros, std::memory_order_relaxed); + } + // Returns the start time of the build that a publish ends, or 0 if the + // publish ended no build (for example, a publish after an install). + uint64_t TakeBuildStartMicros() { + return build_start_micros_.exchange(0, std::memory_order_relaxed); + } + + // Worker API. The census worker changes the worker state only through these + // calls. The worker state is derived: it is never copied into a Version, and + // an install does not copy it. + // + // Locks (see the lock order at the top of this file): + // - A call that takes worker_mutex_ must run without the DB mutex. + // - NeedsWorkerRun(), TryPublishWorker(), GetBuildContext() and + // TrySetBuildContext() take only the log mutex, so DBImpl may call them + // with the DB mutex held. + // - SetBuildWakeup() requires the DB mutex. + // - CaptureCandidateStamp(), SuppressWorkerCandidates(), SnapshotSelected() + // and ReadSelectedSnapshot() serve GC selection, not the worker. They + // take worker_mutex_, so they run without the DB mutex. + Status ResetWorkerState( + uint64_t token, uint64_t version_number, + std::shared_ptr forward_state, + std::shared_ptr block_cache); + Status AddStartingFile(uint64_t token, + std::shared_ptr file); + + // Builds each holder set once, after the initial scan and every delta taken + // during the scan are applied. Call before the first publication. + Status FinalizeWorkerBuild(uint64_t token); + Status SetWorkerPhysicalFiles( + uint64_t token, const std::vector& files); + Status SetWorkerCoarseIndirectOriginFrontier(uint64_t token, + uint64_t frontier); + BlobGcCandidateHeap::Selection SelectWorkerCandidates(uint64_t token); + BlobGcCandidateHeap::Selection SelectWorkerEmptyCandidates(uint64_t token); + uint64_t WorkerRelocationGarbageBytes(uint64_t token) const; + BlobGcCandidateHeap::Totals WorkerCandidateTotals(uint64_t token) const; + uint64_t RetryWorkerCandidates(uint64_t token, uint64_t now_micros); + bool NeedsWorkerRun(uint64_t version_number) const; + + // REQUIRES: DB mutex held. Sets the callback that schedules the census job of + // this column family (`db` and `cfd` are its arguments). The census stores + // only these raw pointers and holds no reference to the DB or the column + // family. Cancel() clears them. The callback only queues work and never + // blocks a LOW pool thread. When the callback queues the column family, + // DBImpl takes its own reference to that column family. The callback takes + // the log mutex (see NeedsWorkerRun()), so callers run it after they release + // the log mutex and metadata_commit_mutex_, still under the DB mutex. + using BuildWakeup = void (*)(void*, void*); + void SetBuildWakeup(BuildWakeup wakeup, void* db, void* cfd); + Status CaptureCandidateStamp(uint64_t version_number, + const std::vector& files, + BlobGcCandidateStamp* stamp) const; + + // Call without the DB mutex. GC reports the files it could not use. A report + // from a replaced worker, or for a file whose revision changed, is ignored. + // Files in `transient_files` get a retry delay; the other files wait for a + // change (see BlobGcCandidateHeap::Suppress()). + void SuppressWorkerCandidates(const BlobGcCandidateStamp& stamp, + const std::vector& files, + const std::set& transient_files, + uint64_t now_micros); + Status ApplyWorkerDelta(uint64_t token, + const BlobReferenceCensusDelta& delta); + uint64_t WorkerVersionNumber() const; + PublishResult TryPublishWorker(uint64_t token, + uint64_t applied_version_number, + uint64_t current_version_number); + std::shared_ptr GetBuildContext() const; + + // Attaches the context only while its token is still the build token. The + // census can be invalidated while the context is prepared without the DB + // mutex. Then this returns false and the caller drops the context without the + // DB mutex. A token whose cursor overflowed still attaches, so the normal + // failure path cleans it up. + bool TrySetBuildContext( + std::shared_ptr context); + + // Call without the DB mutex, after BeginGcJob() registered the job at + // `version_number`. Copies the holders of `generation_numbers` only. The copy + // does not block an install. A non-null `gc_block_cache` is charged for the + // copy under CacheEntryRole::kIndirectBlobGcWorkingMemory. + Status SnapshotSelected(uint64_t token, uint64_t version_number, + const std::set& generation_numbers, + const std::shared_ptr& gc_block_cache, + std::shared_ptr* state); + + // Same copy as SnapshotSelected(), without the GC job checks. + Status ReadSelectedSnapshot( + uint64_t version_number, const std::set& generation_numbers, + const std::shared_ptr& gc_block_cache, + std::shared_ptr* state) const; + + // Test only: builds a full copy of every generation number. Call without the + // DB mutex. + std::shared_ptr TEST_GetReadyState( + uint64_t version_number) const; + + private: + struct WorkerState; + static bool IsCensusGroup(const BlobReferenceGroup& group); + + // True if `worker` is the worker of build `token` and that build is still + // active, so a run of a replaced build cannot change the worker of a newer + // build. REQUIRES: worker_mutex_ held. + bool IsActiveWorker(const WorkerState* worker, uint64_t token) const; + + // Sums the census groups of `file` per generation number, only for + // `selected_names` when it is not null. + static Status SumGroupsByGeneration( + const BlobReferenceSstSummary& file, + const std::set* selected_names, + std::map* totals); + static Status AddFile(const BlobReferenceSstSummary& file, + BlobReferenceCensusState* state); + Status CopyWorkerSnapshot( + const WorkerState& worker, uint64_t version_number, + const std::set& generation_numbers, + const std::shared_ptr& gc_block_cache, + std::shared_ptr* state) const; + static size_t SnapshotMemoryBound(const BlobReferenceCensusState& state); + + // A reader position in the change log. + // - first: the entry where delivery starts while nothing is acknowledged. + // It is set when an entry is appended while nothing is delivered. The + // first acknowledgment clears it; after that, `acknowledged` marks the + // position. + // - delivered: the last entry that TakeBuildDeltas() returned. + // - acknowledged: the last entry that the reader acknowledged. + // - outstanding_bytes: the QueueWeight() of the entries appended to this + // cursor and not yet acknowledged. + // - overflowed: a delta arrived when the cursor already held its limit + // (kMaxLogMemory, or kMaxRestartLogMemory when restart_build is set), and + // the cursor was detached. + struct ChangeLogCursor { + uint64_t token = 0; + std::shared_ptr first; + std::shared_ptr acknowledged; + std::shared_ptr delivered; + size_t outstanding_bytes = 0; + bool overflowed = false; + // Set for the builds after an overflow restart: their limit is + // kMaxRestartLogMemory instead of kMaxLogMemory. A publish clears it. + bool restart_build = false; + }; + static bool HasPending(const ChangeLogCursor& cursor); + static void DetachCursor(ChangeLogCursor* cursor, + BlobReferenceCensusGarbage* garbage, + size_t first_slot); + void AppendChange(const BlobReferenceCensusPreparedInstall& prepared, + BlobReferenceCensusGarbage* garbage); + + // One record per generation number: its totals, its holder set, and the + // physical file that holds its values now. `record_bytes` is the sum + // of BlobReferenceGroup::record_bytes over the census groups with this + // generation number (not kDirect groups, which the census does not count). + struct WorkerGeneration { + BlobCensusSlotSet holders; + // Slot changes made during the initial build. Finalize() applies them to + // `holders` in one pass when the build completes. + std::vector pending_added; + std::vector pending_removed; + uint64_t reference_count = 0; + uint64_t stored_value_bytes = 0; + uint64_t record_bytes = 0; + uint64_t physical_file_number = 0; + }; + struct WorkerState { + uint64_t token = 0; + uint64_t version_number = 0; + std::map generations; + BlobCensusSlotTable slots; + std::shared_ptr forward_state; + std::shared_ptr block_cache; + std::shared_ptr reservation_manager; + // False until Finalize() at the end of the initial build. Before that, a + // removed SST without a slot is a starting SST the scan has not reached. + bool finalized = false; + // Heap bytes of every holder set and pending slot vector, by capacity. + size_t slot_set_bytes = 0; + // The size of the last Reserve() call, for StayingMemoryBytes(). Only the + // worker writes it; a property read loads it without a lock. + std::atomic reserved_bytes{0}; + std::vector scratch; + BlobGcCandidateHeap candidates; + + size_t MemoryUsage(size_t additional_slots, + size_t additional_candidates) const; + Status Reserve(size_t additional_slots); + Status Reserve(size_t additional_slots, size_t additional_candidates); + + // Removes the groups of `removed` and adds the groups of `added` as one + // change. A removal must match what was added; anything else is Corruption. + Status ApplyFiles( + const std::vector>& + removed, + const std::vector>& + added); + Status Finalize(); + static size_t GenerationBytes(const WorkerGeneration& entry); + uint64_t ResolvePhysical(uint64_t generation) const; + Status AdjustGenerationBytes(const WorkerGeneration& entry, bool add); + }; + // Never take this mutex while holding the DB mutex. Changes to the worker + // state and the selected copies for GC take it. Install and the ready checks + // use atomics and the log mutex instead. + mutable std::mutex worker_mutex_; + // See worker_run_mutex(). + std::mutex worker_run_mutex_; + // See "Threads and locks" at the top of this file. + mutable std::mutex metadata_commit_mutex_; + bool metadata_commit_pending_ = false; + bool metadata_commit_ambiguous_ = false; + bool metadata_commit_canceled_ = false; + // Read and replace only with LoadWorkerState() and ExchangeWorkerState(). + std::shared_ptr LoadWorkerState() const; + std::shared_ptr ExchangeWorkerState( + std::shared_ptr next); + // Guards worker_state_ only (see "Threads and locks"). + mutable std::mutex worker_state_mutex_; + std::shared_ptr worker_state_; + std::atomic worker_active_token_{0}; + std::atomic worker_ready_version_{0}; + std::atomic candidate_refresh_requested_{false}; + bool snapshot_pending_ = false; // protected by the log mutex + bool build_waiting_for_install_ = false; // protected by the log mutex + // Protected by the DB mutex; cleared before owner teardown. + BuildWakeup build_wakeup_ = nullptr; + void* build_wakeup_db_ = nullptr; + void* build_wakeup_cfd_ = nullptr; + + mutable std::mutex mutex_; + // One shared, thread-safe reservation for all log entries, so many small + // entries share one cache charge unit. Each entry holds a handle. The build + // cursor releases a handle, always without the DB mutex, and the handle keeps + // the manager alive until it is destroyed. + mutable std::shared_ptr log_reservation_manager_; + mutable std::shared_ptr log_block_cache_; + std::shared_ptr build_context_; + // The last entry. Only its `next` link changes, and this pointer keeps it + // alive. It is detached when the build cursor is drained or closed, and + // released without the DB mutex and the log mutex. The inner links never + // change, so freeing a prefix without a lock cannot race with an append. + std::shared_ptr log_tail_; + ChangeLogCursor build_cursor_; + // The token of the registered GC job, or 0 (see BeginGcJob()). + uint64_t gc_job_token_ = 0; + uint64_t next_build_token_ = 1; + uint64_t next_reservation_id_ = 1; + std::atomic active_install_reservation_{0}; + uint64_t next_gc_token_ = 1; + bool canceled_ = false; + std::vector> rebuild_seeds_; + bool rebuild_requested_ = false; + bool mismatch_reported_ = false; + // The file of the last mismatch report. ClearMismatch() of this file resets + // it to 0 (none). + uint64_t unresolved_mismatch_file_ = 0; + std::atomic gc_mismatch_suppress_count_{0}; + // An overflow restart ran and no build has published since. While it is set, + // BeginBuild() gives the build cursor the restart limit + // (kMaxRestartLogMemory). + bool overflow_restart_used_ = false; + std::atomic overflow_restart_count_{0}; + std::atomic missing_summary_rebuild_count_{0}; + // See StayingMemoryBytes(). Shared with the linked log entries, which may be + // freed after the census. + std::shared_ptr> log_bytes_ = + std::make_shared>(0); + std::atomic build_count_{0}; + std::atomic unready_since_micros_{0}; + std::atomic build_start_micros_{0}; + std::atomic published_candidate_count_{0}; + std::atomic published_collectable_garbage_{0}; + std::atomic published_suppressed_garbage_{0}; + std::atomic published_coarse_pinned_files_{0}; + std::atomic published_coarse_pinned_bytes_{0}; + std::atomic published_coarse_pinned_garbage_{0}; + std::atomic published_relocation_garbage_{0}; +}; + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_reference_metadata.cc b/db/blob/blob_reference_metadata.cc new file mode 100644 index 000000000000..79663ad043ff --- /dev/null +++ b/db/blob/blob_reference_metadata.cc @@ -0,0 +1,1686 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_reference_metadata.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_log_format.h" +#include "db/version_edit.h" +#include "rocksdb/table.h" +#include "table/block_based/block_based_table_reader.h" +#include "test_util/sync_point.h" +#include "util/coding.h" +#include "util/crc32c.h" + +namespace ROCKSDB_NAMESPACE { + +const char kBlobReferenceDescriptorPropertyName[] = + "rocksdb.blob.reference-descriptor"; +const char kBlobReferenceIndexBlockName[] = "rocksdb.blob.reference-index"; +const char kBlobReferenceClassificationPropertyName[] = + "rocksdb.blob.reference-classification"; + +void EncodeBlobReferenceClassification(uint64_t frontier, std::string* output) { + output->clear(); + PutFixed32(output, 1); // Format version 1. + PutFixed64(output, frontier); + PutFixed32(output, crc32c::Value(output->data(), output->size())); +} + +Status DecodeBlobReferenceClassification(const Slice& input, + uint64_t* frontier) { + if (frontier == nullptr) { + return Status::InvalidArgument("Null coarse blob frontier output"); + } + if (input.size() != 16 || DecodeFixed32(input.data()) != 1 || + crc32c::Value(input.data(), 12) != DecodeFixed32(input.data() + 12)) { + return Status::Corruption("Invalid blob reference classification"); + } + *frontier = DecodeFixed64(input.data() + 4); + if (*frontier > kFileNumberMask) { + return Status::Corruption("Invalid coarse blob indirect-origin frontier"); + } + return Status::OK(); +} + +namespace { + +constexpr uint32_t kDescriptorMagic = 0x42475244U; // BGRD +// Groups that took the unsorted fallback, process-wide. +std::atomic unsorted_group_fallbacks{0}; +// Largest encoded (uncompressed) size of a data block that the writer makes. +// See "Compression and block size" at the top of the header for why 16 KB. +constexpr size_t kTargetDataBlockSize = 16 * 1024; +// Largest data block size that the decoders accept. The index decoder checks +// each block's size on disk (before decompression) and each group's run offset +// against this limit, and the run decoder checks the block contents after +// decompression (BlobReferenceGroupDecoder::DecodeRunHeader()). The writer +// keeps every block at or below kTargetDataBlockSize (16 KB), so 64 KB is a +// wide safety margin (estimate, not measured). The static_assert below checks +// that the writer's block buffer (the target size plus one run header and one +// entry) also fits. +constexpr size_t kMaxDataBlockSize = 64 * 1024; +// Largest reference index block that the writer makes and the decoders accept +// (64 MB). It bounds the memory that one index read uses. The count limits +// below do not keep the block under 64 MB. With kMaxGroupCount (2^20) groups, +// the group table alone can take about 71 MB (kMaxIndexEntrySize, 71 bytes +// per group). So EncodeIndex() checks the worst-case size against this limit +// and fails with MemoryLimit. 64 MB is an estimate of "far more than one SST +// needs", not measured. +constexpr size_t kMaxTopLevelMetadataBlockSize = 64 * 1024 * 1024; +// Count limits that both the writer and the index decoder check. They are +// part of the format, so they are fixed numbers and do not depend on C++ type +// sizes. 1M groups, or 1M data blocks of up to 16 KB each (16 GB of encoded +// entries), is far more than one SST needs (estimate, not measured). +constexpr uint64_t kMaxGroupCount = 1ULL << 20; +constexpr uint64_t kMaxDataBlockCount = 1ULL << 20; +// With these limits, the decoded block and group vectors stay under +// kMaxDecodedBlobReferenceBytes, so the index decoder needs no byte check at +// run time. +static_assert(kMaxDataBlockCount * sizeof(BlockHandle) <= + kMaxDecodedBlobReferenceBytes); +static_assert(kMaxGroupCount * sizeof(BlobReferenceIndexEntry) <= + kMaxDecodedBlobReferenceBytes); +// Encoded size bounds used for block fill and size planning. +constexpr size_t kMaxRunHeaderSize = 2 * kMaxVarint64Length + 1; +constexpr size_t kMaxEntrySize = 3 * kMaxVarint64Length + 1 + 8; +constexpr size_t kMaxIndexEntrySize = 1 + 7 * kMaxVarint64Length; +constexpr size_t kMaxBlockTableEntrySize = 2 * kMaxVarint64Length; +// Smallest encoded sizes, which bound a decoded count by the input size +// before any reserve(): a block size takes one varint byte; an index entry +// takes the class byte and at least six one-byte varints; a reference entry +// takes at least the size varint and either a key size varint or a CRC. +constexpr size_t kMinBlockTableEntrySize = 1; +constexpr size_t kMinIndexEntrySize = 7; +constexpr size_t kMinEntrySize = 2; +static_assert(kTargetDataBlockSize + kMaxRunHeaderSize + kMaxEntrySize <= + kMaxDataBlockSize); +// The underlying type is unsigned and the first class is 0, so only the upper +// bound is checked. Update it when a class is added. +bool IsValidEncodingClass(BlobReferenceEncodingClass encoding_class) { + return encoding_class <= BlobReferenceEncodingClass::kIndirectFlattened; +} + +bool AddWithoutOverflow(uint64_t lhs, uint64_t rhs, uint64_t* sum) { + if (sum == nullptr || rhs > std::numeric_limits::max() - lhs) { + return false; + } + *sum = lhs + rhs; + return true; +} + +// A flattened reference names a record in a relocation file, whose size is +// the stored size plus kSimpleGen2BlobTrailerSize (see "Relocation file" in +// blob_index.h). Unflattened and kDirect references name a record in a normal +// blob file and store its user key size in each entry. +bool StoresUserKeySize(BlobReferenceEncodingClass encoding_class) { + return encoding_class != BlobReferenceEncodingClass::kIndirectFlattened; +} + +// Computes the offset that a gap of 0 gives (the gap is defined at the top of +// blob_reference_metadata.h). The gap of an entry is its offset minus this +// base: +// - Unflattened and kDirect: previous offset + previous stored size + +// BlobLogRecord::kHeaderSize (32 bytes) + this entry's user key size. The +// caller passes the last two as `record_overhead`. +// - Flattened: the offset is still the original offset in the origin file, +// and the entry stores no user key size. So the base is previous offset + +// previous stored size, and the caller passes 0. +// The readers (DecodeRunEntriesAt()) and the unsorted fallback +// (SortAndEncodeGroup()) call this function. AppendSorted() computes the same +// base from the group's last offset and stored size. +bool GapBase(const BlobReference& previous, uint64_t record_overhead, + uint64_t* base) { + return AddWithoutOverflow(previous.original_value_offset, + previous.stored_value_size, base) && + AddWithoutOverflow(*base, record_overhead, base); +} + +// The user key size of a reference whose class stores it. Fails when the +// record bytes do not hold a record header. +bool UserKeySize(const BlobReference& reference, uint64_t* key_size) { + if (reference.record_bytes - reference.stored_value_size < + BlobLogRecord::kHeaderSize) { + return false; + } + *key_size = reference.record_bytes - reference.stored_value_size - + BlobLogRecord::kHeaderSize; + return true; +} + +// The per-record overhead of a group's records besides their stored bytes, +// apart from user keys: a record header in a normal blob file, or a trailer +// in a relocation file. +bool GroupRecordOverhead(BlobReferenceEncodingClass encoding_class, + uint64_t reference_count, uint64_t* overhead) { + const uint64_t per_record = StoresUserKeySize(encoding_class) + ? BlobLogRecord::kHeaderSize + : kSimpleGen2BlobTrailerSize; + if (reference_count > std::numeric_limits::max() / per_record) { + return false; + } + *overhead = reference_count * per_record; + return true; +} + +Status ValidateReference(const BlobReference& reference) { + if (!IsValidEncodingClass(reference.encoding_class)) { + return Status::Corruption("Invalid blob reference encoding class"); + } + if (reference.generation_file_number == 0 || + reference.origin_file_number == 0) { + return Status::Corruption("Invalid blob reference file number"); + } + if (!BlobIndex::IsValidCompression(reference.compression)) { + return Status::Corruption("Invalid blob reference compression"); + } + // For unflattened and kDirect, this checks only record_bytes >= stored size. + if (reference.record_bytes < reference.stored_value_size) { + return Status::Corruption("Invalid blob reference record size"); + } + if (reference.encoding_class == BlobReferenceEncodingClass::kDirect) { + if (reference.generation_file_number != reference.origin_file_number || + reference.stable_checksum != 0 || reference.stored_checksum != 0) { + return Status::Corruption("Invalid kDirect blob reference metadata"); + } + } else { + if (reference.compression == kNoCompression && + reference.stored_checksum != + BlobIndex::ExtendChecksumWithCompression(reference.stable_checksum, + kNoCompression)) { + return Status::Corruption("Noncanonical stored blob checksum"); + } + const bool is_unflattened = + reference.generation_file_number == reference.origin_file_number; + if (is_unflattened != (reference.encoding_class == + BlobReferenceEncodingClass::kIndirectUnflattened)) { + return Status::Corruption("Blob reference encoding class mismatch"); + } + // Same rule as kIndirectFlattenedBlob in BlobIndex::DecodeFrom(). + if (!is_unflattened && + reference.generation_file_number < reference.origin_file_number) { + return Status::Corruption("Flattened blob generation is below origin"); + } + if (reference.encoding_class == + BlobReferenceEncodingClass::kIndirectFlattened && + (reference.stored_value_size > std::numeric_limits::max() - + kSimpleGen2BlobTrailerSize || + reference.record_bytes != + reference.stored_value_size + kSimpleGen2BlobTrailerSize)) { + return Status::Corruption("Invalid flattened blob record size"); + } + } + return Status::OK(); +} + +void SetFixed32(std::string* output, size_t offset, uint32_t value) { + assert(output != nullptr); + assert(offset + sizeof(value) <= output->size()); + EncodeFixed32(output->data() + offset, value); +} + +// The group key is not written: readers reach a run through its index group. +void EncodeRunHeader(const BlobReferenceRunHeader& header, + std::string* output) { + PutVarint64(output, header.reference_count); + PutVarint64(output, header.first_offset); + output->push_back(static_cast(header.default_compression)); +} + +// Decodes the run header at `*offset` of an uncompressed data block and moves +// `*offset` past it. The header holds no group key; the run belongs to +// `group`, the index group the reader reached it through. +Status DecodeRunHeaderAt(const Slice& block, size_t* offset, + const BlobReferenceGroup& group, + BlobReferenceRunHeader* header) { + if (*offset >= block.size()) { + return Status::Corruption("Blob reference run offset outside the block"); + } + Slice cursor(block.data() + *offset, block.size() - *offset); + BlobReferenceRunHeader decoded; + decoded.encoding_class = group.encoding_class; + decoded.generation_file_number = group.generation_file_number; + decoded.origin_file_number = group.origin_file_number; + if (!GetCanonicalVarint64(&cursor, &decoded.reference_count) || + !GetCanonicalVarint64(&cursor, &decoded.first_offset) || cursor.empty() || + decoded.reference_count == 0) { + return Status::Corruption("Invalid blob reference run header"); + } + decoded.default_compression = + static_cast(static_cast(cursor[0])); + cursor.remove_prefix(1); + if (!BlobIndex::IsValidCompression(decoded.default_compression)) { + return Status::Corruption("Invalid blob reference run compression"); + } + *offset = block.size() - cursor.size(); + *header = decoded; + return Status::OK(); +} + +// Decodes one entry without its gap from the front of `cursor` into +// `reference`: stored size, compression, record bytes and checksums. The +// caller must set the class before the call, because the class decides which +// fields the entry has. The caller sets the generation, origin and offset. +// Sets `*record_overhead` to the record bytes besides the stored value. +Status DecodeEntryBody(Slice* cursor, CompressionType default_compression, + BlobReference* reference, uint64_t* record_overhead) { + uint64_t size_and_flag = 0; + if (!GetCanonicalVarint64(cursor, &size_and_flag)) { + return Status::Corruption("Invalid blob reference entry"); + } + reference->stored_value_size = size_and_flag >> 1; + reference->compression = default_compression; + if ((size_and_flag & 1) != 0) { + if (cursor->empty()) { + return Status::Corruption("Invalid blob reference entry"); + } + reference->compression = + static_cast(static_cast((*cursor)[0])); + cursor->remove_prefix(1); + if (reference->compression == default_compression) { + return Status::Corruption("Noncanonical blob reference compression"); + } + } + // Record overhead: a record header and the user key in a normal blob + // file, or a trailer in a relocation file. + *record_overhead = kSimpleGen2BlobTrailerSize; + if (StoresUserKeySize(reference->encoding_class)) { + uint64_t key_size = 0; + if (!GetCanonicalVarint64(cursor, &key_size) || + !AddWithoutOverflow(BlobLogRecord::kHeaderSize, key_size, + record_overhead)) { + return Status::Corruption("Invalid blob reference entry"); + } + } + if (!AddWithoutOverflow(reference->stored_value_size, *record_overhead, + &reference->record_bytes)) { + return Status::Corruption("Blob reference record size overflow"); + } + if (reference->HasStableIdentity()) { + if (!GetFixed32(cursor, &reference->stable_checksum)) { + return Status::Corruption("Invalid blob reference checksum"); + } + if (reference->compression == kNoCompression) { + reference->stored_checksum = BlobIndex::ExtendChecksumWithCompression( + reference->stable_checksum, kNoCompression); + } else if (!GetFixed32(cursor, &reference->stored_checksum)) { + return Status::Corruption("Invalid blob reference stored checksum"); + } + } + return Status::OK(); +} + +// Decodes the entries of the run whose header was just decoded, calls +// `visit` for each reference in offset order, and moves `*offset` past the +// run. Checks each entry and the class rules. Offsets strictly increase and +// records do not overlap; with `previous_run_last` (the last reference of the +// group's previous run), this also holds for the run's first reference. +Status DecodeRunEntriesAt( + const Slice& block, size_t* offset, const BlobReferenceRunHeader& header, + const BlobReference* previous_run_last, + const std::function& visit) { + assert(*offset <= block.size()); + Slice cursor(block.data() + *offset, block.size() - *offset); + BlobReference previous; + if (previous_run_last != nullptr) { + previous = *previous_run_last; + } + for (uint64_t i = 0; i < header.reference_count; ++i) { + BlobReference reference; + reference.encoding_class = header.encoding_class; + reference.generation_file_number = header.generation_file_number; + reference.origin_file_number = header.origin_file_number; + uint64_t gap = 0; + if (i > 0 && !GetCanonicalVarint64(&cursor, &gap)) { + return Status::Corruption("Invalid blob reference entry"); + } + uint64_t record_overhead = 0; + Status status = DecodeEntryBody(&cursor, header.default_compression, + &reference, &record_overhead); + if (!status.ok()) { + return status; + } + if (i == 0) { + reference.original_value_offset = header.first_offset; + } + if (i > 0 || previous_run_last != nullptr) { + uint64_t base = 0; + if (!GapBase( + previous, + StoresUserKeySize(header.encoding_class) ? record_overhead : 0, + &base) || + (i > 0 && + !AddWithoutOverflow(base, gap, &reference.original_value_offset))) { + return Status::Corruption("Blob reference offset overflow"); + } + // A run's first offset is stored whole, so check it against the end of + // the previous run's last record. + if (reference.original_value_offset < base || + reference.original_value_offset <= previous.original_value_offset) { + return Status::Corruption("Blob reference offsets are not ordered"); + } + } + status = ValidateReference(reference); + if (status.ok()) { + status = visit(reference); + } + if (!status.ok()) { + return status; + } + previous = reference; + } + *offset = block.size() - cursor.size(); + return Status::OK(); +} + +// The body of BlobReference::FromBlobIndex(). For a kNoCompression indirect +// index, the stored CRC is not in the encoding: BlobIndex::stored_checksum() +// derives it from the logical CRC. With `derive_uncompressed_stored_checksum` +// false, the result's stored_checksum is 0 for such an index instead. The +// collector passes false: it does not encode that CRC (a reader derives it +// again, see DecodeEntryBody()), and it converts every reference of every +// output SST, so it skips the derivation. +Status ConvertBlobIndex(const BlobIndex& blob_index, + bool derive_uncompressed_stored_checksum, + BlobReference* reference, bool* included) { + if (reference == nullptr || included == nullptr) { + return Status::InvalidArgument("Null blob reference output"); + } + *included = false; + Status status = blob_index.Validate(); + if (!status.ok()) { + return status; + } + if (blob_index.IsInlined() || blob_index.IsSameFile()) { + return Status::OK(); + } + if (blob_index.file_number() == 0) { + return Status::Corruption("Invalid blob file number in blob reference"); + } + + BlobReference result; + if (blob_index.IsIndirect()) { + result.generation_file_number = blob_index.generation_file_number(); + result.origin_file_number = blob_index.origin_file_number(); + result.original_value_offset = blob_index.original_value_offset(); + result.stable_checksum = blob_index.stable_checksum(); + result.stored_checksum = derive_uncompressed_stored_checksum || + blob_index.compression() != kNoCompression + ? blob_index.stored_checksum() + : 0; + result.encoding_class = + result.generation_file_number == result.origin_file_number + ? BlobReferenceEncodingClass::kIndirectUnflattened + : BlobReferenceEncodingClass::kIndirectFlattened; + } else { + result.encoding_class = BlobReferenceEncodingClass::kDirect; + result.generation_file_number = blob_index.file_number(); + result.origin_file_number = blob_index.file_number(); + result.original_value_offset = blob_index.offset(); + } + result.stored_value_size = blob_index.size(); + result.compression = blob_index.compression(); + // For unflattened and kDirect, record_bytes is only the stored size here: + // the collector adds the record header and user key later. A flattened + // reference's record bytes need no key. + if (result.encoding_class == BlobReferenceEncodingClass::kIndirectFlattened) { + if (result.stored_value_size > + std::numeric_limits::max() - kSimpleGen2BlobTrailerSize) { + return Status::Corruption("Flattened blob record size overflow"); + } + result.record_bytes = result.stored_value_size + kSimpleGen2BlobTrailerSize; + } else { + result.record_bytes = result.stored_value_size; + } + // Validate() and the construction above make every reference valid; the + // collector does not check it again. Without the derived stored CRC, the + // reference is valid except for that one field. + assert(!derive_uncompressed_stored_checksum || + ValidateReference(result).ok()); + *reference = result; + *included = true; + return Status::OK(); +} + +} // namespace + +Status BlobReference::FromBlobIndex(const BlobIndex& blob_index, + BlobReference* reference, bool* included) { + return ConvertBlobIndex(blob_index, + /*derive_uncompressed_stored_checksum=*/true, + reference, included); +} + +void BlobReferenceMetadataDescriptor::EncodeTo(std::string* output) const { + assert(output != nullptr); + output->clear(); + output->reserve(kEncodedSize); + PutFixed32(output, kDescriptorMagic); + PutFixed32(output, kFormatVersion); + PutFixed64(output, index_handle.offset()); + PutFixed64(output, index_handle.size()); + PutFixed64(output, reference_count); + PutFixed64(output, stored_value_bytes); + PutFixed32(output, column_family_id); + PutFixed64(output, route_snapshot_fingerprint_high); + PutFixed64(output, route_snapshot_fingerprint_low); + PutFixed64(output, sst_file_number); + PutFixed32(output, 0); + assert(output->size() == kEncodedSize); + SetFixed32(output, kEncodedSize - sizeof(uint32_t), + crc32c::Value(output->data(), kEncodedSize - sizeof(uint32_t))); +} + +Status BlobReferenceMetadataDescriptor::DecodeFrom( + const Slice& input, BlobReferenceMetadataDescriptor* descriptor) { + if (descriptor == nullptr) { + return Status::InvalidArgument("Null blob reference descriptor output"); + } + if (input.size() != kEncodedSize) { + return Status::Corruption("Invalid blob reference descriptor size"); + } + Slice cursor = input; + uint32_t magic = 0; + uint32_t version = 0; + uint32_t descriptor_checksum = 0; + BlobReferenceMetadataDescriptor decoded; + uint64_t index_offset = 0; + uint64_t index_size = 0; + if (!GetFixed32(&cursor, &magic) || !GetFixed32(&cursor, &version) || + !GetFixed64(&cursor, &index_offset) || + !GetFixed64(&cursor, &index_size) || + !GetFixed64(&cursor, &decoded.reference_count) || + !GetFixed64(&cursor, &decoded.stored_value_bytes) || + !GetFixed32(&cursor, &decoded.column_family_id) || + !GetFixed64(&cursor, &decoded.route_snapshot_fingerprint_high) || + !GetFixed64(&cursor, &decoded.route_snapshot_fingerprint_low) || + !GetFixed64(&cursor, &decoded.sst_file_number) || + !GetFixed32(&cursor, &descriptor_checksum) || !cursor.empty()) { + return Status::Corruption("Malformed blob reference descriptor"); + } + if (crc32c::Value(input.data(), input.size() - sizeof(uint32_t)) != + descriptor_checksum) { + return Status::Corruption("Blob reference descriptor checksum mismatch"); + } + if (magic != kDescriptorMagic || version != kFormatVersion) { + return Status::Corruption("Unsupported blob reference descriptor"); + } + if (!decoded.HasRepairMetadata()) { + return Status::Corruption("Incomplete blob repair descriptor"); + } + decoded.index_handle = BlockHandle(index_offset, index_size); + uint64_t index_end = 0; + if (decoded.index_handle.IsNull() || index_size == 0 || + index_size > kMaxTopLevelMetadataBlockSize || + !AddWithoutOverflow(index_offset, index_size, &index_end)) { + return Status::Corruption("Invalid blob reference metadata handle"); + } + if (decoded.reference_count == 0 && decoded.stored_value_bytes != 0) { + return Status::Corruption("Invalid empty blob reference descriptor"); + } + *descriptor = decoded; + return Status::OK(); +} + +bool BlobReferenceGroupKey::IsValid() const { + if (!IsValidEncodingClass(encoding_class) || generation_file_number == 0 || + origin_file_number == 0) { + return false; + } + return encoding_class == BlobReferenceEncodingClass::kIndirectFlattened + ? generation_file_number > origin_file_number + : generation_file_number == origin_file_number; +} + +BlobReferenceMetadataCollector::BlobReferenceMetadataCollector( + std::shared_ptr cache_reservation_manager, + const BlobReferenceSstIdentity& identity) + : cache_reservation_manager_(std::move(cache_reservation_manager)) { + descriptor_.column_family_id = identity.column_family_id; + descriptor_.route_snapshot_fingerprint_high = + identity.route_snapshot_fingerprint_high; + descriptor_.route_snapshot_fingerprint_low = + identity.route_snapshot_fingerprint_low; + descriptor_.sst_file_number = identity.sst_file_number; +} + +BlobReferenceMetadataCollector::~BlobReferenceMetadataCollector() { + // The table builder's Abandon() can destroy the collector before any Add() + // or Finish() call reads status_. + status_.PermitUncheckedError(); +} + +uint64_t BlobReferenceMetadataCollector::UnsortedGroupFallbackCount() { + return unsorted_group_fallbacks.load(std::memory_order_relaxed); +} + +uint64_t BlobReferenceMetadataCollector::EstimatedEncodedSize() const { + constexpr uint64_t kFixedOverhead = 4096; + const uint64_t groups = group_records_.size(); + // chunk_bytes_ holds every entry with its gap (a chunk that continues a + // chain starts with its gap) and bounds the unsorted entries. Every group + // adds a run header and an index entry. + const uint64_t run_bytes = chunk_bytes_ + groups * kMaxRunHeaderSize; + // Blocks hold whole chunks, so a block may end up to one chunk short of + // the target, and every block may start with a continuation run header. + const uint64_t blocks = run_bytes / (kTargetDataBlockSize - kChunkSize) + 1; + return kFixedOverhead + run_bytes + groups * kMaxIndexEntrySize + + blocks * (kMaxRunHeaderSize + BlockBasedTable::kBlockTrailerSize + + kMaxBlockTableEntrySize); +} + +void BlobReferenceMetadataCollector::ReleaseBuffers() { + std::vector>().swap(slabs_); + slab_used_ = kSlabSize; + std::vector().swap(chunks_); + std::vector().swap(group_records_); + decltype(group_map_)().swap(group_map_); + last_group_ = kNoChunk; + reference_count_ = 0; + chunk_bytes_ = 0; + std::vector().swap(index_.blocks); + std::vector().swap(index_.groups); + std::vector().swap(groups_); + std::string().swap(index_block_); + block_buffer_bytes_ = 0; +} + +Status BlobReferenceMetadataCollector::Add(const BlobIndex& blob_index) { + return Add(blob_index, /*user_key_size=*/0); +} + +Status BlobReferenceMetadataCollector::Add(const BlobIndex& blob_index, + uint64_t user_key_size) { + if (!status_.ok()) { + return status_; + } + + BlobReference reference; + bool included = false; + status_ = ConvertBlobIndex(blob_index, + /*derive_uncompressed_stored_checksum=*/false, + &reference, &included); + if (!status_.ok() || !included) { + return status_; + } + // FromBlobIndex() already set the record bytes of a flattened reference. + if (StoresUserKeySize(reference.encoding_class)) { + const uint64_t adjustment = + BlobLogRecord::CalculateAdjustmentForRecordHeader(user_key_size); + if (reference.stored_value_size > + std::numeric_limits::max() - adjustment) { + status_ = Status::Corruption("Blob log record size overflow"); + return status_; + } + reference.record_bytes = reference.stored_value_size + adjustment; + } + uint32_t group = kNoChunk; + status_ = FindOrAddGroup(reference, &group); + if (!status_.ok()) { + return status_; + } + GroupRecord& record = group_records_[group]; + if (record.group.reference_count != 0 && + reference.original_value_offset == record.last_offset) { + status_ = Status::Corruption("Repeated blob reference in one SST"); + return status_; + } + if (record.unsorted || + (record.group.reference_count != 0 && + reference.original_value_offset < record.last_offset)) { + // Out of order: encoded once, with its absolute offset, until Finish() + // sorts the group. + status_ = AppendUnsorted(group, reference); + if (!status_.ok()) { + return status_; + } + record.unsorted = true; + } else { + status_ = AppendSorted(group, reference); + if (!status_.ok()) { + return status_; + } + } + BlobReferenceGroup& totals = record.group; + ++totals.reference_count; + if (!AddWithoutOverflow(totals.stored_value_bytes, + reference.stored_value_size, + &totals.stored_value_bytes) || + !AddWithoutOverflow(totals.record_bytes, reference.record_bytes, + &totals.record_bytes)) { + status_ = Status::Corruption("Blob reference byte total overflow"); + return status_; + } + ++reference_count_; + return status_; +} + +Status BlobReferenceMetadataCollector::FindOrAddGroup( + const BlobReference& reference, uint32_t* group) { + // Fast path: most references belong to the same group as the previous one. + if (last_group_ != kNoChunk) { + const BlobReferenceGroup& last = group_records_[last_group_].group; + if (last.group_key() == reference.group_key()) { + *group = last_group_; + return Status::OK(); + } + } + const BlobReferenceGroupKey key = reference.group_key(); + const auto found = group_map_.find(key); + if (found != group_map_.end()) { + *group = found->second; + last_group_ = *group; + return Status::OK(); + } + if (group_records_.size() >= kMaxGroupCount) { + return Status::MemoryLimit("Too many blob reference groups in one SST"); + } + Status status = GrowIfFull(&group_records_, /*initial_capacity=*/16); + if (!status.ok()) { + return status; + } + // Charge one map node. If this insert may rehash, also charge a new bucket + // array of 4 * (bucket count + 16) pointers. ReservedMemory() already counts + // the current bucket array. A rehash allocates a new array of about twice + // the buckets while the old array is still alive, so the extra memory is + // about 2 times the current bucket count. Charging 4 times is a safe upper + // bound (an estimate, not measured). The +16 covers the first insert into an + // empty map. + const bool may_rehash = + group_map_.size() + 1 > + group_map_.bucket_count() * group_map_.max_load_factor(); + status = ResizeReservation( + ReservedMemory() + kGroupMapNodeBytes + + (may_rehash ? 4 * (group_map_.bucket_count() + 16) * sizeof(void*) : 0)); + if (!status.ok()) { + return status; + } + GroupRecord record; + record.group.encoding_class = reference.encoding_class; + record.group.generation_file_number = reference.generation_file_number; + record.group.origin_file_number = reference.origin_file_number; + record.default_compression = reference.compression; + group_records_.push_back(std::move(record)); + indirect_group_count_ += reference.HasStableIdentity(); + *group = static_cast(group_records_.size() - 1); + group_map_.emplace(key, *group); + last_group_ = *group; + return UpdateReservation(); +} + +Status BlobReferenceMetadataCollector::NewChunk(uint32_t* chunk) { + if (chunks_.size() >= kNoChunk - 1) { + return Status::MemoryLimit("Too many blob reference chunks in one SST"); + } + Status status = GrowIfFull(&chunks_, /*initial_capacity=*/128); + if (!status.ok()) { + return status; + } + if (slab_used_ + kChunkSize > kSlabSize) { + status = GrowIfFull(&slabs_, /*initial_capacity=*/4); + if (!status.ok()) { + return status; + } + // Charge the slab before it is allocated. + status = ResizeReservation(ReservedMemory() + kSlabSize); + if (!status.ok()) { + return status; + } + slabs_.emplace_back(new char[kSlabSize]); + slab_used_ = 0; + } + slab_used_ += kChunkSize; + chunks_.emplace_back(); + *chunk = static_cast(chunks_.size() - 1); + // Chunk i is chunk i % kChunksPerSlab of slab i / kChunksPerSlab. + assert(ChunkData(*chunk) == slabs_.back().get() + slab_used_ - kChunkSize); + return Status::OK(); +} + +Status BlobReferenceMetadataCollector::EncodeEntryBody( + const BlobReference& reference, CompressionType default_compression, + char* body, size_t* body_size, uint64_t* key_size) { + *key_size = 0; + if (StoresUserKeySize(reference.encoding_class) && + !UserKeySize(reference, key_size)) { + return Status::Corruption("Blob reference record has no record header"); + } + if (reference.stored_value_size > std::numeric_limits::max() >> 1) { + return Status::Corruption("Blob reference size overflow"); + } + char* end = body; + const bool differs = reference.compression != default_compression; + end = EncodeVarint64(end, + (reference.stored_value_size << 1) | (differs ? 1 : 0)); + if (differs) { + *end++ = static_cast(reference.compression); + } + if (StoresUserKeySize(reference.encoding_class)) { + end = EncodeVarint64(end, *key_size); + } + if (reference.HasStableIdentity()) { + EncodeFixed32(end, reference.stable_checksum); + end += sizeof(uint32_t); + if (reference.compression != kNoCompression) { + EncodeFixed32(end, reference.stored_checksum); + end += sizeof(uint32_t); + } + } + *body_size = static_cast(end - body); + return Status::OK(); +} + +Status BlobReferenceMetadataCollector::AppendUnsorted( + uint32_t group, const BlobReference& reference) { + // The absolute offset, then the entry without its gap. + char entry[kMaxEntrySize]; + char* const body = EncodeVarint64(entry, reference.original_value_offset); + size_t body_size = 0; + uint64_t key_size = 0; + Status status = + EncodeEntryBody(reference, group_records_[group].default_compression, + body, &body_size, &key_size); + if (!status.ok()) { + return status; + } + const size_t entry_size = static_cast(body - entry) + body_size; + GroupRecord& record = group_records_[group]; + if (record.unsorted_tail_chunk == kNoChunk || + chunks_[record.unsorted_tail_chunk].used + entry_size > kChunkSize) { + uint32_t chunk = kNoChunk; + status = NewChunk(&chunk); + if (!status.ok()) { + return status; + } + if (record.unsorted_tail_chunk != kNoChunk) { + chunks_[record.unsorted_tail_chunk].next = chunk; + } else { + record.unsorted_first_chunk = chunk; + } + record.unsorted_tail_chunk = chunk; + } + Chunk& tail = chunks_[record.unsorted_tail_chunk]; + memcpy(ChunkData(record.unsorted_tail_chunk) + tail.used, entry, entry_size); + tail.used = static_cast(tail.used + entry_size); + ++tail.count; + // The final entry has a gap no longer than this absolute offset, so these + // bytes bound its size for EstimatedEncodedSize(). + chunk_bytes_ += entry_size; + return Status::OK(); +} + +Status BlobReferenceMetadataCollector::AppendSorted( + uint32_t group, const BlobReference& reference) { + GroupRecord& record = group_records_[group]; + // Entry without its gap, in the on-disk encoding. + char body[kMaxEntrySize]; + size_t body_size = 0; + uint64_t key_size = 0; + Status body_status = EncodeEntryBody(reference, record.default_compression, + body, &body_size, &key_size); + if (!body_status.ok()) { + return body_status; + } + + char gap_buffer[kMaxVarint64Length]; + size_t gap_size = 0; + if (record.tail_chunk != kNoChunk) { + // This computes the same base as GapBase(), from the group's last offset + // and stored size. + uint64_t base = 0; + if (!AddWithoutOverflow(record.last_offset, record.last_stored_size, + &base) || + (StoresUserKeySize(reference.encoding_class) && + (!AddWithoutOverflow(base, BlobLogRecord::kHeaderSize, &base) || + !AddWithoutOverflow(base, key_size, &base)))) { + return Status::Corruption("Blob reference offset overflow"); + } + if (reference.original_value_offset < base) { + return Status::Corruption("Overlapping blob references in one SST"); + } + gap_size = static_cast( + EncodeVarint64(gap_buffer, reference.original_value_offset - base) - + gap_buffer); + } + + // The gap and the entry, in this order, either continue the tail chunk or + // start a new one. A new chunk that is not the group's first also starts + // with the gap (its join gap). A group's first chunk has no gap. + if (record.tail_chunk == kNoChunk || + chunks_[record.tail_chunk].used + gap_size + body_size > kChunkSize) { + uint32_t chunk = kNoChunk; + Status status = NewChunk(&chunk); + if (!status.ok()) { + return status; + } + if (record.tail_chunk != kNoChunk) { + chunks_[record.tail_chunk].next = chunk; + } else { + record.first_chunk = chunk; + } + record.tail_chunk = chunk; + chunks_[chunk].first_offset = reference.original_value_offset; + } + Chunk& tail = chunks_[record.tail_chunk]; + char* const data = ChunkData(record.tail_chunk) + tail.used; + memcpy(data, gap_buffer, gap_size); + memcpy(data + gap_size, body, body_size); + tail.used = static_cast(tail.used + gap_size + body_size); + ++tail.count; + chunk_bytes_ += gap_size + body_size; + record.last_offset = reference.original_value_offset; + record.last_stored_size = reference.stored_value_size; + return Status::OK(); +} + +Status BlobReferenceMetadataCollector::SortUnsortedGroup(uint32_t group) { + unsorted_group_fallbacks.fetch_add(1, std::memory_order_relaxed); + TEST_SYNC_POINT("BlobReferenceMetadataCollector::SortUnsortedGroup"); + const size_t count = + static_cast(group_records_[group].group.reference_count); + // Temporary memory: one sort key per reference of this one group. It stays + // in ReservedMemory() until the vector is freed, so every charge update + // while the group is encoded again keeps it. + sort_scratch_bytes_ = count * sizeof(SortKey); + Status status = UpdateReservation(); + if (status.ok()) { + std::vector keys; + keys.reserve(count); + status = SortAndEncodeGroup(group, &keys); + TEST_SYNC_POINT( + "BlobReferenceMetadataCollector::SortUnsortedGroup:Encoded"); + } + sort_scratch_bytes_ = 0; + const Status release_status = UpdateReservation(); + if (!status.ok()) { + // The sort error is the one to report. + release_status.PermitUncheckedError(); + return status; + } + return release_status; +} + +Status BlobReferenceMetadataCollector::SortAndEncodeGroup( + uint32_t group, std::vector* keys) { + const GroupRecord& record = group_records_[group]; + BlobReference reference; + reference.encoding_class = record.group.encoding_class; + reference.generation_file_number = record.group.generation_file_number; + reference.origin_file_number = record.group.origin_file_number; + const bool stores_key_size = StoresUserKeySize(reference.encoding_class); + // Sorted chain: gap-encoded entries; each chunk stores its first offset, + // and every chunk but the first starts with its join gap. + for (uint32_t chunk = record.first_chunk; chunk != kNoChunk; + chunk = chunks_[chunk].next) { + const Chunk& header = chunks_[chunk]; + Slice cursor(ChunkData(chunk), header.used); + uint64_t join_gap = 0; + if (chunk != record.first_chunk && + !GetCanonicalVarint64(&cursor, &join_gap)) { + return Status::Corruption("Invalid blob reference chunk join"); + } + for (uint32_t i = 0; i < header.count; ++i) { + uint64_t gap = 0; + if (i > 0 && !GetCanonicalVarint64(&cursor, &gap)) { + return Status::Corruption("Invalid blob reference entry"); + } + const uint32_t position = + header.used - static_cast(cursor.size()); + const BlobReference previous = reference; + uint64_t record_overhead = 0; + Status status = DecodeEntryBody(&cursor, record.default_compression, + &reference, &record_overhead); + if (!status.ok()) { + return status; + } + if (i == 0) { + reference.original_value_offset = header.first_offset; + } else { + uint64_t base = 0; + if (!GapBase(previous, stores_key_size ? record_overhead : 0, &base) || + !AddWithoutOverflow(base, gap, &reference.original_value_offset)) { + return Status::Corruption("Blob reference offset overflow"); + } + } + keys->push_back({reference.original_value_offset, chunk, position}); + } + if (!cursor.empty()) { + return Status::Corruption("Blob reference chunk has trailing bytes"); + } + } + // Unsorted chain: each entry starts with its absolute offset. The entry + // body is decoded here only to check it and to move past it. + for (uint32_t chunk = record.unsorted_first_chunk; chunk != kNoChunk; + chunk = chunks_[chunk].next) { + const Chunk& header = chunks_[chunk]; + Slice cursor(ChunkData(chunk), header.used); + for (uint32_t i = 0; i < header.count; ++i) { + uint64_t offset = 0; + if (!GetCanonicalVarint64(&cursor, &offset)) { + return Status::Corruption("Invalid blob reference entry"); + } + const uint32_t position = + header.used - static_cast(cursor.size()); + uint64_t record_overhead = 0; + Status status = DecodeEntryBody(&cursor, record.default_compression, + &reference, &record_overhead); + if (!status.ok()) { + return status; + } + keys->push_back({offset, chunk, position}); + } + if (!cursor.empty()) { + return Status::Corruption("Blob reference chunk has trailing bytes"); + } + } + if (keys->size() != record.group.reference_count) { + return Status::Corruption("Blob reference group count mismatch"); + } + std::sort(keys->begin(), keys->end(), + [](const SortKey& lhs, const SortKey& rhs) { + return lhs.offset < rhs.offset; + }); + GroupRecord& reset = group_records_[group]; + reset.unsorted = false; + reset.first_chunk = kNoChunk; + reset.tail_chunk = kNoChunk; + reset.unsorted_first_chunk = kNoChunk; + reset.unsorted_tail_chunk = kNoChunk; + // The group keeps its default compression, so no entry grows past the size + // estimate. The old chunks stay allocated but unused until the collector + // frees its buffers. + for (size_t i = 0; i < keys->size(); ++i) { + const SortKey& key = (*keys)[i]; + if (i > 0 && key.offset == (*keys)[i - 1].offset) { + return Status::Corruption("Repeated blob reference in one SST"); + } + // Chunk data stays in place while AppendSorted() adds chunk headers. + Slice cursor(ChunkData(key.chunk) + key.position, + chunks_[key.chunk].used - key.position); + uint64_t record_overhead = 0; + Status status = + DecodeEntryBody(&cursor, group_records_[group].default_compression, + &reference, &record_overhead); + if (status.ok()) { + reference.original_value_offset = key.offset; + // Checks overlaps. + status = AppendSorted(group, reference); + } + if (!status.ok()) { + return status; + } + } + return Status::OK(); +} + +Status BlobReferenceMetadataCollector::Finish( + const BlockWriter& write_block, BlobReferenceMetadataDescriptor* descriptor, + std::shared_ptr* summary) { + const Status status = FinishOnce(write_block, descriptor, summary); + if (status_.ok()) { + // Sticky: a later Add() or Finish() fails instead of writing again. + status_ = Status::InvalidArgument( + "Blob reference metadata collector already finished"); + } + return status; +} + +Status BlobReferenceMetadataCollector::FinishOnce( + const BlockWriter& write_block, BlobReferenceMetadataDescriptor* descriptor, + std::shared_ptr* summary) { + if (!status_.ok()) { + return status_; + } + if (!write_block || descriptor == nullptr) { + status_ = Status::InvalidArgument("Missing blob reference metadata output"); + return status_; + } + status_ = BuildAndWriteDataBlocks(write_block); + if (status_.ok()) { + status_ = EncodeIndex(); + } + if (status_.ok()) { + status_ = UpdateReservation(); + } + if (status_.ok()) { + status_ = write_block(index_block_, /*is_index_block=*/true, + &descriptor_.index_handle); + } + if (status_.ok() && descriptor_.index_handle.IsNull()) { + status_ = + Status::Corruption("Blob reference block writer returned null handle"); + } + if (!status_.ok()) { + return status_; + } + *descriptor = descriptor_; + if (summary == nullptr) { + return Status::OK(); + } + std::shared_ptr result = + std::make_shared(); + result->file_number = descriptor_.sst_file_number; + result->descriptor = descriptor_; + result->groups.swap(groups_); + result->reservation_manager = std::move(cache_reservation_manager_); + result->reservation_handle = std::move(reservation_handle_); + // The moved reservation still covers every buffer while they are freed. + ReleaseBuffers(); + if (result->reservation_manager != nullptr) { + status_ = result->reservation_handle != nullptr + ? result->reservation_handle->Resize(result->MemoryUsage()) + : result->reservation_manager->MakeCacheReservation( + result->MemoryUsage(), &result->reservation_handle); + } + if (status_.ok()) { + *summary = std::move(result); + } + return status_; +} + +size_t BlobReferenceMetadataCollector::ReservedMemory() const { + return slabs_.size() * kSlabSize + + slabs_.capacity() * sizeof(std::unique_ptr) + + chunks_.capacity() * sizeof(Chunk) + + group_records_.capacity() * sizeof(GroupRecord) + sort_scratch_bytes_ + + group_map_.size() * kGroupMapNodeBytes + + group_map_.bucket_count() * sizeof(void*) + + index_.blocks.capacity() * sizeof(BlockHandle) + + index_.groups.capacity() * sizeof(BlobReferenceIndexEntry) + + groups_.capacity() * sizeof(BlobReferenceGroup) + + index_block_.capacity() + block_buffer_bytes_; +} + +Status BlobReferenceMetadataCollector::UpdateReservation() { + return ResizeReservation(ReservedMemory()); +} + +Status BlobReferenceMetadataCollector::ResizeReservation(size_t memory_used) { + if (cache_reservation_manager_ == nullptr) { + return Status::OK(); + } + return reservation_handle_ != nullptr + ? reservation_handle_->Resize(memory_used) + : cache_reservation_manager_->MakeCacheReservation( + memory_used, &reservation_handle_); +} + +Status BlobReferenceMetadataCollector::BuildAndWriteDataBlocks( + const BlockWriter& write_block) { + descriptor_.reference_count = reference_count_; + descriptor_.stored_value_bytes = 0; + if (reference_count_ == 0) { + return Status::OK(); + } + Status status; + for (uint32_t group = 0; group < group_records_.size(); ++group) { + if (group_records_[group].unsorted) { + status = SortUnsortedGroup(group); + if (!status.ok()) { + return status; + } + } + } + // The block buffer (at most one block) and the group order. Charge them + // before they grow. + constexpr size_t kBufferBytes = + kTargetDataBlockSize + kMaxRunHeaderSize + kMaxEntrySize; + block_buffer_bytes_ = kBufferBytes + group_records_.size() * sizeof(uint32_t); + status = UpdateReservation(); + if (!status.ok()) { + return status; + } + // Only the group records are sorted, not the references. + std::vector order(group_records_.size()); + std::iota(order.begin(), order.end(), uint32_t{0}); + std::sort(order.begin(), order.end(), [this](uint32_t lhs, uint32_t rhs) { + return group_records_[lhs].group.group_key() < + group_records_[rhs].group.group_key(); + }); + status = ReserveCapacity(&index_.groups, order.size()); + if (status.ok()) { + // The summary keeps the indirect groups. + status = ReserveCapacity(&groups_, indirect_group_count_); + } + if (!status.ok()) { + return status; + } + + std::string block; + block.reserve(kBufferBytes); + const auto flush_block = [&]() -> Status { + if (index_.blocks.size() >= kMaxDataBlockCount) { + return Status::MemoryLimit( + "Too many blob reference data blocks in one SST"); + } + Status grow_status = GrowIfFull(&index_.blocks, /*initial_capacity=*/1); + if (!grow_status.ok()) { + return grow_status; + } + BlockHandle handle; + Status write_status = write_block(block, /*is_index_block=*/false, &handle); + if (!write_status.ok()) { + return write_status; + } + if (handle.IsNull()) { + return Status::Corruption( + "Blob reference block writer returned null handle"); + } + index_.blocks.push_back(handle); + block.clear(); + return Status::OK(); + }; + + for (uint32_t group : order) { + const GroupRecord& record = group_records_[group]; + assert(record.first_chunk != kNoChunk); + // Bytes of a chunk's join gap; a group's first chunk has none. + const auto gap_bytes = [&](uint32_t chunk, size_t* size) -> Status { + *size = 0; + if (chunk == record.first_chunk) { + return Status::OK(); + } + Slice cursor(ChunkData(chunk), chunks_[chunk].used); + uint64_t gap = 0; + if (!GetCanonicalVarint64(&cursor, &gap)) { + return Status::Corruption("Invalid blob reference chunk join"); + } + *size = chunks_[chunk].used - cursor.size(); + return Status::OK(); + }; + if (!block.empty() && + block.size() + kMaxRunHeaderSize + chunks_[record.first_chunk].used > + kTargetDataBlockSize) { + status = flush_block(); + if (!status.ok()) { + return status; + } + } + BlobReferenceIndexEntry index_entry; + index_entry.group = record.group; + index_entry.start_block = static_cast(index_.blocks.size()); + index_entry.run_offset = static_cast(block.size()); + index_.groups.push_back(index_entry); + if (record.group.encoding_class != BlobReferenceEncodingClass::kDirect) { + groups_.push_back(record.group); + } + if (!AddWithoutOverflow(descriptor_.stored_value_bytes, + record.group.stored_value_bytes, + &descriptor_.stored_value_bytes)) { + return Status::Corruption("Blob reference byte total overflow"); + } + // Each run starts at a chunk, without its join gap, and takes the following + // chunks whole (join gap and entries) while they fit the block. The run + // header comes first and holds the count, so the loop counts the run's + // chunks before it copies them. + uint32_t start = record.first_chunk; + while (start != kNoChunk) { + size_t start_gap = 0; + status = gap_bytes(start, &start_gap); + if (!status.ok()) { + return status; + } + size_t run_size = chunks_[start].used - start_gap; + uint64_t count = chunks_[start].count; + uint32_t end = chunks_[start].next; + while (end != kNoChunk && + block.size() + kMaxRunHeaderSize + run_size + chunks_[end].used <= + kTargetDataBlockSize) { + run_size += chunks_[end].used; + count += chunks_[end].count; + end = chunks_[end].next; + } + BlobReferenceRunHeader run; + run.reference_count = count; + run.first_offset = chunks_[start].first_offset; + run.default_compression = record.default_compression; + EncodeRunHeader(run, &block); + block.append(ChunkData(start) + start_gap, + chunks_[start].used - start_gap); + for (uint32_t chunk = chunks_[start].next; chunk != end; + chunk = chunks_[chunk].next) { + block.append(ChunkData(chunk), chunks_[chunk].used); + } + if (end != kNoChunk) { + // The group continues in the next block with a new run. + status = flush_block(); + if (!status.ok()) { + return status; + } + } + start = end; + } + } + status = flush_block(); + block_buffer_bytes_ = 0; + return status; +} + +Status BlobReferenceMetadataCollector::EncodeIndex() { + const uint64_t max_size = 2 * kMaxVarint64Length + + index_.blocks.size() * kMaxBlockTableEntrySize + + index_.groups.size() * kMaxIndexEntrySize; + if (max_size > kMaxTopLevelMetadataBlockSize) { + return Status::MemoryLimit("Blob reference index exceeds format limit"); + } + index_block_.clear(); + Status status = ReserveCapacity(&index_block_, max_size); + if (!status.ok()) { + return status; + } + // Data blocks are written back to back, so the block table stores only + // the first offset and each block's size. + PutVarint64(&index_block_, index_.blocks.size()); + if (!index_.blocks.empty()) { + PutVarint64(&index_block_, index_.blocks.front().offset()); + } + for (size_t i = 0; i < index_.blocks.size(); ++i) { + const BlockHandle& handle = index_.blocks[i]; + if (i > 0 && handle.offset() != index_.blocks[i - 1].offset() + + index_.blocks[i - 1].size() + + BlockBasedTable::kBlockTrailerSize) { + return Status::Corruption( + "Blob reference data blocks are not back to back"); + } + PutVarint64(&index_block_, handle.size()); + } + PutVarint64(&index_block_, index_.groups.size()); + const BlobReferenceIndexEntry* previous = nullptr; + for (const BlobReferenceIndexEntry& index_entry : index_.groups) { + const BlobReferenceGroup& group = index_entry.group; + const bool same_class = + previous != nullptr && + previous->group.encoding_class == group.encoding_class; + index_block_.push_back(static_cast(group.encoding_class)); + PutVarint64(&index_block_, + group.generation_file_number - + (same_class ? previous->group.generation_file_number : 0)); + if (group.encoding_class == + BlobReferenceEncodingClass::kIndirectFlattened) { + const bool same_generation = + same_class && previous->group.generation_file_number == + group.generation_file_number; + PutVarint64( + &index_block_, + group.origin_file_number - + (same_generation ? previous->group.origin_file_number : 0)); + } + PutVarint64(&index_block_, + index_entry.start_block - + (previous != nullptr ? previous->start_block : 0)); + PutVarint64(&index_block_, index_entry.run_offset); + PutVarint64(&index_block_, group.reference_count); + PutVarint64(&index_block_, group.stored_value_bytes); + if (StoresUserKeySize(group.encoding_class)) { + // Every entry has a record header, so this does not underflow. + uint64_t overhead = 0; + if (!GroupRecordOverhead(group.encoding_class, group.reference_count, + &overhead) || + group.record_bytes - group.stored_value_bytes < overhead) { + return Status::Corruption("Invalid blob reference group record bytes"); + } + PutVarint64(&index_block_, + group.record_bytes - group.stored_value_bytes - overhead); + } + previous = &index_entry; + } + return Status::OK(); +} + +Status DecodeBlobReferenceIndex( + const Slice& input, const BlobReferenceMetadataDescriptor& descriptor, + BlobReferenceIndex* index) { + if (index == nullptr) { + return Status::InvalidArgument("Null blob reference index output"); + } + if (input.size() != descriptor.index_handle.size() || + input.size() > kMaxTopLevelMetadataBlockSize) { + return Status::Corruption("Blob reference index size mismatch"); + } + Slice cursor = input; + BlobReferenceIndex decoded; + uint64_t block_count = 0; + if (!GetCanonicalVarint64(&cursor, &block_count) || + block_count > kMaxDataBlockCount || + block_count > cursor.size() / kMinBlockTableEntrySize) { + return Status::Corruption("Invalid blob reference block count"); + } + // The data blocks are back to back and the index block follows the last + // one, so each offset is rebuilt from the first offset and the sizes. + decoded.blocks.reserve(static_cast(block_count)); + uint64_t next_offset = 0; + if (block_count > 0 && !GetCanonicalVarint64(&cursor, &next_offset)) { + return Status::Corruption("Invalid blob reference block offset"); + } + for (uint64_t i = 0; i < block_count; ++i) { + uint64_t size = 0; + uint64_t end = 0; + if (!GetCanonicalVarint64(&cursor, &size) || size == 0 || + size > kMaxDataBlockSize || + !AddWithoutOverflow(next_offset, + size + BlockBasedTable::kBlockTrailerSize, &end) || + end > descriptor.index_handle.offset()) { + return Status::Corruption("Invalid blob reference block handle"); + } + decoded.blocks.emplace_back(next_offset, size); + next_offset = end; + } + if (block_count > 0 && next_offset != descriptor.index_handle.offset()) { + return Status::Corruption( + "Blob reference index does not follow the last data block"); + } + // There is no fixed limit on the reference count. As a sanity check, the + // count must fit in the data blocks: an entry takes at least kMinEntrySize + // (2) bytes, so a block of kMaxDataBlockSize (64 KB) bytes holds at most + // 32K entries. The writer keeps every block at or below + // kTargetDataBlockSize (16 KB) uncompressed bytes, so a valid SST always + // passes. The size check above is on the size on disk, so for a compressed + // block this is a loose bound, not an exact one. + if (descriptor.reference_count > + block_count * (kMaxDataBlockSize / kMinEntrySize)) { + return Status::Corruption( + "Blob reference count does not fit its data blocks"); + } + + uint64_t group_count = 0; + if (!GetCanonicalVarint64(&cursor, &group_count) || + group_count > kMaxGroupCount || + group_count > cursor.size() / kMinIndexEntrySize || + (group_count == 0) != (block_count == 0) || + (group_count == 0) != (descriptor.reference_count == 0)) { + return Status::Corruption("Invalid blob reference group count"); + } + decoded.groups.reserve(static_cast(group_count)); + uint64_t reference_count = 0; + uint64_t stored_value_bytes = 0; + for (uint64_t i = 0; i < group_count; ++i) { + const BlobReferenceIndexEntry* previous = + decoded.groups.empty() ? nullptr : &decoded.groups.back(); + BlobReferenceIndexEntry entry; + BlobReferenceGroup& group = entry.group; + if (cursor.empty()) { + return Status::Corruption("Truncated blob reference index"); + } + group.encoding_class = static_cast( + static_cast(cursor[0])); + cursor.remove_prefix(1); + if (!IsValidEncodingClass(group.encoding_class)) { + return Status::Corruption("Invalid blob reference encoding class"); + } + const bool same_class = + previous != nullptr && + previous->group.encoding_class == group.encoding_class; + uint64_t value = 0; + if (!GetCanonicalVarint64(&cursor, &value) || + !AddWithoutOverflow( + same_class ? previous->group.generation_file_number : 0, value, + &group.generation_file_number)) { + return Status::Corruption("Invalid blob reference group generation"); + } + group.origin_file_number = group.generation_file_number; + if (group.encoding_class == + BlobReferenceEncodingClass::kIndirectFlattened) { + const bool same_generation = + same_class && previous->group.generation_file_number == + group.generation_file_number; + if (!GetCanonicalVarint64(&cursor, &value) || + !AddWithoutOverflow( + same_generation ? previous->group.origin_file_number : 0, value, + &group.origin_file_number)) { + return Status::Corruption("Invalid blob reference group origin"); + } + } + if (!group.group_key().IsValid()) { + return Status::Corruption("Invalid blob reference group"); + } + if (previous != nullptr && + !(previous->group.group_key() < group.group_key())) { + return Status::Corruption("Blob reference groups are not ordered"); + } + uint64_t start_delta = 0; + uint64_t run_offset = 0; + uint64_t start_block = 0; + uint64_t user_key_bytes = 0; + uint64_t overhead = 0; + if (!GetCanonicalVarint64(&cursor, &start_delta) || + !GetCanonicalVarint64(&cursor, &run_offset) || + !GetCanonicalVarint64(&cursor, &group.reference_count) || + !GetCanonicalVarint64(&cursor, &group.stored_value_bytes) || + (StoresUserKeySize(group.encoding_class) && + !GetCanonicalVarint64(&cursor, &user_key_bytes)) || + group.reference_count == 0 || + !GroupRecordOverhead(group.encoding_class, group.reference_count, + &overhead) || + !AddWithoutOverflow(group.stored_value_bytes, overhead, + &group.record_bytes) || + !AddWithoutOverflow(group.record_bytes, user_key_bytes, + &group.record_bytes) || + !AddWithoutOverflow(reference_count, group.reference_count, + &reference_count) || + !AddWithoutOverflow(stored_value_bytes, group.stored_value_bytes, + &stored_value_bytes)) { + return Status::Corruption("Invalid blob reference group totals"); + } + if (!AddWithoutOverflow(previous != nullptr ? previous->start_block : 0, + start_delta, &start_block) || + start_block >= block_count || run_offset >= kMaxDataBlockSize || + (previous == nullptr && (start_block != 0 || run_offset != 0)) || + (previous != nullptr && start_delta == 0 && + run_offset <= previous->run_offset)) { + return Status::Corruption("Invalid blob reference group position"); + } + entry.start_block = static_cast(start_block); + entry.run_offset = static_cast(run_offset); + decoded.groups.push_back(entry); + } + if (!cursor.empty()) { + return Status::Corruption("Trailing bytes in blob reference index"); + } + if (reference_count != descriptor.reference_count || + stored_value_bytes != descriptor.stored_value_bytes) { + return Status::Corruption("Blob reference index does not match totals"); + } + *index = std::move(decoded); + return Status::OK(); +} + +Status CopyIndirectBlobReferenceGroups( + const BlobReferenceIndex& index, + const std::function& before_allocation, + std::vector* groups) { + assert(groups != nullptr); + size_t count = 0; + for (const BlobReferenceIndexEntry& entry : index.groups) { + count += entry.group.encoding_class != BlobReferenceEncodingClass::kDirect; + } + if (before_allocation) { + Status status = before_allocation(count * sizeof(BlobReferenceGroup)); + if (!status.ok()) { + return status; + } + } + std::vector result; + result.reserve(count); + for (const BlobReferenceIndexEntry& entry : index.groups) { + if (entry.group.encoding_class != BlobReferenceEncodingClass::kDirect) { + result.push_back(entry.group); + } + } + *groups = std::move(result); + return Status::OK(); +} + +size_t FindBlobReferenceGroup(const BlobReferenceIndex& index, + const BlobReferenceGroupKey& key) { + const auto found = + std::lower_bound(index.groups.begin(), index.groups.end(), key, + [](const BlobReferenceIndexEntry& entry, + const BlobReferenceGroupKey& target) { + return entry.group.group_key() < target; + }); + if (found == index.groups.end() || !(found->group.group_key() == key)) { + return index.groups.size(); + } + return static_cast(found - index.groups.begin()); +} + +BlobReferenceGroupDecoder::BlobReferenceGroupDecoder( + const BlobReferenceIndex& index, size_t group) + : index_(index), + group_(group), + block_(index.groups[group].start_block), + offset_(index.groups[group].run_offset), + remaining_(index.groups[group].group.reference_count) { + assert(group < index.groups.size()); +} + +Status BlobReferenceGroupDecoder::DecodeRunHeader(const Slice& contents, + uint64_t* reference_count) { + assert(!done()); + // The index decoder checks each block's size on disk. A compressed block + // is smaller on disk than its contents, so the contents are checked here, + // after decompression. The count bounds assume this limit for every block. + if (contents.size() > kMaxDataBlockSize) { + return Status::Corruption( + "Blob reference data block is larger than 64 KB after decompression"); + } + const BlobReferenceGroup& group = index_.groups[group_].group; + contents_ = contents; + Status status = DecodeRunHeaderAt(contents_, &offset_, group, &header_); + if (status.ok() && header_.reference_count > remaining_) { + status = Status::Corruption("Blob reference run count exceeds group"); + } + if (status.ok()) { + *reference_count = header_.reference_count; + } + return status; +} + +Status BlobReferenceGroupDecoder::DecodeRunEntries( + const std::function& visit) { + Status status = DecodeRunEntriesAt( + contents_, &offset_, header_, has_previous_ ? &previous_ : nullptr, + [&](const BlobReference& reference) { + if (!AddWithoutOverflow(stored_value_bytes_, + reference.stored_value_size, + &stored_value_bytes_) || + !AddWithoutOverflow(record_bytes_, reference.record_bytes, + &record_bytes_)) { + return Status::Corruption("Blob reference byte total overflow"); + } + previous_ = reference; + has_previous_ = true; + return visit(reference); + }); + if (!status.ok()) { + return status; + } + remaining_ -= header_.reference_count; + const bool at_block_end = offset_ == contents_.size(); + if (remaining_ != 0) { + // The group continues at the start of the next block. + if (!at_block_end || block_ + 1 >= index_.blocks.size()) { + return Status::Corruption("Blob reference group is cut short"); + } + ++block_; + offset_ = 0; + return Status::OK(); + } + const BlobReferenceGroup& group = index_.groups[group_].group; + if (stored_value_bytes_ != group.stored_value_bytes || + record_bytes_ != group.record_bytes) { + return Status::Corruption("Blob reference group totals mismatch"); + } + // The last run must end where the next group starts, or at the end of its + // block (the last group: of the last block). + bool fits = false; + if (group_ + 1 < index_.groups.size()) { + const BlobReferenceIndexEntry& next = index_.groups[group_ + 1]; + fits = (next.start_block == block_ && next.run_offset == offset_) || + (at_block_end && next.start_block == block_ + 1 && + next.run_offset == 0); + } else { + fits = at_block_end && block_ + 1 == index_.blocks.size(); + } + if (!fits) { + return Status::Corruption("Blob reference run does not end at next run"); + } + return Status::OK(); +} + +Status BlobReferenceGroupDecoder::DecodeRun( + const Slice& contents, + const std::function& visit) { + uint64_t reference_count = 0; + Status status = DecodeRunHeader(contents, &reference_count); + return status.ok() ? DecodeRunEntries(visit) : status; +} + +namespace { + +// Decodes group `group` with blocks from `read_block`. `*loaded_block` is the +// block already in `*contents`, so a block shared by two groups is read once. +Status DecodeGroup(const BlobReferenceIndex& index, size_t group, + const BlobReferenceBlockReader& read_block, + size_t* loaded_block, Slice* contents, + const std::function& visit) { + BlobReferenceGroupDecoder decoder(index, group); + while (!decoder.done()) { + if (*loaded_block != decoder.next_block()) { + Status status = read_block(decoder.next_block(), contents); + if (!status.ok()) { + return status; + } + *loaded_block = decoder.next_block(); + } + Status status = decoder.DecodeRun(*contents, visit); + if (!status.ok()) { + return status; + } + } + return Status::OK(); +} + +} // namespace + +Status ForEachBlobReferenceInGroup( + const BlobReferenceIndex& index, size_t group, + const BlobReferenceBlockReader& read_block, + const std::function& visit) { + if (group >= index.groups.size()) { + return Status::InvalidArgument("Blob reference group out of range"); + } + size_t loaded_block = std::numeric_limits::max(); + Slice contents; + return DecodeGroup(index, group, read_block, &loaded_block, &contents, visit); +} + +Status ForEachBlobReference( + const BlobReferenceIndex& index, const BlobReferenceBlockReader& read_block, + const std::function& visit) { + // Groups are visited in index order. Each group's runs end where the next + // group starts, so the walk covers every block from start to end; each + // block is read once. + size_t loaded_block = std::numeric_limits::max(); + Slice contents; + for (size_t group = 0; group < index.groups.size(); ++group) { + Status status = + DecodeGroup(index, group, read_block, &loaded_block, &contents, visit); + if (!status.ok()) { + return status; + } + } + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_reference_metadata.h b/db/blob/blob_reference_metadata.h new file mode 100644 index 000000000000..560ab8ec8d78 --- /dev/null +++ b/db/blob/blob_reference_metadata.h @@ -0,0 +1,1105 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// Exact blob reference metadata +// +// When an SST is built, it saves a sorted list of every blob reference it keeps +// that points to a blob file (inlined and same-file blob indexes are not +// included). This list is the "exact reference metadata". Its intended users +// are the census (live bytes and garbage per blob file), indirect blob GC and +// RepairDB: they learn from it which blob values an SST keeps without reading +// its key-value data. See "Readers" below. It is called "exact" because it +// lists every such reference, not an estimate, so a reader can tell exactly +// which values the SST keeps. +// +// The caller of the table builder decides which SSTs get it +// (TableBuilderOptions::collect_blob_reference_metadata). When +// enable_blob_indirection is on, or the DB has used it before: +// - a compaction output at L1 or deeper gets exact metadata; +// - a compaction output at L0 gets it only when an input has it, or when +// the compaction is bottommost (see +// Compaction::CollectExactBlobReferences()); +// - flush output gets only a coarse classification (see "Coarse SST" and +// "Coarse classification property" below). +// SstFileWriter output gets neither. It can hold only same-file (embedded) +// blob references (SstFileWriter::OpenWithEmbeddedBlobs), never a reference to +// a blob file, so it has nothing to list. It stays in the legacy form. +// +// "Legacy" means "written the way base RocksDB writes it, without blob +// indirection". For example, a legacy reference is a kBlob or kBlobTTL index, +// and a legacy SST has no exact reference metadata. +// +// Reference cap and epoch runs: the collector keeps all references of an SST +// in memory until the SST is finished. So compaction starts a new output once +// an exact output holds blob_indirection_max_references_per_sst references +// (the reference cap; the code calls it the blob reference limit). The option +// comment in include/rocksdb/advanced_options.h gives the memory numbers. +// A compaction to L0 is a special case. It happens only in a few cases: +// universal compaction with num_levels = 1 (every compaction writes to L0); +// universal compaction when the sorted run right after the picked runs is an +// L0 file; leveled intra-L0 compaction; and CompactFiles() with output level +// 0. In base RocksDB such an output is always one L0 file. With blob +// indirection, the cap can cut it into several exact L0 files that share one +// epoch number (an epoch run). Every compaction picker takes all files of an +// epoch run or none of them (see VersionStorageInfo::ExpandExactL0Inputs() for +// why). +// +// Terms (generation number, origin, original offset, relocation file, old file +// and route are defined in blob_index.h): +// - encoding class (class): the kind of reference, stored as one byte. +// kDirect (0): a legacy kBlob or kBlobTTL index (file number and offset +// only). +// kIndirectUnflattened (1, "unflattened"): a kIndirectBlob index with +// generation == origin. +// kIndirectFlattened (2, "flattened"): a kIndirectFlattenedBlob index, +// whose generation is a relocation file. +// - stored size: the size of the value's stored (maybe compressed) bytes. +// - record bytes: the bytes the value's record takes in the file named by +// its generation: +// unflattened, kDirect: stored size + BlobLogRecord::kHeaderSize +// (32 bytes) + user key size, as in a normal blob file. +// flattened: stored size + kSimpleGen2BlobTrailerSize (5 bytes). This +// sum is the size of one record in a relocation file. +// The census and GC need record bytes, not just stored sizes, because a +// file's size includes every record's header and key (or trailer). +// - reference group (group): all references of one SST with the same +// (encoding class, generation number, origin). +// - run: the part of one group inside one data block. A group that fits in +// one block is one run; a larger group continues at the start of the +// next block with a new run. A run never crosses a block. +// - reference index block (index block): the block that lists the data +// blocks and the groups. It is not the SST's own index block. +// - relocation-state fingerprint: a 128-bit hash of the routes and +// relocation files of the Version that the SST was built from (see +// BlobReferenceMetadataDescriptor). +// +// Sort order: (encoding class, generation, origin, original offset). So each +// group is contiguous, and inside a group the references are in offset order. +// +// Layout. All blocks are written back to back through the table's meta-block +// path, each with the normal block trailer (compression type and checksum). +// They are written before the SST tail (tail_start_offset), so a normal +// table open does not prefetch them. +// +// +---------------------+ +// | data block 0 | one or more runs per block +// | ... | +// | data block N-1 | +// +---------------------+ +// | index block | block table, then one entry per group +// +---------------------+ +// | descriptor property | magic, format version, index handle, totals, +// +---------------------+ CF ID, relocation-state fingerprint, SST +// file number, CRC32C (72 bytes) +// +// An SST with no references still has an index block (block count 0, +// group count 0) and the descriptor; DecodeFrom() requires a non-null +// index handle. +// +// The next diagram shows an example SST with four groups, A, B, C and D, in +// three data blocks. The group names and the offsets x, y and z are example +// values. +// +// index block data blocks +// +-------------------------+ +// | block table | block 0 +// | first offset, sizes --+-------->+-------------------------------+ +// +-------------------------+ | run A | run B | run C1 ... | +// | group table | +-------------------------------+ +// | A: block 0, offset 0 | block 1 +// | B: block 0, offset x | +-------------------------------+ +// | C: block 0, offset y -+-------->| run C2 (all of block 1) | +// | D: block 2, offset z | +-------------------------------+ +// +-------------------------+ block 2 +// +-------------------------------+ +// | run C3 | run D | +// +-------------------------------+ +// +// Group C is large. Its first run, C1, starts at offset y in block 0. C2 +// fills block 1, and C3 starts at offset 0 of block 2. Group D starts in +// block 2 right after C3. +// +// Data block contents (after decompression; all varints are canonical). +// A block holds one or more runs, back to back. A run is a run header +// followed by `count` entries, one entry per reference: +// +// +------------+---------+---------+-----+---------+------------+----- +// | run header | entry 1 | entry 2 | ... | entry n | run header | ... +// +------------+---------+---------+-----+---------+------------+----- +// |<------------------- one run ------------------>|<-- next run +// +// Run header: +// count varint; number of entries in this run +// first offset varint; original offset of entry 1 +// default compression 1 byte; the compression of the first +// reference that the collector received for +// this group (not always entry 1 of the run) +// +// Entry: +// gap varint; entries 2..n only +// size and flag varint; (stored size << 1) | differs (stored +// size < 2^63) +// compression 1 byte; only when differs = 1 +// user key size varint; unflattened and kDirect only (records in +// a normal blob file) +// logical CRC fixed 4 bytes; unflattened and flattened only +// stored CRC fixed 4 bytes; unflattened and flattened only, +// and only when compression != kNoCompression +// +// A run header does not repeat the group key. A reader always reaches a +// run through its index entry, which gives the class, generation and +// origin. +// +// What the fields mean: +// - compression: the CompressionType the blob value itself was written +// with in its blob file (the same field as in the BlobIndex). It is not +// the compression of this metadata block, which is in the block trailer. +// - differs: a 1-bit flag. 1 means this entry's compression is not the +// run's default compression, and a compression byte follows. The flag is +// kept in the lowest bit of the stored size varint, so the many entries +// that use the default pay no extra byte for it. +// - user key size: the size of the user key in the value's record in its +// normal blob file. With BlobLogRecord::kHeaderSize it gives the record +// bytes and the gap. +// - gap: stores an offset as a small number. Offsets are sorted inside a +// run, so each entry stores only how far it is from where it would be if +// the records were back to back: +// +// blob file: [hdr][key 1][value 1][hdr][key 2][value 2] +// ^ ^ +// offset 1 offset 2 +// ||<--------->| +// kHeaderSize + key size 2 +// +// unflattened, kDirect: +// offset 2 = offset 1 + stored size 1 + BlobLogRecord::kHeaderSize +// + user key size 2 + gap +// flattened (the user key size in the origin file is not known): +// offset 2 = offset 1 + stored size 1 + gap +// +// So the gap is 0 for back-to-back records. When it is not 0, it is the +// bytes of the records in between that this SST does not reference. +// - logical CRC, stored CRC: copies of the two checksums in the SST's +// BlobIndex (see blob_index.h). GC reads the live set from this +// metadata, not from the SST's entries, so they must be here. +// stored CRC: GC checks every payload it copies against it, without +// decompressing. +// logical CRC: GC does not check it (it never decompresses), but +// needs it to rebuild the stored CRC for kNoCompression (which is +// not written) and to build a full BlobIndex for the reader APIs. +// RepairDB compares both when it matches references to copies. +// +// Index block contents (never compressed; all varints are canonical): +// +// Block table: +// block count varint +// first block offset varint; file offset of data block 0 (only when +// block count > 0) +// size varint, per data block in file order; the block's +// size on disk without its trailer (compressed size +// if compressed), as in a BlockHandle +// +// Group table: +// group count varint +// then, per group in sort order: +// class 1 byte +// generation varint; a delta from the previous entry's +// generation when the class is the same, else the +// full value +// origin varint; flattened only. A delta from the previous +// entry's origin when the class and generation are +// the same, else the full value +// start block delta varint; the number of the block holding the +// group's first run, minus the previous group's. +// 0 means the same block as the previous group. +// run offset varint; where the group's first run starts inside +// that block (after decompression) +// count varint; references in the group +// stored bytes varint; sum of the stored sizes +// user key bytes varint; sum of the user key sizes; unflattened and +// kDirect only +// +// What the fields are for: +// - Block table: data blocks are back to back, so the reader rebuilds every +// offset in memory: offset[0] = first block offset, offset[i+1] = +// offset[i] + size[i] + block trailer size. The end of the last data +// block must equal the index block's offset. Knowing every block's exact +// size before reading any data lets a reader plan, merge and batch its +// block reads. +// - Deltas (generation, origin, start block): the sorted order makes +// neighbors close, so most deltas fit in one byte. +// - start block delta + run offset: let GC jump straight to a group's +// first run. Groups that start in the same block have increasing run +// offsets. The group's later runs, if any, start at offset 0 of the +// following blocks. +// - count, stored bytes, user key bytes: the group totals. The group's +// record bytes are stored bytes + count * BlobLogRecord::kHeaderSize + +// user key bytes (unflattened, kDirect), or stored bytes + count * +// kSimpleGen2BlobTrailerSize (flattened). The census reads only these, +// so it never reads a data block. +// - kDirect groups are in the group table too. The census skips them. +// +// Writer: +// The table builder (BlockBasedTableBuilder) owns one +// BlobReferenceMetadataCollector per SST, only in the cases listed at the +// top. It calls Add() for each BlobIndex that the SST keeps, in plain blob +// values and in wide-column entities. Before it writes the SST tail, it +// calls Finish(), which writes the data blocks and the index block. The +// builder then adds the index block to the meta-index +// (kBlobReferenceIndexBlockName) and stores the descriptor as a table +// property (kBlobReferenceDescriptorPropertyName). When the table has a +// block cache and charging for kBlobReferenceMetadataBuildingBuffer is not +// disabled, the collector's memory is charged to the block cache. Details +// are in BlobReferenceMetadataCollector. +// +// Readers: the readers below are the intended users of this format. The +// decoders in this file are the only code that reads it. A reader first +// decodes the descriptor (BlobReferenceMetadataDescriptor::DecodeFrom()), then +// the index (DecodeBlobReferenceIndex()), and then the runs of a group +// (BlobReferenceGroupDecoder, or the ForEachBlobReference* helpers that use +// it). +// - The census reads only the index. +// - GC sorts the groups it needs from one SST, finds them in the index, +// reads only the blocks that hold their runs (merged and batched), and +// decodes only those runs. +// - Repair and remote compaction verify an SST's metadata by walking every +// group in index order, which visits every run and every block. +// A reader always reads, checks and decompresses a whole data block, then +// decodes only the runs it needs. Every run is checked against its group: +// entries must decode with the fields of the group's class, offsets must +// increase and gaps cannot be negative, a run must end exactly at the next +// run or the block end, and each group's count, stored bytes and user key +// bytes must equal its index totals. +// +// Compression and block size: +// - Data blocks are compressed with the compressor that the SST uses for +// its own index blocks (the basic compressor of the output level, no +// dictionary), so a reader needs no dictionary block. A block that does +// not compress well is stored raw. The trailer records the type. +// - The index block is never compressed. It is small: it has one entry per +// reference group, so its size grows with the number of blob files that +// the SST points to. Compression could save some bytes, but the census +// reads this block at every open, and a raw block needs no decompressor +// and no extra buffer. +// - The data block target is 16 KB of encoded bytes, before compression +// (kTargetDataBlockSize). The writer never makes a larger block. GC +// usually needs only one small group from a block but must read, check +// and decompress the whole block, so smaller blocks waste less I/O and +// CPU. There is no point lookup and no block cache here, so larger blocks +// bring no benefit. +// +// Format notes: +// - The record bytes of unflattened and kDirect references assume the +// current blob record header size (BlobLogRecord::kHeaderSize). If the +// blob log record format ever changes it, this metadata format version +// must change too. +// - Record bytes (see Terms) are the size of the record in the file named +// by the generation number. When that file is an old file whose values a +// route sends to a relocation file, the value of an unflattened reference +// takes the flattened record bytes (stored size + +// kSimpleGen2BlobTrailerSize) there. A reader that +// counts bytes per blob file or relocation file must use that size. +// +// Integrity: every block, including the index, is protected by the table's +// block checksum. The descriptor has its own CRC, and the index totals must +// match it. With BlockBasedTableOptions::checksum = kNoChecksum, corrupt +// metadata is detected only when it breaks the format or order checks. +// +// Coarse SST +// +// Background: exact reference metadata needs collector memory while the SST is +// built (see "Writer" above). To save that memory, a table builder (the +// intended writer) can skip the collector. A "coarse SST" is an SST built +// without exact reference metadata. Instead it records one file number, its +// frontier. An SST that has the full list is an "exact SST". +// +// Frontier: the oldest indirect origin that the coarse SST references +// (FileMetaData::oldest_coarse_indirect_origin_file_number). For example, +// frontier 12 (an example file number) means that the SST may reference blob +// file 12 or any newer indirect origin, but indirect blob GC cannot know which +// ones. +// - kInvalidBlobFileNumber (0): the SST is not a coarse SST. A coarse SST +// always has a frontier above 0; VersionEdit decoding rejects a coarse +// frontier of 0. +// - Any other value: the SST pins every indirect-origin file number at or +// above the frontier. +// "Pin" here means that GC must treat the file as used and must not collect +// it. It is not the base RocksDB meaning (a Version, block or iterator that +// stays alive while something references it). +// +// A coarse SST never holds a flattened reference: its writer must not write +// one, and the code in this file does not check it. So each indirect reference +// of a coarse SST has generation number == origin, and one number (the oldest +// origin) is enough. +// +// The Version keeps the lowest frontier of all its SSTs +// (VersionStorageInfo::OldestCoarseIndirectOriginFileNumber()). An +// indirect-origin blob file at or above it is "coarse pinned" +// (VersionStorageInfo::IsCoarseIndirectOriginBlobPinned()), and the census +// keeps it out of the GC candidate heap. IsCoarseIndirectOriginBlobPinned() +// compares only the file number with the frontier, so it also returns true +// for a relocation file at or above the frontier. The census heap applies the +// pin only to indirect-origin blob files. Indirect blob GC skips a normal +// input whose file number is pinned, and a relocation input with a pinned +// routed generation number. In normal operation a coarse SST never references +// a relocation file or a routed number (a flush references only its own new +// blob files, and a coarse L0 compaction has only coarse inputs and does not +// flatten), so the relocation check is a safety margin. +// +// Coarse classification property +// +// "Coarse SST" above defines a coarse SST and its frontier: the one file number +// it records instead of a reference list, the oldest indirect origin that it +// may reference. This section says where the frontier is stored and which +// builders write coarse SSTs. +// +// Property: a coarse SST stores its frontier in the table property +// kBlobReferenceClassificationPropertyName ("coarse classification", 16 bytes; +// see EncodeBlobReferenceClassification()). Compaction and flush read it back +// into FileMetaData::oldest_coarse_indirect_origin_file_number. The builder +// writes the property for every SST that it builds without exact metadata +// when track_blob_reference_classification is set, also when the SST has no +// indirect reference. So the property alone does not make an SST coarse. +// With frontier 0 (no indirect reference), the SST is stored as a legacy SST +// (see ExtractBlobReferenceClassificationFromTableProperties() in +// db/builder.cc). With any other frontier, it is a coarse SST. +// +// Builders: the table builder writes only this property, and no exact +// metadata, in three cases: +// - flush output; +// - WAL recovery flush output; +// - a compaction output at L0 when no input has exact metadata and the +// compaction is not bottommost (see +// Compaction::CollectExactBlobReferences()). + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache/cache_reservation_manager.h" +#include "db/blob/blob_constants.h" +#include "db/blob/blob_index.h" +#include "rocksdb/status.h" +#include "table/format.h" +#include "util/hash_containers.h" + +namespace ROCKSDB_NAMESPACE { + +// Table property that holds the encoded BlobReferenceMetadataDescriptor. +extern const char kBlobReferenceDescriptorPropertyName[]; +// Meta-index name of the index block. +extern const char kBlobReferenceIndexBlockName[]; +// Table property that holds the coarse classification. +extern const char kBlobReferenceClassificationPropertyName[]; + +// Encodes the coarse classification (see "Coarse classification property" +// above) as 16 bytes: format version (Fixed32, value 1), frontier (Fixed64), +// and a CRC32C of the first 12 bytes (Fixed32). +void EncodeBlobReferenceClassification(uint64_t frontier, std::string* output); +// Returns Corruption if the size, version or CRC is wrong, or if the frontier +// is above kFileNumberMask. Returns InvalidArgument for a null `frontier`. +Status DecodeBlobReferenceClassification(const Slice& input, + uint64_t* frontier); + +// Valid range of blob_indirection_max_references_per_sst. +// - Upper bound: the metadata of one SST has at most 2^20 data blocks +// (kMaxDataBlockCount in the .cc file) of about 16 KB, so it holds about +// 16 GB of entries. On disk, one entry uses about 8 to 12 bytes +// (estimate), so that is about 1.4G to 2G references. The upper bound 2^28 +// (256M) stays well under that, so an output at the cap plus one key's +// history still fits the format. In memory, the collector uses about the +// on-disk entry size plus a small chunk overhead, about 8 to 14 bytes per +// reference (an estimate; see "Memory usage" below), so 256M references +// use about 2 to 3.5 GB of collector memory. +// - Lower bound: 4096, a fixed choice (not measured). +inline constexpr uint64_t kMinBlobIndirectionMaxReferencesPerSst = 4096; +inline constexpr uint64_t kMaxBlobIndirectionMaxReferencesPerSst = 1ULL << 28; + +// How a reference is encoded in its BlobIndex. It is stored on disk and is +// the first part of the group key and sort order. +enum class BlobReferenceEncodingClass : uint8_t { + // Legacy kBlob or kBlobTTL index; generation == origin == its file number. + kDirect = 0, + // kIndirectBlob index with generation == origin (unflattened reference). + kIndirectUnflattened = 1, + // kIndirectFlattenedBlob index (generation > origin). Only flatten writes it: + // compaction rewrote the generation to the file that held the value at + // flatten time (a relocation file). + kIndirectFlattened = 2, +}; + +// The key of a group: (encoding class, generation, origin). Groups are ordered +// and looked up by it. kDirect and unflattened groups have origin == +// generation; flattened groups have generation > origin. +struct BlobReferenceGroupKey { + BlobReferenceEncodingClass encoding_class = + BlobReferenceEncodingClass::kDirect; + uint64_t generation_file_number = 0; + uint64_t origin_file_number = 0; + + auto Tie() const { + return std::tie(encoding_class, generation_file_number, origin_file_number); + } + bool operator==(const BlobReferenceGroupKey& other) const { + return Tie() == other.Tie(); + } + bool operator<(const BlobReferenceGroupKey& other) const { + return Tie() < other.Tie(); + } + + // True if the class is known, the file numbers are not 0, and the relation + // between the generation and the origin fits the class (see above). + bool IsValid() const; + + struct Hash { + size_t operator()(const BlobReferenceGroupKey& key) const { + size_t hash = static_cast(key.encoding_class); + hash = hash * 31 + std::hash()(key.generation_file_number); + return hash * 31 + std::hash()(key.origin_file_number); + } + }; +}; + +// One blob reference kept by an SST that points to a blob file (inlined and +// same-file indexes are not included). Terms follow blob_index.h. +struct BlobReference { + BlobReferenceEncodingClass encoding_class = + BlobReferenceEncodingClass::kDirect; + // Generation. For kDirect, the blob file number. + uint64_t generation_file_number = 0; + // Origin. For kDirect, the same as the generation. + uint64_t origin_file_number = 0; + // Original offset in the origin file. For kDirect, the offset. + uint64_t original_value_offset = 0; + // Size of the stored (maybe compressed) value bytes. + uint64_t stored_value_size = 0; + // Record bytes: the bytes this record takes in the file named by its + // generation. Stored size + BlobLogRecord::kHeaderSize + user key size for + // unflattened and kDirect, or stored size + kSimpleGen2BlobTrailerSize for + // flattened. The census and indirect blob GC need record bytes because a + // file's size includes each record's header and key (or trailer). For + // unflattened and kDirect references, FromBlobIndex() does not know the user + // key size, so it sets this field to the stored size only, and Add() adds the + // record header and the user key. For flattened references, FromBlobIndex() + // sets the full value. + uint64_t record_bytes = 0; + // Compression of the stored value bytes. + CompressionType compression = kNoCompression; + // stable_checksum: the logical CRC (the name follows BlobIndex). 0 for + // kDirect. + uint32_t stable_checksum = 0; + // stored_checksum: the stored CRC (the name follows BlobIndex). For + // kNoCompression it equals the logical CRC extended with the compression + // byte. 0 for kDirect. + uint32_t stored_checksum = 0; + + // Converts `blob_index` into `*reference`. For an index that does not point + // to a blob file (kInlinedTTL, or a same-file index), returns OK and sets + // `*included` to false. A kBlob or kBlobTTL index becomes kDirect. Returns + // Corruption for an invalid index and InvalidArgument for null outputs. + // Except for kIndirectFlattened, the result's record_bytes is only the + // stored size; the collector adds the record header and user key. + static Status FromBlobIndex(const BlobIndex& blob_index, + BlobReference* reference, bool* included); + + // True for indirect references (kIndirectBlob or kIndirectFlattenedBlob), + // which have a stable identity (origin, original offset) and checksums. + bool HasStableIdentity() const { + return encoding_class != BlobReferenceEncodingClass::kDirect; + } + + BlobReferenceGroupKey group_key() const { + return {encoding_class, generation_file_number, origin_file_number}; + } +}; + +// Memory limit (128 MB) for one vector that a reader decodes from this +// metadata. The index decoder's count limits keep its vectors under it. The +// data blocks may hold more references than fit, so a caller that decodes +// references into one vector must check this limit and return an error, +// instead of letting bad metadata use unbounded memory. +inline constexpr size_t kMaxDecodedBlobReferenceBytes = 128 * 1024 * 1024; + +// Totals of one group: all references with the same (encoding class, +// generation, origin). Summaries keep only indirect (non-kDirect) groups. +// Readers such as the census and indirect blob GC use these totals (mainly +// record_bytes) for live-byte and garbage math without reading each reference. +// For a kDirect group, origin_file_number equals generation_file_number. +struct BlobReferenceGroup { + BlobReferenceEncodingClass encoding_class = + BlobReferenceEncodingClass::kDirect; + uint64_t generation_file_number = 0; + uint64_t origin_file_number = 0; + uint64_t reference_count = 0; + // Sum of stored_value_size: value bytes only. + uint64_t stored_value_bytes = 0; + // Sum of record_bytes: the bytes these references keep in use in the + // generation file. From the index totals it is stored bytes + count * + // BlobLogRecord::kHeaderSize + user key bytes (unflattened, kDirect), or + // stored bytes + count * kSimpleGen2BlobTrailerSize (flattened). + uint64_t record_bytes = 0; + + BlobReferenceGroupKey group_key() const { + return {encoding_class, generation_file_number, origin_file_number}; + } +}; + +// One index entry: a group, the data block where it starts, and the offset +// of its first run in that block's uncompressed contents. +struct BlobReferenceIndexEntry { + BlobReferenceGroup group; + uint32_t start_block = 0; + uint32_t run_offset = 0; +}; + +// The decoded index block. +struct BlobReferenceIndex { + // Data blocks in file order. + std::vector blocks; + // Every group, kDirect ones included, in key order. + std::vector groups; +}; + +// Header of one run: the references of one group in one data block. Only +// the count, first offset and default compression are stored. The key fields +// (encoding class, generation, origin) are not on disk; the decoder copies +// them from the run's index group. +struct BlobReferenceRunHeader { + BlobReferenceEncodingClass encoding_class = + BlobReferenceEncodingClass::kDirect; + uint64_t generation_file_number = 0; + // Equals the generation for unflattened and kDirect runs. + uint64_t origin_file_number = 0; + uint64_t reference_count = 0; + uint64_t first_offset = 0; + CompressionType default_compression = kNoCompression; +}; + +// Top-level record of the exact reference metadata, stored as a table +// property. It points to the index block and holds the totals, the column +// family ID, the SST file number and the relocation-state fingerprint. +// Encoding (EncodeTo()), 72 bytes: +// magic (fixed32) | format version (fixed32) | index offset (fixed64) | +// index size (fixed64) | reference count (fixed64) | stored bytes +// (fixed64) | CF ID (fixed32) | fingerprint high (fixed64) | fingerprint +// low (fixed64) | SST file number (fixed64) | CRC32C of the rest (fixed32) +struct BlobReferenceMetadataDescriptor { + // The only readable format. + static constexpr uint32_t kFormatVersion = 1; + // Size of the encoded descriptor, including its trailing CRC32C. + static constexpr size_t kEncodedSize = 72; + + // Location of the index block. + BlockHandle index_handle; + // Number of references in the SST, kDirect ones included. + uint64_t reference_count = 0; + // Sum of stored_value_size over all references, kDirect ones included. + uint64_t stored_value_bytes = 0; + // Column family of the SST. max() means unknown; DecodeFrom() rejects it. + uint32_t column_family_id = std::numeric_limits::max(); + // High and low halves of a 128-bit fingerprint of the relocation state + // (relocation files and routes) that the SST was built with. A compaction + // output stores the fingerprint of the compaction's input Version (see + // VersionStorageInfo::ComputeBlobRelocationStateFingerprint()). 0 means not + // set. DecodeFrom() does not check it. + uint64_t route_snapshot_fingerprint_high = 0; + uint64_t route_snapshot_fingerprint_low = 0; + // The file number the SST had when it was built. It can differ from the + // current file name: CompactionService output is renamed when it is + // installed, but its contents, this number included, do not change. + uint64_t sst_file_number = 0; + + // True for the metadata of an SST that keeps no blob reference to a blob + // file. + bool IsEmpty() const { + return reference_count == 0 && stored_value_bytes == 0; + } + + // True if the repair fields are set: a known column family and a nonzero + // SST file number. + bool HasRepairMetadata() const { + return column_family_id != std::numeric_limits::max() && + sst_file_number != 0; + } + + // Writes exactly kEncodedSize bytes, ending with a CRC32C of the rest. + void EncodeTo(std::string* output) const; + + // Checks the size, CRC, magic, format version and HasRepairMetadata(). Also + // checks that the index handle is not null, that its size is more than 0 + // and at most 64 MB, that its end does not overflow, and that a descriptor + // with no references has 0 stored bytes. Returns Corruption on any check + // failure, and InvalidArgument when `descriptor` is null. On failure, + // `*descriptor` is unchanged. + static Status DecodeFrom(const Slice& input, + BlobReferenceMetadataDescriptor* descriptor); +}; + +// The group totals of one finished SST, kept in memory. Finish() returns it +// when the caller asks for it. It holds the indirect groups but not the +// references, which stay on disk. It is immutable and shared by pointer, so +// the cache charge belongs to this one object, not to each owner. +struct BlobReferenceSstSummary { + // Same as descriptor.sst_file_number: the build file number, which may + // differ from the current file name. + uint64_t file_number = 0; + BlobReferenceMetadataDescriptor descriptor; + // Cache charge for MemoryUsage(), moved from the collector. Both are null + // when charging is disabled. + std::shared_ptr reservation_manager; + std::unique_ptr + reservation_handle; + // Indirect groups in key order. Declared after the reservation members, so + // this buffer is freed before its charge is released. + std::vector groups; + + // Bytes charged to the cache for this summary. + size_t MemoryUsage() const { + return sizeof(*this) + groups.capacity() * sizeof(BlobReferenceGroup); + } +}; + +// The identity of the SST that a BlobReferenceMetadataCollector builds; it is +// written into the descriptor. Callers set the fields by name, so the SST file +// number and the two fingerprint halves (all uint64_t) cannot be swapped by +// mistake. +struct BlobReferenceSstIdentity { + uint32_t column_family_id = 0; + uint64_t sst_file_number = 0; + // The relocation-state fingerprint (two 64-bit halves); 0 when not set. + uint64_t route_snapshot_fingerprint_high = 0; + uint64_t route_snapshot_fingerprint_low = 0; +}; + +// Builds the exact reference metadata of one SST (see the top of this file). +// +// Background: the census, indirect blob GC and RepairDB must know which blob +// values an SST keeps without reading its key-value data. The table builder +// therefore records each blob reference while it writes the SST, and stores the +// sorted list in metadata blocks before the SST tail. The table builder owns +// one collector per SST and calls it during flush and compaction. +// +// Write path: +// 1. Add() receives each BlobIndex that the SST keeps. It puts the +// reference into its group, (encoding class, generation, origin), and +// encodes it once, in the on-disk entry encoding. +// 2. Finish() sorts the groups by group key, packs their entries into data +// blocks of about 16 KB, writes the blocks, and then writes the index +// block. +// +// Chunks: each group keeps its encoded entries in a chain of 512-byte chunks. +// A chunk holds only whole entries. The group's first chunk starts with an +// entry. Every later chunk starts with the gap that joins its first entry to +// the previous entry of the group (the "join gap"). Chunks come from 64 KB +// slabs (kSlabSize, 128 chunks). 64 KB was chosen by judgment, not by +// measurement. It is large enough that slab allocations are rare (one per 128 +// chunks), and small enough that a small SST wastes little memory. +// +// In one group, references normally arrive in rising offset order: one job +// writes each blob file in key order, and the SST is also built in key order. +// The collector then needs no sort. Finish() copies the chunks whole into +// data blocks. Consecutive chunks of one group in one block form one run. +// When a chunk starts a new run, Finish() drops its join gap. +// +// Why 512 bytes: the chunk size balances three costs. +// - Chunk header: each chunk has a 16-byte header in memory, which is 3% +// of 512 bytes. Smaller chunks need more headers (6% at 256 bytes). +// - Partly filled chunks: the last chunk of each group is on average half +// empty. This waste grows with the chunk size and the group count (256 +// bytes per group at 512, 2 KB per group at 4 KB). +// - Block fill: a run takes only whole chunks, so a 16 KB data block can +// end with up to one chunk of unused space. At 512 bytes this is about +// 3% of a block. +// One entry, with its gap, is at most kMaxEntrySize (39 bytes), so a +// 512-byte chunk holds at least 13 entries, and about 50 typical small +// entries (estimate). +// For an SST with few groups, the chunk size changes memory very little. For +// a small SST with many groups, the partly filled chunks dominate, so larger +// chunks cost much more memory. Chunks smaller than 512 bytes use a little +// more memory and make Finish() slower when an SST has many groups. +// +// Unsorted fallback: a group takes the fallback when it receives an offset +// lower than its last one. That reference and all later references of the +// group go into a second chunk chain. Each entry there stores its absolute +// offset instead of a gap. In Finish(), the collector sorts the group with +// one 16-byte key (offset and entry position) per reference, checks it for +// repeated and overlapping references, and encodes it again from the encoded +// entries. It decodes one entry at a time and never builds a vector of +// BlobReference. +// +// Memory usage: the collector buffers use about the size of the encoded +// entries, plus the chunk headers and the unused end of each group's last +// chunk. A compressed value needs more bytes because its entry also stores +// the CRC of the stored bytes. Many groups in a small SST cost more, because +// each group has a partly filled last chunk. A fallback group needs up to 16 +// more bytes per reference for its sort keys, plus a second copy of its +// entries while they are encoded again. The cache is charged for all of this +// memory. +// +// Memory charge: when charging is on (see "Writer" at the top of this file), +// the collector charges its buffers to the block cache through its +// reservation handle. The rule is: charge first, then allocate, then set the +// charge to the real usage. +// - ReserveCapacity() raises the charge to the current usage plus the new +// buffer, because the old buffer is still alive while reserve() runs. +// Then it allocates, and UpdateReservation() sets the charge to +// ReservedMemory(). +// - NewChunk() charges each 64 KB slab before it allocates the slab. +// - ReservedMemory() is the one list of charged buffers. A new buffer must +// be added there. The group map is counted as an estimate per node +// (kGroupMapNodeBytes) plus its bucket array (one pointer per bucket). The +// temporary buffers of Finish() (the sort keys of a fallback group and the +// block buffer) are counted through sort_scratch_bytes_ and +// block_buffer_bytes_ while they exist. +// +// Lifecycle: +// 1. Construct the collector with the SST's column family, file number and +// relocation-state fingerprint. +// 2. Call Add() for each BlobIndex in each entry that the SST keeps. +// 3. Call Finish() at most once. +// 4. Destroy the collector. The destructor frees every buffer and then +// releases the cache charge, also when the build is abandoned. +// +// Errors: the first error is sticky, so later Add() and Finish() calls return +// it. After Finish(), later Add() and Finish() calls return InvalidArgument. +class BlobReferenceMetadataCollector { + public: + // Writes one metadata block (uncompressed contents) into the SST and + // returns its handle, which must not be null on success. The writer may + // compress a data block; it must write the index block (`is_index_block`) + // raw. + using BlockWriter = std::function; + + BlobReferenceMetadataCollector( + std::shared_ptr cache_reservation_manager, + const BlobReferenceSstIdentity& identity); + ~BlobReferenceMetadataCollector(); + BlobReferenceMetadataCollector(const BlobReferenceMetadataCollector&) = + delete; + BlobReferenceMetadataCollector& operator=( + const BlobReferenceMetadataCollector&) = delete; + BlobReferenceMetadataCollector(BlobReferenceMetadataCollector&&) = delete; + BlobReferenceMetadataCollector& operator=(BlobReferenceMetadataCollector&&) = + delete; + + // `user_key_size` is the size stored in a normal blob-log record. It is + // ignored for flattened references. An index that does not point to a blob + // file is skipped (see FromBlobIndex()). + Status Add(const BlobIndex& blob_index, uint64_t user_key_size); + + // Same as Add() with user_key_size 0: record_bytes of an unflattened or + // kDirect reference then counts the record header but not the key. + Status Add(const BlobIndex& blob_index); + + // Writes the data blocks and then the index block through `write_block`, + // and fills `*descriptor`, index handle included. When `summary` is not + // null, it also returns the SST summary, which takes over the group buffer + // and the cache charge; the other buffers are freed first. The summary lives + // only in memory; it is never written to the SST or the MANIFEST. + Status Finish(const BlockWriter& write_block, + BlobReferenceMetadataDescriptor* descriptor, + std::shared_ptr* summary); + + // A conservative O(1) upper bound on the uncompressed metadata size, used + // by output-file size planning before Finish(): the encoded chunk bytes + // plus bounds for the gaps at chunk joins, the run headers, the index and + // the block trailers. + uint64_t EstimatedEncodedSize() const; + + // References added so far. + uint64_t reference_count() const { return reference_count_; } + + // Process-wide count of groups that took the unsorted fallback in + // Finish(), for tests and stress runs. + static uint64_t UnsortedGroupFallbackCount(); + + size_t TEST_ReservedMemory() const { return ReservedMemory(); } + + private: + // Grows `container` to at least `target_capacity`. Charges the cache for + // the new buffer first, then sets the charge to the real usage. + template + Status ReserveCapacity(Container* container, size_t target_capacity) { + if (target_capacity <= container->capacity()) { + return Status::OK(); + } + const size_t current_memory = ReservedMemory(); + if (target_capacity > + (std::numeric_limits::max() - current_memory) / + sizeof(typename Container::value_type)) { + return Status::MemoryLimit("Blob reference metadata capacity overflow"); + } + // The old allocation remains alive while reserve() allocates the new one. + const Status status = ResizeReservation( + current_memory + + target_capacity * sizeof(typename Container::value_type)); + if (!status.ok()) { + return status; + } + container->reserve(target_capacity); + return UpdateReservation(); + } + // Doubles the capacity of a full `container` (to `initial_capacity` when + // it has none) through ReserveCapacity(). + template + Status GrowIfFull(Container* container, size_t initial_capacity) { + if (container->size() < container->capacity()) { + return Status::OK(); + } + return ReserveCapacity(container, container->capacity() == 0 + ? initial_capacity + : 2 * container->capacity()); + } + + // Frees the collection buffers. Used when the summary takes the charge. + void ReleaseBuffers(); + + // The work of Finish(), which then makes the collector finished. + Status FinishOnce(const BlockWriter& write_block, + BlobReferenceMetadataDescriptor* descriptor, + std::shared_ptr* summary); + + // Sets the cache charge to ReservedMemory(). + Status UpdateReservation(); + + // Sets the cache charge to `memory_used`, making the handle on first use. + // Does nothing without a reservation manager. + Status ResizeReservation(size_t memory_used); + + // Heap memory of all collector buffers. + size_t ReservedMemory() const; + + // Packs the group chunks into data blocks of about 16 KB and writes them. + // Fills index_, groups_, and the descriptor totals. + Status BuildAndWriteDataBlocks(const BlockWriter& write_block); + + // Encodes index_ into index_block_. + Status EncodeIndex(); + + // Bytes of one chunk and of one slab of chunks. + static constexpr size_t kChunkSize = 512; + static constexpr size_t kSlabSize = 64 * 1024; + static constexpr uint32_t kNoChunk = std::numeric_limits::max(); + static constexpr size_t kChunksPerSlab = kSlabSize / kChunkSize; + // Header of one chunk, kept in memory only; the chunk bytes are at + // ChunkData(). A chunk of a group's sorted chain holds whole entries in the + // on-disk encoding. The group's first chunk starts with an entry (no gap). + // Every later chunk starts with the gap that joins its first entry to the + // last entry of the previous chunk. So Finish() copies a chunk as is when it + // continues a run, and skips the gap when the chunk starts a new run. + // `first_offset` is the original offset of the chunk's first entry. + struct Chunk { + uint64_t first_offset = 0; + // Next chunk of the same chain, or kNoChunk. + uint32_t next = kNoChunk; + uint16_t count = 0; + uint16_t used = 0; + }; + static_assert(sizeof(Chunk) == 16, "Chunk headers are 3% of chunk memory"); + static_assert(kChunkSize <= std::numeric_limits::max(), + "Chunk counts and sizes fit uint16_t"); + char* ChunkData(uint32_t chunk) const { + return slabs_[chunk / kChunksPerSlab].get() + + (chunk % kChunksPerSlab) * kChunkSize; + } + // One group: its key and totals, and its chunk chain. + struct GroupRecord { + BlobReferenceGroup group; + // The compression of the first reference that Add() received for this + // group; the default of its runs. + CompressionType default_compression = kNoCompression; + uint64_t last_offset = 0; + uint64_t last_stored_size = 0; + uint32_t first_chunk = kNoChunk; + uint32_t tail_chunk = kNoChunk; + // Set when an offset arrived lower than the group's last offset. Such + // references, and all later ones of the group, wait in the unsorted + // chunk chain until Finish(). Each unsorted entry is its absolute offset + // and then the entry without its gap. + bool unsorted = false; + uint32_t unsorted_first_chunk = kNoChunk; + uint32_t unsorted_tail_chunk = kNoChunk; + }; + // One reference of a fallback group while it is sorted: its offset and + // where its entry (without gap or offset) starts in the chunks. + struct SortKey { + uint64_t offset; + uint32_t chunk; + uint32_t position; + }; + static_assert(sizeof(SortKey) == 16, "SortKey is the fallback's memory"); + // Charged heap bytes of one node of group_map_: an estimate of the key, the + // value and two pointers. + static constexpr size_t kGroupMapNodeBytes = + sizeof(BlobReferenceGroupKey) + sizeof(uint32_t) + 2 * sizeof(void*); + + // Returns the record of the reference's group, making it on first use. + Status FindOrAddGroup(const BlobReference& reference, uint32_t* group); + + // Encodes the entry of `reference` without its gap, in the on-disk + // encoding, into `body` (kMaxEntrySize bytes). Sets the user key size for + // classes that store it. + static Status EncodeEntryBody(const BlobReference& reference, + CompressionType default_compression, char* body, + size_t* body_size, uint64_t* key_size); + // Encodes a reference whose offset is above the group's last offset, with + // its gap, into the group's tail chunk. Starts a new chunk when the gap and + // the entry do not fit. + Status AppendSorted(uint32_t group, const BlobReference& reference); + + // Encodes an out-of-order reference into the group's unsorted chain. + Status AppendUnsorted(uint32_t group, const BlobReference& reference); + + // Gets one empty chunk, allocating (and charging) a slab when needed. + Status NewChunk(uint32_t* chunk); + + // Sorts all references of a fallback group, sorted and unsorted chains, + // through one 16-byte key each, checks for repeats and overlaps, and + // re-encodes the group from its encoded entries. + Status SortUnsortedGroup(uint32_t group); + + // The body of SortUnsortedGroup(), with the sort keys in `keys`, whose + // charge the caller holds. + Status SortAndEncodeGroup(uint32_t group, std::vector* keys); + + // Null when cache charging is disabled. + std::shared_ptr cache_reservation_manager_; + // Charge for ReservedMemory(). + std::unique_ptr + reservation_handle_; + // Slabs of chunk memory, and the chunk headers. + std::vector> slabs_; + size_t slab_used_ = kSlabSize; + std::vector chunks_; + // Group records in Add() order, and the map from group key to record. + std::vector group_records_; + UnorderedMapH + group_map_; + // The group of the last Add(), for a fast path. + uint32_t last_group_ = kNoChunk; + uint64_t reference_count_ = 0; + // Bytes of entries in chunks, for EstimatedEncodedSize(). + uint64_t chunk_bytes_ = 0; + // Bytes of the sort keys while one group takes the fallback. + size_t sort_scratch_bytes_ = 0; + // Groups that are not kDirect, for the summary. + size_t indirect_group_count_ = 0; + // The index made by Finish(). + BlobReferenceIndex index_; + // Indirect groups for the summary. + std::vector groups_; + // Charge for the block buffers while Finish() runs. + size_t block_buffer_bytes_ = 0; + std::string index_block_; + BlobReferenceMetadataDescriptor descriptor_; + // The first error; sticky. + Status status_; +}; + +// Decodes the index block. Checks canonical varints, that the data blocks fit +// back to back before the index block and end exactly at its offset, that +// groups are valid and strictly increasing, that start blocks and run offsets +// do not go backward, and that the group totals add up to the descriptor +// totals. Returns Corruption on any failure; `*index` changes only on success. +// The caller reads the block with checksum verification. +Status DecodeBlobReferenceIndex( + const Slice& input, const BlobReferenceMetadataDescriptor& descriptor, + BlobReferenceIndex* index); + +// Sets `*groups` to the groups of a decoded index that are not kDirect, in key +// order. These are the groups that a BlobReferenceSstSummary holds and the +// census reads. DecodeBlobReferenceIndex() has already validated the index. +// When set, `before_allocation` gets the size in bytes of the group buffer +// before the buffer is allocated. If it returns non-OK, this function returns +// that status and leaves `*groups` unchanged. +Status CopyIndirectBlobReferenceGroups( + const BlobReferenceIndex& index, + const std::function& before_allocation, + std::vector* groups); + +// Position of the group with key `key` in `index.groups`, or +// index.groups.size() if the SST has no such group. +size_t FindBlobReferenceGroup(const BlobReferenceIndex& index, + const BlobReferenceGroupKey& key); + +// Decodes the runs of one index group, one data block at a time. This is the +// only decoder of reference runs, so every check of a group walk is here: +// - each run's header and entries are valid for the group's class; +// - offsets strictly increase and records do not overlap, inside a run and +// from the last reference of one run to the first of the next; +// - a run continued in the next block ends its block; +// - the run counts never pass the group count, and the counts and bytes add +// up to the group totals; +// - the last run ends where the next group starts (the last group: at the end +// of the last block). +// The caller reads block next_block() and passes its uncompressed contents. +class BlobReferenceGroupDecoder { + public: + // `index` must outlive the decoder. Requires group < index.groups.size(). + BlobReferenceGroupDecoder(const BlobReferenceIndex& index, size_t group); + + // True once the group's last run is decoded. + bool done() const { return remaining_ == 0; } + // The data block that holds the next run. + size_t next_block() const { return block_; } + + // Decodes the header of the next run in `contents`, the contents of block + // next_block(), and returns its reference count, which is at most the + // references left in the group. `contents` must stay valid until + // DecodeRunEntries() returns. Returns Corruption when `contents` (after + // decompression) are larger than the 64 KB block limit. + Status DecodeRunHeader(const Slice& contents, uint64_t* reference_count); + + // Decodes the entries of the run whose header was just decoded and calls + // `visit` for each reference in offset order. + Status DecodeRunEntries( + const std::function& visit); + // DecodeRunHeader(), then DecodeRunEntries(). + Status DecodeRun(const Slice& contents, + const std::function& visit); + + private: + const BlobReferenceIndex& index_; + const size_t group_; + size_t block_; + size_t offset_; + uint64_t remaining_; + uint64_t stored_value_bytes_ = 0; + uint64_t record_bytes_ = 0; + Slice contents_; + BlobReferenceRunHeader header_; + bool has_previous_ = false; + // The last decoded reference, for the order check across runs. + BlobReference previous_; +}; + +// Returns the uncompressed contents of data block `block_ordinal`. The +// contents stay valid until the next call. +using BlobReferenceBlockReader = + std::function; + +// Visits the references of `index.groups[group]` in offset order through +// BlobReferenceGroupDecoder. +Status ForEachBlobReferenceInGroup( + const BlobReferenceIndex& index, size_t group, + const BlobReferenceBlockReader& read_block, + const std::function& visit); + +// Visits every reference in key order by walking the index groups in order +// with BlobReferenceGroupDecoder, which reads every block once. +Status ForEachBlobReference( + const BlobReferenceIndex& index, const BlobReferenceBlockReader& read_block, + const std::function& visit); + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_relocation_file.cc b/db/blob/blob_relocation_file.cc new file mode 100644 index 000000000000..2e66cefbc3a2 --- /dev/null +++ b/db/blob/blob_relocation_file.cc @@ -0,0 +1,514 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_relocation_file.h" + +#include +#include + +#include "db/blob/blob_gen2_format.h" +#include "util/coding.h" +#include "util/compression.h" +#include "util/crc32c.h" +#include "util/math.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +constexpr uint32_t kRelocationDirectoryMagic = 0x524C4344U; // RLCD +// Header: magic (4) | format version (4) | origin count (8) | served file +// count (8) | total entries (8) | total record bytes (8) | directory digest +// (4) | header CRC (4). +constexpr size_t kDirectoryDigestOffset = 40; +constexpr size_t kHeaderChecksumOffset = 44; +constexpr size_t kDirectoryHeaderSize = 48; +// Origin entry: origin, first and last original offset, entry count and +// record bytes (8 each). Then one 8-byte number per served file. +constexpr size_t kOriginEntrySize = 40; +constexpr size_t kServedFileSize = sizeof(uint64_t); +// The digest covers the counts and totals (header bytes 8..39) and the body. +constexpr size_t kDigestedHeaderOffset = 8; + +uint32_t ComputeDirectoryDigest(const char* header, const char* body, + size_t body_size) { + const uint32_t header_crc = + crc32c::Value(header + kDigestedHeaderOffset, + kDirectoryDigestOffset - kDigestedHeaderOffset); + return crc32c::Extend(header_crc, body, body_size); +} + +bool AddRelocationBytesWithoutOverflow(uint64_t lhs, uint64_t rhs, + uint64_t* sum) { + if (sum == nullptr || rhs > std::numeric_limits::max() - lhs) { + return false; + } + *sum = lhs + rhs; + return true; +} + +} // namespace + +void BlobRelocationDirectory::EncodeTo(std::string* output) const { + assert(output != nullptr); + output->clear(); + output->reserve(kDirectoryHeaderSize + origins_.size() * kOriginEntrySize + + served_file_numbers_.size() * kServedFileSize); + PutFixed32(output, kRelocationDirectoryMagic); + PutFixed32(output, kFormatVersion); + PutFixed64(output, origins_.size()); + PutFixed64(output, served_file_numbers_.size()); + PutFixed64(output, total_entries_); + PutFixed64(output, total_record_bytes_); + // Directory digest and header CRC, filled in below. + PutFixed32(output, 0); + PutFixed32(output, 0); + assert(output->size() == kDirectoryHeaderSize); + + for (const BlobRelocationOrigin& origin : origins_) { + PutFixed64(output, origin.origin_file_number); + PutFixed64(output, origin.first_original_offset); + PutFixed64(output, origin.last_original_offset); + PutFixed64(output, origin.entry_count); + PutFixed64(output, origin.record_bytes); + } + for (uint64_t file_number : served_file_numbers_) { + PutFixed64(output, file_number); + } + const uint32_t digest = ComputeDirectoryDigest( + output->data(), output->data() + kDirectoryHeaderSize, + output->size() - kDirectoryHeaderSize); + EncodeFixed32(output->data() + kDirectoryDigestOffset, digest); + EncodeFixed32(output->data() + kHeaderChecksumOffset, + crc32c::Value(output->data(), kHeaderChecksumOffset)); +} + +Status BlobRelocationDirectory::DecodeFrom(const Slice& input, + BlobRelocationDirectory* directory) { + if (directory == nullptr) { + return Status::InvalidArgument("Null blob relocation directory output"); + } + if (input.size() < kDirectoryHeaderSize) { + return Status::Corruption("Invalid blob relocation directory size"); + } + + Slice cursor = input; + uint32_t magic = 0; + uint32_t version = 0; + uint64_t origin_count = 0; + uint64_t served_count = 0; + uint32_t directory_digest = 0; + uint32_t header_checksum = 0; + BlobRelocationDirectory decoded; + if (!GetFixed32(&cursor, &magic) || !GetFixed32(&cursor, &version) || + !GetFixed64(&cursor, &origin_count) || + !GetFixed64(&cursor, &served_count) || + !GetFixed64(&cursor, &decoded.total_entries_) || + !GetFixed64(&cursor, &decoded.total_record_bytes_) || + !GetFixed32(&cursor, &directory_digest) || + !GetFixed32(&cursor, &header_checksum)) { + return Status::Corruption("Truncated blob relocation directory"); + } + if (magic != kRelocationDirectoryMagic || version != kFormatVersion) { + return Status::Corruption("Unsupported blob relocation directory"); + } + // The size must be exactly what the counts say; this bounds both counts + // by the input size, so no fixed limit is needed. + if (origin_count == 0 || served_count == 0 || + origin_count > cursor.size() / kOriginEntrySize || + served_count > cursor.size() / kServedFileSize || + cursor.size() != + origin_count * kOriginEntrySize + served_count * kServedFileSize) { + return Status::Corruption("Invalid blob relocation directory counts"); + } + if (crc32c::Value(input.data(), kHeaderChecksumOffset) != header_checksum) { + return Status::Corruption( + "Blob relocation directory header checksum mismatch"); + } + if (ComputeDirectoryDigest(input.data(), cursor.data(), cursor.size()) != + directory_digest) { + return Status::Corruption("Blob relocation directory checksum mismatch"); + } + + decoded.origins_.reserve(static_cast(origin_count)); + uint64_t computed_entries = 0; + uint64_t computed_record_bytes = 0; + for (uint64_t i = 0; i < origin_count; ++i) { + BlobRelocationOrigin origin; + if (!GetFixed64(&cursor, &origin.origin_file_number) || + !GetFixed64(&cursor, &origin.first_original_offset) || + !GetFixed64(&cursor, &origin.last_original_offset) || + !GetFixed64(&cursor, &origin.entry_count) || + !GetFixed64(&cursor, &origin.record_bytes)) { + return Status::Corruption("Truncated blob relocation origin entry"); + } + if (origin.origin_file_number == 0 || origin.entry_count == 0 || + origin.first_original_offset > origin.last_original_offset || + origin.entry_count - 1 > + origin.last_original_offset - origin.first_original_offset || + origin.entry_count > origin.record_bytes / kSimpleGen2BlobTrailerSize || + (!decoded.origins_.empty() && + decoded.origins_.back().origin_file_number >= + origin.origin_file_number) || + !AddRelocationBytesWithoutOverflow(computed_entries, origin.entry_count, + &computed_entries) || + !AddRelocationBytesWithoutOverflow(computed_record_bytes, + origin.record_bytes, + &computed_record_bytes)) { + return Status::Corruption("Invalid blob relocation origin entry"); + } + decoded.origins_.push_back(origin); + } + decoded.served_file_numbers_.reserve(static_cast(served_count)); + for (uint64_t i = 0; i < served_count; ++i) { + uint64_t file_number = 0; + if (!GetFixed64(&cursor, &file_number) || file_number == 0 || + (!decoded.served_file_numbers_.empty() && + decoded.served_file_numbers_.back() >= file_number)) { + return Status::Corruption("Invalid blob relocation served file"); + } + decoded.served_file_numbers_.push_back(file_number); + } + if (!cursor.empty() || computed_entries != decoded.total_entries_ || + computed_record_bytes != decoded.total_record_bytes_) { + return Status::Corruption("Blob relocation directory totals mismatch"); + } + decoded.directory_digest_ = directory_digest; + decoded.encoded_size_ = input.size(); + *directory = std::move(decoded); + return Status::OK(); +} + +Status BlobRelocationDirectory::ValidateForFileSize(uint64_t file_size) const { + if (file_size < kBlobRelocationFilePrefixSize || + total_record_bytes_ > file_size - kBlobRelocationFilePrefixSize) { + return Status::Corruption( + "Blob relocation record bytes exceed physical file size"); + } + if (encoded_size_ > file_size) { + return Status::Corruption( + "Blob relocation directory exceeds physical file size"); + } + return Status::OK(); +} + +const BlobRelocationOrigin* BlobRelocationDirectory::Find( + uint64_t origin_file_number) const { + const auto it = + std::lower_bound(origins_.begin(), origins_.end(), origin_file_number, + [](const BlobRelocationOrigin& origin, uint64_t value) { + return origin.origin_file_number < value; + }); + return it != origins_.end() && it->origin_file_number == origin_file_number + ? &*it + : nullptr; +} + +Status BlobRelocationDirectoryBuilder::Add(uint64_t origin_file_number, + uint64_t original_value_offset, + uint64_t stored_value_size, + CompressionType compression) { + if (finished_) { + return Status::InvalidArgument( + "Cannot add to finished blob relocation directory"); + } + if (origin_file_number == 0 || !CompressionTypeSupported(compression) || + stored_value_size > + std::numeric_limits::max() - kSimpleGen2BlobTrailerSize) { + return Status::InvalidArgument("Invalid blob relocation record"); + } + if (has_current_ && origin_file_number < current_.origin_file_number) { + return Status::InvalidArgument("Blob relocation origins are not sorted"); + } + if (has_current_ && origin_file_number == current_.origin_file_number && + original_value_offset <= current_.last_original_offset) { + return Status::InvalidArgument("Blob relocation offsets are not unique"); + } + if (has_current_ && origin_file_number == current_.origin_file_number && + original_value_offset < current_value_end_) { + return Status::InvalidArgument("Blob relocation values overlap"); + } + if (stored_value_size > + std::numeric_limits::max() - original_value_offset) { + return Status::InvalidArgument("Blob relocation value end overflows"); + } + const uint64_t value_end = original_value_offset + stored_value_size; + const uint64_t record_bytes = stored_value_size + kSimpleGen2BlobTrailerSize; + if (!has_current_ || origin_file_number != current_.origin_file_number) { + Status status = FinishCurrentOrigin(); + if (!status.ok()) { + return status; + } + BlobRelocationOrigin next; + next.origin_file_number = origin_file_number; + next.first_original_offset = original_value_offset; + next.last_original_offset = original_value_offset; + next.entry_count = 1; + next.record_bytes = record_bytes; + current_ = next; + current_value_end_ = value_end; + has_current_ = true; + return Status::OK(); + } + + BlobRelocationOrigin next = current_; + if (!AddRelocationBytesWithoutOverflow(current_.entry_count, 1, + &next.entry_count) || + !AddRelocationBytesWithoutOverflow(current_.record_bytes, record_bytes, + &next.record_bytes)) { + return Status::InvalidArgument("Blob relocation directory overflow"); + } + next.last_original_offset = original_value_offset; + current_ = next; + current_value_end_ = value_end; + return Status::OK(); +} + +Status BlobRelocationDirectoryBuilder::AddServedFile(uint64_t file_number) { + if (finished_) { + return Status::InvalidArgument( + "Cannot add to finished blob relocation directory"); + } + if (file_number == 0) { + return Status::InvalidArgument("Invalid blob relocation served file"); + } + if (!served_file_numbers_.empty() && + served_file_numbers_.back() == file_number) { + return Status::OK(); + } + served_file_numbers_.push_back(file_number); + return Status::OK(); +} + +Status BlobRelocationDirectoryBuilder::FinishCurrentOrigin() { + if (!has_current_) { + return Status::OK(); + } + uint64_t total_entries = 0; + uint64_t total_record_bytes = 0; + if (!AddRelocationBytesWithoutOverflow( + directory_.total_entries_, current_.entry_count, &total_entries) || + !AddRelocationBytesWithoutOverflow(directory_.total_record_bytes_, + current_.record_bytes, + &total_record_bytes)) { + return Status::InvalidArgument("Blob relocation directory overflow"); + } + directory_.origins_.push_back(current_); + directory_.total_entries_ = total_entries; + directory_.total_record_bytes_ = total_record_bytes; + has_current_ = false; + return Status::OK(); +} + +Status BlobRelocationDirectoryBuilder::Finish( + BlobRelocationDirectory* directory, std::string* encoded) { + if (directory == nullptr) { + return Status::InvalidArgument("Null blob relocation directory output"); + } + if (finished_) { + return Status::InvalidArgument( + "Blob relocation directory already finished"); + } + Status status = FinishCurrentOrigin(); + if (!status.ok()) { + return status; + } + if (directory_.origins_.empty()) { + return Status::InvalidArgument("Empty blob relocation directory"); + } + std::sort(served_file_numbers_.begin(), served_file_numbers_.end()); + served_file_numbers_.erase( + std::unique(served_file_numbers_.begin(), served_file_numbers_.end()), + served_file_numbers_.end()); + if (served_file_numbers_.empty()) { + return Status::InvalidArgument("Blob relocation file serves no file"); + } + directory_.served_file_numbers_ = served_file_numbers_; + std::string encoded_directory; + directory_.EncodeTo(&encoded_directory); + status = BlobRelocationDirectory::DecodeFrom(encoded_directory, directory); + if (status.ok()) { + finished_ = true; + if (encoded != nullptr) { + *encoded = std::move(encoded_directory); + } + } + return status; +} + +void EncodeBlobRelocationKeyTo(uint64_t origin_file_number, + uint64_t original_value_offset, char* out) { + EncodeFixed64(out, EndianSwapValue(origin_file_number)); + EncodeFixed64(out + sizeof(uint64_t), EndianSwapValue(original_value_offset)); +} + +std::string EncodeBlobRelocationKey(uint64_t origin_file_number, + uint64_t original_value_offset) { + std::string key(kBlobRelocationKeySize, '\0'); + EncodeBlobRelocationKeyTo(origin_file_number, original_value_offset, + key.data()); + return key; +} + +Status DecodeBlobRelocationKey(const Slice& key, uint64_t* origin_file_number, + uint64_t* original_value_offset) { + if (origin_file_number == nullptr || original_value_offset == nullptr) { + return Status::InvalidArgument("Null blob relocation key output"); + } + if (key.size() != + sizeof(*origin_file_number) + sizeof(*original_value_offset)) { + return Status::Corruption("Invalid blob relocation key size"); + } + *origin_file_number = EndianSwapValue(DecodeFixed64(key.data())); + *original_value_offset = + EndianSwapValue(DecodeFixed64(key.data() + sizeof(*origin_file_number))); + if (*origin_file_number == 0) { + return Status::Corruption("Invalid blob relocation origin"); + } + return Status::OK(); +} + +BlockBasedTableOptions MakeBlobRelocationTableOptions( + const BlockBasedTableOptions& source) { + BlockBasedTableOptions options(source); + // format_version 6 adds the per-file context to block and record checksums, + // so a record copied to another offset fails its check. Relocation files are + // a new file kind, so this has no compatibility cost. + options.format_version = std::max(options.format_version, 6); + // Relocation files use kCRC32c for block and record checksums, whatever + // checksum type the table options set, unless they set kNoChecksum. A record + // trailer can then reuse the stored CRC32C of the value, so streaming a value + // needs no second checksum pass over its payload. Normal SSTs and blob files + // keep their configured checksum type. + if (options.checksum != kNoChecksum) { + options.checksum = kCRC32c; + } + options.index_type = BlockBasedTableOptions::kBinarySearch; + // A lookup key usually lies between two location entry keys, so a data + // block hash index (exact keys only) cannot help, and the index key of a + // data block must be its last key (see "Table options" in the header). The + // CF's restart interval is kept. + options.data_block_index_type = + BlockBasedTableOptions::kDataBlockBinarySearch; + options.index_shortening = + BlockBasedTableOptions::IndexShorteningMode::kNoShortening; + // 4 KB is an estimate, not a measured best value. A small block makes each + // lookup read fewer bytes (the records are outside the data blocks), but it + // makes the index larger. + options.block_size = 4 * 1024; + // Alignment padding only wastes space between records and blocks here. + options.block_align = false; + options.super_block_alignment_size = 0; + options.flush_block_policy_factory.reset(); + options.filter_policy.reset(); + options.partition_filters = false; + options.user_defined_index_factory.reset(); + options.index_mode = BlockBasedTableOptions::IndexMode::kStandardOnly; + options.use_udi_as_primary_index = false; + options.fail_if_no_udi_on_open = false; + // Index caching and pinning follow the CF's table options + // (cache_index_and_filter_blocks, metadata_cache_options): a relocation + // file is opened without a level, so it pins like a non-L0 SST. + return options; +} + +void EncodeBlobRelocationGroup(uint64_t first_record_offset, + const BlobRelocationGroupValue* values, + size_t count, std::string* output) { + assert(values != nullptr); + assert(count >= 1 && count <= kBlobRelocationGroupSize); + assert(output != nullptr); + output->clear(); + PutVarint32(output, static_cast(count)); + PutVarint64(output, first_record_offset); + // The end of the value before (0 before the first value). + uint64_t previous_end = 0; + for (size_t i = 0; i < count; ++i) { + assert(values[i].original_value_offset >= previous_end); + assert(values[i].stored_value_size >= 1 && + values[i].stored_value_size <= kMaxSimpleGen2BlobPayloadSize); + PutVarint64(output, values[i].original_value_offset - previous_end); + PutVarint64(output, values[i].stored_value_size); + previous_end = + values[i].original_value_offset + values[i].stored_value_size; + } +} + +const BlobRelocationGroup::Value* BlobRelocationGroup::Find( + uint64_t original_value_offset) const { + const Value* const end = values.data() + count; + const Value* const found = + std::lower_bound(values.data(), end, original_value_offset, + [](const Value& value, uint64_t offset) { + return value.original_value_offset < offset; + }); + return found != end && found->original_value_offset == original_value_offset + ? found + : nullptr; +} + +Status DecodeBlobRelocationGroup(const Slice& value, + uint64_t last_original_value_offset, + BlobRelocationGroup* group) { + assert(group != nullptr); + Slice input = value; + uint32_t count = 0; + if (!GetVarint32(&input, &count) || count < 1 || + count > kBlobRelocationGroupSize) { + return Status::Corruption("Invalid blob relocation location entry count"); + } + BlobRelocationGroup decoded; + uint64_t record_offset = 0; + if (!GetVarint64(&input, &record_offset)) { + return Status::Corruption("Invalid blob relocation location entry"); + } + decoded.count = count; + // The end of the value before (0 before the first value), and the original + // offset of the current value. + uint64_t previous_end = 0; + uint64_t offset = 0; + for (uint32_t i = 0; i < count; ++i) { + uint64_t gap = 0; + uint64_t size = 0; + if (!GetVarint64(&input, &gap) || !GetVarint64(&input, &size)) { + return Status::Corruption("Invalid blob relocation location entry"); + } + if (gap > std::numeric_limits::max() - previous_end) { + return Status::Corruption( + "Invalid blob relocation location entry offset gap"); + } + if (size == 0 || size > kMaxSimpleGen2BlobPayloadSize) { + return Status::Corruption( + "Invalid blob relocation location entry stored size"); + } + offset = previous_end + gap; + if (size > std::numeric_limits::max() - offset) { + return Status::Corruption( + "Invalid blob relocation location entry offset gap"); + } + previous_end = offset + size; + BlobRelocationGroup::Value& entry = decoded.values[i]; + entry.original_value_offset = offset; + entry.record_offset = record_offset; + entry.stored_value_size = size; + const uint64_t record_bytes = size + kSimpleGen2BlobTrailerSize; + if (record_offset > std::numeric_limits::max() - record_bytes) { + return Status::Corruption( + "Invalid blob relocation location entry record offset"); + } + record_offset += record_bytes; + } + if (!input.empty()) { + return Status::Corruption("Blob relocation location entry has extra bytes"); + } + if (offset != last_original_value_offset) { + return Status::Corruption( + "Blob relocation location entry key does not match its last value"); + } + *group = decoded; + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_relocation_file.h b/db/blob/blob_relocation_file.h new file mode 100644 index 000000000000..d3bef7f3a099 --- /dev/null +++ b/db/blob/blob_relocation_file.h @@ -0,0 +1,404 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "rocksdb/compression_type.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "rocksdb/table.h" + +// Blob relocation file +// ==================== +// +// Background: indirect blob GC removes garbage from a blob file without +// rewriting any SST. It copies the live values of mostly-garbage blob files +// into one new file, the relocation file. A route (old file number -> +// relocation file) then sends reads of the old file number to the relocation +// file. The old file number is the generation number that the SST's BlobIndex +// stores. The reader finds the value in the relocation file by its stable +// identity (origin, original offset), which never changes when the value moves. +// This file defines the relocation file format, its directory, and its key +// encoding. +// +// Terms: generation number ("Generation" in db/blob/blob_index.h), origin, +// original offset, stable identity, logical CRC and stored CRC are defined in +// db/blob/blob_index.h. Its Generation paragraph also explains routes. This +// file adds: +// origin partition All values in this file that share one origin file. +// Each has one directory entry (BlobRelocationOrigin). +// record One stored value in the SimpleGen2Blob format +// (db/blob/blob_gen2_format.h). Normal SSTs use the +// same format for embedded (same-file) blobs, so +// "embedded blob", "same-file blob" and "SimpleGen2 +// record" name the same record format. +// record bytes Stored value size + kSimpleGen2BlobTrailerSize +// (5 bytes: 1 compression byte and a 4-byte checksum). +// +// Layout +// ------ +// A relocation file is a block-based table with an 8-byte prefix in front. +// The table builder writes each value itself, as a record in the file, before +// the data block that holds the value's location entry. +// +// offset 0 +// +---------------------------+ +// | "RDBRLC01" | prefix (kBlobRelocationFilePrefix) +// +---------------------------+ +// | record | each record: stored bytes | +// | | compression (1) | checksum (4) +// | record | +// | ... | +// | data block | location entries of the records above +// | record ... | +// | data block | +// +---------------------------+ <- tail start +// | index, properties, | properties hold the directory +// | metaindex, footer | +// +---------------------------+ +// +// - Offsets: every block handle and record offset is an absolute file offset. +// The prefix only moves the first record to offset 8. +// - Prefix: relocation files use the normal blob file name (.blob). +// The prefix tells them apart from blob log files, which start with a +// BlobLogHeader, for a tool that scans the DB directory without the +// MANIFEST. A reader that is told the file is a relocation file does not +// read the prefix. It checks the table footer, file number, tail size and +// directory digest (see "Directory" below) instead +// (BlobRelocationFileReader::Open). +// - Record: the stored bytes are copied verbatim (never recompressed). The +// compression byte (the record's compression marker) tells how the stored +// bytes are compressed. The checksum uses the table's checksum type. It +// covers the stored bytes and the compression byte, and it is modified by +// the record's file offset, so a record copied to another offset fails the +// check. +// - Location entry: one key-value pair in a data block. It locates a group: up +// to kBlobRelocationGroupSize (32) records of one origin, in original offset +// order. The table builder starts a new group when the origin changes or the +// group is full. A group shares one key and one entry header among up to +// 32 values, so it takes far fewer bytes per value than one entry per value. +// - Key: 16 bytes (kBlobRelocationKeySize): the big-endian origin file number, +// then the big-endian original offset of the LAST value in the group. +// Big-endian makes bytewise order equal to numeric (origin, offset) order. +// So the first entry at or after (origin, offset) is the only group that +// can hold that value. The internal key has sequence 0 and type kTypeValue: +// the value is a location entry, not a BlobIndex, so code that decodes +// BlobIndex values (for example a plain table read) does not try to decode +// it. The comparator is always bytewise, without user timestamps. +// - Value (EncodeBlobRelocationGroup()), all numbers as varints: +// count (varint32, 1..32) | first record offset (varint64) | +// count x { offset gap (varint64) | stored size (varint64) } +// The offset gap of a value is the number of bytes between the end of the +// value before it and its own start: original offset - (previous original +// offset + previous stored size). For the first value of an entry, the end +// before it is 0, so its gap is its original offset. For the other values, +// a record header and the user key lie between two values in a normal blob +// file, so a gap is small and takes one or two bytes. Values of one origin +// never overlap, so a gap is never negative, and it may be 0. +// The records of one group are written back to back, so the record offset +// of a value is the first record offset plus the record bytes of the values +// before it. Each stored size is 1..kMaxSimpleGen2BlobPayloadSize (an empty +// value is never written to a blob file). The entry has no compression +// type: a reader expects the compression of the indirect BlobIndex, and the +// record read checks it against the record's compression marker. +// - Table options (MakeBlobRelocationTableOptions): binary search index, no +// filter, no UDI. Every lookup comes from a live BlobIndex, so the value +// always exists and a filter cannot help. A lookup key usually lies between +// two entry keys, so data blocks have no hash index (it finds only exact +// keys), and the index uses no key shortening (kNoShortening): the index key +// of a data block is its last key, so the block that the index finds always +// holds the first entry at or after the lookup key. The batched lookup +// (BlockBasedTable::MultiGetRawEntries) does not step to the next block. The +// block size is 4 KB. This is an estimate, not a measured best value. The +// column family's restart interval is kept. Block alignment is off, because +// padding only wastes space. The table builder also turns off compression +// sampling (BlockBasedTableBuilder::StartBlobRelocationFile()), because +// sampling reports stats for user data and these data blocks hold only +// location entries. The data blocks that hold location entries (location +// blocks) use the column family's SST compression (see +// NewBlobRelocationTableBuilder()). +// +// Directory +// --------- +// The directory is stored in the table property +// kBlobRelocationDirectoryPropertyName. It starts with a 48-byte header: +// magic (4) | format version (4) | origin count (8) | served file count (8) | +// total entries (8) | total record bytes (8) | directory digest (4) | +// header CRC (4) +// The header CRC covers the first 44 bytes. The decoder first checks that the +// counts match the directory size, then checks the header CRC, and only then +// reads any entry. So a damaged header fails before any entry is read. The +// digest covers the counts, the totals and the body, but not the magic and the +// version, so it identifies the file contents only. After the header, the +// directory has one entry per origin partition, sorted by origin: +// origin file number | first and last original offset | entry count | +// record bytes +// The entries are followed by the served files. These are the old file +// numbers that may be routed to this file. The writer adds them with +// AddBlobRelocationServedFile(). The list lets the file alone tell which old +// file numbers it serves. The table builder computes the entries from the +// records it writes. +// +// The table property num_entries is the number of groups. With the group rule +// above it equals, for each origin partition, its entry count divided by 32 +// and rounded up, summed over the partitions; BlobRelocationFileReader::Open() +// checks this. +// +// The directory digest (directory_digest()) is the CRC32C of the counts, the +// totals and the directory body. It is the file's identity: the code that +// installs the file keeps the digest, and BlobRelocationFileReader::Open() +// rejects a file whose directory digest does not match the expected one. +// +// Reads +// ----- +// A read seeks the key (origin, original offset). It checks that the entry +// found has the same origin, decodes its group, finds the exact original +// offset in it (Corruption if it is not there), and reads the record. The +// table below says which checksums a read checks. The record checksum is +// checked by BlockBasedTable::MultiGetSameFileBlob(). The logical CRC is +// checked by BlobSource::FinishIndirectBlobRead(), with the same rule for +// indirect reads of normal blob files. As in base RocksDB, a range of a +// compressed value is not supported: a read of one returns Corruption. +// +// Read kind Record checksum Logical CRC +// ------------------------ -------------------- ------------------------- +// whole value when verify_checksums when verify_checksums is +// is set (the verify set or the value goes into +// policy) the stable blob cache +// range of an uncompressed never: only the never +// value (callers send only range bytes are +// strict parts) read, not the record +// trailer +// raw read (GC copies the stored CRC, and the never (the value is not +// stored bytes) record checksum decompressed) +// unless the file uses +// kNoChecksum +// +// Relocation files always use kCRC32c or kNoChecksum (see +// MakeBlobRelocationTableOptions()). + +namespace ROCKSDB_NAMESPACE { + +// For how the MANIFEST adds relocation files and routes reads to them, see the +// overview in db/blob/blob_relocation_manifest.h. + +// Table property that holds the encoded BlobRelocationDirectory. +inline constexpr char kBlobRelocationDirectoryPropertyName[] = + "rocksdb.blob.relocation-directory"; + +// The first 8 bytes of every relocation file. They tell a relocation file +// apart from a blob log file with the same .blob name. Block handles +// and record offsets stay absolute file offsets. The prefix only moves the +// first record to offset 8. +inline constexpr char kBlobRelocationFilePrefix[] = "RDBRLC01"; +inline constexpr size_t kBlobRelocationFilePrefixSize = + sizeof(kBlobRelocationFilePrefix) - 1; + +// Directory entry of one origin partition. +struct BlobRelocationOrigin { + // Blob file where the values were first written. + uint64_t origin_file_number = 0; + // Smallest and largest original offset of the partition's values. + uint64_t first_original_offset = 0; + uint64_t last_original_offset = 0; + // Number of values (records) in the partition. Each location entry holds up + // to kBlobRelocationGroupSize (32) values, so the partition has entry_count + // / 32 location entries, rounded up. For example, 70 values use 3 location + // entries. + uint64_t entry_count = 0; + // Sum of stored value size + kSimpleGen2BlobTrailerSize over the entries. + uint64_t record_bytes = 0; +}; + +// The directory of a relocation file (see "Directory" at the top of this +// file). It holds one checked entry per origin partition, sorted by origin, +// the list of served files, the totals and the digest that identifies the +// file. The table builder and BlobRelocationFileReader::Open check that every +// origin and served file is older (has a smaller number) than the relocation +// file itself. The directory is immutable after DecodeFrom() or +// BlobRelocationDirectoryBuilder::Finish(), so several threads can read it. +class BlobRelocationDirectory { + public: + // Version of the directory encoding. Readers reject any other version. + static constexpr uint32_t kFormatVersion = 2; + + // Encodes the directory (header, entries, served files, digest, header CRC) + // into `*output`, replacing its contents. + void EncodeTo(std::string* output) const; + + // Decodes and checks an encoded directory. Returns Corruption on any + // failure. `*directory` is changed only on success. It checks, in order: + // - the magic and the version; + // - at least one entry and one served file, and a size of exactly the + // header plus the stated entries and served files (this bounds the + // counts, so there is no fixed count limit); + // - the header CRC and the digest; + // - each entry and the entry order; + // - served files in strictly increasing order; + // - the totals. + static Status DecodeFrom(const Slice& input, + BlobRelocationDirectory* directory); + + // Rejects a directory that cannot fit in the physical relocation file: its + // encoded size must fit in `file_size`, and its record bytes must fit in + // `file_size` minus the prefix. This is a loose bound: a file that passes + // may still be too small. + Status ValidateForFileSize(uint64_t file_size) const; + + // Entry of `origin_file_number`, or nullptr. Binary search. + const BlobRelocationOrigin* Find(uint64_t origin_file_number) const; + + const std::vector& origins() const { return origins_; } + + // The old file numbers that may be routed to this file, increasing. + const std::vector& served_file_numbers() const { + return served_file_numbers_; + } + uint64_t total_entries() const { return total_entries_; } + uint64_t total_record_bytes() const { return total_record_bytes_; } + // The directory digest (see "Directory" at the top of this file). + uint32_t directory_digest() const { return directory_digest_; } + // Size of the encoding. + uint64_t encoded_size() const { return encoded_size_; } + + private: + friend class BlobRelocationDirectoryBuilder; + + std::vector origins_; + std::vector served_file_numbers_; + uint64_t total_entries_ = 0; + uint64_t total_record_bytes_ = 0; + uint32_t directory_digest_ = 0; + // Size of the encoding this directory was decoded from or encodes to. + uint64_t encoded_size_ = 0; +}; + +// Builds the directory while the table builder appends records in key order +// (origin, original offset). It keeps one entry per origin partition, not one +// object per value. It is not thread-safe. One table builder uses it. +class BlobRelocationDirectoryBuilder { + public: + // Records that this file serves the old file number `file_number`, so a + // route from `file_number` to this file may exist. Call it at any time + // before Finish(). Duplicate numbers are allowed: a repeat of the last one is + // skipped, and Finish() removes the other duplicates. Returns InvalidArgument + // for file number 0 or after Finish(). + Status AddServedFile(uint64_t file_number); + + // Adds one record. Calls must be in strictly increasing (origin, original + // offset) order, and a value must not start before the end (original offset + // + stored size) of the value before it of the same origin; otherwise + // InvalidArgument. + Status Add(uint64_t origin_file_number, uint64_t original_value_offset, + uint64_t stored_value_size, CompressionType compression); + // Produces the checked directory and, when `encoded` is non-null, its + // encoding. InvalidArgument when there is no record or no served file. Can + // be called once. + Status Finish(BlobRelocationDirectory* directory, + std::string* encoded = nullptr); + + private: + // Appends the origin partition being built to the directory. + Status FinishCurrentOrigin(); + + BlobRelocationDirectory directory_; + // Origin partition being built; valid when has_current_. + BlobRelocationOrigin current_; + // The end (original offset + stored size) of the last value of current_. + uint64_t current_value_end_ = 0; + // From AddServedFile; sorted and deduplicated in Finish(). + std::vector served_file_numbers_; + bool has_current_ = false; + bool finished_ = false; +}; + +// Metadata of a finished relocation file: its encoded directory (the same +// bytes as the table property) and the size of its tail (index, properties, +// metaindex, footer). The caller that installs the file takes the totals and +// the digest from the directory. +struct BlobRelocationFileMetadata { + std::string encoded_directory; + uint64_t tail_size = 0; +}; + +// Encodes the relocation key (see "Key" at the top of this file). Both parts +// are fixed-width and big-endian, so the bytewise comparator orders keys in +// numeric (origin, original offset) order. +inline constexpr size_t kBlobRelocationKeySize = 16; +std::string EncodeBlobRelocationKey(uint64_t origin_file_number, + uint64_t original_value_offset); +// Writes the same 16-byte key into `out` (kBlobRelocationKeySize bytes), +// without a heap allocation. +void EncodeBlobRelocationKeyTo(uint64_t origin_file_number, + uint64_t original_value_offset, char* out); +// Decodes a relocation key. Returns Corruption for a key that is not 16 bytes +// or has origin 0. +Status DecodeBlobRelocationKey(const Slice& key, uint64_t* origin_file_number, + uint64_t* original_value_offset); + +// Returns the table options of a relocation file. It starts from the column +// family's options, so the block cache and the restart interval are kept. It +// removes the features that serve user keys (filter, UDI) and the data block +// hash index. It sets the binary search index, turns off index key shortening +// and sets the 4 KB block size. See "Table options" at the top of this file +// for the reasons. It forces format_version >= 6, so record checksums depend +// on the file offset. +BlockBasedTableOptions MakeBlobRelocationTableOptions( + const BlockBasedTableOptions& source); + +// Most values in one location entry (see "Location entry" at the top of this +// file). 32 is a design choice. Larger groups would save only a little more +// space, because the shared key and header are already small per value. But a +// lookup decodes every value of its entry, so a cached lookup gets slower. +inline constexpr size_t kBlobRelocationGroupSize = 32; + +// One value of a location entry. +struct BlobRelocationGroupValue { + uint64_t original_value_offset = 0; + uint64_t stored_value_size = 0; +}; + +// Encodes the value of a location entry (see "Value" at the top of this file) +// into `*output`, replacing its contents. `values` must hold 1..32 values in +// original offset order that do not overlap (each value starts at or after +// the end of the value before it), each with a stored size of +// 1..kMaxSimpleGen2BlobPayloadSize. +void EncodeBlobRelocationGroup(uint64_t first_record_offset, + const BlobRelocationGroupValue* values, + size_t count, std::string* output); + +// A decoded location entry. Readers decode an entry once and look up several +// values in it. +struct BlobRelocationGroup { + struct Value { + uint64_t original_value_offset = 0; + uint64_t record_offset = 0; + uint64_t stored_value_size = 0; + }; + size_t count = 0; + std::array values; + + // The value at `original_value_offset`, or nullptr. Binary search. + const Value* Find(uint64_t original_value_offset) const; +}; + +// Decodes and checks the value of a location entry whose key holds +// `last_original_value_offset`. Returns Corruption when the value ends early, +// the count is not 1..32, a stored size is 0 or above +// kMaxSimpleGen2BlobPayloadSize, an offset or a value end overflows, bytes are +// left over, or the last original offset is not `last_original_value_offset`. +// `*group` is changed only on success. +Status DecodeBlobRelocationGroup(const Slice& value, + uint64_t last_original_value_offset, + BlobRelocationGroup* group); + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_relocation_gc.cc b/db/blob/blob_relocation_gc.cc new file mode 100644 index 000000000000..f7f84478d330 --- /dev/null +++ b/db/blob/blob_relocation_gc.cc @@ -0,0 +1,303 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_relocation_gc.h" + +#include +#include +#include +#include +#include +#include + +#include "cache/cache_reservation_manager.h" + +namespace ROCKSDB_NAMESPACE { + +namespace { + +// The record overhead is not compared: one value can be referenced unflattened +// (a record header and key in its origin) and flattened (a trailer in a +// relocation file) at the same time. +auto ImmutableFields(const BlobRelocationLiveValue& value) { + return std::tie(value.stored_value_size, value.compression, + value.stable_checksum, value.stored_checksum); +} + +} // namespace + +BlobRelocationMemoryTracker::BlobRelocationMemoryTracker( + std::shared_ptr cache_reservation_manager) + : cache_reservation_manager_(std::move(cache_reservation_manager)) {} + +Status BlobRelocationMemoryTracker::SetReservation(size_t* current_bytes, + size_t requested_bytes) { + if (current_bytes == nullptr || *current_bytes > reserved_bytes_) { + return Status::Corruption("Invalid blob relocation memory reservation"); + } + const size_t retained = reserved_bytes_ - *current_bytes; + if (requested_bytes > kMaxBytes - retained) { + return Status::MemoryLimit("Blob relocation live sets exceed limit"); + } + const size_t target = retained + requested_bytes; + if (cache_reservation_manager_ != nullptr) { + Status status = cache_reservation_manager_->UpdateCacheReservation(target); + if (!status.ok()) { + return status; + } + } + reserved_bytes_ = target; + *current_bytes = requested_bytes; + return Status::OK(); +} + +BlobRelocationLiveSet::BlobRelocationLiveSet( + std::shared_ptr cache_reservation_manager, + std::vector generation_numbers) + : BlobRelocationLiveSet(std::make_shared( + std::move(cache_reservation_manager)), + std::move(generation_numbers)) {} + +BlobRelocationLiveSet::BlobRelocationLiveSet( + std::shared_ptr memory_tracker, + std::vector generation_numbers) + : memory_tracker_(std::move(memory_tracker)), + generation_numbers_(std::move(generation_numbers)) { + assert(memory_tracker_ != nullptr); +} + +BlobRelocationLiveSet::~BlobRelocationLiveSet() { + ReleaseWorkingMemory().PermitUncheckedError(); +} + +Status BlobRelocationLiveSet::Reserve(size_t additional) { + if (finalized_) { + return Status::InvalidArgument("Blob relocation live set is finalized"); + } + if (generation_numbers_.empty() || generation_numbers_.front() == 0 || + !std::is_sorted(generation_numbers_.begin(), generation_numbers_.end()) || + std::adjacent_find(generation_numbers_.begin(), + generation_numbers_.end()) != + generation_numbers_.end()) { + return Status::InvalidArgument( + "Invalid blob relocation input generation numbers"); + } + if (additional == 0) { + return Status::OK(); + } + if (additional > values_.max_size() - values_.size()) { + return Status::MemoryLimit("Blob relocation live set is too large"); + } + const size_t target = values_.size() + additional; + if (target <= values_.capacity()) { + return Status::OK(); + } + // Reserve the exact size, not a doubled capacity: the caller reserves the + // full census total in one call, so doubling would only double the charge. + const size_t capacity = target; + if (capacity > + std::numeric_limits::max() / sizeof(BlobRelocationLiveValue)) { + return Status::MemoryLimit("Blob relocation live set overflow"); + } + Status status = memory_tracker_->SetReservation( + &charged_memory_, capacity * sizeof(BlobRelocationLiveValue)); + if (!status.ok()) { + return status; + } + values_.reserve(capacity); + return UpdateReservation(); +} + +Status BlobRelocationLiveSet::Add( + const std::vector& references) { + Status status = Reserve(references.size()); + for (size_t i = 0; status.ok() && i < references.size(); ++i) { + status = Append(references[i]); + } + return status; +} + +Status BlobRelocationLiveSet::Append(const BlobReference& reference) { + if (finalized_ || values_.size() == values_.capacity()) { + return Status::Corruption("Blob relocation live set has no room"); + } + // The generation number of an unflattened reference is its origin. The + // generation number of a flattened one is the relocation file it was + // flattened to. + const bool unflattened = reference.encoding_class == + BlobReferenceEncodingClass::kIndirectUnflattened; + if (!reference.HasStableIdentity() || reference.origin_file_number == 0 || + unflattened != + (reference.generation_file_number == reference.origin_file_number) || + !std::binary_search(generation_numbers_.begin(), + generation_numbers_.end(), + reference.generation_file_number) || + reference.record_bytes < reference.stored_value_size || + reference.stored_value_size > std::numeric_limits::max() || + reference.record_bytes - reference.stored_value_size > + std::numeric_limits::max()) { + return Status::Corruption("Unexpected blob relocation reference"); + } + BlobRelocationLiveValue value; + value.origin_file_number = reference.origin_file_number; + value.original_value_offset = reference.original_value_offset; + value.stored_value_size = static_cast(reference.stored_value_size); + value.record_overhead = static_cast(reference.record_bytes - + reference.stored_value_size); + value.stable_checksum = reference.stable_checksum; + value.stored_checksum = reference.stored_checksum; + value.compression = reference.compression; + values_.push_back(value); + return Status::OK(); +} + +Status BlobRelocationLiveSet::Finalize() { + if (finalized_) { + return Status::InvalidArgument("Blob relocation live set is finalized"); + } + finalized_ = true; + const auto sort_key = [](const BlobRelocationLiveValue& value) { + return std::make_tuple(value.origin_file_number, + value.original_value_offset, value.stored_value_size, + value.record_overhead, value.compression, + value.stable_checksum, value.stored_checksum); + }; + // Copies of one value can differ only in record_overhead (see Finalize() in + // the header for why the smallest one is correct). The sort puts the smallest + // record_overhead first, and the loop below keeps it. In a normal input every + // copy is unflattened, with the same record_overhead. + // + // The choice of record_overhead does not change the indirect blob GC + // decision: for a relocation input, the GC job computes the live record bytes + // again as stored bytes plus one trailer per value. + std::sort(values_.begin(), values_.end(), + [&](const BlobRelocationLiveValue& lhs, + const BlobRelocationLiveValue& rhs) { + return sort_key(lhs) < sort_key(rhs); + }); + + size_t output = 0; + for (size_t i = 0; i < values_.size(); ++i) { + const BlobRelocationLiveValue& value = values_[i]; + if (output != 0 && + values_[output - 1].origin_file_number == value.origin_file_number && + values_[output - 1].original_value_offset == + value.original_value_offset) { + if (ImmutableFields(values_[output - 1]) != ImmutableFields(value)) { + return Status::Corruption( + "Conflicting metadata for one stable blob identity"); + } + continue; + } + if (value.record_bytes() > + std::numeric_limits::max() - live_record_bytes_ || + value.stored_value_size > + std::numeric_limits::max() - live_stored_value_bytes_) { + return Status::Corruption("Blob relocation live byte overflow"); + } + live_record_bytes_ += value.record_bytes(); + live_stored_value_bytes_ += value.stored_value_size; + if (origin_runs_.empty() || + origin_runs_.back().origin_file_number != value.origin_file_number) { + origin_runs_.push_back({value.origin_file_number, output, output}); + } + values_[output++] = value; + origin_runs_.back().end = output; + } + values_.resize(output); + return UpdateReservation(); +} + +Status BlobRelocationLiveSet::ReleaseWorkingMemory() { + std::vector().swap(values_); + std::vector().swap(origin_runs_); + return memory_tracker_->SetReservation(&charged_memory_, 0); +} + +size_t BlobRelocationLiveSet::ReservedMemory() const { + return values_.capacity() * sizeof(BlobRelocationLiveValue) + + origin_runs_.capacity() * sizeof(BlobRelocationOriginRun); +} + +Status BlobRelocationLiveSet::UpdateReservation() { + return memory_tracker_->SetReservation(&charged_memory_, ReservedMemory()); +} + +Status PlanBlobRelocationPacking( + const std::vector& items, + uint64_t target_output_bytes, size_t max_outputs, + std::vector* bins) { + if (bins == nullptr || target_output_bytes == 0 || max_outputs == 0) { + return Status::InvalidArgument("Invalid blob relocation packing limits"); + } + bins->clear(); + std::vector order(items.size()); + for (size_t i = 0; i < items.size(); ++i) { + if (items[i].input_file_number == 0 || items[i].estimated_bytes == 0) { + return Status::InvalidArgument("Invalid blob relocation packing item"); + } + order[i] = i; + } + std::sort(order.begin(), order.end(), [&](size_t lhs, size_t rhs) { + const BlobRelocationPackingItem& left = items[lhs]; + const BlobRelocationPackingItem& right = items[rhs]; + if (left.estimated_bytes != right.estimated_bytes) { + return left.estimated_bytes > right.estimated_bytes; + } + return left.input_file_number < right.input_file_number; + }); + + const auto shares_origin = [&](const BlobRelocationPackingBin& bin, + const BlobRelocationPackingItem& item) { + for (size_t existing_index : bin.item_indexes) { + const std::vector& existing = items[existing_index].origins; + auto lhs = existing.begin(); + auto rhs = item.origins.begin(); + while (lhs != existing.end() && rhs != item.origins.end()) { + if (*lhs == *rhs) { + return true; + } + if (*lhs < *rhs) { + ++lhs; + } else { + ++rhs; + } + } + } + return false; + }; + for (size_t item_index : order) { + const BlobRelocationPackingItem& item = items[item_index]; + size_t destination = bins->size(); + for (size_t i = 0; i < bins->size(); ++i) { + BlobRelocationPackingBin& bin = (*bins)[i]; + if (item.estimated_bytes <= + target_output_bytes - + std::min(bin.estimated_bytes, target_output_bytes) && + !shares_origin(bin, item)) { + destination = i; + break; + } + } + if (destination == bins->size()) { + if (bins->size() == max_outputs) { + bins->clear(); + return Status::MemoryLimit("Too many blob relocation outputs"); + } + bins->emplace_back(); + } + BlobRelocationPackingBin& bin = (*bins)[destination]; + if (item.estimated_bytes > + std::numeric_limits::max() - bin.estimated_bytes) { + bins->clear(); + return Status::Corruption("Blob relocation packing byte overflow"); + } + bin.item_indexes.push_back(item_index); + bin.estimated_bytes += item.estimated_bytes; + } + return Status::OK(); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_relocation_gc.h b/db/blob/blob_relocation_gc.h new file mode 100644 index 000000000000..dfa7574ccc03 --- /dev/null +++ b/db/blob/blob_relocation_gc.h @@ -0,0 +1,422 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include +#include +#include + +#include "db/blob/blob_reference_metadata.h" +#include "db/blob/blob_relocation_manifest.h" +#include "rocksdb/status.h" +#include "table/table_reader.h" + +namespace ROCKSDB_NAMESPACE { + +class CacheReservationManager; + +// Indirect blob GC building blocks +// +// Intended user: indirect blob GC. +// +// Indirect blob GC copies the live values of mostly-garbage blob files into +// new relocation files, and rewrites no SST. One GC job collects a few files of +// one column family. A job is built from five parts that work without the DB. +// This file holds the first three: +// 1. Live set (BlobRelocationLiveSet): the live references of one input. +// 2. Memory budget (BlobRelocationMemoryTracker): bounds the working memory of +// the job. +// 3. Packing plan (PlanBlobRelocationPacking()): puts the inputs into outputs. +// 4. Batch reference reader (TableReader::ReadBlobReferences() in +// table/table_reader.h): reads the references of many groups from one SST. +// 5. Stored reads and writes: BlobFileReader and BlobRelocationFileReader in +// db/blob/blob_file_reader.h read the stored bytes of values. The two +// BlockBasedTableBuilder::AddBlobRelocation() overloads write them into an +// output. The streaming overload uses WriteSimpleGen2BlobRecordStream() in +// db/blob/blob_gen2_format.h. +// +// Terms: +// - Input: a file that one GC job collects. An input is a normal blob file or +// a relocation file. A normal input is an indirect-origin blob file: a normal +// blob file written with enable_blob_indirection on +// (BlobFileMetaData::IsIndirectOrigin()), so it is the origin of +// kIndirectBlob references. +// - Route: a MANIFEST record that maps an old file number to the relocation +// file that now holds its live values (see +// db/blob/blob_relocation_manifest.h). +// - Generation numbers of an input: the file numbers that SSTs use to reference +// the input's live values. They are the input's own file number and, for a +// relocation file, every old file number that has a route to it. +// - Output: a relocation file that the job writes. +// - Live set: the references to one input that some SST still keeps (see +// BlobRelocationLiveSet below). Each input has its own live set. +// - Record bytes: the bytes of one value's record in its file. In a normal +// blob file they are the blob log record header, the user key and the +// stored value. In a relocation file they are the stored value plus the +// 5-byte trailer. The record bytes of an input are the sum over all its +// values. +// - Stored CRC: the CRC32C of the stored (maybe compressed) bytes of a value +// plus its compression byte. +// - Sink: an object that receives the references of a batch read in a form it +// chooses (BlobReferenceSink in table/table_reader.h). A live set is a sink. +// - Location block: a data block of a relocation file. It holds location +// entries (where each value is in the file), not values. +// - Coalesced read: one read that covers several neighbor values and the bytes +// between them (see ReadCoalescedRanges() in db/blob/blob_gen2_format.h). +// +// A GC job uses the parts in this order: +// +// census totals --> LiveSet::Reserve() charge the memory first +// each SST --> ReadBlobReferences() fill the live set (a sink) +// each live set --> LiveSet::Finalize() sort, merge, origin runs +// all inputs --> PlanBlobRelocationPacking() inputs -> outputs +// each output --> MultiGetStored*(), Stream*() read the stored bytes and +// check the stored CRC +// --> AddBlobRelocation() write each record +// +// The census totals are the reference counts that the census (see +// db/blob/blob_reference_census.h) keeps for the input's generation numbers. +// They give the size of the live set before any SST is read. LiveSet is short +// for BlobRelocationLiveSet. Origin runs are described at +// BlobRelocationOriginRun below. +// +// Indirect blob GC job +// +// Background: compaction-time blob GC (enable_blob_garbage_collection) moves a +// value only when compaction rewrites the SST that references it. So a blob +// file that is mostly garbage can stay as long as its SSTs are not compacted. +// Indirect blob GC is a background job that is not part of any compaction. It +// runs on the same LOW-pool workers as compaction (see "Job steps"). +// +// One job has 1 to kIndirectBlobGCMaxInputs (8) inputs of one column family, +// and writes at most kIndirectBlobGCMaxOutputs (8) outputs. +// +// Job terms (the terms above also apply): +// - Stale job: the job found that an input or the census changed while it ran. +// The causes are: the census was not ready or was invalidated before the +// job took its census snapshot, a normal input left the Version or its +// metadata changed, an input became pinned by an SST without exact +// reference metadata (a coarse SST), the census disagrees with the SSTs, +// the column family was dropped, or the job's census registration ended. +// The coarse pin can appear during a job: a flush takes its blob file +// number before GC selects the input, and installs its coarse SST after. +// A stale job installs nothing. It returns TryAgain, and +// BackgroundCompaction() turns that into Busy and enqueues the column +// family again. Two states that the GC rules exclude are invariant checks +// instead (an assert in debug builds, a WARN line and a stale job in +// release builds): a relocation input that changed, and a live input +// without an output. +// (A coarse SST pins every indirect-origin file number at or above its +// frontier: GC must treat each such file number as used. See "Coarse SST" +// in db/blob/blob_reference_metadata.h.) +// - Input Version: the Version that the job holds a reference to when it +// starts. GC reads the inputs, their routes and the SST list from it. +// - Candidate-local failure: a failure that affects only one input, such as +// Corruption in the input's records or in the exact SST metadata. GC then +// suppresses that input and sets no DB-wide background error. +// - Deferred input: an input that this job leaves for a later job because it +// does not fit. When the inputs need more than 8 outputs or the disk space +// check fails, the largest input waits first. When the GC memory budget is +// used by earlier inputs, the input that does not fit and all later inputs +// wait. +// - Census entry: the SSTs that hold references with one generation number, +// and their totals. See db/blob/blob_reference_census.h. +// - Census snapshot: GC's copy of the census entries of the inputs' +// generation numbers at the input Version (BlobReferenceCensusState). The +// job does not update it with the Version changes made while it runs (see +// "Install protocol" in DBImpl::IndirectBlobGCJob::Install()). +// - Census heap: the per-column-family list of GC candidates that the census +// ranks by garbage ratio (BlobGcCandidateHeap in +// db/blob/blob_reference_census.h). +// - Blob file retirement (kBlobFileRetirement): a MANIFEST record that removes +// a collected normal blob file (see db/blob/blob_relocation_manifest.h). +// +// Job steps (step 1 is in BackgroundCompaction(); steps 2-8 are in +// DBImpl::IndirectBlobGCJob, which DBImpl::RunIndirectBlobGCBatch() runs): +// 1. BackgroundCompaction() takes up to 8 candidates that the census publishes +// to the Version +// (VersionStorageInfo::GetBlobPhysicalFilesForIndirectBlobGC()), on a +// LOW-pool worker that found no LSM compaction. +// 2. Plan. GC takes the census snapshot +// (BlobReferenceCensus::SnapshotSelected()) and reserves each live set from +// the census reference counts. +// 3. Read. GC reads the needed SSTs one at a time. For each SST it gets a +// reader with TableCache::GetBlobReferenceReader() (the SST's table cache +// reader, or an ephemeral reader on a miss) and reads all needed generation +// numbers in one batch (TableReader::ReadBlobReferences()) into the live +// sets. +// 4. Check. GC compares the SSTs and totals that it found with the census +// snapshot. On a mismatch, the census rebuilds once +// (BlobReferenceCensus::ReportMismatch()). +// 5. Decide. Each input is relocated, gets only its removal (it has no live +// value), or is suppressed. GC relocates an input only if its exact garbage +// is at least 50% of its record bytes. The 50% threshold is a fixed choice, +// not measured. +// 6. Pack. GC puts whole inputs into outputs, largest first +// (PlanBlobRelocationPacking()). +// 7. Copy. GC copies the stored bytes of each live value into its output and +// checks each value with its stored CRC. +// 8. Install. GC builds one MANIFEST edit from its census snapshot and +// installs it with one LogAndApply() call. When the edit is first in the +// MANIFEST writer queue, GC checks under the DB mutex that the job is still +// registered with the census, that each input is unchanged, and that no +// coarse SST pins an input (see "Stale job"). The edit adds the +// outputs and routes each live generation number of an input to the +// input's output. It re-points the routes into a relocation input to the +// output, deletes the routes that no SST uses, writes the blob file +// retirement of each normal input, and deletes each relocation input. +// +// Suppression: GC skips an input that it cannot or should not collect. It +// suppresses the input at two levels: +// - In the census heap (BlobReferenceCensus::SuppressWorkerCandidates()), so +// that later Versions do not select it again. A normal suppression lasts +// until the file changes (its total bytes, its live bytes, its +// is_indirect_origin or is_lost flag, or its coarse pin) or until the next +// census rebuild (see BlobGcCandidateHeap::Suppress()). A rise of the coarse +// frontier also brings back every suppressed relocation file above the old +// frontier, whatever the reason for its suppression, because the heap does +// not store the reason (see +// BlobGcCandidateHeap::SetCoarseIndirectOriginFrontier()). A transient +// suppression lasts until a retry delay passes. The delay starts at 1 s and +// doubles after each failure, up to 32 s (fixed choices, not measured): a +// short failure retries soon, and a lasting failure costs at most one attempt +// every 32 s (see BlobGcCandidateHeap::SuppressTransient()). Each census +// suppression carries a stamp (BlobGcCandidateStamp) taken at the input +// Version. The census ignores a stamp from an older census worker, or from an +// older revision of the file. A file gets a new revision when it changes. +// - In the current Version +// (VersionStorageInfo::SuppressBlobFileForIndirectBlobGC()), so that the next +// pick in this Version takes other candidates. When the job ends without +// error and no input is left to install, GC does this at once. When the job +// installs, GC does this only if no other Version was installed during the +// job, because a newer Version can hold SST metadata that makes the input +// eligible. A stale or failed job does not suppress anything in the Version. +// When a job fails with CompactionTooLarge and the Version did not change, +// the deferred inputs move to the end of the Version's candidate list +// (VersionStorageInfo::DeferBlobFilesForIndirectBlobGC()). +// +// The main reasons, and the kind of census suppression for each (see +// DBImpl::IndirectBlobGCJob for the full list): +// +// Reason Census suppression +// ----------------------------------------------- ------------------ +// invalid input metadata normal +// table format is not block-based normal +// a coarse SST pins a generation number of the normal +// input +// census mismatch remains after a rebuild normal +// corrupt exact SST metadata normal +// exact live bytes above the input's record bytes normal +// a lost relocation file that SSTs still normal +// reference +// enable_blob_files, enable_blob_indirection or normal +// enable_blob_garbage_collection is off in the +// latest options of the column family, and the +// input has live values +// exact garbage below 50% normal +// the output size estimate overflows normal +// outputs would not be smaller than the inputs normal, for every +// eligible input +// an output file is larger than its inputs normal, for the first +// input of that output +// the input cannot be opened for relocation normal +// (Corruption, MemoryLimit or NotSupported), or +// a source value is corrupt +// live set over the 64 MB GC memory budget, and transient +// no earlier input holds a share +// census snapshot cannot be charged to the block transient, for every +// cache, or the GC read and write buffers input of the job +// cannot be charged +// not enough disk space even for one input (the transient, for every +// job returns CompactionTooLarge) input the job deferred +// +// 64 MB is BlobRelocationMemoryTracker::kMaxBytes. Output, I/O and MANIFEST +// errors are not candidate-local. They use the normal background-error +// handling. + +// Bounds the memory of one indirect blob GC job, so that many inputs together +// cannot use more than kMaxBytes (64 MB, an estimate). The budget is charged to +// the block cache. Each user (for example, the live set of one input) keeps its +// own capacity, but updates this total reservation before it grows and when it +// releases its memory. +// +// In a GC job, the live sets of all inputs and the GC buffers (I/O workspace, +// SST readers, group reads) share one tracker (see DBImpl::IndirectBlobGCJob). +class BlobRelocationMemoryTracker { + public: + explicit BlobRelocationMemoryTracker( + std::shared_ptr cache_reservation_manager); + + // Changes one user's share of the budget. `*current_bytes` is that user's + // current share. On success, the total becomes total - *current_bytes + + // requested_bytes, and *current_bytes is set to requested_bytes. Pass 0 to + // release the share. Returns MemoryLimit if the new total would be more than + // kMaxBytes, the block cache error if the charge fails, and Corruption if + // *current_bytes is larger than the total. + Status SetReservation(size_t* current_bytes, size_t requested_bytes); + + size_t reserved_bytes() const { return reserved_bytes_; } + + private: + static constexpr size_t kMaxBytes = 64 * 1024 * 1024; + + std::shared_ptr cache_reservation_manager_; + size_t reserved_bytes_ = 0; +}; + +// One live value of an indirect blob GC input's live set. stable_checksum is +// the logical CRC (CRC32C of the uncompressed value). stored_checksum is the +// stored CRC (CRC32C of the stored bytes plus the compression byte). +struct BlobRelocationLiveValue { + uint64_t origin_file_number = 0; + uint64_t original_value_offset = 0; + uint32_t stored_value_size = 0; + // Record bytes minus stored size: the record header and key, or trailer. + uint32_t record_overhead = 0; + uint32_t stable_checksum = 0; + uint32_t stored_checksum = 0; + CompressionType compression = kNoCompression; + + uint64_t record_bytes() const { + return uint64_t{stored_value_size} + record_overhead; + } +}; +static_assert(sizeof(BlobRelocationLiveValue) <= 40, + "A live value is 40 bytes"); + +// An origin run: the values of a finalized live set that have one origin, +// values [begin, end). An output stores the values of one origin run together. +// It is not the same as a run of the reference metadata (see "run" in +// db/blob/blob_reference_metadata.h). +struct BlobRelocationOriginRun { + uint64_t origin_file_number = 0; + size_t begin = 0; + size_t end = 0; +}; + +// Exact live set of one indirect blob GC input, charged to the GC memory +// budget. It is read from the SST references of the input's generation numbers +// `generation_numbers` (sorted, unique): the input's own file number and every +// old file number routed to it. Each appended reference is checked: its +// generation number must be one of `generation_numbers`, and its encoding class +// must match its generation number (unflattened if and only if generation +// number == origin). It is then kept as a 40-byte BlobRelocationLiveValue. +class BlobRelocationLiveSet : public BlobReferenceSink { + public: + // Makes a private memory tracker, for a live set used alone (tests). + BlobRelocationLiveSet( + std::shared_ptr cache_reservation_manager, + std::vector generation_numbers); + + // Uses a shared memory tracker, so that several live sets share one 64 MB + // budget. A GC job uses this constructor for all its live sets. + BlobRelocationLiveSet( + std::shared_ptr memory_tracker, + std::vector generation_numbers); + + ~BlobRelocationLiveSet() override; + BlobRelocationLiveSet(const BlobRelocationLiveSet&) = delete; + BlobRelocationLiveSet& operator=(const BlobRelocationLiveSet&) = delete; + BlobRelocationLiveSet(BlobRelocationLiveSet&&) = delete; + BlobRelocationLiveSet& operator=(BlobRelocationLiveSet&&) = delete; + + // Reserves charged capacity for `additional` more values. + Status Reserve(size_t additional); + + // Calls Reserve() once, then Append() for each reference. + Status Add(const std::vector& references); + + // Sorts by (origin, original offset) and merges duplicates of one identity + // (the same value can be referenced under several generation numbers). + // Copies must agree on size, compression and checksums. + // + // Copies may differ in record_overhead, and only in a relocation input. An + // unflattened reference (generation number = origin) reaches the input + // through a route from its normal blob file, so it carries the record size + // of that file (the blob log record header and the user key). A flattened + // reference carries the size of a relocation file record trailer (5 bytes). + // The value really lives in a relocation file, so the trailer size is the + // correct one. It is always the smaller one, because a blob log record + // header alone is 32 bytes. So Finalize() keeps the smallest + // record_overhead. + Status Finalize(); + + // Frees the values and releases their share of the GC memory budget. + Status ReleaseWorkingMemory(); + + // BlobReferenceSink. + size_t size() const override { return values_.size(); } + size_t Room() const override { return values_.capacity() - values_.size(); } + Status Append(const BlobReference& reference) override; + void Truncate(size_t size) override { values_.resize(size); } + + const std::vector& references() const { + return values_; + } + + // After Finalize(): one run per origin, in origin order. + const std::vector& origin_runs() const { + return origin_runs_; + } + uint64_t live_record_bytes() const { return live_record_bytes_; } + uint64_t live_stored_value_bytes() const { return live_stored_value_bytes_; } + + size_t TEST_ReservedMemory() const { return ReservedMemory(); } + + private: + Status UpdateReservation(); + size_t ReservedMemory() const; + + std::shared_ptr memory_tracker_; + const std::vector generation_numbers_; + std::vector values_; + std::vector origin_runs_; + size_t charged_memory_ = 0; + uint64_t live_record_bytes_ = 0; + uint64_t live_stored_value_bytes_ = 0; + bool finalized_ = false; +}; + +// One whole indirect blob GC input to pack. +struct BlobRelocationPackingItem { + uint64_t input_file_number = 0; + + // The expected size of the input's values in an output. The planner only + // compares and adds these values. The indirect blob GC job estimates them + // from the live record bytes and the live value count of the input (see + // EstimateIndirectBlobGCPartitionBytes() in + // db/db_impl/db_impl_compaction_flush.cc). + uint64_t estimated_bytes = 0; + + // The origins of the input's live values, sorted and unique: the origin of + // each origin run of the input's finalized live set. + std::vector origins; +}; + +// One planned output ("bin" means output here): the inputs that go into one +// relocation file, as indexes into `items` of PlanBlobRelocationPacking(), and +// the sum of their estimated_bytes. +struct BlobRelocationPackingBin { + std::vector item_indexes; + uint64_t estimated_bytes = 0; +}; + +// Packs whole inputs into outputs, deterministically. Inputs are sorted by +// estimated size, largest first. Each goes to the first output that has room +// and shares no origin with it. Each input goes to exactly one output and is +// never split. An input larger than `target_output_bytes` gets an output of its +// own. Two inputs that hold the same origin (possible after RepairDB) never +// share an output, since an output holds one entry per (origin, original +// offset). Returns MemoryLimit if the inputs need more than `max_outputs` +// outputs. Here MemoryLimit means "over a resource limit", not a memory error. +// The indirect blob GC job then defers the largest input and plans again (see +// "Deferred input" above). +Status PlanBlobRelocationPacking( + const std::vector& items, + uint64_t target_output_bytes, size_t max_outputs, + std::vector* bins); + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_relocation_manifest.cc b/db/blob/blob_relocation_manifest.cc new file mode 100644 index 000000000000..aef8785829ba --- /dev/null +++ b/db/blob/blob_relocation_manifest.cc @@ -0,0 +1,857 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "db/blob/blob_relocation_manifest.h" + +#include +#include +#include +#include +#include +#include + +#ifdef __SSE2__ +#include +#endif // __SSE2__ + +#include "cache/cache_reservation_manager.h" +#include "db/blob/blob_file_meta.h" +#include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_read_request.h" +#include "port/lang.h" +#include "util/coding.h" +#include "util/hash.h" + +namespace ROCKSDB_NAMESPACE { + +// BlobFileTable + +namespace { + +// Tests set this to check the plain scan on hosts with SSE2. +std::atomic blob_file_table_plain_scan{false}; + +uint64_t BlobFileTableHash(uint64_t key) { + // fmix64 of MurmurHash3: consecutive file numbers spread over the shards. + key ^= key >> 33; + key *= 0xff51afd7ed558ccdULL; + key ^= key >> 33; + key *= 0xc4ceb9fe1a85ec53ULL; + key ^= key >> 33; + return key; +} + +// Scans return the 1-based index of `key`, or 0. Keys are unique and the +// padding is 0, which is never a key, so the scans may read the padding. +template +uint32_t PlainScan(const Key* keys, uint32_t padded, Key key) { + uint32_t index = 0; + for (uint32_t i = 0; i < padded; ++i) { + index += keys[i] == key ? i + 1 : 0; + } + return index; +} + +#ifdef __SSE2__ +uint32_t Sse2Scan(const uint32_t* keys, uint32_t padded, uint32_t key) { + const __m128i needle = _mm_set1_epi32(static_cast(key)); + for (uint32_t i = 0; i < padded; i += 4) { + const __m128i equal = _mm_cmpeq_epi32( + _mm_loadu_si128(reinterpret_cast(keys + i)), needle); + const int mask = _mm_movemask_ps(_mm_castsi128_ps(equal)); + if (mask != 0) { + return i + static_cast(__builtin_ctz(mask)) + 1; + } + } + return 0; +} + +uint32_t Sse2Scan(const uint64_t* keys, uint32_t padded, uint64_t key) { + const __m128i needle = _mm_set1_epi64x(static_cast(key)); + for (uint32_t i = 0; i < padded; i += 2) { + const __m128i equal = _mm_cmpeq_epi32( + _mm_loadu_si128(reinterpret_cast(keys + i)), needle); + // Both 32-bit halves must match: AND with the halves swapped. + const __m128i both = + _mm_and_si128(equal, _mm_shuffle_epi32(equal, _MM_SHUFFLE(2, 3, 0, 1))); + const int mask = _mm_movemask_pd(_mm_castsi128_pd(both)); + if (mask != 0) { + return i + static_cast(__builtin_ctz(mask)) + 1; + } + } + return 0; +} +#endif // __SSE2__ + +template +uint32_t ScanKeys(const Key* keys, uint32_t padded, Key key) { +#ifdef __SSE2__ + if (!blob_file_table_plain_scan.load(std::memory_order_relaxed)) { + return Sse2Scan(keys, padded, key); + } +#endif // __SSE2__ + return PlainScan(keys, padded, key); +} + +} // namespace + +void BlobFileTable::TEST_UsePlainScan(bool plain) { + blob_file_table_plain_scan.store(plain, std::memory_order_relaxed); +} + +BlobFileTable::BlobFileTable(const BlobFileTable& other) + : shards_(other.shards_), size_(other.size_), charge_(other.charge_) { + for (Shard* shard : shards_) { + if (shard != nullptr) { + Ref(shard); + } + } + Charge(TopBytes(), 0); +} + +BlobFileTable& BlobFileTable::operator=(const BlobFileTable& other) { + if (this != &other) { + BlobFileTable copy(other); + std::swap(shards_, copy.shards_); + std::swap(size_, copy.size_); + std::swap(charge_, copy.charge_); + } + return *this; +} + +BlobFileTable::~BlobFileTable() { + size_t freed = TopBytes(); + for (Shard* shard : shards_) { + if (shard != nullptr) { + freed += Unref(shard); + } + } + Charge(0, freed); +} + +void BlobFileTable::SetCacheReservationManager( + std::shared_ptr manager) { + // A table with shards may share them with tables of another manager. + if (shards_.empty()) { + charge_ = std::move(manager); + } +} + +void BlobFileTable::Charge(size_t added, size_t freed) const { + if (charge_ == nullptr || added == freed) { + return; + } + // Best effort: a failed reservation never fails an edit. The manager still + // tracks the usage when the cache refuses the reservation. + Status s = added > freed + ? charge_->UpdateCacheReservation(added - freed, + /*increase=*/true) + : charge_->UpdateCacheReservation(freed - added, + /*increase=*/false); + s.PermitUncheckedError(); +} + +void BlobFileTable::Ref(Shard* shard) { + shard->refs.fetch_add(1, std::memory_order_relaxed); +} + +size_t BlobFileTable::Unref(Shard* shard) { + if (shard->refs.fetch_sub(1, std::memory_order_acq_rel) != 1) { + return 0; + } + const size_t bytes = shard->Bytes(); + shard->~Shard(); + ::operator delete(shard); + return bytes; +} + +BlobFileTable::Shard* BlobFileTable::MakeShard(const std::vector& run) { + assert(!run.empty()); + const uint32_t count = static_cast(run.size()); + const uint32_t padded = (count + 3) & ~uint32_t{3}; + const bool wide = std::any_of(run.begin(), run.end(), [](const Entry& entry) { + return (entry.key >> 32) != 0; + }); + const size_t bytes = sizeof(Shard) + size_t{padded} * (wide ? 8 : 4) + + size_t{count} * (sizeof(uint64_t) + sizeof(uint32_t)); + Shard* shard = static_cast(::operator new(bytes)); + new (shard) Shard(); + shard->refs.store(1, std::memory_order_relaxed); + shard->count = count; + shard->padded = padded; + shard->wide = wide ? 1 : 0; + for (uint32_t i = 0; i < padded; ++i) { + const uint64_t key = i < count ? run[i].key : 0; + assert(i >= count || key != kInvalidBlobFileNumber); + if (wide) { + shard->keys64()[i] = key; + } else { + shard->keys32()[i] = static_cast(key); + } + } + for (uint32_t i = 0; i < count; ++i) { + assert(run[i].value != 0); + shard->values()[i] = run[i].value; + shard->counts()[i] = run[i].count; + } + return shard; +} + +uint64_t BlobFileTable::Find(uint64_t file_number, uint32_t* count) const { + if (count != nullptr) { + *count = 0; + } + if (shards_.empty() || file_number == kInvalidBlobFileNumber) { + return 0; + } + const Shard* shard = + shards_[BlobFileTableHash(file_number) & (shards_.size() - 1)]; + if (shard == nullptr) { + return 0; + } + uint32_t index = 0; + if (shard->wide) { + index = ScanKeys(shard->keys64(), shard->padded, file_number); + } else if ((file_number >> 32) == 0) { + index = ScanKeys(shard->keys32(), shard->padded, + static_cast(file_number)); + } + if (index == 0) { + return 0; + } + if (count != nullptr) { + *count = shard->counts()[index - 1]; + } + return shard->values()[index - 1]; +} + +void BlobFileTable::Apply(const std::vector& ops) { + if (ops.empty()) { + return; + } + size_t added = 0; + size_t freed = 0; + if (shards_.empty()) { + shards_.assign(1, nullptr); + added += TopBytes(); + } + const size_t mask = shards_.size() - 1; + // Ops grouped by shard, in their original order within a shard. + std::map> ops_by_shard; + for (const Op& op : ops) { + assert(op.file_number != kInvalidBlobFileNumber); + ops_by_shard[BlobFileTableHash(op.file_number) & mask].push_back(&op); + } + for (const auto& shard_ops : ops_by_shard) { + Shard*& slot = shards_[shard_ops.first]; + std::map> entries; + if (slot != nullptr) { + for (uint32_t i = 0; i < slot->count; ++i) { + entries.emplace(slot->Key(i), + std::make_pair(slot->values()[i], slot->counts()[i])); + } + size_ -= slot->count; + // Other tables may share the old shard; this table drops it. + freed += Unref(slot); + slot = nullptr; + } + for (const Op* op : shard_ops.second) { + if (op->value == 0) { + entries.erase(op->file_number); + } else { + entries[op->file_number] = std::make_pair(op->value, op->count); + } + } + if (!entries.empty()) { + std::vector run; + run.reserve(entries.size()); + for (const auto& entry : entries) { + run.push_back({entry.first, entry.second.first, entry.second.second}); + } + slot = MakeShard(run); + added += slot->Bytes(); + size_ += run.size(); + } + } + const size_t shard_count = shards_.size(); + if (size_ > 2 * kTargetEntriesPerShard * shard_count || + (shard_count > 1 && size_ * 4 < kTargetEntriesPerShard * shard_count)) { + size_t new_count = 1; + while (new_count * kTargetEntriesPerShard < size_) { + new_count <<= 1; + } + Rebuild(new_count, &added, &freed); + } + Charge(added, freed); +} + +void BlobFileTable::Rebuild(size_t shard_count, size_t* added, size_t* freed) { + assert(shard_count > 0 && (shard_count & (shard_count - 1)) == 0); + std::vector> runs(shard_count); + ForEach([&](uint64_t file_number, uint64_t value, uint32_t count) { + runs[BlobFileTableHash(file_number) & (shard_count - 1)].push_back( + {file_number, value, count}); + }); + std::vector shards(shard_count, nullptr); + for (size_t i = 0; i < shard_count; ++i) { + if (!runs[i].empty()) { + std::sort( + runs[i].begin(), runs[i].end(), + [](const Entry& lhs, const Entry& rhs) { return lhs.key < rhs.key; }); + shards[i] = MakeShard(runs[i]); + *added += shards[i]->Bytes(); + } + } + *freed += TopBytes(); + std::swap(shards_, shards); + *added += TopBytes(); + for (Shard* shard : shards) { + if (shard != nullptr) { + *freed += Unref(shard); + } + } +} + +size_t BlobFileTable::ApproximateMemoryUsage() const { + size_t bytes = TopBytes(); + for (const Shard* shard : shards_) { + if (shard != nullptr) { + bytes += shard->Bytes(); + } + } + return bytes; +} + +namespace { + +// Each relocation file and each route has its own 128-bit digest: Hash2x64 of +// an encoding that starts with a type tag (1 = file, 2 = route). +// BlobRelocationState adds the digest of an item when the item is set and +// subtracts it when the item is replaced or deleted. Sums do not depend on +// order, so the state fingerprint does not depend on the order of edits. +// AddDigest() adds the digest of `encoded` when `add` is true, else subtracts +// it. +void AddDigest(const std::string& encoded, bool add, uint64_t* high, + uint64_t* low) { + uint64_t digest_high = 0; + uint64_t digest_low = 0; + Hash2x64(encoded.data(), encoded.size(), &digest_high, &digest_low); + if (add) { + *high += digest_high; + *low += digest_low; + } else { + *high -= digest_high; + *low -= digest_low; + } +} + +std::string EncodeFileForDigest(const BlobRelocationFileMetaData& file) { + std::string encoded; + PutFixed32(&encoded, 1U /* relocation file */); + PutFixed64(&encoded, file.GetFileNumber()); + PutFixed64(&encoded, file.GetFileSize()); + PutLengthPrefixedSlice(&encoded, file.GetChecksumMethod()); + PutLengthPrefixedSlice(&encoded, file.GetChecksumValue()); + PutFixed32(&encoded, file.GetDirectoryDigest()); + PutFixed32(&encoded, file.IsMarkedLost() ? 1U : 0U); + return encoded; +} + +std::string EncodeRouteForDigest(uint64_t source, uint64_t target) { + std::string encoded; + PutFixed32(&encoded, 2U /* route */); + PutFixed64(&encoded, source); + PutFixed64(&encoded, target); + return encoded; +} + +} // namespace + +// BlobRelocationState + +const BlobRelocationState::Files& BlobRelocationState::files() const { + STATIC_AVOID_DESTRUCTION(Files, empty); + return files_ != nullptr ? *files_ : empty; +} + +BlobRelocationState::Files& BlobRelocationState::MutableFiles() { + if (files_ == nullptr) { + files_ = std::make_shared(); + } + if (files_.use_count() != 1) { + files_ = std::make_shared(*files_); + } + return const_cast(*files_); +} + +uint64_t BlobRelocationState::Lookup(uint64_t file_number, + uint64_t* count) const { + const auto it = pending_.find(file_number); + if (it == pending_.end()) { + uint32_t table_count = 0; + const uint64_t value = table_.Find(file_number, &table_count); + if (count != nullptr) { + *count = table_count; + } + return value; + } + if (count != nullptr) { + *count = it->second.second; + } + return it->second.first; +} + +void BlobRelocationState::Stage(uint64_t file_number, uint64_t value, + uint64_t count) { + pending_[file_number] = std::make_pair(value, count); + validated_.store(false, std::memory_order_relaxed); +} + +void BlobRelocationState::Seal() { + if (pending_.empty()) { + return; + } + std::vector ops; + ops.reserve(pending_.size()); + for (const auto& entry : pending_) { + uint32_t count = 0; + const uint64_t value = table_.Find(entry.first, &count); + // VersionBuilder rejects more than 2^32 - 1 routes to one target. + assert(entry.second.second <= std::numeric_limits::max()); + const uint32_t final_count = static_cast(entry.second.second); + if (entry.second.first != value || final_count != count) { + ops.push_back({entry.first, entry.second.first, final_count}); + } + } + pending_.clear(); + table_.Apply(ops); +} + +size_t BlobRelocationState::IncrementalRetentionBytes( + const BlobRelocationState& base, + const std::vector& changed_files) const { + // A census change log entry that holds this state is charged the result. + // + // Counted: + // - The BlobRelocationState object itself (sizeof(*this)). + // - The files vectors of both states, when this state has its own copy. + // - The metadata of each changed relocation file, old and new. No other + // reservation charges this metadata, and an earlier change log entry can + // still hold the old one. Each object counts as its size plus 8 pointers + // (an estimate of the allocation overhead) plus its checksum strings. + // + // Not counted: the file table. Its shards and top arrays are charged by + // ownership to the file metadata reservation (see + // BlobFileTable::SetCacheReservationManager()). Each shard is charged once + // and released when its last holder frees it, so the table stays charged + // while any entry or Version holds it. Counting it per entry would charge the + // table copy of every flush again. + size_t bytes = sizeof(*this); + const auto add = [&bytes](size_t count, size_t item_bytes) { + if (item_bytes != 0 && + count > (std::numeric_limits::max() - bytes) / item_bytes) { + bytes = std::numeric_limits::max(); + } else { + bytes += count * item_bytes; + } + }; + if (files_ != base.files_) { + add(files().capacity() + base.files().capacity(), + sizeof(std::shared_ptr)); + } + for (uint64_t file_number : changed_files) { + const BlobRelocationFileMetaData* const old_file = + base.PeekFile(file_number); + const BlobRelocationFileMetaData* const new_file = PeekFile(file_number); + if (old_file == new_file) { + continue; + } + for (const BlobRelocationFileMetaData* file : {old_file, new_file}) { + if (file != nullptr) { + add(1, sizeof(BlobRelocationFileMetaData) + 8 * sizeof(void*)); + add(file->GetChecksumMethod().capacity(), 1); + add(file->GetChecksumValue().capacity(), 1); + } + } + } + return bytes; +} + +std::shared_ptr BlobRelocationState::FindFile( + uint64_t file_number) const { + const BlobRelocationFileMetaData* file = PeekFile(file_number); + if (file == nullptr) { + return nullptr; + } + const Files& all = files(); + const auto it = std::lower_bound( + all.begin(), all.end(), file_number, + [](const std::shared_ptr& value, + uint64_t number) { return value->GetFileNumber() < number; }); + assert(it != all.end() && it->get() == file); + return *it; +} + +const BlobRelocationFileMetaData* BlobRelocationState::PeekFile( + uint64_t file_number) const { + const uint64_t value = Lookup(file_number); + return BlobFileTable::IsRelocationFile(value) + ? BlobFileTable::AsRelocationFile(value) + : nullptr; +} + +uint64_t BlobRelocationState::FindRouteTarget(uint64_t old_file_number) const { + const uint64_t value = Lookup(old_file_number); + return BlobFileTable::IsRoute(value) ? BlobFileTable::RouteTarget(value) + : kInvalidBlobFileNumber; +} + +size_t BlobRelocationState::RouteCount(uint64_t file_number) const { + uint64_t count = 0; + const uint64_t value = Lookup(file_number, &count); + return BlobFileTable::IsRelocationFile(value) ? count : 0; +} + +void BlobRelocationState::SetFile( + std::shared_ptr file) { + assert(file != nullptr); + const uint64_t file_number = file->GetFileNumber(); + const BlobRelocationFileMetaData* previous = PeekFile(file_number); + if (previous != nullptr) { + AddDigest(EncodeFileForDigest(*previous), /*add=*/false, &digest_high_, + &digest_low_); + if (previous->IsMarkedLost()) { + --lost_file_count_; + } + } + AddDigest(EncodeFileForDigest(*file), /*add=*/true, &digest_high_, + &digest_low_); + if (file->IsMarkedLost()) { + ++lost_file_count_; + } + if (previous != nullptr && !aggregate_overflow_) { + physical_file_bytes_ -= previous->GetFileSize(); + record_bytes_ -= previous->GetTotalRecordBytes(); + } + if (file->GetFileSize() > + std::numeric_limits::max() - physical_file_bytes_ || + file->GetTotalRecordBytes() > + std::numeric_limits::max() - record_bytes_) { + aggregate_overflow_ = true; + } else if (!aggregate_overflow_) { + physical_file_bytes_ += file->GetFileSize(); + record_bytes_ += file->GetTotalRecordBytes(); + } + // A file set again keeps the routes to it. + uint64_t route_count = 0; + if (previous != nullptr) { + Lookup(file_number, &route_count); + } + Stage(file_number, + BlobFileTable::RelocationFileValue(file.get(), file->IsMarkedLost()), + route_count); + Files& files = MutableFiles(); + const auto it = std::lower_bound( + files.begin(), files.end(), file_number, + [](const std::shared_ptr& value, + uint64_t number) { return value->GetFileNumber() < number; }); + if (it != files.end() && (*it)->GetFileNumber() == file_number) { + *it = std::move(file); + } else { + files.insert(it, std::move(file)); + } +} + +void BlobRelocationState::DeleteFile(uint64_t file_number) { + const BlobRelocationFileMetaData* file = PeekFile(file_number); + if (file == nullptr) { + return; + } + AddDigest(EncodeFileForDigest(*file), /*add=*/false, &digest_high_, + &digest_low_); + if (file->IsMarkedLost()) { + --lost_file_count_; + } + if (!aggregate_overflow_) { + physical_file_bytes_ -= file->GetFileSize(); + record_bytes_ -= file->GetTotalRecordBytes(); + } + Stage(file_number, 0, /*count=*/0); + Files& files = MutableFiles(); + const auto it = std::lower_bound( + files.begin(), files.end(), file_number, + [](const std::shared_ptr& value, + uint64_t number) { return value->GetFileNumber() < number; }); + assert(it != files.end() && (*it)->GetFileNumber() == file_number); + files.erase(it); +} + +void BlobRelocationState::AdjustRouteCount(uint64_t target, int64_t delta) { + uint64_t count = 0; + const uint64_t value = Lookup(target, &count); + if (!BlobFileTable::IsRelocationFile(value)) { + // The target is not a live relocation file. Either the route names a + // missing target (validation reports it), or the target was deleted earlier + // in the same build. + return; + } + assert(delta >= 0 || count >= static_cast(-delta)); + Stage(target, value, + static_cast(static_cast(count) + delta)); +} + +void BlobRelocationState::SetRoute(const BlobRelocationRoute& route) { + const uint64_t source = route.GetOldFileNumber(); + const uint64_t target = route.GetTargetFileNumber(); + const uint64_t previous = FindRouteTarget(source); + if (previous == target) { + return; + } + if (previous != kInvalidBlobFileNumber) { + AdjustRouteCount(previous, -1); + AddDigest(EncodeRouteForDigest(source, previous), /*add=*/false, + &digest_high_, &digest_low_); + } else { + ++route_count_; + } + AddDigest(EncodeRouteForDigest(source, target), /*add=*/true, &digest_high_, + &digest_low_); + Stage(source, BlobFileTable::RouteValue(target), /*count=*/0); + AdjustRouteCount(target, 1); +} + +void BlobRelocationState::DeleteRoute(uint64_t old_file_number) { + const uint64_t target = FindRouteTarget(old_file_number); + if (target == kInvalidBlobFileNumber) { + return; + } + AddDigest(EncodeRouteForDigest(old_file_number, target), /*add=*/false, + &digest_high_, &digest_low_); + Stage(old_file_number, 0, /*count=*/0); + --route_count_; + AdjustRouteCount(target, -1); +} + +void BlobRelocationState::SetBlobFile(const BlobFileMetaData* file) { + assert(file != nullptr); + Stage(file->GetBlobFileNumber(), BlobFileTable::BlobFileValue(file), + /*count=*/0); +} + +void BlobRelocationState::DeleteBlobFile(uint64_t file_number) { + if (BlobFileTable::IsBlobFile(Lookup(file_number))) { + Stage(file_number, 0, /*count=*/0); + } +} + +void BlobRelocationState::ComputeFingerprint(uint32_t format_version, + uint64_t* high, + uint64_t* low) const { + std::string encoded; + PutFixed32(&encoded, 3U /* fingerprint encoding version */); + PutFixed32(&encoded, format_version); + PutFixed64(&encoded, files().size()); + PutFixed64(&encoded, route_count_); + PutFixed64(&encoded, digest_high_); + PutFixed64(&encoded, digest_low_); + Hash2x64(encoded.data(), encoded.size(), high, low); +} + +// BlobRelocationFileAddition + +BlobRelocationFileAddition::BlobRelocationFileAddition( + uint64_t file_number, uint64_t file_size, std::string checksum_method, + std::string checksum_value, const Slice& encoded_directory, + uint64_t tail_size) + : file_number_(file_number), + file_size_(file_size), + tail_size_(tail_size), + checksum_method_(std::move(checksum_method)), + checksum_value_(std::move(checksum_value)) { + BlobRelocationDirectory directory; + directory_status_ = + BlobRelocationDirectory::DecodeFrom(encoded_directory, &directory); + if (directory_status_.ok()) { + directory_status_ = directory.ValidateForFileSize(file_size_); + } + if (directory_status_.ok()) { + total_entries_ = directory.total_entries(); + total_record_bytes_ = directory.total_record_bytes(); + directory_digest_ = directory.directory_digest(); + } +} + +BlobRelocationFileAddition::BlobRelocationFileAddition( + uint64_t file_number, uint64_t file_size, std::string checksum_method, + std::string checksum_value, uint64_t total_entries, + uint64_t total_record_bytes, uint32_t directory_digest, uint64_t tail_size) + : file_number_(file_number), + file_size_(file_size), + tail_size_(tail_size), + checksum_method_(std::move(checksum_method)), + checksum_value_(std::move(checksum_value)), + total_entries_(total_entries), + total_record_bytes_(total_record_bytes), + directory_digest_(directory_digest) {} + +Status BlobRelocationFileAddition::Validate() const { + if (!directory_status_.ok()) { + return directory_status_; + } + if (file_number_ == kInvalidBlobFileNumber || + file_number_ > kMaxBlobRelocationFileNumber) { + return Status::Corruption("Invalid blob relocation file number"); + } + if (file_size_ <= kBlobRelocationFilePrefixSize) { + return Status::Corruption("Invalid blob relocation file size"); + } + // An open reads the table metadata range that the builder recorded, so the + // tail size must be nonzero and fit in the file. + if (tail_size_ == 0 || tail_size_ > file_size_) { + return Status::Corruption("Invalid blob relocation metadata tail size"); + } + if (checksum_method_.empty() != checksum_value_.empty()) { + return Status::Corruption("Incomplete blob relocation file checksum"); + } + // Every record holds at least its trailer (kSimpleGen2BlobTrailerSize), and + // all records fit in the file after the prefix. + if (total_entries_ == 0 || + total_entries_ > total_record_bytes_ / kSimpleGen2BlobTrailerSize || + total_record_bytes_ > file_size_ - kBlobRelocationFilePrefixSize) { + return Status::Corruption("Invalid blob relocation file totals"); + } + return Status::OK(); +} + +void BlobRelocationFileAddition::EncodeTo(std::string* output) const { + PutVarint64(output, file_number_); + PutVarint64(output, file_size_); + PutLengthPrefixedSlice(output, checksum_method_); + PutLengthPrefixedSlice(output, checksum_value_); + PutVarint64(output, total_entries_); + PutVarint64(output, total_record_bytes_); + PutFixed32(output, directory_digest_); + PutVarint64(output, tail_size_); +} + +Status BlobRelocationFileAddition::DecodeFrom(Slice* input) { + if (input == nullptr) { + return Status::InvalidArgument("Null blob relocation addition input"); + } + + Slice checksum_method; + Slice checksum_value; + if (!GetVarint64(input, &file_number_) || !GetVarint64(input, &file_size_) || + !GetLengthPrefixedSlice(input, &checksum_method) || + !GetLengthPrefixedSlice(input, &checksum_value) || + !GetVarint64(input, &total_entries_) || + !GetVarint64(input, &total_record_bytes_) || + !GetFixed32(input, &directory_digest_) || + !GetVarint64(input, &tail_size_) || !input->empty()) { + return Status::Corruption("Malformed blob relocation file addition"); + } + + checksum_method_.assign(checksum_method.data(), checksum_method.size()); + checksum_value_.assign(checksum_value.data(), checksum_value.size()); + directory_status_ = Status::OK(); + return Validate(); +} + +std::string BlobRelocationFileAddition::DebugString() const { + std::ostringstream stream; + stream << "file=" << file_number_ << " size=" << file_size_ + << " tail_size=" << tail_size_ << " entries=" << total_entries_ + << " record_bytes=" << total_record_bytes_ + << " directory_digest=" << directory_digest_; + return stream.str(); +} + +BlobPhysicalFile BlobRelocationFileAddition::ToPhysicalFile() const { + return BlobPhysicalFile{ + BlobFileOpenInfo{file_number_, checksum_value_, checksum_method_}, + file_size_, /*is_relocation=*/true, tail_size_, directory_digest_}; +} + +bool operator==(const BlobRelocationFileAddition& lhs, + const BlobRelocationFileAddition& rhs) { + return lhs.GetFileNumber() == rhs.GetFileNumber() && + lhs.GetFileSize() == rhs.GetFileSize() && + lhs.GetTailSize() == rhs.GetTailSize() && + lhs.GetChecksumMethod() == rhs.GetChecksumMethod() && + lhs.GetChecksumValue() == rhs.GetChecksumValue() && + lhs.GetTotalEntries() == rhs.GetTotalEntries() && + lhs.GetTotalRecordBytes() == rhs.GetTotalRecordBytes() && + lhs.GetDirectoryDigest() == rhs.GetDirectoryDigest(); +} + +bool operator!=(const BlobRelocationFileAddition& lhs, + const BlobRelocationFileAddition& rhs) { + return !(lhs == rhs); +} + +// Routes + +Status BlobRelocationRoute::Validate() const { + if (old_file_number_ == kInvalidBlobFileNumber || + target_file_number_ == kInvalidBlobFileNumber || + old_file_number_ > kMaxBlobRelocationFileNumber || + target_file_number_ > kMaxBlobRelocationFileNumber) { + return Status::Corruption("Invalid blob relocation route file number"); + } + if (old_file_number_ == target_file_number_) { + return Status::Corruption("Blob relocation route targets its own file"); + } + return Status::OK(); +} + +void BlobRelocationRoute::EncodeTo(std::string* output) const { + PutVarint64(output, old_file_number_); + PutVarint64(output, target_file_number_); +} + +Status BlobRelocationRoute::DecodeFrom(Slice* input) { + if (input == nullptr) { + return Status::InvalidArgument("Null blob relocation route input"); + } + if (!GetVarint64(input, &old_file_number_) || + !GetVarint64(input, &target_file_number_) || !input->empty()) { + return Status::Corruption("Malformed blob relocation route"); + } + return Validate(); +} + +void BlobRelocationRouteDeletion::EncodeTo(std::string* output) const { + PutVarint64(output, old_file_number_); +} + +Status BlobRelocationRouteDeletion::DecodeFrom(Slice* input) { + if (input == nullptr) { + return Status::InvalidArgument("Null blob relocation route deletion input"); + } + if (!GetVarint64(input, &old_file_number_) || !input->empty() || + old_file_number_ == kInvalidBlobFileNumber) { + return Status::Corruption("Malformed blob relocation route deletion"); + } + return Status::OK(); +} + +std::string BlobRelocationRoute::DebugString() const { + std::ostringstream stream; + stream << "old=" << old_file_number_ << " target=" << target_file_number_; + return stream.str(); +} + +bool operator==(const BlobRelocationRoute& lhs, + const BlobRelocationRoute& rhs) { + return lhs.GetOldFileNumber() == rhs.GetOldFileNumber() && + lhs.GetTargetFileNumber() == rhs.GetTargetFileNumber(); +} + +bool operator!=(const BlobRelocationRoute& lhs, + const BlobRelocationRoute& rhs) { + return !(lhs == rhs); +} + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_relocation_manifest.h b/db/blob/blob_relocation_manifest.h new file mode 100644 index 000000000000..7f5a791eab04 --- /dev/null +++ b/db/blob/blob_relocation_manifest.h @@ -0,0 +1,841 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// Blob relocation: MANIFEST records and Version state +// =================================================== +// +// Background +// ---------- +// Indirect blob GC removes garbage from blob files without rewriting any SST. +// It copies the live values of whole blob files into a relocation file. The +// SSTs keep their old references, so every reader must still find each value, +// also after a crash and reopen. Files on disk cannot answer this: a relocation +// file on disk may be half written or not installed yet. So the MANIFEST +// records where values live, and each Version owns that state, as it owns SST +// and blob files. +// This file holds: +// - the MANIFEST record codecs for relocation files and routes. A route sends +// reads of a file that GC removed to the relocation file that now holds its +// values (see Terms). VersionEdit encodes these records with the codecs in +// this file, and VersionBuilder checks and applies them. +// - the per-Version state built from these records (BlobFileTable and +// BlobRelocationState). Version::GetBlob reads through this state. +// The code that creates relocation files and routes is not in this file. +// +// Terms +// ----- +// Origin, original offset, stable identity, generation number, live file, old +// file and route are defined in db/blob/blob_index.h. This file adds: +// - relocation file: a block-based table that holds copied values. A reader +// finds a value in it by its stable identity; see blob_relocation_file.h. +// - target: the live relocation file that a route points to. It holds all live +// values of the old file. +// - physical file: a file that exists on disk under its file number: an SST, a +// blob file or a relocation file. A generation number is not always a +// physical file: after GC removes a blob file, its number can stay only as +// the key of a route. +// - indirect origin: a normal blob file written with enable_blob_indirection on +// (BlobFileAddition::IsIndirectOrigin()). Only such a file can be the origin +// of a kIndirectBlob reference. +// - format fence: a required MANIFEST record that says a column family or the +// whole DB may hold data that only blob indirection writes (feature data, +// defined below). A binary that does not know the record fails to open the +// DB. The per-CF format fence marks one column family, and the DB-global +// format fence marks the whole DB (see "MANIFEST records" below). +// - feature data: persisted state that only blob indirection writes: a format +// fence, a relocation file, a route, a blob file retirement or accepted-loss +// record, an indirect-origin blob file, or an SST with exact reference +// metadata (db/blob/blob_reference_metadata.h). +// - fenced column family: a column family whose per-CF format fence is set. +// Only a fenced column family has a file table (BlobFileTable). +// - legacy column family: a column family without the per-CF format fence. It +// has no file table and no relocation state. +// - activation: the first edit in the DB that sets a per-CF format fence. From +// then on the VersionSet keeps the DB-global format fence. +// +// Lifecycle of one value +// ---------------------- +// This example follows one value from its write until its route is deleted. +// It uses key "k", blob file 12, relocation files 20 and 30, and offset 4096. +// All of these are example values. +// +// Base RocksDB +// 1. Write and flush. Put("k", v) goes to the memtable. With +// enable_blob_files, a flush writes the SST, and a value of at least +// min_blob_size goes to a blob file instead, here file 12 at offset 4096. +// The SST keeps only a small BlobIndex that points to it. Later +// compactions copy the BlobIndex, not the value. +// 2. Garbage. When "k" is overwritten or deleted, a compaction drops its old +// BlobIndex, and the value in file 12 becomes garbage. File 12 cannot be +// deleted while it holds other live values. Compaction-time blob GC (base +// RocksDB, enable_blob_garbage_collection) can move a value only when a +// compaction rewrites the SST entry that points to it, because a kBlob +// index holds the file number and offset. So file 12 goes away only after +// compactions have rewritten every SST entry that points to it. +// +// Blob indirection enables moving values in blob files without rewriting SSTs +// 3. With blob indirection, the SST stores a kIndirectBlob index (see +// db/blob/blob_index.h) with origin = 12 and original offset = 4096. Its +// generation number is 12: for kIndirectBlob it equals the origin and is not +// stored. Origin and original offset never change, so they name the value +// for its whole life (its stable identity). The generation number tells a +// reader where to start looking. +// 4. Indirect blob GC. A background job picks input files that hold a lot of +// garbage. An input is a blob file written with enable_blob_indirection on +// (an indirect origin), or a relocation file from an earlier GC round. +// Here the only input is blob file 12. The job copies only the live values +// of its inputs into a new relocation file 20, then installs the result +// with one VersionEdit: +// - adds relocation file 20; +// - adds a route 12 -> 20, so that references that still say +// generation number 12 find their value in file 20; +// - re-points every route whose target is an input to the new relocation +// file. No route targets file 12 here, so nothing is re-pointed (step +// 4b shows a round that re-points a route); +// - removes the input files from the Version: a kBlobFileRetirement +// record for a blob file (here file 12), a kBlobRelocationFileDeletion +// record for a relocation file. The files are deleted from disk later, +// as other obsolete files are. The routes stay; step 7 removes them. +// No SST changes. The SSTs still say generation number 12. +// 4b. A later GC round. Relocation file 20 is now an input, and its live +// values move to relocation file 30. The edit adds file 30, changes route +// 12 -> 20 to 12 -> 30 (so a read still follows at most one route), adds +// route 20 -> 30 for references that a compaction already flattened to 20 +// (step 6), and deletes file 20. If an input has no live values left, no +// reference can need it, so the edit deletes the routes into the input +// instead of re-pointing them. +// 5. Read. A read of key "k" decodes its BlobIndex (generation number 12, +// origin 12, original offset 4096). The blob cache key is the stable +// identity (origin 12, original offset 4096), not the file the value is in. +// A whole-value read that misses fills the cache under that key (when a blob +// cache is set and ReadOptions::fill_cache is true), so a cached value still +// hits after indirect blob GC moves it, and a hit needs no file lookup +// (BlobSource::GetIndirectBlobFromCache()). On a cache miss, the Version +// looks up generation number 12 in its file table (BlobFileTable, +// Version::ResolveBlobPhysicalFile()). One entry per number says what the +// number is now: +// - a live blob file: the value has not moved; read it there; +// - a live relocation file: a flattened reference (step 6); read it +// there; +// - a route: here 12 -> 20, so look up 20, which must be a live +// relocation file. A route target is never itself a route key, so a +// read follows at most one route (one hop). +// Anything else is Corruption: no entry, a route whose target is not a +// live relocation file, or a live blob file that is not the origin of +// the reference or is not an indirect origin. How the value is found +// inside a relocation file is in "Reads" in +// db/blob/blob_relocation_file.h. +// 6. Flatten. A later compaction may rewrite the generation number of a +// reference to the route target (generation number = 20, a flattened +// reference). Origin and original offset do not change. Such a reference +// finds file 20 in the file table directly. Flatten is a side effect of +// compaction: no job rewrites an SST only to flatten it. +// 7. Route deletion. When no SST uses old file number 12 any more, a later +// MANIFEST edit removes its route (12 -> 20, or 12 -> 30 after step 4b). +// +// MANIFEST records (codecs in this file and in VersionEdit) +// --------------------------------------------------------- +// All tags are required tags (below kTagSafeIgnoreMask), so a binary that does +// not know them fails to open the DB instead of misreading it. +// - Per-CF format fence (kBlobIndirectionFormatVersion): marks a column family +// that may hold feature data. Every edit that adds an SST built with +// enable_blob_indirection on sets it; the first such edit is the activation +// (see Terms). It is sticky: it stays after the option is turned off. The +// relocation records below need it. +// - DB-global fence (kDbBlobIndirectionFormatVersion): marks the DB, not a +// column family. Each MANIFEST snapshot writes it first when the DB's format +// is active. +// - Relocation file addition and deletion: a Version owns the live relocation +// files of its column family. The addition record keeps only numbers +// (BlobRelocationFileAddition); the directory stays in the file. File numbers +// are never reused. +// - Route and route deletion: one route per old file. The old file is not a +// live file, the target is a live relocation file of the same column family, +// and no route points to a deleted file. So a target is never itself a route +// key: no chains, one hop. A route deletion record removes the route of an +// old file that no SST uses any more, so the Version does not keep unused +// routes forever. +// - Blob file retirement (kBlobFileRetirement): removes a live indirect-origin +// blob file that GC has collected. A file with live values also gets its +// route in the same edit. +// - Accepted loss (kBlobPhysicalFileLoss): records that the bytes of a live +// relocation file are gone. It never names a normal or indirect-origin blob +// file; a missing one uses the base dropped-file record (kDroppedBlobFile). +// VersionBuilder keeps the file in the Version and marks it lost, and reads +// of its values return NotFound. Only tests create a loss. +// WriteCurrentStateToManifest() writes the record again for each file that +// is marked lost. +// - The DB-global fence stays set even after the last fenced column family is +// dropped. +// +// File protection +// --------------- +// New file numbers come from one DB-wide counter, so two column families do +// not get the same file number. Nothing checks this across column families. +// These checks keep a wrong MANIFEST record from deleting or misreading a file: +// - Per column family: VersionBuilder applies relocation, route, retirement and +// loss records to the state of the edit's own column family. It rejects a +// relocation file whose number is already an SST, a blob file or a different +// relocation file there (an identical repeated record, as at replay, is +// accepted); a new SST whose number is a relocation file there; a new blob +// file whose number is a relocation file or a route there; a route whose +// target is not a live relocation file there; a relocation file deletion that +// names no live relocation file there; and a loss record that names no live +// relocation file there. +// - Purge: the obsolete file purge is DB-wide. A .blob file stays while any +// column family's Version lists it as a blob file or a relocation file +// (Version::RemoveLiveFiles() for a normal purge, Version::AddLiveFiles() +// for a full scan). +// - Read: when a reader opens a file, it checks the column family ID in the +// blob file header, or in the table properties of a relocation file, and +// returns Corruption on a mismatch. Each read in a relocation file also +// checks the value's key (origin, original offset), so a wrong route cannot +// return another value. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/blob/blob_constants.h" +#include "db/blob/blob_relocation_file.h" +#include "rocksdb/rocksdb_namespace.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" + +namespace ROCKSDB_NAMESPACE { + +// The format version that both MANIFEST format fences store: the per-CF fence +// (kBlobIndirectionFormatVersion) and the DB-global fence +// (kDbBlobIndirectionFormatVersion). A fence is not set when its value is 0, so +// VersionEdit decoding returns Corruption for a fence record that holds 0. It +// returns NotSupported for any other value that is not this one, so a binary +// does not open a DB that uses a format it does not know. +inline constexpr uint32_t kBlobIndirectionManifestFormatVersion = 2; + +class BlobFileMetaData; + +// The largest relocation file number and route file number. It equals +// kFileNumberMask (2^62 - 1), the limit of all table and blob file numbers, +// because relocation files take their numbers from the same counter; a +// static_assert in version_edit.cc checks this. BlobFileTable (below) also +// relies on this limit: it stores a route as target << 1 | 1, so bit 0 marks a +// route and the other 63 bits hold the target. For example, a route to file 20 +// is stored as 41, and 41 >> 1 gives 20 back. Any number up to 2^62 - 1 fits, +// with one bit to spare. See "Tagged value" in the BlobFileTable comment for +// the full encoding. +constexpr uint64_t kMaxBlobRelocationFileNumber = (uint64_t{1} << 62) - 1; +class CacheReservationManager; + +// The MANIFEST record that adds one relocation file to a column family. It +// keeps only numbers: file number, file size, checksum, tail size, total +// entries, total record bytes and the directory digest. The directory itself +// (see blob_relocation_file.h) stays in the file; a reader checks the file's +// directory against the digest kept here. The tail size is the byte size of the +// table metadata at the end of the file (index, properties, metaindex and +// footer); an open uses it to read that range. The record has no version or +// column family field: the format fence gives the version, and the VersionEdit +// names the column family. +struct BlobPhysicalFile; + +class BlobRelocationFileAddition { + public: + BlobRelocationFileAddition() = default; + BlobRelocationFileAddition(const BlobRelocationFileAddition&) = default; + BlobRelocationFileAddition& operator=(const BlobRelocationFileAddition&) = + default; + BlobRelocationFileAddition(BlobRelocationFileAddition&&) noexcept = default; + BlobRelocationFileAddition& operator=(BlobRelocationFileAddition&&) noexcept = + default; + ~BlobRelocationFileAddition() { directory_status_.PermitUncheckedError(); } + + // From a finished file: takes the totals and digest from its encoded + // directory. A directory that does not decode makes Validate() fail. + BlobRelocationFileAddition(uint64_t file_number, uint64_t file_size, + std::string checksum_method, + std::string checksum_value, + const Slice& encoded_directory, + uint64_t tail_size); + // From stored numbers. + BlobRelocationFileAddition(uint64_t file_number, uint64_t file_size, + std::string checksum_method, + std::string checksum_value, uint64_t total_entries, + uint64_t total_record_bytes, + uint32_t directory_digest, uint64_t tail_size); + + uint64_t GetFileNumber() const { return file_number_; } + uint64_t GetFileSize() const { return file_size_; } + uint64_t GetTailSize() const { return tail_size_; } + const std::string& GetChecksumMethod() const { return checksum_method_; } + const std::string& GetChecksumValue() const { return checksum_value_; } + uint64_t GetTotalEntries() const { return total_entries_; } + uint64_t GetTotalRecordBytes() const { return total_record_bytes_; } + uint32_t GetDirectoryDigest() const { return directory_digest_; } + + // The physical file that a reader opens for this relocation file. The + // result points to this addition's checksum strings, so this addition must + // outlive it. + BlobPhysicalFile ToPhysicalFile() const; + + Status Validate() const; + void EncodeTo(std::string* output) const; + Status DecodeFrom(Slice* input); + std::string DebugString() const; + + private: + uint64_t file_number_ = kInvalidBlobFileNumber; + uint64_t file_size_ = 0; + uint64_t tail_size_ = 0; + std::string checksum_method_; + std::string checksum_value_; + uint64_t total_entries_ = 0; + uint64_t total_record_bytes_ = 0; + uint32_t directory_digest_ = 0; + // Set when the directory given to the constructor did not decode. + Status directory_status_; +}; + +bool operator==(const BlobRelocationFileAddition& lhs, + const BlobRelocationFileAddition& rhs); +bool operator!=(const BlobRelocationFileAddition& lhs, + const BlobRelocationFileAddition& rhs); + +// The in-memory metadata of one live relocation file. Versions share it through +// shared_ptr and never change it. When the last holder drops it, the deleter +// given to Create() runs: VersionBuilder's deleter marks the file obsolete, so +// the file can then be deleted from disk. +class BlobRelocationFileMetaData { + public: + template + static std::shared_ptr Create( + BlobRelocationFileAddition addition, Deleter deleter) { + return std::shared_ptr( + new BlobRelocationFileMetaData(std::move(addition)), + std::move(deleter)); + } + + static std::shared_ptr Create( + BlobRelocationFileAddition addition) { + return std::shared_ptr( + new BlobRelocationFileMetaData(std::move(addition))); + } + + BlobRelocationFileMetaData(const BlobRelocationFileMetaData&) = delete; + BlobRelocationFileMetaData& operator=(const BlobRelocationFileMetaData&) = + delete; + BlobRelocationFileMetaData(BlobRelocationFileMetaData&&) = delete; + BlobRelocationFileMetaData& operator=(BlobRelocationFileMetaData&&) = delete; + ~BlobRelocationFileMetaData() = default; + + uint64_t GetFileNumber() const { return addition_.GetFileNumber(); } + uint64_t GetFileSize() const { return addition_.GetFileSize(); } + uint64_t GetTailSize() const { return addition_.GetTailSize(); } + const std::string& GetChecksumMethod() const { + return addition_.GetChecksumMethod(); + } + const std::string& GetChecksumValue() const { + return addition_.GetChecksumValue(); + } + uint64_t GetTotalEntries() const { return addition_.GetTotalEntries(); } + uint64_t GetTotalRecordBytes() const { + return addition_.GetTotalRecordBytes(); + } + uint32_t GetDirectoryDigest() const { return addition_.GetDirectoryDigest(); } + const BlobRelocationFileAddition& GetAddition() const { return addition_; } + + // An accepted loss is recorded per Version, not on the physical file: older + // Versions without the loss can still read the file. This returns a copy of + // `backing` that is marked lost. The copy keeps `backing` alive, so the file + // still has exactly one owner object, whose release makes the file obsolete. + // A Version with the loss never reads the file, even if a copy of the file + // appears on disk again later. + static std::shared_ptr CreateMarkedLost( + std::shared_ptr backing) { + std::shared_ptr marked = + Create(backing->GetAddition()); + marked->lost_ = true; + marked->backing_metadata_ = std::move(backing); + return marked; + } + bool IsMarkedLost() const { return lost_; } + + private: + explicit BlobRelocationFileMetaData(BlobRelocationFileAddition addition) + : addition_(std::move(addition)) {} + + BlobRelocationFileAddition addition_; + std::shared_ptr backing_metadata_; + bool lost_ = false; +}; + +// A route: sends reads of an old file number (a reference's generation number) +// to the relocation file that holds its live values now. The MANIFEST stores +// it, and the Version owns it. Validate() checks only the record itself; +// VersionBuilder checks it against the Version (see the top of this file). +class BlobRelocationRoute { + public: + BlobRelocationRoute() = default; + BlobRelocationRoute(uint64_t old_file_number, uint64_t target_file_number) + : old_file_number_(old_file_number), + target_file_number_(target_file_number) {} + + // The old file number. + uint64_t GetOldFileNumber() const { return old_file_number_; } + // The target: a live relocation file. + uint64_t GetTargetFileNumber() const { return target_file_number_; } + + Status Validate() const; + void EncodeTo(std::string* output) const; + Status DecodeFrom(Slice* input); + std::string DebugString() const; + + private: + uint64_t old_file_number_ = kInvalidBlobFileNumber; + uint64_t target_file_number_ = kInvalidBlobFileNumber; +}; + +// The MANIFEST record that deletes a route. It holds only the old file number. +class BlobRelocationRouteDeletion { + public: + BlobRelocationRouteDeletion() = default; + explicit BlobRelocationRouteDeletion(uint64_t old_file_number) + : old_file_number_(old_file_number) {} + + uint64_t GetOldFileNumber() const { return old_file_number_; } + + void EncodeTo(std::string* output) const; + Status DecodeFrom(Slice* input); + + private: + uint64_t old_file_number_ = kInvalidBlobFileNumber; +}; + +// The BlobFileTable of one Version of a column family that has used blob +// indirection: its format fence is set (the fence stays after +// enable_blob_indirection is turned off, because feature data may remain). The +// read path and VersionBuilder use it to look up a file number. It maps a file +// number to one of: +// - a live blob file, +// - a live relocation file, or +// - a route (old file -> target). +// The three key sets never overlap. File number 0 (kInvalidBlobFileNumber) is +// never a key. A read of a reference looks up its generation number once, plus +// once more for the target when it finds a route. +// +// Shards: the table is an array of shards. The shard of a key is fmix64(key) & +// (shard count - 1). The shard count is a power of two, chosen so that a shard +// holds about kTargetEntriesPerShard (32) entries. A shard with 32 keys of 4 +// bytes has 128 bytes of keys (about two cache lines). A lookup scans these +// keys. The SSE2 scan stops at the first match; the plain scan reads all keys. +// The value 32 is an estimate, not a measured best. Apply() picks a new shard +// count when the average shard holds more than 2 * kTargetEntriesPerShard (64) +// entries, or fewer than kTargetEntriesPerShard / 4 (8) entries while there is +// more than one shard. The new count is the smallest power of two that gives at +// most 32 entries per shard on average. The gap between 8 and 64 keeps a table +// from resizing again after a few edits. +// +// Shard layout (one allocation per shard): +// +// +--------+-------------------+-----------------+-----------------+ +// | Shard | keys | values | route counts | +// | header | (padded count) | (count) | (count) | +// +--------+-------------------+-----------------+-----------------+ +// +// - keys: sorted file numbers. They are 32-bit when every key of the shard +// fits in 32 bits, else 64-bit. They are padded with 0 to a multiple of 4, +// so the SSE2 scan can always load whole 16-byte groups; 0 never matches a +// key. +// - values: one 64-bit tagged value per key (see below). +// - route counts: one 32-bit count per key: the number of routes to this +// relocation file, 0 for other entries. VersionBuilder uses it to reject +// the deletion of a file that a route still targets, without a scan of all +// routes. +// +// Tagged value: metadata pointers are 8-byte aligned, so the low 3 bits are +// free for tags. +// - route: target << 1 | 1 (bit 0 set). File numbers are at most +// kMaxBlobRelocationFileNumber (2^62 - 1), so the target always fits. +// - live blob file: the BlobFileMetaData pointer, low 3 bits 0. +// - live relocation file: the BlobRelocationFileMetaData pointer, with bit 1 +// (value 2) set. Bit 2 (value 4) marks a relocation file that a loss +// record (kBlobPhysicalFileLoss) names. +// +// Ownership: the table does not own the metadata. The Version's blob file and +// relocation file vectors own it, and they always hold the same files as the +// table. +// +// Copy-on-write: a table is immutable once its Version is published, so +// Versions share memory at two levels. +// 1. A new Version whose edits change no table entry shares the whole +// BlobRelocationState of its base Version (one shared_ptr copy; see +// VersionStorageInfo::MutableBlobRelocationState()). For example, a +// compaction that writes no blob file and drops no blob reference. A +// dropped reference adds garbage to a blob file, which gives the file +// new BlobFileMetaData and so changes its table entry. +// 2. Otherwise the new Version copies the BlobRelocationState. The copy of +// the table copies only the top array of shard pointers and adds one +// reference to each shard. Apply() then builds a new shard for each shard +// that the edit touches. The new table drops its reference to the old +// shard, but other tables keep using it. A shard is freed when its last +// table drops it. +// +// Example: a flush adds blob file 205 to Version 100, which makes Version 101. +// The table has 4 shards, and file 205 falls in shard 2. All numbers are +// example values. +// +// Version 100 table Version 101 table +// top array top array (a copy) +// +-----+ +-----+ +// | [0] |-----> shard A <-------| [0] | +// | [1] |-----> shard B <-------| [1] | +// | [2] |-----> shard C | [2] |-----> shard C2 (C plus file 205) +// | [3] |-----> shard D <-------| [3] | +// +-----+ +-----+ +// +// References: A, B and D have 2 each. C and C2 have 1 each. When Version 100 +// is freed, A, B and D go down to 1, and C is freed. +// +// Memory per shard (from the struct sizes; allocator overhead is not +// counted): +// - Shard header: 16 bytes (refs, count, padded, wide; four 32-bit fields). +// - Each entry: a 4-byte key (8 bytes when a key of the shard does not fit +// in 32 bits), an 8-byte tagged value and a 4-byte route count, so 16 +// bytes. The keys are padded to a multiple of 4. +// - A full shard of 32 entries: 16 + 32 * 16 = 528 bytes. +// - The top array: 8 bytes per shard. +// +// Memory example. The numbers are estimates computed from the sizes above, +// not measurements. A column family has 10,000 entries (live blob files, +// relocation files and routes together), in a table built in one step (for +// example, the table of the first fenced Version): +// - The shard count is 512, the smallest power of two that gives at most 32 +// entries per shard on average. A shard holds about 20 entries and takes +// about 330 bytes. +// - A table that grows edit by edit is rebuilt only when the average shard +// holds more than 64 entries, so at 10,000 entries it can have 256 shards +// of about 39 entries (about 640 bytes each). Its total size is about the +// same. +// - One full table takes about 175 KB: about 171 KB of shards and a 4 KB +// top array. +// - A new Version of case 1 above adds no table memory. +// - A new Version of case 2 adds the 4 KB top array plus about 330 bytes for +// each touched shard. A flush that adds one blob file touches one shard +// (about 4.3 KB). An indirect blob GC edit that adds relocation file 20 +// and turns blob file 12 into route 12 -> 20 touches at most 2 shards +// (about 4.7 KB). It also copies the relocation file vector (16 bytes per +// relocation file). +// - Ten live Versions that each differ from the previous one by such an +// edit take about 175 KB + 10 * 4.7 KB = 222 KB. Ten full copies would +// take about 1.75 MB. +// +// Worst case: when Apply() picks a new shard count, or one edit touches every +// shard, the new Version builds all shards, as a full copy would. A resize +// happens only when the average shard holds more than 64 or fewer than 8 +// entries, so it is rare (see "Shards" above). +class BlobFileTable { + public: + static constexpr size_t kTargetEntriesPerShard = 32; + + // One change for Apply(): set the entry of `file_number` to the tagged + // `value` and the route count `count`. A `value` of 0 erases the entry. + struct Op { + uint64_t file_number = kInvalidBlobFileNumber; + uint64_t value = 0; + uint32_t count = 0; + }; + + static uint64_t BlobFileValue(const BlobFileMetaData* file) { + const uint64_t value = reinterpret_cast(file); + assert((value & 7) == 0); + return value; + } + static uint64_t RelocationFileValue(const BlobRelocationFileMetaData* file, + bool lost) { + const uint64_t value = reinterpret_cast(file); + assert((value & 7) == 0); + return value | 2 | (lost ? 4 : 0); + } + static uint64_t RouteValue(uint64_t target) { return target << 1 | 1; } + static bool IsRoute(uint64_t value) { return (value & 1) != 0; } + static uint64_t RouteTarget(uint64_t value) { return value >> 1; } + static bool IsRelocationFile(uint64_t value) { + return value != 0 && !IsRoute(value) && (value & 2) != 0; + } + static bool IsBlobFile(uint64_t value) { + return value != 0 && !IsRoute(value) && (value & 2) == 0; + } + static bool IsLost(uint64_t value) { + return IsRelocationFile(value) && (value & 4) != 0; + } + static const BlobFileMetaData* AsBlobFile(uint64_t value) { + assert(IsBlobFile(value)); + assert((value & 7) == 0); + return reinterpret_cast(value & ~uint64_t{7}); + } + static const BlobRelocationFileMetaData* AsRelocationFile(uint64_t value) { + assert(IsRelocationFile(value)); + return reinterpret_cast(value & + ~uint64_t{7}); + } + + BlobFileTable() = default; + BlobFileTable(const BlobFileTable& other); + BlobFileTable& operator=(const BlobFileTable& other); + ~BlobFileTable(); + + // The tagged value of `file_number`, or 0. `count`, when not null, gets the + // entry's route count (0 when there is no entry). + uint64_t Find(uint64_t file_number, uint32_t* count = nullptr) const; + + // Applies `ops` in order. It builds new copies of the touched shards, or of + // all shards when the shard count changes. Other copies of the table keep the + // old shards. + void Apply(const std::vector& ops); + size_t size() const { return size_; } + // Bytes of the top array and the shards, shared ones included. + size_t ApproximateMemoryUsage() const; + + // Charges the table memory to `manager`, the cache reservation for file + // metadata: the top array of every table copy, and each shard once, when it + // is built. The charge is best effort: a failed charge never fails an edit. + // The call has an effect only on a table without shards, because a table with + // shards may share them with tables of another manager. Copies of the table + // share the manager. + void SetCacheReservationManager( + std::shared_ptr manager); + // Visits every (file number, value, count), in no particular order. + template + void ForEach(Fn&& fn) const { + for (const Shard* shard : shards_) { + if (shard != nullptr) { + for (uint32_t i = 0; i < shard->count; ++i) { + fn(shard->Key(i), shard->values()[i], shard->counts()[i]); + } + } + } + } + + // Tests choose the key scan; production uses SSE2 when it is available. + static void TEST_UsePlainScan(bool plain); + + private: + struct Shard; + struct Entry { + uint64_t key; + uint64_t value; + uint32_t count; + }; + static Shard* MakeShard(const std::vector& run); + static void Ref(Shard* shard); + + // Returns the bytes freed: the shard's size if this was the last reference. + static size_t Unref(Shard* shard); + void Rebuild(size_t shard_count, size_t* added, size_t* freed); + void Charge(size_t added, size_t freed) const; + size_t TopBytes() const { return shards_.capacity() * sizeof(Shard*); } + + std::vector shards_; + size_t size_ = 0; + std::shared_ptr charge_; + + struct Shard { + std::atomic refs; + uint32_t count; + uint32_t padded; + uint32_t wide; + const uint32_t* keys32() const { + return reinterpret_cast(this + 1); + } + uint32_t* keys32() { return reinterpret_cast(this + 1); } + const uint64_t* keys64() const { + return reinterpret_cast(this + 1); + } + uint64_t* keys64() { return reinterpret_cast(this + 1); } + size_t KeyBytes() const { return size_t{padded} * (wide ? 8 : 4); } + const uint64_t* values() const { + return reinterpret_cast( + reinterpret_cast(this + 1) + KeyBytes()); + } + uint64_t* values() { + return reinterpret_cast(reinterpret_cast(this + 1) + + KeyBytes()); + } + const uint32_t* counts() const { + return reinterpret_cast(values() + count); + } + uint32_t* counts() { return reinterpret_cast(values() + count); } + uint64_t Key(uint32_t i) const { return wide ? keys64()[i] : keys32()[i]; } + size_t Bytes() const { + return sizeof(Shard) + KeyBytes() + + size_t{count} * (sizeof(uint64_t) + sizeof(uint32_t)); + } + }; +}; + +// The relocation state of one Version of a fenced column family: its live +// relocation files, its routes and the file table (BlobFileTable), which also +// lists the live blob files. The read path and VersionBuilder use the table for +// lookups. A legacy column family has no relocation state (VersionStorageInfo +// holds a null pointer). Routes live only in the table. Relocation files are +// also kept in a vector sorted by file number, for iteration. Deleted files and +// routes are not kept (no deletion history). +// +// Ownership: while a VersionBuilder builds a new Version, it owns the state +// alone and changes it. A published Version exposes the state only through +// const handles. +class BlobRelocationState { + public: + using Files = std::vector>; + + BlobRelocationState() = default; + BlobRelocationState(const BlobRelocationState& base) + : files_(base.files_), + table_(base.table_), + route_count_(base.route_count_), + physical_file_bytes_(base.physical_file_bytes_), + record_bytes_(base.record_bytes_), + lost_file_count_(base.lost_file_count_), + digest_high_(base.digest_high_), + digest_low_(base.digest_low_), + aggregate_overflow_(base.aggregate_overflow_), + validated_(base.validated()) { + assert(base.pending_.empty()); + } + + // Sorted by file number. + const Files& files() const; + const BlobFileTable& table() const { + assert(pending_.empty()); + return table_; + } + size_t route_count() const { return route_count_; } + // Visits every route, in no particular order. + template + void ForEachRoute(Fn&& fn) const { + if (route_count_ == 0) { + return; + } + table().ForEach([&](uint64_t file_number, uint64_t value, uint32_t) { + if (BlobFileTable::IsRoute(value)) { + fn(BlobRelocationRoute(file_number, BlobFileTable::RouteTarget(value))); + } + }); + } + + // The total file size and the total record bytes of the live relocation + // files. SetFile() and DeleteFile() keep them up to date. + uint64_t PhysicalFileBytes() const { return physical_file_bytes_; } + uint64_t RecordBytes() const { return record_bytes_; } + // True once one of the two totals above would overflow. The totals then stop + // changing, and Version validation fails with Corruption. + bool AggregateOverflow() const { return aggregate_overflow_; } + // True when at least one relocation file is marked lost (an accepted loss). + bool HasLostFiles() const { return lost_file_count_ != 0; } + std::shared_ptr FindFile( + uint64_t file_number) const; + // Like FindFile(), without copying the shared pointer (read path). + const BlobRelocationFileMetaData* PeekFile(uint64_t file_number) const; + + // The target of the route of `old_file_number`, or kInvalidBlobFileNumber. + uint64_t FindRouteTarget(uint64_t old_file_number) const; + + // The number of routes to relocation file `file_number`. + size_t RouteCount(uint64_t file_number) const; + + // Mutators of a state that is not published yet. They buffer the table + // changes in `pending_`. Seal() applies the buffer to the table; it must run + // before table() is read or the state is published. + void SetFile(std::shared_ptr file); + void DeleteFile(uint64_t file_number); + void SetRoute(const BlobRelocationRoute& route); + void DeleteRoute(uint64_t old_file_number); + + // Live blob files of a fenced column family, for lookups. The Version's blob + // file vector owns them. + void SetBlobFile(const BlobFileMetaData* file); + void DeleteBlobFile(uint64_t file_number); + void Seal(); + + // Drops the relocation file metadata of a state that no Version uses any + // more. The metadata deleters use the column family, so the caller runs this + // under the DB mutex. The rest (the file table) can then be freed without the + // mutex. Only destruction may follow. + void ReleaseFiles() { files_.reset(); } + void SetCacheReservationManager( + std::shared_ptr manager) { + assert(pending_.empty()); + table_.SetCacheReservationManager(std::move(manager)); + } + bool sealed() const { return pending_.empty(); } + // Computes a 128-bit fingerprint of the relocation files and routes in O(1). + // SetFile(), DeleteFile(), SetRoute() and DeleteRoute() keep two running + // digest sums up to date, so the result does not depend on the order of the + // changes. The fingerprint also covers `format_version` and the file and + // route counts. + void ComputeFingerprint(uint32_t format_version, uint64_t* high, + uint64_t* low) const; + // Returns an estimate, meant as an upper bound, of the memory that this state + // keeps alive beyond `base` and that no other reservation charges. + // `changed_files` lists the relocation files that can differ between the two + // states. The file table is not counted (the reason is in + // IncrementalRetentionBytes() in blob_relocation_manifest.cc). + size_t IncrementalRetentionBytes( + const BlobRelocationState& base, + const std::vector& changed_files) const; + + // True when this state was fully checked, or when VersionBuilder checked + // every change on top of a checked base + // (VersionStorageInfo::MarkBlobRelocationChangesValidated()). Release builds + // then skip the full scan in + // VersionStorageInfo::ValidateBlobRelocationState(). A copy keeps the flag, + // and any staged change (Stage()) clears it. It is atomic because many + // threads read a published Version. + bool validated() const { return validated_.load(std::memory_order_relaxed); } + void MarkValidated() const { + validated_.store(true, std::memory_order_relaxed); + } + + private: + Files& MutableFiles(); + + // The tagged value of `file_number` and its route count, pending edits + // included. + uint64_t Lookup(uint64_t file_number, uint64_t* count = nullptr) const; + void Stage(uint64_t file_number, uint64_t value, uint64_t count); + void AdjustRouteCount(uint64_t target, int64_t delta); + + std::shared_ptr files_; + BlobFileTable table_; + // Table changes not applied yet (see Seal()): file number -> (new tagged + // value, route count). A value of 0 erases the entry. The count is 64-bit + // here, while the table keeps 32 bits: a build applies route changes one by + // one, so a target's count can for a moment be larger than its final count. + std::map> pending_; + size_t route_count_ = 0; + uint64_t physical_file_bytes_ = 0; + uint64_t record_bytes_ = 0; + size_t lost_file_count_ = 0; + // Sums of the digests of every relocation file and route. + uint64_t digest_high_ = 0; + uint64_t digest_low_ = 0; + bool aggregate_overflow_ = false; + mutable std::atomic validated_{false}; +}; + +bool operator==(const BlobRelocationRoute& lhs, const BlobRelocationRoute& rhs); +bool operator!=(const BlobRelocationRoute& lhs, const BlobRelocationRoute& rhs); + +} // namespace ROCKSDB_NAMESPACE diff --git a/db/blob/blob_source.cc b/db/blob/blob_source.cc index d4d11f54f006..bb74612f31ec 100644 --- a/db/blob/blob_source.cc +++ b/db/blob/blob_source.cc @@ -5,6 +5,7 @@ #include "db/blob/blob_source.h" +#include #include #include #include @@ -14,18 +15,44 @@ #include "db/blob/blob_contents.h" #include "db/blob/blob_file_reader.h" #include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" +#include "db/blob/same_file_blob_reader.h" +#include "db/version_edit.h" #include "file/random_access_file_reader.h" +#include "logging/logging.h" #include "memory/memory_allocator_impl.h" #include "monitoring/statistics_impl.h" #include "options/cf_options.h" #include "table/get_context.h" #include "table/multiget_context.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/math.h" +#include "util/mutexlock.h" namespace ROCKSDB_NAMESPACE { namespace { +Status ValidateIndirectBlobIndexForRead(const BlobIndex& blob_index, + bool is_relocation_file) { + if (!blob_index.IsIndirect()) { + return Status::InvalidArgument("Expected indirect blob index"); + } + Status status = blob_index.Validate(); + if (!status.ok()) { + return status; + } + if (!CompressionTypeSupported(blob_index.compression()) || + (is_relocation_file && + blob_index.size() > kMaxSimpleGen2BlobPayloadSize)) { + return Status::Corruption( + "Indirect blob size or compression is not supported"); + } + return Status::OK(); +} + Status AppendBlobRefreshRetryFailure(const Status& stale_status, const Status& retry_status) { assert(stale_status.IsCorruption()); @@ -37,6 +64,86 @@ Status AppendBlobRefreshRetryFailure(const Status& stale_status, stale_status, "; refresh retry failed: ", retry_status.ToString()); } +// Start and length of the file read for a whole value at `value_offset` in a +// blob log file. With checksum verification the read also covers the record +// header in front of the value. +std::pair WholeBlobReadExtent( + const ReadOptions& read_options, size_t user_key_size, + uint64_t value_offset, uint64_t value_size) { + const uint64_t adjustment = + read_options.verify_checksums + ? BlobLogRecord::CalculateAdjustmentForRecordHeader(user_key_size) + : 0; + return {value_offset >= adjustment ? value_offset - adjustment : 0, + value_size > std::numeric_limits::max() - adjustment + ? std::numeric_limits::max() + : value_size + adjustment}; +} + +constexpr size_t kMaxIndirectBatch = MultiGetContext::MAX_BATCH_SIZE; + +// One planned file read: its (start, length) and the index of its request. +struct PlannedRead { + uint64_t start = 0; + uint64_t length = 0; + size_t request = 0; + // A copy shares the source's record check, which includes the user key, so + // only reads for the same user key are duplicates. + Slice user_key; +}; +using PlannedReads = autovector; +using ReadBatch = autovector; +using ReadBatches = autovector; +// (request, request whose result it copies) for exact duplicate reads. +using DuplicateReads = autovector, kMaxIndirectBatch>; + +// Sorts `reads` by (start, length, user key) and splits them into the fewest +// batches whose reads do not overlap, because FileSystem::MultiRead requires +// non-overlapping requests. When no reads overlap (the usual case) there is one +// batch. The sort puts equal reads next to each other, so an exact duplicate of +// the previous read (same start, length and user key) is not read again; it +// goes to `duplicates`. Batches hold request indexes. +void PlanReadBatches(PlannedReads* reads, ReadBatches* batches, + DuplicateReads* duplicates) { + std::sort(reads->begin(), reads->end(), + [](const PlannedRead& lhs, const PlannedRead& rhs) { + if (lhs.start != rhs.start || lhs.length != rhs.length) { + return std::make_pair(lhs.start, lhs.length) < + std::make_pair(rhs.start, rhs.length); + } + return lhs.user_key.compare(rhs.user_key) < 0; + }); + autovector batch_ends; + const PlannedRead* previous = nullptr; + for (const PlannedRead& read : *reads) { + if (previous != nullptr && previous->start == read.start && + previous->length == read.length && + previous->user_key == read.user_key) { + duplicates->emplace_back(read.request, previous->request); + continue; + } + previous = &read; + const uint64_t end = + read.length > std::numeric_limits::max() - read.start + ? std::numeric_limits::max() + : read.start + read.length; + size_t batch = batch_ends.size(); + for (size_t i = 0; i < batch_ends.size(); ++i) { + if (batch_ends[i] <= read.start) { + batch = i; + break; + } + } + if (batch == batch_ends.size()) { + batches->emplace_back(); + batch_ends.push_back(end); + } else { + batch_ends[batch] = end; + } + (*batches)[batch].push_back(read.request); + } +} + // Runs read(reader) against the blob-file reader for `file_number`. On a // Corruption -- which a stale cached reader (e.g. the physical file was // replaced) can surface -- evicts the cached reader, reopens it uncached, @@ -47,11 +154,15 @@ Status AppendBlobRefreshRetryFailure(const Status& stale_status, // `read` is invoked with a BlobFileReader* and returns a Status; because it may // run twice, it must re-initialize any output it populates on each call (and is // taken by const reference rather than forwarded, since it is not moved-from). -template +// +// `check(reader)` runs first on the cached reader. A metadata error it returns +// (for example a compression mismatch) is returned as is, without the evict and +// retry. +template Status ReadBlobWithReaderRetry(BlobFileCache* blob_file_cache, const ReadOptions& read_options, const BlobFileOpenInfo& blob_file, - const ReadFn& read) { + const CheckFn& check, const ReadFn& read) { CacheHandleGuard blob_file_reader; Status s = blob_file_cache->GetBlobFileReader(read_options, blob_file, &blob_file_reader); @@ -60,6 +171,10 @@ Status ReadBlobWithReaderRetry(BlobFileCache* blob_file_cache, } assert(blob_file_reader.GetValue()); + s = check(blob_file_reader.GetValue()); + if (!s.ok()) { + return s; + } s = read(blob_file_reader.GetValue()); if (!s.IsCorruption()) { return s; @@ -89,6 +204,59 @@ Status ReadBlobWithReaderRetry(BlobFileCache* blob_file_cache, return s; } +template +Status ReadBlobWithReaderRetry(BlobFileCache* blob_file_cache, + const ReadOptions& read_options, + const BlobFileOpenInfo& blob_file, + const ReadFn& read) { + return ReadBlobWithReaderRetry( + blob_file_cache, read_options, blob_file, + [](BlobFileReader*) { return Status::OK(); }, read); +} + +// Reads `requests` in the batches planned from `reads` (see PlanReadBatches) +// with read_batch(batch_reads, &bytes_read). Each value goes to +// *value_of(request); an exact duplicate gets its own copy, allocated from +// `allocator`, and is reported to on_duplicate(request) (it was not read from +// storage). Returns the bytes read. +template +uint64_t ReadInPlannedBatches(autovector* requests, + PlannedReads* reads, MemoryAllocator* allocator, + const ReadBatchFn& read_batch, + const ValueFn& value_of, + const DuplicateFn& on_duplicate) { + ReadBatches batches; + DuplicateReads duplicates; + PlanReadBatches(reads, &batches, &duplicates); + uint64_t total_bytes_read = 0; + for (const ReadBatch& batch : batches) { + autovector>> batch_reads; + for (size_t request : batch) { + batch_reads.emplace_back(&(*requests)[request], + std::unique_ptr()); + } + uint64_t batch_bytes_read = 0; + read_batch(batch_reads, &batch_bytes_read); + total_bytes_read += batch_bytes_read; + for (size_t i = 0; i < batch.size(); ++i) { + if ((*requests)[batch[i]].status->ok()) { + *value_of(batch[i]) = std::move(batch_reads[i].second); + } + } + } + for (const auto& [duplicate, source] : duplicates) { + on_duplicate(duplicate); + *(*requests)[duplicate].status = *(*requests)[source].status; + if ((*requests)[source].status->ok()) { + const Slice value = (*value_of(source))->data(); + *value_of(duplicate) = std::make_unique( + AllocateAndCopyBlock(value, allocator), value.size()); + } + } + return total_bytes_read; +} + // Records lazy wide-column read metrics for one actual storage read issued // while resolving a lazy result (attributed via Env::IOActivity::kLazyResolve): // `bytes_read` bytes were read from storage. Covers both whole-column and @@ -134,8 +302,137 @@ void PinCacheHitSubRange(CacheHandleGuard* blob_handle, blob_handle->TransferTo(value); } +namespace { +// Returns the domain (1, 2 or 3) of the stable cache keys of this DB and +// session. A stable key must never equal a physical blob cache key (the key of +// a physical file and offset) of the same DB and session. +// +// Why: stable keys live in the blob cache next to physical blob keys and +// embedded blob keys. When the blob cache is also the block cache, they also +// live next to SST block keys and cache reservation keys. A collision would +// return the bytes of another value. SST block keys and embedded blob keys are +// built the same way as physical blob keys: OffsetableCacheKey with a real file +// number. For keys of this DB and session, this proof applies to all of them +// ("physical key" below). Keys of another session differ as for any other cache +// key (see the end of this comment). For the key layout, see OffsetableCacheKey +// in cache/cache_key.h. +// +// Stable keys and physical blob keys are both built by OffsetableCacheKey from +// an internal id `id`, where id[0] = session_lower and id[1] = H ^ file +// number. H depends only on db_id and the session. +// +// Terms: +// - First word: the first 64 bits of the cache key. OffsetableCacheKey sets it +// to DownwardInvolution(id[0]) ^ ReverseBits(id[1]). The offset changes only +// the second word. +// - P: the low two bits of a physical key's first word. ReverseBits moves the +// top two bits of id[1] to the low two bits. File numbers are at most +// kFileNumberMask (62 bits), so they never set those top two bits, and P is +// the same for every file of this DB and session. +// - Domain (1, 2 or 3): a value that the stable key xors into the top two bits +// of id[1], like a file number that no real file can have. The low two bits +// of a stable first word are then P ^ domain. +// +// OffsetableCacheKey swaps a zero first word with the second word, +// ReverseBits(session_lower). So when P is 0, a physical first word can also +// have the low bits of that swapped word. One fixed domain is therefore not +// enough. The loop picks the first domain whose stable low bits are not 0 and, +// when P is 0, also differ from the low bits of the swapped word. At most one +// of the three domains is rejected: the domain equal to P when P != 0, or the +// domain that gives the swapped word's low bits when P == 0. The choice depends +// only on db_id and the session, not on the origin. A stable first word is +// never 0, so it is never swapped, and distinct origins give distinct first +// words. Reserved keys (CacheKey::CreateUnique*) have a first word of 0, so a +// stable key never equals one either. Keys of different sessions differ as for +// any other cache key. +uint64_t ChooseStableKeyDomain(const UniqueId64x2& id) { + const uint64_t physical_low_bits = + (DownwardInvolution(id[0]) ^ ReverseBits(id[1])) & 3; + const uint64_t swapped_low_bits = ReverseBits(id[0]) & 3; + for (uint64_t domain = 1; domain <= 3; ++domain) { + const uint64_t stable_low_bits = physical_low_bits ^ domain; + if (stable_low_bits != 0 && + (physical_low_bits != 0 || stable_low_bits != swapped_low_bits)) { + return domain; + } + } + assert(false); + return 1; +} } // namespace +CacheKey GetStableBlobCacheKey(const std::string& db_id, + const std::string& db_session_id, + uint64_t origin_file_number, + uint64_t original_value_offset) { + static_assert(kFileNumberMask == 0x3fffffffffffffffULL); + static_assert(BlobIndex::kMaxFileNumber == kFileNumberMask); + assert(origin_file_number != 0 && origin_file_number <= kFileNumberMask); + OffsetableCacheKey physical_base(db_id, db_session_id, origin_file_number); + UniqueId64x2 id = physical_base.ToInternalUniqueId(); + id[1] ^= ReverseBits(ChooseStableKeyDomain(id)); + return OffsetableCacheKey::FromInternalUniqueId(&id).WithOffset( + original_value_offset); +} + +// Applies the configured blob-cache memory charging policy, so reads and flush +// prepopulation are accounted against the block cache when charging is enabled. +std::shared_ptr GetBlobCacheWithReservation( + const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options) { + std::shared_ptr blob_cache = immutable_options.blob_cache; + const BlockBasedTableOptions* const bbto = + mutable_cf_options.table_factory->GetOptions(); + if (blob_cache && bbto && + bbto->cache_usage_options.options_overrides.at(CacheEntryRole::kBlobCache) + .charged == CacheEntryRoleOptions::Decision::kEnabled) { + blob_cache = std::make_shared(blob_cache, bbto->block_cache); + } + return blob_cache; +} + +} // namespace + +CacheKey TEST_GetStableBlobCacheKey(const std::string& db_id, + const std::string& db_session_id, + uint64_t origin_file_number, + uint64_t original_value_offset) { + return GetStableBlobCacheKey(db_id, db_session_id, origin_file_number, + original_value_offset); +} + +StableBlobCacheKeyBuilder::StableBlobCacheKeyBuilder( + const std::string& db_id, const std::string& db_session_id) { + UniqueId64x2 id; + Status s = GetSstInternalUniqueId(db_id, db_session_id, /*file_number=*/0, + &id, /*force=*/true); + assert(s.ok()); + s.PermitUncheckedError(); + session_lower_ = id[0]; + db_hash_ = id[1]; + if (session_lower_ == 0) { + slow_db_id_ = db_id; + slow_db_session_id_ = db_session_id; + use_slow_path_ = true; + return; + } + // The domain depends only on db_id and the session: file numbers are at most + // kFileNumberMask, so they do not change the top two bits of id[1]. + domain_bits_ = ReverseBits(ChooseStableKeyDomain(id)); +} + +CacheKey StableBlobCacheKeyBuilder::Get(uint64_t origin_file_number, + uint64_t original_value_offset) const { + assert(origin_file_number != 0 && origin_file_number <= kFileNumberMask); + if (use_slow_path_) { + return GetStableBlobCacheKey(slow_db_id_, slow_db_session_id_, + origin_file_number, original_value_offset); + } + UniqueId64x2 id{session_lower_, db_hash_ ^ origin_file_number ^ domain_bits_}; + return OffsetableCacheKey::FromInternalUniqueId(&id).WithOffset( + original_value_offset); +} + BlobSource::BlobSource(const ImmutableOptions& immutable_options, const MutableCFOptions& mutable_cf_options, const std::string& db_id, @@ -144,17 +441,33 @@ BlobSource::BlobSource(const ImmutableOptions& immutable_options, : db_id_(db_id), db_session_id_(db_session_id), statistics_(immutable_options.statistics.get()), + clock_(immutable_options.clock), blob_file_cache_(blob_file_cache), - blob_cache_(immutable_options.blob_cache), - lowest_used_cache_tier_(immutable_options.lowest_used_cache_tier) { - auto bbto = - mutable_cf_options.table_factory->GetOptions(); - if (bbto && - bbto->cache_usage_options.options_overrides.at(CacheEntryRole::kBlobCache) - .charged == CacheEntryRoleOptions::Decision::kEnabled) { - blob_cache_ = SharedCacheInterface{std::make_shared( - immutable_options.blob_cache, bbto->block_cache)}; - } + stable_cache_admission_mutex_(kNumberOfAdmissionMutexStripes), + blob_cache_storage_( + GetBlobCacheWithReservation(immutable_options, mutable_cf_options)), + blob_cache_(blob_cache_storage_), + stable_blob_cache_(blob_cache_storage_), + simple_gen2_decompressor_( + GetBuiltinV2CompressionManager()->GetDecompressor()), + lowest_used_cache_tier_(immutable_options.lowest_used_cache_tier) {} + +CacheKey BlobSource::StableCacheKey(uint64_t origin_file_number, + uint64_t original_value_offset) const { + assert(stable_blob_cache_); + std::call_once(stable_key_builder_once_, [this]() { + stable_key_builder_ = + std::make_unique(db_id_, db_session_id_); + }); + CacheKey key = + stable_key_builder_->Get(origin_file_number, original_value_offset); + // The builder's key must equal the slow path's key for the current db_id and + // session id. + assert(key.AsSlice() == GetStableBlobCacheKey(db_id_, db_session_id_, + origin_file_number, + original_value_offset) + .AsSlice()); + return key; } BlobSource::~BlobSource() = default; @@ -221,6 +534,142 @@ Status BlobSource::PutBlobIntoCache( return s; } +Status BlobSource::GetStableBlobFromCache( + const Slice& cache_key, CacheHandleGuard* cached_blob, + bool record_statistics, bool primary_only) const { + assert(stable_blob_cache_); + assert(cached_blob != nullptr); + assert(cached_blob->IsEmpty()); + + StableTypedHandle* const cache_handle = + primary_only + ? stable_blob_cache_.Lookup(cache_key) + : stable_blob_cache_.LookupFull(cache_key, nullptr /* context */, + Cache::Priority::BOTTOM, statistics_, + lowest_used_cache_tier_); + if (cache_handle == nullptr) { + if (record_statistics) { + RecordTick(statistics_, BLOB_DB_CACHE_MISS); + } + return Status::NotFound("Stable blob not found in cache"); + } + + *cached_blob = CacheHandleGuard(stable_blob_cache_.get(), + cache_handle); + const StableBlobContents* const contents = cached_blob->GetValue(); + assert(contents != nullptr); + Status status = contents->validation_status(); + if (!status.ok()) { + cached_blob->Reset(); + stable_blob_cache_.get()->Erase(cache_key); + return status; + } + if (record_statistics) { + PERF_COUNTER_ADD(blob_cache_hit_count, 1); + PERF_COUNTER_ADD(blob_cache_read_byte, contents->size()); + RecordTick(statistics_, BLOB_DB_CACHE_HIT); + RecordTick(statistics_, BLOB_DB_CACHE_BYTES_READ, contents->size()); + } + return Status::OK(); +} + +Status BlobSource::PutStableBlobIntoCache( + const Slice& cache_key, const BlobIndex& blob_index, + std::unique_ptr* logical_value, + CacheHandleGuard* cached_blob) const { + assert(stable_blob_cache_); + assert(blob_index.IsIndirect()); + assert(logical_value != nullptr && *logical_value != nullptr); + assert(cached_blob != nullptr); + assert(cached_blob->IsEmpty()); + + // FinishIndirectBlobRead has checked the logical CRC. The entry adopts the + // read buffer, so there is no copy, and it is built outside the lock. + const size_t value_size = (*logical_value)->size(); + std::unique_ptr contents; + Status status = StableBlobContents::CreateWithVerifiedAllocation( + (*logical_value)->ReleaseAllocation(), value_size, + blob_index.fingerprint(), &contents); + logical_value->reset(); + if (!status.ok()) { + return status; + } + + // This lock is taken only on a miss, after a file read. Between the check + // for an existing entry and the insert, it keeps a second read of the same + // value from inserting another entry, so readers keep using the first one. + // The first lookup already checked the secondary cache, so this lookup checks + // only the primary cache. + MutexLock admission_lock(&stable_cache_admission_mutex_.Get(cache_key)); + status = GetStableBlobFromCache(cache_key, cached_blob, + /*record_statistics=*/false, + /*primary_only=*/true); + if (status.ok() || !status.IsNotFound()) { + return status; + } + + return InsertStableBlob(cache_key, &contents, cached_blob); +} + +Status BlobSource::InsertStableBlob( + const Slice& cache_key, std::unique_ptr* contents, + CacheHandleGuard* cached_blob) const { + assert(contents != nullptr && *contents != nullptr); + const size_t value_size = (*contents)->size(); + StableTypedHandle* cache_handle = nullptr; + const Status status = stable_blob_cache_.InsertFull( + cache_key, contents->get(), (*contents)->ApproximateMemoryUsage(), + &cache_handle, Cache::Priority::BOTTOM, lowest_used_cache_tier_); + if (!status.ok()) { + RecordTick(statistics_, BLOB_DB_CACHE_ADD_FAILURES); + return status; + } + + [[maybe_unused]] StableBlobContents* const released = contents->release(); + assert(cache_handle != nullptr); + CacheHandleGuard guard(stable_blob_cache_.get(), + cache_handle); + if (cached_blob != nullptr) { + *cached_blob = std::move(guard); + } + RecordTick(statistics_, BLOB_DB_CACHE_ADD); + RecordTick(statistics_, BLOB_DB_CACHE_BYTES_WRITE, value_size); + return Status::OK(); +} + +Status BlobSource::PrepopulateFlushBlob( + const Slice& value, uint64_t blob_file_number, uint64_t blob_offset, + const StableBlobFingerprint* fingerprint) const { + if (!blob_cache_) { + return Status::OK(); + } + if (fingerprint == nullptr) { + const CacheKey cache_key = + GetCacheKey(blob_file_number, /*file_size=*/0, blob_offset); + const Status s = blob_cache_.InsertSaved( + cache_key.AsSlice(), value, nullptr /*context*/, + Cache::Priority::BOTTOM, lowest_used_cache_tier_); + if (s.ok()) { + RecordTick(statistics_, BLOB_DB_CACHE_ADD); + RecordTick(statistics_, BLOB_DB_CACHE_BYTES_WRITE, value.size()); + } else { + RecordTick(statistics_, BLOB_DB_CACHE_ADD_FAILURES); + } + return s; + } + std::unique_ptr contents; + Status s = StableBlobContents::CreateWithVerifiedValue( + value, *fingerprint, stable_blob_cache_.get()->memory_allocator(), + &contents); + if (!s.ok()) { + RecordTick(statistics_, BLOB_DB_CACHE_ADD_FAILURES); + return s; + } + const CacheKey cache_key = StableCacheKey(blob_file_number, blob_offset); + return InsertStableBlob(cache_key.AsSlice(), &contents, + /*cached_blob=*/nullptr); +} + BlobSource::TypedHandle* BlobSource::GetEntryFromCache(const Slice& key) const { return blob_cache_.LookupFull(key, nullptr /* context */, Cache::Priority::BOTTOM, statistics_, @@ -264,6 +713,112 @@ void BlobSource::PinOwnedBlob(std::unique_ptr* owned_blob, blob, nullptr); } +Status BlobSource::PinStableCacheHit( + CacheHandleGuard* cached_blob, uint64_t range_offset, + size_t range_length, PinnableSlice* value) { + const size_t logical_size = cached_blob->GetValue()->size(); + if (range_length != kWholeBlobLength && + (range_offset > logical_size || + range_length > logical_size - static_cast(range_offset))) { + return Status::InvalidArgument("Stable blob range is out of bounds"); + } + PinStableCachedBlob(cached_blob, range_offset, range_length, value); + return Status::OK(); +} + +Status BlobSource::GetIndirectBlobFromCache(const BlobIndex& blob_index, + uint64_t range_offset, + size_t range_length, + PinnableSlice* value) const { + assert(value != nullptr); + assert(blob_index.IsIndirect()); + value->Reset(); + if (range_length != kWholeBlobLength && + blob_index.compression() != kNoCompression) { + // A range of a compressed value is not supported, also on a cache hit (see + // GetIndirectBlob()). Checking here means that no caller can miss it. + return Status::Corruption("Cannot range-read a compressed blob"); + } + if (!stable_blob_cache_) { + return Status::NotFound("No stable blob cache"); + } + const CacheKey stable_key = StableCacheKey( + blob_index.origin_file_number(), blob_index.original_value_offset()); + CacheHandleGuard cached_blob; + const Status status = GetStableBlobFromCache( + stable_key.AsSlice(), &cached_blob, + /*record_statistics=*/true, /*primary_only=*/false); + if (!status.ok()) { + return status; + } + return PinStableCacheHit(&cached_blob, range_offset, range_length, value); +} + +void BlobSource::PinStableCachedBlob( + CacheHandleGuard* cached_blob, uint64_t range_offset, + size_t range_length, PinnableSlice* value) { + assert(cached_blob != nullptr); + assert(cached_blob->GetValue() != nullptr); + assert(value != nullptr); + + const Slice full = cached_blob->GetValue()->data(); + Slice selected = full; + if (range_length != kWholeBlobLength) { + assert(range_offset <= full.size()); + assert(range_length <= full.size() - range_offset); + selected = + Slice(full.data() + static_cast(range_offset), range_length); + } + value->Reset(); + value->PinSlice(selected, static_cast(nullptr)); + cached_blob->TransferTo(value); +} + +Status BlobSource::FinishIndirectBlobRead( + const ReadOptions& read_options, const BlobIndex& blob_index, + const CacheKey* stable_cache_key, bool materialized_whole, + std::unique_ptr* physical_value, + const std::optional& value_crc32c, PinnableSlice* result) const { + assert(blob_index.IsIndirect()); + assert(physical_value != nullptr && *physical_value != nullptr); + assert(result != nullptr); + if (!materialized_whole) { + PinOwnedBlob(physical_value, result); + return Status::OK(); + } + const bool fill_stable_cache = + stable_cache_key != nullptr && read_options.fill_cache; + const Slice value = (*physical_value)->data(); + if (read_options.verify_checksums || fill_stable_cache) { + // The reader may have computed the CRC while it checked the record. + const uint32_t checksum = value_crc32c.has_value() + ? *value_crc32c + : crc32c::Value(value.data(), value.size()); + if (checksum != blob_index.stable_checksum()) { + return Status::Corruption( + "Stable blob checksum mismatch", + "generation " + std::to_string(blob_index.generation_file_number()) + + ", origin " + std::to_string(blob_index.origin_file_number()) + + ", original offset " + + std::to_string(blob_index.original_value_offset())); + } + } + + if (fill_stable_cache) { + CacheHandleGuard cached_blob; + Status status = PutStableBlobIntoCache( + stable_cache_key->AsSlice(), blob_index, physical_value, &cached_blob); + if (!status.ok()) { + return status; + } + PinStableCachedBlob(&cached_blob, /*range_offset=*/0, kWholeBlobLength, + result); + return Status::OK(); + } + PinOwnedBlob(physical_value, result); + return Status::OK(); +} + Status BlobSource::InsertEntryIntoCache(const Slice& key, BlobContents* value, TypedHandle** cache_handle, Cache::Priority priority) const { @@ -465,6 +1020,519 @@ Status BlobSource::GetBlobRange(const ReadOptions& read_options, return s; } +Status BlobSource::GetIndirectBlob( + const ReadOptions& read_options, const Slice& user_key, + const BlobIndex& blob_index, const BlobPhysicalFile& file, + const MutableCFOptions& mutable_cf_options, uint64_t range_offset, + size_t range_length, FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* value, uint64_t* bytes_read, bool cache_checked) { + assert(value != nullptr); + value->Reset(); + if (bytes_read != nullptr) { + *bytes_read = 0; + } + if (range_length != kWholeBlobLength && + blob_index.compression() != kNoCompression) { + // As in base RocksDB, a range of a compressed value is not supported, also + // when the stable cache holds the value: a byte range cannot be + // decompressed alone. Callers read the whole value and slice it. + return Status::Corruption("Cannot range-read a compressed blob"); + } + Status status; + if (!cache_checked) { + status = + GetIndirectBlobFromCache(blob_index, range_offset, range_length, value); + if (!status.IsNotFound()) { + return status; + } + } + status = ValidateIndirectBlobIndexForRead(blob_index, file.is_relocation); + if (!status.ok()) { + return status; + } + if (range_length != kWholeBlobLength && + (range_offset > blob_index.size() || + static_cast(range_length) > + blob_index.size() - range_offset)) { + return Status::InvalidArgument("Stable blob range is out of bounds"); + } + if (file.open_info.file_number == 0 || file.file_size == 0) { + return Status::Corruption("Invalid indirect blob physical file"); + } + + // The stable cache key is derived only when there is a stable cache. It is + // used to fill the cache after the read. + CacheKey stable_key; + if (stable_blob_cache_) { + stable_key = StableCacheKey(blob_index.origin_file_number(), + blob_index.original_value_offset()); + } + if (read_options.read_tier == kBlockCacheTier) { + return Status::Incomplete("Cannot read indirect blob: no disk I/O allowed"); + } + + const bool materialize_whole = range_length == kWholeBlobLength; + // A value that will be cached ends up in a buffer from the cache's allocator, + // so the cache entry can adopt the buffer without another copy. A relocation + // record is read directly into that buffer. A normal blob file value is read + // into a local buffer and copied (or decompressed) into it once, as in base. + MemoryAllocator* const allocator = + stable_blob_cache_ && read_options.fill_cache + ? stable_blob_cache_.get()->memory_allocator() + : nullptr; + // The reader computes the value CRC while it checks the record, when it can + // (whole uncompressed values). + std::optional value_crc32c; + std::optional* const value_crc32c_out = + materialize_whole && blob_index.compression() == kNoCompression + ? &value_crc32c + : nullptr; + std::unique_ptr physical_value; + uint64_t physical_bytes_read = 0; + if (file.is_relocation) { + CacheHandleGuard reader; + status = blob_file_cache_->GetRelocationFileReader( + read_options, file, mutable_cf_options, &reader); + if (!status.ok()) { + return status; + } + SameFileBlobStandaloneRead output; + RelocationBlobReadRequest request; + request.blob_index = &blob_index; + request.range_offset = materialize_whole ? 0 : range_offset; + request.range_length = materialize_whole ? kWholeBlobLength : range_length; + request.status = &status; + request.output = &output; + reader.GetValue()->relocation()->MultiGetBlob(read_options, allocator, + /*num_reads=*/1, &request, + &physical_bytes_read); + physical_value = std::move(output.contents); + if (value_crc32c_out != nullptr) { + value_crc32c = output.gen2.payload_crc32c; + } + } else { + status = ReadBlobWithReaderRetry( + blob_file_cache_, read_options, file.open_info, + [&](BlobFileReader* reader) { + // A metadata error: an evict and reopen cannot fix it. + return reader->GetCompressionType() == blob_index.compression() + ? Status::OK() + : Status::Corruption( + "Compression type mismatch when reading indirect " + "blob"); + }, + [&](BlobFileReader* reader) { + physical_value.reset(); + value_crc32c.reset(); + physical_bytes_read = 0; + if (materialize_whole) { + return reader->GetBlob(read_options, user_key, + blob_index.original_value_offset(), + blob_index.size(), blob_index.compression(), + prefetch_buffer, allocator, &physical_value, + &physical_bytes_read, value_crc32c_out); + } + return reader->GetBlobRange( + read_options, user_key, blob_index.original_value_offset(), + blob_index.size(), range_offset, range_length, + /*allocator=*/nullptr, &physical_value, &physical_bytes_read); + }); + } + if (!status.ok()) { + return status; + } + if (bytes_read != nullptr) { + *bytes_read = physical_bytes_read; + } + // BlobFileReader records these for blob log files. The relocation reader + // reads through the table reader, which does not record them, so BlobSource + // records them here. + if (file.is_relocation) { + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, physical_bytes_read); + PERF_COUNTER_ADD(blob_read_count, 1); + PERF_COUNTER_ADD(blob_read_byte, physical_bytes_read); + } + // BlobFileReader and the relocation table reader do not record lazy-read + // statistics, so BlobSource records them here. A stable cache hit returns + // earlier, so it is not counted. + if (read_options.io_activity == Env::IOActivity::kLazyResolve) { + RecordLazyRead(statistics_, physical_bytes_read); + if (!materialize_whole) { + RecordLazyPartialRead(statistics_, physical_bytes_read, + blob_index.size()); + } + } + + return FinishIndirectBlobRead( + read_options, blob_index, stable_blob_cache_ ? &stable_key : nullptr, + materialize_whole, &physical_value, value_crc32c, value); +} + +void BlobSource::MultiGetIndirectBlobFromOneFile( + const ReadOptions& read_options, const BlobPhysicalFile& file, + const MutableCFOptions& mutable_cf_options, size_t num_requests, + IndirectBlobReadRequest* requests, uint64_t* bytes_read) { + if (bytes_read != nullptr) { + *bytes_read = 0; + } + if (num_requests == 0) { + return; + } + assert(requests != nullptr); + if (file.open_info.file_number == 0 || file.file_size == 0) { + for (size_t i = 0; i < num_requests; ++i) { + assert(requests[i].status != nullptr); + *requests[i].status = + Status::Corruption("Invalid indirect blob physical file"); + } + return; + } + // One caller, Version::MultiGetBlob, does not limit the number of requests + // per file. The fixed-size arrays below hold at most kMaxIndirectBatch + // (MultiGetContext::MAX_BATCH_SIZE, 32) requests, so a larger call is split + // into chunks of that size. A chunk cannot overflow the uint64_t byte counts + // below: a relocation value is at most kMaxSimpleGen2BlobPayloadSize (about + // 4 GiB), and a normal blob file read must fit inside the file. + if (num_requests > kMaxIndirectBatch) { + uint64_t total_bytes_read = 0; + for (size_t begin = 0; begin < num_requests; begin += kMaxIndirectBatch) { + const size_t count = std::min(num_requests - begin, kMaxIndirectBatch); + uint64_t chunk_bytes_read = 0; + MultiGetIndirectBlobFromOneFile(read_options, file, mutable_cf_options, + count, requests + begin, + &chunk_bytes_read); + total_bytes_read += chunk_bytes_read; + } + if (bytes_read != nullptr) { + *bytes_read = total_bytes_read; + } + return; + } + + // A request that needs a file read, with its physical result. + struct Pending { + size_t request = 0; + // Whole value, else an uncompressed range read alone. + bool whole = false; + // Set only with a stable cache. + CacheKey stable_key; + // The read outputs: + // - out.contents: the physical value. + // - out.gen2.payload_crc32c: set by the reader when it computed the value + // CRC while checking the record. + // - out.gen2.copied: true when the value was copied from an identical read + // in the batch, not read from storage. + SameFileBlobStandaloneRead out; + }; + autovector pending; + const bool no_io = read_options.read_tier == kBlockCacheTier; + for (size_t i = 0; i < num_requests; ++i) { + IndirectBlobReadRequest& request = requests[i]; + assert(request.blob_index != nullptr); + assert(request.result != nullptr); + assert(request.status != nullptr); + const BlobIndex& blob_index = *request.blob_index; + request.result->Reset(); + if (request.range_length != kWholeBlobLength && + blob_index.compression() != kNoCompression) { + // A range of a compressed value is not supported (see GetIndirectBlob()). + *request.status = + Status::Corruption("Cannot range-read a compressed blob"); + continue; + } + if (!request.cache_checked) { + *request.status = + GetIndirectBlobFromCache(blob_index, request.range_offset, + request.range_length, request.result); + if (!request.status->IsNotFound()) { + continue; + } + } + *request.status = + ValidateIndirectBlobIndexForRead(blob_index, file.is_relocation); + if (!request.status->ok()) { + continue; + } + CacheKey stable_key; + if (stable_blob_cache_) { + stable_key = StableCacheKey(blob_index.origin_file_number(), + blob_index.original_value_offset()); + } + const bool whole = request.range_length == kWholeBlobLength; + if (!whole && (request.range_offset > blob_index.size() || + static_cast(request.range_length) > + blob_index.size() - request.range_offset)) { + *request.status = + Status::InvalidArgument("Stable blob range is out of bounds"); + continue; + } + if (no_io) { + *request.status = + Status::Incomplete("Cannot read indirect blob: no disk I/O allowed"); + continue; + } + *request.status = Status::OK(); + pending.emplace_back(); + pending.back().request = i; + pending.back().whole = whole; + pending.back().stable_key = stable_key; + } + if (pending.empty()) { + return; + } + + // A value that will be cached ends up in a buffer from the cache's allocator, + // so the cache entry can adopt the buffer without another copy. A relocation + // record is read directly into that buffer. A normal blob file value is read + // into a local buffer and copied (or decompressed) into it once, as in base. + MemoryAllocator* const allocator = + stable_blob_cache_ && read_options.fill_cache + ? stable_blob_cache_.get()->memory_allocator() + : nullptr; + const auto value_crc32c_out = [requests](Pending& p) { + return p.whole && requests[p.request].blob_index->compression() == + kNoCompression + ? &p.out.gen2.payload_crc32c + : nullptr; + }; + uint64_t physical_bytes_read = 0; + if (file.is_relocation) { + CacheHandleGuard reader; + Status open_status = blob_file_cache_->GetRelocationFileReader( + read_options, file, mutable_cf_options, &reader); + if (!open_status.ok()) { + for (const Pending& p : pending) { + *requests[p.request].status = open_status; + } + return; + } + // At most kMaxIndirectBatch requests, kept contiguous for the reader. + std::array + relocation_requests; + size_t relocation_count = 0; + for (Pending& p : pending) { + const IndirectBlobReadRequest& request = requests[p.request]; + RelocationBlobReadRequest& relocation_request = + relocation_requests[relocation_count++]; + relocation_request.blob_index = request.blob_index; + relocation_request.range_offset = p.whole ? 0 : request.range_offset; + relocation_request.range_length = + p.whole ? kWholeBlobLength : request.range_length; + relocation_request.status = request.status; + relocation_request.output = &p.out; + } + reader.GetValue()->relocation()->MultiGetBlob( + read_options, allocator, relocation_count, relocation_requests.data(), + &physical_bytes_read); + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, physical_bytes_read); + PERF_COUNTER_ADD(blob_read_byte, physical_bytes_read); + } else { + CacheHandleGuard reader; + Status open_status = blob_file_cache_->GetBlobFileReader( + read_options, file.open_info, &reader); + if (!open_status.ok()) { + for (const Pending& p : pending) { + *requests[p.request].status = open_status; + } + return; + } + // Requests and their planned reads; *_owner maps a request to its index in + // `pending`. + autovector whole_requests; + autovector range_requests; + autovector whole_owner; + autovector range_owner; + PlannedReads whole_reads; + PlannedReads range_reads; + for (size_t position = 0; position < pending.size(); ++position) { + Pending& p = pending[position]; + IndirectBlobReadRequest& request = requests[p.request]; + const BlobIndex& blob_index = *request.blob_index; + if (reader.GetValue()->GetCompressionType() != blob_index.compression()) { + *request.status = Status::Corruption( + "Compression type mismatch when reading indirect blob"); + continue; + } + if (blob_index.size() > std::numeric_limits::max()) { + *request.status = Status::Corruption("Indirect blob size is too large"); + continue; + } + const uint64_t offset = blob_index.original_value_offset(); + if (p.whole) { + whole_requests.emplace_back( + request.user_key, offset, static_cast(blob_index.size()), + blob_index.compression(), /*result=*/nullptr, request.status); + if (read_options.verify_checksums) { + whole_requests.back().value_crc32c = value_crc32c_out(p); + } + const auto extent = WholeBlobReadExtent( + read_options, request.user_key.size(), offset, blob_index.size()); + whole_reads.push_back({extent.first, extent.second, + whole_requests.size() - 1, request.user_key}); + whole_owner.push_back(position); + } else { + range_requests.emplace_back(request.user_key, offset, blob_index.size(), + request.range_offset, request.range_length, + /*result=*/nullptr, request.status); + range_reads.push_back({offset + request.range_offset, + request.range_length, range_requests.size() - 1, + request.user_key}); + range_owner.push_back(position); + } + } + physical_bytes_read += ReadInPlannedBatches( + &whole_requests, &whole_reads, allocator, + [&](autovector>>& batch_reads, + uint64_t* batch_bytes_read) { + reader.GetValue()->MultiGetBlob(read_options, allocator, batch_reads, + batch_bytes_read); + }, + [&](size_t request) { + return &pending[whole_owner[request]].out.contents; + }, + [&](size_t request) { + Pending& p = pending[whole_owner[request]]; + p.out.gen2.copied = true; + // The copy has no CRC from the reader; FinishIndirectBlobRead + // computes the value CRC again. + p.out.gen2.payload_crc32c.reset(); + }); + physical_bytes_read += ReadInPlannedBatches( + &range_requests, &range_reads, /*allocator=*/nullptr, + [&](autovector>>& batch_reads, + uint64_t* batch_bytes_read) { + reader.GetValue()->MultiGetBlobRange(read_options, batch_reads, + batch_bytes_read); + }, + [&](size_t request) { + return &pending[range_owner[request]].out.contents; + }, + [&](size_t request) { + pending[range_owner[request]].out.gen2.copied = true; + }); + // A Corruption of a whole read may come from a stale cached reader (for + // example, the file was replaced). Like MultiGetBlobFromOneFile, those + // reads are retried once with a newly opened reader. Range reads are not + // retried, like MultiGetBlobRangeFromOneFile. + autovector stale; + for (size_t r = 0; r < whole_requests.size(); ++r) { + if (whole_requests[r].status->IsCorruption()) { + stale.push_back(r); + } + } + if (!stale.empty()) { + reader.Reset(); + blob_file_cache_->Evict(file.open_info.file_number); + std::unique_ptr fresh_reader; + const Status reopen_status = blob_file_cache_->OpenBlobFileReaderUncached( + read_options, file.open_info, &fresh_reader, + /*allow_footer_skip_retry=*/false); + if (!reopen_status.ok()) { + for (size_t r : stale) { + *whole_requests[r].status = AppendBlobRefreshRetryFailure( + *whole_requests[r].status, reopen_status); + } + } else { + autovector retry_requests; + autovector stale_statuses; + PlannedReads retry_reads; + for (size_t r : stale) { + BlobReadRequest& request = whole_requests[r]; + stale_statuses.push_back(*request.status); + // Only a failed retry reads the stale status. + stale_statuses.back().PermitUncheckedError(); + *request.status = Status::OK(); + // The first pass may have copied this read from an equal read. The + // retry reads it again, so it counts as a read unless the retry + // itself copies it. + pending[whole_owner[r]].out.gen2.copied = false; + pending[whole_owner[r]].out.gen2.payload_crc32c.reset(); + retry_requests.push_back(request); + const auto extent = + WholeBlobReadExtent(read_options, request.user_key->size(), + request.offset, request.len); + retry_reads.push_back({extent.first, extent.second, + retry_requests.size() - 1, *request.user_key}); + } + physical_bytes_read += ReadInPlannedBatches( + &retry_requests, &retry_reads, allocator, + [&](autovector< + std::pair>>& + batch_reads, + uint64_t* batch_bytes_read) { + fresh_reader->MultiGetBlob(read_options, allocator, batch_reads, + batch_bytes_read); + }, + [&](size_t k) { + return &pending[whole_owner[stale[k]]].out.contents; + }, + [&](size_t k) { + Pending& p = pending[whole_owner[stale[k]]]; + p.out.gen2.copied = true; + p.out.gen2.payload_crc32c.reset(); + }); + bool install_fresh_reader = false; + for (size_t k = 0; k < stale.size(); ++k) { + Status* const status = retry_requests[k].status; + if (status->ok()) { + install_fresh_reader = true; + } else { + *status = AppendBlobRefreshRetryFailure(stale_statuses[k], *status); + } + } + if (install_fresh_reader) { + CacheHandleGuard ignored_reader; + blob_file_cache_ + ->RefreshBlobFileReader(file.open_info, &fresh_reader, + &ignored_reader) + .PermitUncheckedError(); + } + } + } + } + + size_t successful_reads = 0; + for (Pending& p : pending) { + IndirectBlobReadRequest& request = requests[p.request]; + if (!request.status->ok()) { + continue; + } + successful_reads += p.out.gen2.copied ? 0 : 1; + if (read_options.io_activity == Env::IOActivity::kLazyResolve && + !p.out.gen2.copied) { + // For a relocation file this is an estimate: the range length for an + // uncompressed range read, else the record size (value plus + // kSimpleGen2BlobTrailerSize, 5 bytes). It does not count the index and + // data block reads. + const BlobIndex& blob_index = *request.blob_index; + const uint64_t read_size = + !p.whole ? request.range_length + : file.is_relocation + ? blob_index.size() + kSimpleGen2BlobTrailerSize + : WholeBlobReadExtent(read_options, request.user_key.size(), + blob_index.original_value_offset(), + blob_index.size()) + .second; + RecordLazyRead(statistics_, read_size); + if (!p.whole) { + RecordLazyPartialRead(statistics_, read_size, blob_index.size()); + } + } + *request.status = FinishIndirectBlobRead( + read_options, *request.blob_index, + stable_blob_cache_ ? &p.stable_key : nullptr, p.whole, &p.out.contents, + p.out.gen2.payload_crc32c, request.result); + } + if (file.is_relocation) { + PERF_COUNTER_ADD(blob_read_count, successful_reads); + } + if (bytes_read != nullptr) { + *bytes_read = physical_bytes_read; + } +} + Status BlobSource::GetSimpleGen2Blob( const ReadOptions& read_options, const OffsetableCacheKey& base_cache_key, RandomAccessFileReader* file, uint64_t record_offset, uint64_t payload_size, @@ -511,8 +1579,9 @@ Status BlobSource::GetSimpleGen2Blob( // Cache miss (or no cache configured). Read the record into a buffer // allocated from the blob cache's memory allocator when we intend to insert - // it, exposing the uncompressed payload as BlobContents (the trailer just - // sits unused at the tail of the buffer). + // it. An uncompressed payload is exposed as BlobContents in place (the + // trailer sits unused at the tail of the buffer); a compressed payload is + // decompressed into a new buffer. MemoryAllocator* const allocator = (blob_cache_ && read_options.fill_cache) ? blob_cache_.get()->memory_allocator() : nullptr; @@ -527,8 +1596,19 @@ Status BlobSource::GetSimpleGen2Blob( return s; } - std::unique_ptr blob_contents( - new BlobContents(std::move(buf), static_cast(payload_size))); + std::unique_ptr blob_contents; + if (expected_compression == kNoCompression) { + blob_contents.reset( + new BlobContents(std::move(buf), static_cast(payload_size))); + } else { + s = UncompressBlobContentsIfNeeded( + Slice(buf.get(), static_cast(payload_size)), + expected_compression, simple_gen2_decompressor_.get(), allocator, + clock_, statistics_, &blob_contents); + if (!s.ok()) { + return s; + } + } // Record the per-read statistics (mirrors BlobFileReader::GetBlob). RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, record_size); @@ -660,6 +1740,9 @@ void BlobSource::MultiGetSimpleGen2Blob( size_t idx; CacheAllocationPtr buf; uint64_t record_size; + // Outputs of the read. extras.copied marks a copy of a duplicate request's + // read, which counts no read statistics. + SimpleGen2ReadExtras extras; }; std::vector pending; std::vector read_reqs; @@ -697,6 +1780,9 @@ void BlobSource::MultiGetSimpleGen2Blob( req.status}); pending.push_back(Pending{i, std::move(buf), record_size}); } + for (size_t k = 0; k < pending.size(); ++k) { + read_reqs[k].extras = &pending[k].extras; + } if (read_reqs.empty()) { return; @@ -712,14 +1798,27 @@ void BlobSource::MultiGetSimpleGen2Blob( continue; } const uint64_t record_size = pending[k].record_size; - std::unique_ptr blob_contents(new BlobContents( - std::move(pending[k].buf), static_cast(req.payload_size))); + std::unique_ptr blob_contents; + if (req.expected_compression == kNoCompression) { + blob_contents.reset(new BlobContents( + std::move(pending[k].buf), static_cast(req.payload_size))); + } else { + *req.status = UncompressBlobContentsIfNeeded( + Slice(pending[k].buf.get(), static_cast(req.payload_size)), + req.expected_compression, simple_gen2_decompressor_.get(), allocator, + clock_, statistics_, &blob_contents); + if (!req.status->ok()) { + continue; + } + } - RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, record_size); - PERF_COUNTER_ADD(blob_read_count, 1); - PERF_COUNTER_ADD(blob_read_byte, record_size); - if (read_options.io_activity == Env::IOActivity::kLazyResolve) { - RecordLazyRead(statistics_, record_size); + if (!pending[k].extras.copied) { + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, record_size); + PERF_COUNTER_ADD(blob_read_count, 1); + PERF_COUNTER_ADD(blob_read_byte, record_size); + if (read_options.io_activity == Env::IOActivity::kLazyResolve) { + RecordLazyRead(statistics_, record_size); + } } if (blob_cache_ && read_options.fill_cache) { @@ -756,6 +1855,9 @@ void BlobSource::MultiGetSimpleGen2BlobRange( struct Pending { size_t idx; CacheAllocationPtr buf; + // Outputs of the read. extras.copied marks a copy of a duplicate request's + // read, which counts no read statistics. + SimpleGen2ReadExtras extras; }; std::vector pending; std::vector read_reqs; @@ -795,6 +1897,9 @@ void BlobSource::MultiGetSimpleGen2BlobRange( req.status}); pending.push_back(Pending{i, std::move(buf)}); } + for (size_t k = 0; k < pending.size(); ++k) { + read_reqs[k].extras = &pending[k].extras; + } if (read_reqs.empty()) { return; @@ -811,11 +1916,13 @@ void BlobSource::MultiGetSimpleGen2BlobRange( std::unique_ptr blob_contents( new BlobContents(std::move(pending[k].buf), req.range_length)); - RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, req.range_length); - PERF_COUNTER_ADD(blob_read_count, 1); - PERF_COUNTER_ADD(blob_read_byte, req.range_length); - RecordLazyRead(statistics_, req.range_length); - RecordLazyPartialRead(statistics_, req.range_length, req.payload_size); + if (!pending[k].extras.copied) { + RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, req.range_length); + PERF_COUNTER_ADD(blob_read_count, 1); + PERF_COUNTER_ADD(blob_read_byte, req.range_length); + RecordLazyRead(statistics_, req.range_length); + RecordLazyPartialRead(statistics_, req.range_length, req.payload_size); + } PinOwnedBlob(&blob_contents, req.result); } diff --git a/db/blob/blob_source.h b/db/blob/blob_source.h index 802f69c23859..0256d4d77bc6 100644 --- a/db/blob/blob_source.h +++ b/db/blob/blob_source.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include "cache/cache_key.h" #include "cache/typed_cache.h" @@ -26,8 +28,54 @@ class Status; class FilePrefetchBuffer; class Slice; class RandomAccessFileReader; +class Decompressor; +class CompressionManager; +class SystemClock; enum ChecksumType : char; +// Indirect values use the stable blob cache; see the "Stable blob cache" +// comment at stable_blob_cache_ below. + +// Returns the stable blob cache key of an indirect value (a value that a +// kIndirectBlob or kIndirectFlattenedBlob BlobIndex points to). The key is +// built from db_id, the session id, the origin file number and the original +// offset. It does not include the generation or the physical file and offset, +// so moving a value into a relocation file neither duplicates nor invalidates +// its cache entry. This is the reference (slow) derivation, for tests. Reads +// use StableBlobCacheKeyBuilder, which gives the same keys faster. +CacheKey TEST_GetStableBlobCacheKey(const std::string& db_id, + const std::string& db_session_id, + uint64_t origin_file_number, + uint64_t original_value_offset); + +// Gives the same keys as TEST_GetStableBlobCacheKey, but hashes db_id and the +// session and chooses the domain (see ChooseStableKeyDomain in blob_source.cc) +// once, at construction. Each Get() only mixes in the origin and the offset. +// The builder does not change after construction, so several threads can use it +// at the same time. +class StableBlobCacheKeyBuilder { + public: + StableBlobCacheKeyBuilder(const std::string& db_id, + const std::string& db_session_id); + CacheKey Get(uint64_t origin_file_number, + uint64_t original_value_offset) const; + + private: + // GetSstInternalUniqueId(db_id, session, file_number) is + // {session_lower_, db_hash_ ^ file_number}. + uint64_t session_lower_ = 0; + uint64_t db_hash_ = 0; + // ReverseBits(domain), xored into the second word of the internal id. + uint64_t domain_bits_ = 0; + // OffsetableCacheKey treats a zero session_lower in a special way, so the + // fast path would give keys that differ from the reference derivation. A + // session id that decodes to session_lower == 0 (DBImpl does not create one) + // therefore uses the reference derivation. + std::string slow_db_id_; + std::string slow_db_session_id_; + bool use_slow_path_ = false; +}; + // One whole-record request for BlobSource::MultiGetSimpleGen2Blob (the batched // counterpart of GetSimpleGen2Blob). The per-file context (base cache key, // file, checksum context) is passed once to the batch call; these are the @@ -114,6 +162,82 @@ class BlobSource { size_t range_length, PinnableSlice* value, uint64_t* bytes_read); + // Looks up the value of one indirect BlobIndex in the stable blob cache only. + // Readers (Version) call it first, before they look up any file, so a hit + // needs no blob file lookup. On a hit it pins the cached value (or the range + // at `range_offset` of `range_length` bytes; kWholeBlobLength selects the + // whole value) into `*value` and returns OK. A range of a compressed value + // returns Corruption first, before the cache is checked, also when there is + // no stable cache. Otherwise it returns NotFound when there is no stable + // cache or the value is not cached, InvalidArgument for a range outside the + // value, and Corruption for a damaged entry from the secondary cache. + // + // A hit is not checked against `blob_index` (no fingerprint, file or + // relocation check). This is safe for these reasons: + // - The stable key includes db_id and db_session_id, so entries from before a + // reopen never match. + // - A blob file never changes after it is written, and the DB never reuses + // a file number. So (origin, original offset) names the same bytes for the + // life of the DB. + // - Outside tests, a relocation file is marked lost only when the open + // replays a loss record, so no entry of this session can belong to a lost + // file. + Status GetIndirectBlobFromCache(const BlobIndex& blob_index, + uint64_t range_offset, size_t range_length, + PinnableSlice* value) const; + + // Reads the value of one indirect BlobIndex from `file`, the physical file + // that the caller picked: a normal blob file, or a relocation file when + // `file.is_relocation` is set. The caller (Version) picks the file; this + // class never chooses one. The stable cache key uses only the stable identity + // of the value (origin, original offset). + // - Range of a compressed value: returns Corruption, as in base RocksDB, + // before any cache lookup or read. A byte range of a compressed value + // cannot be decompressed alone, so callers read the whole value and slice + // it. + // - Stable cache hit: the same as GetIndirectBlobFromCache(). With + // `cache_checked`, the caller already looked up the cache and missed, so + // this function does not look it up again. + // - Whole value: reads and decompresses the whole value. Its logical CRC is + // checked when verify_checksums is set or when it is inserted into the + // stable cache (fill_cache with a blob cache). + // - Uncompressed range: reads only the range bytes; no CRC check and no cache + // fill. Callers send only strict parts of a value (see + // ReadPathBlobResolver::ResolveColumnRange), and a range is never turned + // into a whole read. + // - A kBlockCacheTier read that misses the cache returns Incomplete. + // `*bytes_read` is the size read from the physical file (0 on a cache hit). + // For a relocation file it is an estimate: the record size (or the range + // length for an uncompressed range read), without the index and data block + // reads. `mutable_cf_options` configure the table reader when a relocation + // file is opened. + Status GetIndirectBlob(const ReadOptions& read_options, const Slice& user_key, + const BlobIndex& blob_index, + const BlobPhysicalFile& file, + const MutableCFOptions& mutable_cf_options, + uint64_t range_offset, size_t range_length, + FilePrefetchBuffer* prefetch_buffer, + PinnableSlice* value, uint64_t* bytes_read, + bool cache_checked); + + // Batched counterpart of GetIndirectBlob for requests that all read the same + // physical file. Each request gets the same contract as GetIndirectBlob and + // its own status (IndirectBlobReadRequest::cache_checked plays the role of + // `cache_checked`), with one difference. When a read from a normal blob file + // fails with Corruption, GetIndirectBlob drops the cached file reader and + // tries once more with a new reader, for whole-value reads and for + // uncompressed range reads. This function does that only for whole-value + // reads, so an uncompressed range read returns the Corruption. Both fail + // closed. For a normal blob file, the reads are sorted and split into + // batches whose file ranges do not overlap. `*bytes_read` is the sum of the + // physical read sizes. For a relocation file it is an estimate: the sum of + // the record sizes (or range lengths for uncompressed range reads), without + // the index and data block reads. + void MultiGetIndirectBlobFromOneFile( + const ReadOptions& read_options, const BlobPhysicalFile& file, + const MutableCFOptions& mutable_cf_options, size_t num_requests, + IndirectBlobReadRequest* requests, uint64_t* bytes_read); + // Reads a SimpleGen2Blob payload (see db/blob/blob_gen2_format.h) through the // blob value cache and BLOB_DB_* statistics. This is the counterpart to // GetBlob() for the second-generation blob record format, which is read @@ -138,9 +262,11 @@ class BlobSource { // On a cache hit, pins the cached value into `*value` (no copy). On a miss, // reads + verifies the record into a cache-allocator buffer, records the // per-read stats, inserts it into the cache (when configured and fill_cache - // is set), and pins it into `*value`. If blob_cache_ is not configured, the - // record is still read and read stats recorded, just without a cache - // lookup/insert. + // is set), and pins it into `*value`. A compressed record is decompressed + // into a new buffer, and the cache holds the decompressed value. If + // blob_cache_ is not configured, the record is still read and read stats + // recorded, just without a cache lookup/insert. Incomplete on a + // kBlockCacheTier miss. Status GetSimpleGen2Blob(const ReadOptions& read_options, const OffsetableCacheKey& base_cache_key, RandomAccessFileReader* file, uint64_t record_offset, @@ -175,8 +301,8 @@ class BlobSource { PinnableSlice* value, uint64_t* bytes_read); // Batched counterpart of GetSimpleGen2Blob: resolves `num_records` whole - // embedded (SimpleGen2Blob) records from one file, coalescing all cache-miss - // reads into a single MultiRead. The per-file context (base cache key, file, + // embedded (SimpleGen2Blob) records from one file, coalescing cache-miss + // reads into MultiRead batches. The per-file context (base cache key, file, // checksum type, base context checksum) is shared by all records; per-record // inputs/outputs are in `reqs`. Each record independently probes the blob // cache and, on a miss, is read + verified from the file; whole records are @@ -192,7 +318,7 @@ class BlobSource { // Batched counterpart of GetSimpleGen2BlobRange: resolves `num_records` // uncompressed embedded sub-ranges from one file, coalescing cache-miss reads - // into a single MultiRead. Like GetSimpleGen2BlobRange, each record probes + // into MultiRead batches. Like GetSimpleGen2BlobRange, each record probes // the whole-payload cache (slicing on a hit) and a partial read is never // inserted into the cache. Per-record outcome is in *reqs[i].status. void MultiGetSimpleGen2BlobRange(const ReadOptions& read_options, @@ -269,6 +395,20 @@ class BlobSource { inline Cache* GetBlobCache() const { return blob_cache_.get(); } + // Inserts a value that flush just wrote into the blob cache + // (prepopulate_blob_cache = kFlushOnly). A legacy value (a value that a + // kBlob BlobIndex points to) is keyed by its physical file and offset. An + // indirect value, given its BlobIndex `fingerprint`, is keyed by its stable + // identity (origin = the new file, original offset). The caller computed + // the logical CRC in `fingerprint` from this same value, so the insert does + // not compute it again. Unlike a read, the insert does not check for an + // existing entry under the per-key admission mutex + // (stable_cache_admission_mutex_): no other entry can exist yet for a value + // in a new file. No-op without a blob cache. + Status PrepopulateFlushBlob(const Slice& value, uint64_t blob_file_number, + uint64_t blob_offset, + const StableBlobFingerprint* fingerprint) const; + bool TEST_BlobInCache(uint64_t file_number, uint64_t file_size, uint64_t offset, size_t* charge = nullptr) const; @@ -281,6 +421,11 @@ class BlobSource { using TypedHandle = SharedCacheInterface::TypedHandle; private: + using StableSharedCacheInterface = + FullTypedSharedCacheInterface; + using StableTypedHandle = StableSharedCacheInterface::TypedHandle; + Status GetBlobFromCache(const Slice& cache_key, CacheHandleGuard* cached_blob) const; @@ -288,12 +433,76 @@ class BlobSource { std::unique_ptr* blob, CacheHandleGuard* cached_blob) const; + // Looks up the stable cache entry for `cache_key`. Returns NotFound on a + // miss. An invalid entry (promoted from damaged secondary-cache bytes) is + // erased and returns Corruption. The entry is not compared with the reader's + // BlobIndex (see GetIndirectBlobFromCache() for why). With `primary_only`, + // the secondary cache is not probed. + Status GetStableBlobFromCache( + const Slice& cache_key, CacheHandleGuard* cached_blob, + bool record_statistics, bool primary_only) const; + + // Inserts `*logical_value`, whose logical CRC the caller has checked, under + // `cache_key` and pins it in `cached_blob`. The entry adopts the buffer (no + // copy); `*logical_value` is empty afterwards. The entry is built outside the + // per-key admission mutex; under it there is only a primary-cache lookup and + // the insert, so a good entry is never replaced by a conflicting one. If the + // primary cache already has an entry under `cache_key`, that entry is + // returned, and the new one is dropped. No comparison is needed, because one + // stable key names one immutable value. + Status PutStableBlobIntoCache( + const Slice& cache_key, const BlobIndex& blob_index, + std::unique_ptr* logical_value, + CacheHandleGuard* cached_blob) const; + + // Inserts `*contents` under `cache_key` and records the cache statistics. + // With `cached_blob`, pins the entry there; otherwise releases it. On success + // `*contents` is empty. + Status InsertStableBlob( + const Slice& cache_key, std::unique_ptr* contents, + CacheHandleGuard* cached_blob) const; + + // Finishes one indirect read after the physical read, and pins the result + // into `result`. Takes ownership of `*physical_value`. + // - `materialized_whole` is false: `*physical_value` holds the requested + // uncompressed range and is pinned as is. + // - `materialized_whole` is true: `*physical_value` holds the whole value. + // 1. Check the logical CRC when verify_checksums is set or when the value + // goes into the stable cache. Use `value_crc32c` if the reader already + // computed the CRC32C of the uncompressed value in the same pass as its + // record check: BlobFileReader::GetBlob() for blob log files + // (BlobReadRequest::value_crc32c), or the no-BlobSource + // BlockBasedTable::MultiGetSameFileBlob() for relocation files + // (SimpleGen2ReadExtras::payload_crc32c in SameFileBlobStandaloneRead). + // Then the value is not read a second time. + // 2. If fill_cache is set and `stable_cache_key` is not null (it is null + // without a stable cache), insert the value and pin the cache entry. + // 3. Otherwise pin the whole value. + Status FinishIndirectBlobRead(const ReadOptions& read_options, + const BlobIndex& blob_index, + const CacheKey* stable_cache_key, + bool materialized_whole, + std::unique_ptr* physical_value, + const std::optional& value_crc32c, + PinnableSlice* result) const; + static void PinCachedBlob(CacheHandleGuard* cached_blob, PinnableSlice* value); static void PinOwnedBlob(std::unique_ptr* owned_blob, PinnableSlice* value); + // Moves the cache handle into `value` and points it at the whole stable value + // or at [range_offset, range_offset + range_length). + static void PinStableCachedBlob( + CacheHandleGuard* cached_blob, uint64_t range_offset, + size_t range_length, PinnableSlice* value); + // Checks that the range fits in the cached value, then pins it into + // `*value`. Returns InvalidArgument for a range outside the value. + static Status PinStableCacheHit( + CacheHandleGuard* cached_blob, uint64_t range_offset, + size_t range_length, PinnableSlice* value); + TypedHandle* GetEntryFromCache(const Slice& key) const; Status InsertEntryIntoCache(const Slice& key, BlobContents* value, @@ -310,13 +519,75 @@ class BlobSource { const std::string& db_session_id_; Statistics* statistics_; + // Times the decompression of SimpleGen2 records. + SystemClock* clock_; // A cache to store blob file reader. BlobFileCache* blob_file_cache_; + // Serializes the final insert into the stable cache for one stable identity + // (see PutStableBlobIntoCache). A hash of the cache key picks one of 128 + // mutex stripes (1 << 7). 128 is an estimate that keeps lock contention low + // at a small memory cost; it was not measured. + static constexpr size_t kNumberOfAdmissionMutexStripes = 1 << 7; + mutable Striped> + stable_cache_admission_mutex_; + + // The blob cache that both typed interfaces below use: the configured blob + // cache, or a ChargedCache wrapper of it when blob cache memory is charged to + // the block cache. + std::shared_ptr blob_cache_storage_; + // A cache to store uncompressed blobs. mutable SharedCacheInterface blob_cache_; + // Stable blob cache: the cache entries of indirect values. + // + // Background: an indirect value keeps its stable identity (origin, original + // offset) when indirect blob GC moves it. So its cache key (the stable cache + // key) uses only that identity, plus db_id and the session + // (StableBlobCacheKeyBuilder), and a cached value stays valid when the value + // moves. The entries have their own type (StableBlobContents, which can be + // saved to a secondary cache). They share the configured blob cache and its + // charge policy with legacy (kBlob) values, which are keyed by physical file + // and offset. + // + // Life of an entry: + // 1. Read: GetIndirectBlobFromCache() looks up the key before any blob + // file. A hit is used without a check against the BlobIndex (see + // there for why). + // 2. Insert after a file read: PutStableBlobIntoCache() runs after the + // logical CRC check. It locks the admission mutex that a hash of the + // key picks (one of kNumberOfAdmissionMutexStripes (128) stripes; see + // stable_cache_admission_mutex_). Then it looks up the primary cache + // again and inserts only on a miss. So an entry that another thread + // inserted first is kept. + // 3. Insert at flush: PrepopulateFlushBlob() inserts a new value directly + // (InsertStableBlob()). + // 4. Secondary cache: when an entry moves to a secondary cache, its cache + // item helper writes a header (with its own CRC) and then the value. + // When the secondary cache loads an entry back into the primary cache + // (a promoted entry), StableBlobContents::CreateFromSaved() checks the + // header CRC and the value CRC. If a check fails, the entry is marked + // invalid, and GetStableBlobFromCache() erases it and returns + // Corruption. + mutable StableSharedCacheInterface stable_blob_cache_; + + // Returns the stable cache key of (origin, original offset). Requires a + // stable cache. + CacheKey StableCacheKey(uint64_t origin_file_number, + uint64_t original_value_offset) const; + + // Derives stable cache keys. Built on first use, not at construction: + // DBImpl sets db_id_ during recovery, after the column families (and their + // BlobSource) are created, and reads start only after recovery. + mutable std::once_flag stable_key_builder_once_; + mutable std::unique_ptr stable_key_builder_; + + // SimpleGen2 records can mix built-in blob compression types in one table. + // Created once; Decompressor is safe for concurrent use. + std::shared_ptr simple_gen2_decompressor_; + // The control option of how the cache tiers will be used. Currently rocksdb // support block/blob cache (volatile tier) and secondary cache (this tier // isn't strictly speaking a non-volatile tier since the compressed cache in diff --git a/db/blob/blob_source_test.cc b/db/blob/blob_source_test.cc index e0817b307a68..218bcf47774b 100644 --- a/db/blob/blob_source_test.cc +++ b/db/blob/blob_source_test.cc @@ -5,25 +5,37 @@ #include "db/blob/blob_source.h" +#include +#include #include #include #include +#include #include #include +#include #include "cache/charged_cache.h" #include "cache/compressed_secondary_cache.h" #include "db/blob/blob_contents.h" #include "db/blob/blob_file_cache.h" #include "db/blob/blob_file_reader.h" +#include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" #include "db/blob/blob_log_writer.h" #include "db/db_test_util.h" #include "file/filename.h" #include "file/read_write_util.h" +#include "memory/memory_allocator_impl.h" #include "options/cf_options.h" #include "rocksdb/options.h" +#include "table/unique_id_impl.h" +#include "test_util/sync_point.h" +#include "util/coding.h" #include "util/compression.h" +#include "util/crc32c.h" +#include "util/math.h" #include "util/random.h" namespace ROCKSDB_NAMESPACE { @@ -108,6 +120,30 @@ BlobFileOpenInfo OpenInfoWithoutChecksum(uint64_t file_number) { return BlobFileOpenInfo{file_number, Slice(), Slice()}; } +// A physical file for indirect reads. A relocation file here has no tail size, +// so it can be read only from the stable cache. +// Sets a callback on `sync_point`, which passes the batched file reads +// (std::vector*). The callback counts the calls in `*calls` and +// checks that the reads are sorted and do not overlap. +void ExpectSortedNonOverlappingReads(const char* sync_point, + std::atomic* calls) { + SyncPoint::GetInstance()->SetCallBack(sync_point, [calls](void* arg) { + const auto* const reads = static_cast*>(arg); + ++*calls; + for (size_t i = 1; i < reads->size(); ++i) { + EXPECT_LE((*reads)[i - 1].offset + (*reads)[i - 1].len, + (*reads)[i].offset); + } + }); +} + +BlobPhysicalFile TestPhysicalFile(uint64_t file_number, uint64_t file_size, + bool is_relocation, + uint32_t directory_digest) { + return BlobPhysicalFile{OpenInfoWithoutChecksum(file_number), file_size, + is_relocation, /*tail_size=*/0, directory_digest}; +} + } // anonymous namespace class BlobSourceTest : public DBTestBase { @@ -137,6 +173,856 @@ class BlobSourceTest : public DBTestBase { std::string db_session_id_; }; +TEST(StableBlobCacheKeyTest, SeparatesFinalKeysIncludingZeroWordSwap) { + const std::string db_id = "stable-cache-domain-test"; + constexpr uint64_t file_number = 100; + UniqueId64x2 initial_id; + ASSERT_OK(GetSstInternalUniqueId(db_id, EncodeSessionId(0, 1), file_number, + &initial_id, true)); + // GetSstInternalUniqueId gives id = {session_lower, H ^ file number}, where H + // depends on db_id and the upper session bits, not on session_lower. The raw + // first word of a cache key is DownwardInvolution(id[0]) ^ + // ReverseBits(id[1]). Each round xors `domain` into id[1] and picks a real + // encoded session whose session_lower makes this word 0, so + // OffsetableCacheKey swaps it: + // - 0: the physical key of `file_number` is swapped. + // - 1 << 63 (ReverseBits(1)): a stable key with GetStableBlobCacheKey's + // domain 1 would be swapped, so GetStableBlobCacheKey must reject domain 1. + // In both rounds, stable keys must differ from physical and reserved keys. + for (uint64_t domain : {uint64_t{0}, uint64_t{1} << 63}) { + const uint64_t lower = + DownwardInvolution(ReverseBits(initial_id[1] ^ domain)); + ASSERT_NE(lower, 0U); + const std::string session = EncodeSessionId(0, lower); + UniqueId64x2 forced_id; + ASSERT_OK(GetSstInternalUniqueId(db_id, session, file_number | domain, + &forced_id, true)); + ASSERT_EQ(DownwardInvolution(forced_id[0]) ^ ReverseBits(forced_id[1]), 0U); + for (uint64_t origin : + {file_number, file_number + 1, uint64_t{0x3fffffffffffffff}}) { + const CacheKey stable = + TEST_GetStableBlobCacheKey(db_id, session, origin, 42); + ASSERT_EQ(stable.AsSlice().size(), 16U); + ASSERT_FALSE(stable.IsEmpty()); + const CacheKey other_offset = + TEST_GetStableBlobCacheKey(db_id, session, origin, 43); + EXPECT_NE(stable.AsSlice(), other_offset.AsSlice()); + const CacheKey other_origin = TEST_GetStableBlobCacheKey( + db_id, session, origin == file_number ? file_number + 1 : file_number, + 42); + EXPECT_NE(stable.AsSlice(), other_origin.AsSlice()); + for (uint64_t physical : + {file_number, file_number + 1, uint64_t{0x3fffffffffffffff}}) { + const OffsetableCacheKey base(db_id, session, physical); + const CacheKey physical_key = base.WithOffset(42); + // Comparing only the first 8 bytes is stronger than comparing a few + // offsets: an offset changes only the second word, so no offset + // (including the shifted offsets that SST blocks use) can make keys + // equal. + EXPECT_NE(Slice(stable.AsSlice().data(), 8), + Slice(physical_key.AsSlice().data(), 8)); + } + } + const std::shared_ptr cache = NewLRUCache(1024); + const CacheKey reservation = + CacheKey::CreateUniqueForCacheLifetime(cache.get()); + const CacheKey stable = + TEST_GetStableBlobCacheKey(db_id, session, file_number, 0); + EXPECT_NE(Slice(stable.AsSlice().data(), 8), + Slice(reservation.AsSlice().data(), 8)); + } +} + +TEST(StableBlobCacheKeyTest, BuilderMatchesSlowPath) { + // The builder hashes db_id and the session once. Each of its keys must equal + // the slow path's key, also for a malformed session id, which + // GetSstInternalUniqueId hashes into a nonzero session_lower. + const uint32_t seed = 20261107; + SCOPED_TRACE("seed " + std::to_string(seed)); + Random rnd(seed); + const std::vector> ids{ + {"db-id-1", EncodeSessionId(1, 2)}, + {"another-db-id", EncodeSessionId(0x1234, 0xfedcba)}, + {"", EncodeSessionId(7, 1)}, + {"db-id-2", "not a session id"}}; + for (const auto& [db_id, session] : ids) { + const StableBlobCacheKeyBuilder builder(db_id, session); + for (int i = 0; i < 200; ++i) { + const uint64_t origin = + 1 + rnd.Uniform(1 << 30) + + (rnd.OneIn(10) ? uint64_t{0x3fffffff00000000} : 0); + const uint64_t offset = rnd.Next64() >> 8; + ASSERT_EQ( + builder.Get(origin, offset).AsSlice(), + TEST_GetStableBlobCacheKey(db_id, session, origin, offset).AsSlice()) + << db_id << " " << origin << " " << offset; + } + } +} + +TEST(StableBlobContentsTest, AdoptsAllocationWithoutCopy) { + // A cache entry adopts the read buffer: the value bytes stay where the reader + // put them, and a trailer after the value is ignored. + const std::string value = "stable-value-bytes"; + CacheAllocationPtr allocation = + AllocateBlock(value.size() + 5, /*allocator=*/nullptr); + memcpy(allocation.get(), value.data(), value.size()); + memcpy(allocation.get() + value.size(), "TRAIL", 5); + const char* const buffer = allocation.get(); + const uint32_t logical = crc32c::Value(value.data(), value.size()); + std::unique_ptr contents; + ASSERT_OK(StableBlobContents::CreateWithVerifiedAllocation( + std::move(allocation), value.size(), + StableBlobFingerprint{ + value.size(), kNoCompression, logical, + BlobIndex::ExtendChecksumWithCompression(logical, kNoCompression)}, + &contents)); + ASSERT_NE(contents, nullptr); + EXPECT_EQ(contents->data(), value); + EXPECT_EQ(contents->data().data(), buffer); +} + +TEST(StableBlobContentsTest, ValidatesSavedFingerprint) { + const std::string value = "stable-value"; + const uint32_t checksum = crc32c::Value(value.data(), value.size()); + const uint32_t stored_checksum = + BlobIndex::ExtendChecksumWithCompression(checksum, kNoCompression); + const StableBlobFingerprint fingerprint{value.size(), kNoCompression, + checksum, stored_checksum}; + std::unique_ptr contents; + ASSERT_OK(StableBlobContents::CreateWithVerifiedValue( + value, fingerprint, /*allocator=*/nullptr, &contents)); + ASSERT_EQ(contents->data(), value); + const auto save = [](const StableBlobContents& entry) { + std::string saved(entry.SavedSize(), '\0'); + entry.SaveTo(0, saved.size(), saved.data()); + return saved; + }; + const std::string saved = save(*contents); + ASSERT_EQ(saved.size(), StableBlobContents::kSavedHeaderSize + value.size()); + // A save split at any offset gives the same bytes. + for (size_t split = 0; split <= saved.size(); ++split) { + std::string pieces(saved.size(), '\0'); + contents->SaveTo(0, split, pieces.data()); + contents->SaveTo(split, saved.size() - split, pieces.data() + split); + ASSERT_EQ(pieces, saved) << split; + } + std::unique_ptr promoted = + StableBlobContents::CreateFromSaved(saved, /*allocator=*/nullptr); + ASSERT_OK(promoted->validation_status()); + EXPECT_EQ(promoted->data(), value); + EXPECT_TRUE(promoted->fingerprint() == fingerprint); + + // A damaged value, a damaged header, an unsupported compression type with a + // valid header CRC and a truncated header are all rejected. + std::string damaged_value = saved; + damaged_value.back() ^= 1; + EXPECT_TRUE(StableBlobContents::CreateFromSaved(damaged_value, nullptr) + ->validation_status() + .IsCorruption()); + std::string damaged_header = saved; + damaged_header[2] ^= 1; + EXPECT_TRUE(StableBlobContents::CreateFromSaved(damaged_header, nullptr) + ->validation_status() + .IsCorruption()); + std::string unsupported = saved; + unsupported[16] = static_cast(kFirstCustomCompression); + EncodeFixed32(unsupported.data() + 17, crc32c::Value(unsupported.data(), 17)); + EXPECT_TRUE(StableBlobContents::CreateFromSaved(unsupported, nullptr) + ->validation_status() + .IsCorruption()); + EXPECT_TRUE(StableBlobContents::CreateFromSaved( + Slice(saved.data(), StableBlobContents::kSavedHeaderSize - 1), + nullptr) + ->validation_status() + .IsCorruption()); + + std::unique_ptr invalid; + StableBlobFingerprint wrong_size = fingerprint; + ++wrong_size.stored_value_size; + EXPECT_TRUE(StableBlobContents::CreateWithVerifiedValue( + value, wrong_size, /*allocator=*/nullptr, &invalid) + .IsInvalidArgument()); + StableBlobFingerprint unsupported_compression = fingerprint; + unsupported_compression.compression = kFirstCustomCompression; + EXPECT_TRUE( + StableBlobContents::CreateWithVerifiedValue( + value, unsupported_compression, /*allocator=*/nullptr, &invalid) + .IsInvalidArgument()); + // The primary cache trusts an entry made by CreateWithVerifiedValue without a + // check. Its saved bytes are still checked on promotion from a secondary + // cache, so damaged bytes are rejected. + std::string trusted = save(*contents); + trusted.back() ^= 1; + EXPECT_TRUE(StableBlobContents::CreateFromSaved(trusted, nullptr) + ->validation_status() + .IsCorruption()); +} + +namespace { +// Keeps the saved form of every evicted entry, optionally with the last byte +// flipped, and promotes it through the entry's cache item helper. +class SavingSecondaryCache : public SecondaryCache { + public: + const char* Name() const override { return "SavingSecondaryCache"; } + Status Insert(const Slice& key, Cache::ObjectPtr obj, + const Cache::CacheItemHelper* helper, + bool /*force_insert*/) override { + std::string saved(helper->size_cb(obj), '\0'); + Status status = helper->saveto_cb(obj, 0, saved.size(), saved.data()); + if (!status.ok()) { + return status; + } + if (corrupt && !saved.empty()) { + saved.back() ^= 1; + } + ++inserts; + saved_[key.ToString()] = std::move(saved); + return Status::OK(); + } + Status InsertSaved(const Slice& key, const Slice& saved, + CompressionType /*type*/, CacheTier /*source*/) override { + saved_[key.ToString()] = saved.ToString(); + return Status::OK(); + } + std::unique_ptr Lookup( + const Slice& key, const Cache::CacheItemHelper* helper, + Cache::CreateContext* create_context, bool /*wait*/, + bool /*advise_erase*/, Statistics* /*stats*/, + bool& kept_in_sec_cache) override { + kept_in_sec_cache = false; + const auto found = saved_.find(key.ToString()); + if (found == saved_.end()) { + return nullptr; + } + Cache::ObjectPtr obj = nullptr; + size_t charge = 0; + Status status = helper->create_cb(found->second, kNoCompression, + CacheTier::kVolatileTier, create_context, + /*allocator=*/nullptr, &obj, &charge); + saved_.erase(found); + if (!status.ok()) { + return nullptr; + } + return std::make_unique(obj, charge); + } + bool SupportForceErase() const override { return true; } + void Erase(const Slice& key) override { saved_.erase(key.ToString()); } + void WaitAll(std::vector /*handles*/) override {} + std::string GetPrintableOptions() const override { return ""; } + + bool corrupt = false; + int inserts = 0; + + private: + class Handle : public SecondaryCacheResultHandle { + public: + Handle(Cache::ObjectPtr value, size_t size) : value_(value), size_(size) {} + bool IsReady() override { return true; } + void Wait() override {} + Cache::ObjectPtr Value() override { return value_; } + size_t Size() override { return size_; } + + private: + Cache::ObjectPtr value_; + size_t size_; + }; + std::map saved_; +}; +} // namespace + +TEST_F(BlobSourceTest, StableBlobCacheSecondaryCacheRoundTrip) { + // Primary hit, eviction to a secondary cache and promotion with good bytes, + // then promotion of damaged saved bytes, which fails and is erased. + auto secondary = std::make_shared(); + LRUCacheOptions primary_options; + primary_options.capacity = 64 << 10; + primary_options.num_shard_bits = 0; + primary_options.metadata_charge_policy = kDontChargeCacheMetadata; + primary_options.secondary_cache = secondary; + options_.blob_cache = NewLRUCache(primary_options); + options_.lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier; + options_.statistics = CreateDBStatistics(); + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_StableSecondaryCache"), 0); + DestroyAndReopen(options_); + const ImmutableOptions immutable_options(options_); + const MutableCFOptions mutable_cf_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr uint64_t origin_file_number = 10; + constexpr ExpirationRange expiration_range; + // Value 0 takes the good round trip, value 1 the damaged one. + const std::vector user_keys{"key0", "key1"}; + const std::vector values{std::string(200, 'a'), + std::string(200, 'b')}; + std::vector keys{user_keys[0], user_keys[1]}; + std::vector blobs{values[0], values[1]}; + std::vector offsets(2); + std::vector sizes(2); + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, origin_file_number, keys, + blobs, kNoCompression, offsets, sizes); + const uint64_t file_size = BlobLogHeader::kSize + + 2 * (BlobLogRecord::kHeaderSize + + user_keys[0].size() + values[0].size()) + + BlobLogFooter::kSize; + std::shared_ptr file_cache = NewLRUCache(1024); + FileOptions file_options; + BlobFileCache blob_file_cache( + file_cache.get(), &immutable_options, &file_options, column_family_id, + /*blob_file_read_hist=*/nullptr, /*io_tracer=*/nullptr); + BlobSource blob_source(immutable_options, mutable_cf_options, db_id_, + db_session_id_, &blob_file_cache); + std::vector indexes(2); + for (size_t i = 0; i < indexes.size(); ++i) { + const uint32_t stable_checksum = + crc32c::Value(values[i].data(), values[i].size()); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin_file_number, + {.origin_file_number = origin_file_number, + .original_value_offset = offsets[i], + .stored_value_size = sizes[i], + .compression = kNoCompression, + .stable_checksum = stable_checksum, + .stored_checksum = BlobIndex::ExtendChecksumWithCompression( + stable_checksum, kNoCompression)}); + ASSERT_OK(indexes[i].DecodeFrom(encoded)); + } + const auto read = [&](size_t i, PinnableSlice* value) { + ReadOptions read_options; + read_options.verify_checksums = true; + uint64_t bytes_read = 0; + value->Reset(); + return blob_source.GetIndirectBlob( + read_options, user_keys[i], indexes[i], + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, value, &bytes_read, + /*cache_checked=*/false); + }; + // Fills the primary cache so its stable entries move to the secondary cache. + static const Cache::CacheItemHelper kFillerHelper{ + CacheEntryRole::kMisc, [](Cache::ObjectPtr, MemoryAllocator*) {}}; + int filler = 0; + const auto evict = [&]() { + const std::string key = "filler" + std::to_string(filler++); + static char filler_object; + // Below the capacity, so the LRU cache keeps it and evicts the rest. + ASSERT_OK(options_.blob_cache->Insert(key, &filler_object, &kFillerHelper, + (64 << 10) - 100)); + }; + + Statistics* const statistics = options_.statistics.get(); + PinnableSlice value; + ASSERT_OK(read(0, &value)); + ASSERT_EQ(value, values[0]); + const uint64_t hits = statistics->getTickerCount(BLOB_DB_CACHE_HIT); + ASSERT_OK(read(0, &value)); + ASSERT_EQ(value, values[0]); + EXPECT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), hits + 1); + + // The pinned value holds the cache handle; release it before eviction. + value.Reset(); + ASSERT_NO_FATAL_FAILURE(evict()); + EXPECT_EQ(secondary->inserts, 1); + const uint64_t misses = statistics->getTickerCount(BLOB_DB_CACHE_MISS); + ASSERT_OK(read(0, &value)); + ASSERT_EQ(value, values[0]); + // Served by promotion, not by a file read. + EXPECT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), misses); + + value.Reset(); + ASSERT_OK(read(1, &value)); + ASSERT_EQ(value, values[1]); + value.Reset(); + secondary->corrupt = true; + const int inserts = secondary->inserts; + ASSERT_NO_FATAL_FAILURE(evict()); + EXPECT_GT(secondary->inserts, inserts); + EXPECT_TRUE(read(1, &value).IsCorruption()); + const CacheKey stable_key = TEST_GetStableBlobCacheKey( + db_id_, db_session_id_, origin_file_number, offsets[1]); + Cache::Handle* const handle = + options_.blob_cache->Lookup(stable_key.AsSlice()); + EXPECT_EQ(handle, nullptr); + if (handle != nullptr) { + options_.blob_cache->Release(handle); + } +} + +class StableBlobCacheTest : public BlobSourceTest, + public testing::WithParamInterface {}; + +INSTANTIATE_TEST_CASE_P(CacheImplementations, StableBlobCacheTest, + testing::Bool()); + +TEST_P(StableBlobCacheTest, IndirectBlobUsesStableLogicalCacheKey) { + options_.statistics = CreateDBStatistics(); + options_.blob_cache = GetParam() + ? HyperClockCacheOptions(1 << 20).MakeSharedCache() + : NewLRUCache(1 << 20); + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_StableLogicalCache"), 0); + DestroyAndReopen(options_); + const ImmutableOptions immutable_options(options_); + const MutableCFOptions mutable_cf_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr uint64_t origin_file_number = 10; + constexpr bool has_ttl = false; + constexpr ExpirationRange expiration_range; + const std::string user_key = "key"; + const std::string logical_value = "stable-logical-value"; + std::vector keys{user_key}; + std::vector blobs{logical_value}; + std::vector offsets(1); + std::vector sizes(1); + WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range, + expiration_range, origin_file_number, keys, blobs, + kNoCompression, offsets, sizes); + const uint64_t file_size = BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + + user_key.size() + logical_value.size() + + BlobLogFooter::kSize; + + std::shared_ptr file_cache = NewLRUCache(1024); + FileOptions file_options; + BlobFileCache blob_file_cache( + file_cache.get(), &immutable_options, &file_options, column_family_id, + /*blob_file_read_hist=*/nullptr, /*io_tracer=*/nullptr); + BlobSource blob_source(immutable_options, mutable_cf_options, db_id_, + db_session_id_, &blob_file_cache); + + const uint32_t stable_checksum = + crc32c::Value(logical_value.data(), logical_value.size()); + const auto make_index = [&](uint64_t generation, uint32_t checksum) { + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, generation, + {.origin_file_number = origin_file_number, + .original_value_offset = offsets[0], + .stored_value_size = sizes[0], + .compression = kNoCompression, + .stable_checksum = checksum, + .stored_checksum = 0}); + BlobIndex index; + EXPECT_OK(index.DecodeFrom(encoded)); + return index; + }; + const BlobIndex unflattened_index = + make_index(origin_file_number, stable_checksum); + + ReadOptions read_options; + read_options.verify_checksums = true; + read_options.io_activity = Env::IOActivity::kLazyResolve; + PinnableSlice value; + uint64_t bytes_read = 0; + const BlobIndex wrong_disk_checksum = + make_index(origin_file_number, stable_checksum ^ 1); + read_options.fill_cache = false; + EXPECT_TRUE(blob_source + .GetIndirectBlob( + read_options, user_key, wrong_disk_checksum, + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false) + .IsCorruption()); + EXPECT_TRUE(value.empty()); + + const uint64_t lazy_before_batch = + options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT); + const uint64_t lazy_bytes_before_batch = + options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_BYTES); + + std::array duplicate_values; + std::array duplicate_statuses; + std::array duplicate_requests; + for (size_t i = 0; i < duplicate_requests.size(); ++i) { + duplicate_requests[i].user_key = user_key; + duplicate_requests[i].blob_index = &unflattened_index; + duplicate_requests[i].result = &duplicate_values[i]; + duplicate_requests[i].status = &duplicate_statuses[i]; + } + std::atomic duplicate_multiread_calls{0}; + ExpectSortedNonOverlappingReads("BlobFileReader::MultiGetBlob:Requests", + &duplicate_multiread_calls); + SyncPoint::GetInstance()->EnableProcessing(); + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, duplicate_requests.size(), duplicate_requests.data(), + &bytes_read); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(duplicate_statuses[0]); + ASSERT_OK(duplicate_statuses[1]); + EXPECT_EQ(duplicate_values[0], logical_value); + EXPECT_EQ(duplicate_values[1], logical_value); + // An exact duplicate is read once: one storage read, shared by both. + EXPECT_EQ(duplicate_multiread_calls.load(), 1U); + EXPECT_EQ(options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT), + lazy_before_batch + 1); + EXPECT_EQ(options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_BYTES), + lazy_bytes_before_batch + bytes_read); + + // Different user keys at the same offset are not shared: each request checks + // its own key against the record, so only the real key succeeds. + { + const std::string other_key = "other"; + std::array key_values; + std::array key_statuses; + std::array key_requests; + for (size_t i = 0; i < key_requests.size(); ++i) { + key_requests[i].user_key = i == 0 ? Slice(user_key) : Slice(other_key); + key_requests[i].blob_index = &unflattened_index; + key_requests[i].result = &key_values[i]; + key_requests[i].status = &key_statuses[i]; + } + // Not a lazy read, so the lazy read counts below stay unchanged. + ReadOptions key_read_options = read_options; + key_read_options.io_activity = Env::IOActivity::kUnknown; + blob_source.MultiGetIndirectBlobFromOneFile( + key_read_options, + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, key_requests.size(), key_requests.data(), + &bytes_read); + ASSERT_OK(key_statuses[0]); + EXPECT_EQ(key_values[0], logical_value); + EXPECT_TRUE(key_statuses[1].IsCorruption()) << key_statuses[1].ToString(); + EXPECT_TRUE(key_values[1].empty()); + } + + std::array overlapping_values; + std::array overlapping_statuses; + std::array overlapping_requests; + for (size_t i = 0; i < overlapping_requests.size(); ++i) { + overlapping_requests[i].user_key = user_key; + overlapping_requests[i].blob_index = &unflattened_index; + overlapping_requests[i].range_offset = i * 3; + overlapping_requests[i].range_length = 5; + overlapping_requests[i].result = &overlapping_values[i]; + overlapping_requests[i].status = &overlapping_statuses[i]; + } + std::atomic overlapping_multiread_calls{0}; + ExpectSortedNonOverlappingReads("BlobFileReader::MultiGetBlobRange:Requests", + &overlapping_multiread_calls); + SyncPoint::GetInstance()->EnableProcessing(); + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, overlapping_requests.size(), + overlapping_requests.data(), &bytes_read); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(overlapping_statuses[0]); + ASSERT_OK(overlapping_statuses[1]); + EXPECT_EQ(overlapping_values[0], Slice(logical_value.data(), 5)); + EXPECT_EQ(overlapping_values[1], Slice(logical_value.data() + 3, 5)); + EXPECT_EQ(overlapping_multiread_calls.load(), 2U); + EXPECT_EQ(options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT), + lazy_before_batch + 3); + EXPECT_EQ( + options_.statistics->getTickerCount(BLOB_DB_LAZY_PARTIAL_READ_COUNT), 2U); + EXPECT_EQ( + options_.statistics->getTickerCount(BLOB_DB_LAZY_PARTIAL_BYTES_SAVED), + 2 * (logical_value.size() - 5)); + + read_options.fill_cache = true; + ASSERT_OK(blob_source.GetIndirectBlob( + read_options, user_key, unflattened_index, + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false)); + EXPECT_EQ(value, logical_value); + EXPECT_GT(bytes_read, 0U); + EXPECT_EQ(options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT), + lazy_before_batch + 4); + + // The stable key does not include the generation or the physical file, so a + // read with another generation from another file hits the same entry. + const BlobIndex flattened_index = + make_index(/*generation=*/999, stable_checksum); + value.Reset(); + read_options.read_tier = kBlockCacheTier; + ASSERT_OK(blob_source.GetIndirectBlob( + read_options, user_key, flattened_index, + TestPhysicalFile(/*file_number=*/777, file_size, true, 123), + mutable_cf_options, /*range_offset=*/2, + /*range_length=*/6, /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false)); + EXPECT_EQ(value, Slice(logical_value.data() + 2, 6)); + EXPECT_EQ(bytes_read, 0U); + EXPECT_EQ(options_.statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT), + lazy_before_batch + 4); + + BlobFileCache other_cf_file_cache(file_cache.get(), &immutable_options, + &file_options, column_family_id + 1, + /*blob_file_read_hist=*/nullptr, + /*io_tracer=*/nullptr); + BlobSource other_cf_source(immutable_options, mutable_cf_options, db_id_, + db_session_id_, &other_cf_file_cache); + value.Reset(); + // File numbers are unique in the whole DB. This test-only source of another + // column family uses the same origin, so it hits the same entry. In a real + // DB, a Version passes only blob files of its own column family to + // BlobSource. + ASSERT_OK(other_cf_source.GetIndirectBlob( + read_options, user_key, unflattened_index, + TestPhysicalFile(/*file_number=*/778, file_size, false, 0), + mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false)); + EXPECT_EQ(value, logical_value); + EXPECT_EQ(bytes_read, 0U); + + // A hit is not checked against the BlobIndex: an index with the same stable + // identity but another generation and CRC gets the cached value, also from + // the cache-only lookup. + const BlobIndex conflicting_index = + make_index(/*generation=*/999, stable_checksum ^ 1); + value.Reset(); + ASSERT_OK(blob_source.GetIndirectBlob( + read_options, user_key, conflicting_index, + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false)); + EXPECT_EQ(value, logical_value); + EXPECT_EQ(bytes_read, 0U); + value.Reset(); + ASSERT_OK(blob_source.GetIndirectBlobFromCache( + conflicting_index, /*range_offset=*/2, /*range_length=*/4, &value)); + EXPECT_EQ(value, logical_value.substr(2, 4)); + value.Reset(); + EXPECT_TRUE(blob_source + .GetIndirectBlobFromCache(conflicting_index, + /*range_offset=*/2, + logical_value.size(), &value) + .IsInvalidArgument()); + + // The lookups with the conflicting index do not evict or replace the cached + // entry. + value.Reset(); + ASSERT_OK(blob_source.GetIndirectBlob( + read_options, user_key, unflattened_index, + TestPhysicalFile(/*file_number=*/779, file_size, true, 456), + mutable_cf_options, /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false)); + EXPECT_EQ(value, logical_value); + EXPECT_EQ(bytes_read, 0U); + + std::string oversized_encoding; + BlobIndex::EncodeIndirectBlob( + &oversized_encoding, origin_file_number + 1, + {.origin_file_number = origin_file_number + 1, + .original_value_offset = offsets[0], + .stored_value_size = kMaxSimpleGen2BlobPayloadSize + 1, + .compression = kNoCompression, + .stable_checksum = stable_checksum, + .stored_checksum = 0}); + BlobIndex oversized_index; + ASSERT_OK(oversized_index.DecodeFrom(oversized_encoding)); + + std::string unsupported_encoding; + BlobIndex::EncodeIndirectBlob(&unsupported_encoding, origin_file_number + 2, + {.origin_file_number = origin_file_number + 2, + .original_value_offset = offsets[0], + .stored_value_size = sizes[0], + .compression = kFirstCustomCompression, + .stable_checksum = stable_checksum, + .stored_checksum = 0}); + BlobIndex unsupported_index; + ASSERT_OK(unsupported_index.DecodeFrom(unsupported_encoding)); + + // These indexes are not cached, so the read checks them before any I/O. + const auto verify_rejected_before_io = [&](const BlobIndex& index, + bool is_relocation_file) { + value.Reset(); + EXPECT_TRUE( + blob_source + .GetIndirectBlob(read_options, user_key, index, + TestPhysicalFile(origin_file_number, file_size, + is_relocation_file, 0), + mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false) + .IsCorruption()); + EXPECT_TRUE(value.empty()); + + Status multi_status; + IndirectBlobReadRequest request; + request.user_key = user_key; + request.blob_index = &index; + request.result = &value; + request.status = &multi_status; + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, + TestPhysicalFile(origin_file_number, file_size, is_relocation_file, 0), + mutable_cf_options, /*num_requests=*/1, &request, &bytes_read); + EXPECT_TRUE(multi_status.IsCorruption()); + EXPECT_TRUE(value.empty()); + }; + verify_rejected_before_io(oversized_index, /*is_relocation_file=*/true); + verify_rejected_before_io(unsupported_index, /*is_relocation_file=*/false); + + value.Reset(); + EXPECT_TRUE(blob_source + .GetIndirectBlob( + read_options, user_key, oversized_index, + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false) + .IsIncomplete()); + Status oversized_normal_status; + IndirectBlobReadRequest oversized_normal_request; + oversized_normal_request.user_key = user_key; + oversized_normal_request.blob_index = &oversized_index; + oversized_normal_request.result = &value; + oversized_normal_request.status = &oversized_normal_status; + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, /*num_requests=*/1, &oversized_normal_request, + &bytes_read); + EXPECT_TRUE(oversized_normal_status.IsIncomplete()); + EXPECT_TRUE(value.empty()); + + std::string uncached_encoding; + BlobIndex::EncodeIndirectBlob(&uncached_encoding, origin_file_number, + {.origin_file_number = origin_file_number, + .original_value_offset = offsets[0] + 1, + .stored_value_size = sizes[0], + .compression = kNoCompression, + .stable_checksum = stable_checksum, + .stored_checksum = 0}); + BlobIndex uncached_offset_index; + ASSERT_OK(uncached_offset_index.DecodeFrom(uncached_encoding)); + value.Reset(); + EXPECT_TRUE(blob_source + .GetIndirectBlob( + read_options, user_key, uncached_offset_index, + TestPhysicalFile(origin_file_number, file_size, false, 0), + mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, + /*prefetch_buffer=*/nullptr, &value, &bytes_read, + /*cache_checked=*/false) + .IsIncomplete()); + EXPECT_TRUE(value.empty()); + // A cache-only lookup of a value that is not cached returns NotFound. + EXPECT_TRUE(blob_source + .GetIndirectBlobFromCache(uncached_offset_index, + /*range_offset=*/0, + kWholeBlobLength, &value) + .IsNotFound()); + EXPECT_TRUE(value.empty()); +} + +TEST_F(BlobSourceTest, CompressedIndirectRangeIsRejectedOnCacheHit) { + // As in base RocksDB, a range of a compressed value is not supported. A whole + // read first puts the value into the stable blob cache. A range of the same + // value then still returns Corruption, from the cache lookup, the scalar read + // and the batched read. A whole read still returns the value. + if (!Snappy_Supported()) { + return; + } + options_.blob_cache = NewLRUCache(1 << 20); + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_CompressedIndirectRange"), 0); + DestroyAndReopen(options_); + const ImmutableOptions immutable_options(options_); + const MutableCFOptions mutable_cf_options(options_); + + constexpr uint32_t column_family_id = 1; + constexpr uint64_t origin_file_number = 10; + constexpr ExpirationRange expiration_range; + const std::string user_key = "key"; + const std::string logical_value(1024, 'c'); + std::vector keys{user_key}; + std::vector blobs{logical_value}; + std::vector offsets(1); + std::vector sizes(1); + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, origin_file_number, keys, + blobs, kSnappyCompression, offsets, sizes); + const uint64_t file_size = BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + + user_key.size() + sizes[0] + BlobLogFooter::kSize; + + std::shared_ptr file_cache = NewLRUCache(1024); + FileOptions file_options; + BlobFileCache blob_file_cache( + file_cache.get(), &immutable_options, &file_options, column_family_id, + /*blob_file_read_hist=*/nullptr, /*io_tracer=*/nullptr); + BlobSource blob_source(immutable_options, mutable_cf_options, db_id_, + db_session_id_, &blob_file_cache); + + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, origin_file_number, + {.origin_file_number = origin_file_number, + .original_value_offset = offsets[0], + .stored_value_size = sizes[0], + .compression = kSnappyCompression, + .stable_checksum = + crc32c::Value(logical_value.data(), logical_value.size()), + .stored_checksum = 0}); + BlobIndex index; + ASSERT_OK(index.DecodeFrom(encoded)); + const BlobPhysicalFile file = + TestPhysicalFile(origin_file_number, file_size, false, 0); + + ReadOptions read_options; + read_options.fill_cache = true; + PinnableSlice value; + uint64_t bytes_read = 0; + ASSERT_OK(blob_source.GetIndirectBlob( + read_options, user_key, index, file, mutable_cf_options, + /*range_offset=*/0, kWholeBlobLength, /*prefetch_buffer=*/nullptr, &value, + &bytes_read, /*cache_checked=*/false)); + ASSERT_EQ(value, logical_value); + // The value is in the stable cache now. + value.Reset(); + ASSERT_OK(blob_source.GetIndirectBlobFromCache(index, /*range_offset=*/0, + kWholeBlobLength, &value)); + ASSERT_EQ(value, logical_value); + + value.Reset(); + EXPECT_TRUE(blob_source + .GetIndirectBlobFromCache(index, /*range_offset=*/1, + /*range_length=*/10, &value) + .IsCorruption()); + EXPECT_TRUE(blob_source + .GetIndirectBlob(read_options, user_key, index, file, + mutable_cf_options, /*range_offset=*/1, + /*range_length=*/10, + /*prefetch_buffer=*/nullptr, &value, + &bytes_read, /*cache_checked=*/false) + .IsCorruption()); + + std::array values; + std::array statuses; + std::array requests; + for (size_t i = 0; i < requests.size(); ++i) { + requests[i].user_key = user_key; + requests[i].blob_index = &index; + requests[i].result = &values[i]; + requests[i].status = &statuses[i]; + } + requests[1].range_offset = 1; + requests[1].range_length = 10; + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, file, mutable_cf_options, requests.size(), requests.data(), + &bytes_read); + ASSERT_OK(statuses[0]); + EXPECT_EQ(values[0], logical_value); + EXPECT_TRUE(statuses[1].IsCorruption()) << statuses[1].ToString(); +} + TEST_F(BlobSourceTest, GetBlobsFromCache) { options_.cf_paths.emplace_back( test::PerThreadDBPath(env_, "BlobSourceTest_GetBlobsFromCache"), 0); @@ -1323,6 +2209,244 @@ TEST_F(BlobSourceTest, MultiGetBlobReturnsGoodBlobsWhenRefreshOpenFails) { } } +TEST_F(BlobSourceTest, MultiGetIndirectBlobRetriesStaleReader) { + // Like BlobSource::MultiGetBlobFromOneFile, a batched indirect read of whole + // values from a normal blob file retries a Corruption once with a newly + // opened reader. + options_.statistics = CreateDBStatistics(); + options_.cf_paths.emplace_back( + test::PerThreadDBPath(env_, "BlobSourceTest_IndirectStaleReader"), 0); + DestroyAndReopen(options_); + const ImmutableOptions immutable_options(options_); + const MutableCFOptions mutable_cf_options(options_); + constexpr uint32_t column_family_id = 1; + constexpr uint64_t blob_file_number = 1; + constexpr ExpirationRange expiration_range; + const std::vector user_keys{"key0", "key1"}; + const std::vector values{"blob-value-0", "blob-value-1"}; + std::vector keys{user_keys[0]}; + std::vector blobs{values[0]}; + std::vector offsets(1); + std::vector sizes(1); + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, blob_file_number, keys, + blobs, kNoCompression, offsets, sizes); + + std::shared_ptr file_cache = NewLRUCache(1024); + FileOptions file_options; + BlobFileCache blob_file_cache(file_cache.get(), &immutable_options, + &file_options, column_family_id, + /*blob_file_read_hist=*/nullptr, + /*io_tracer=*/nullptr); + BlobSource blob_source(immutable_options, mutable_cf_options, db_id_, + db_session_id_, &blob_file_cache); + const auto make_index = [&](size_t i, uint64_t offset) { + const uint32_t checksum = crc32c::Value(values[i].data(), values[i].size()); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, blob_file_number, + {.origin_file_number = blob_file_number, + .original_value_offset = offset, + .stored_value_size = values[i].size(), + .compression = kNoCompression, + .stable_checksum = checksum, + .stored_checksum = BlobIndex::ExtendChecksumWithCompression( + checksum, kNoCompression)}); + BlobIndex index; + EXPECT_OK(index.DecodeFrom(encoded)); + return index; + }; + ReadOptions read_options; + read_options.verify_checksums = true; + read_options.fill_cache = false; + const auto read = [&](size_t i, const BlobIndex& index, uint64_t file_size, + PinnableSlice* value) { + Status status; + IndirectBlobReadRequest request; + request.user_key = user_keys[i]; + request.blob_index = &index; + request.result = value; + request.status = &status; + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, + TestPhysicalFile(blob_file_number, file_size, /*is_relocation=*/false, + /*directory_digest=*/0), + mutable_cf_options, /*num_requests=*/1, &request, + /*bytes_read=*/nullptr); + return status; + }; + const auto file_size = [&]() { + uint64_t size = 0; + EXPECT_OK(env_->GetFileSize( + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number), + &size)); + return size; + }; + + // The first read caches a reader of the one-record file. + PinnableSlice value; + ASSERT_OK(read(0, make_index(0, offsets[0]), file_size(), &value)); + ASSERT_EQ(value, values[0]); + + // The file is replaced by a larger one. The cached reader rejects the new + // record's offset as past its file size; the retry reads it. + keys = {user_keys[0], user_keys[1]}; + blobs = {values[0], values[1]}; + offsets.resize(2); + sizes.resize(2); + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, blob_file_number, keys, + blobs, kNoCompression, offsets, sizes); + const BlobIndex second_index = make_index(1, offsets[1]); + Statistics* const statistics = options_.statistics.get(); + const uint64_t opens = statistics->getTickerCount(NO_FILE_OPENS); + value.Reset(); + ASSERT_OK(read(1, second_index, file_size(), &value)); + EXPECT_EQ(value, values[1]); + EXPECT_EQ(statistics->getTickerCount(NO_FILE_OPENS), opens + 1); + + // Each batch has kOldRecords (18) reads of old records, which the stale + // reader still serves, and two equal lazy reads of a new record, which fail + // on the stale reader. The first pass reads one of the two and copies it to + // the other. The retry may pick the other one as its source. Either way, the + // new record is read again once and counted as one lazy read. + { + constexpr size_t kOldRecords = 18; + std::vector batch_keys; + std::vector batch_values; + for (size_t i = 0; i <= kOldRecords; ++i) { + batch_keys.push_back("batch" + std::to_string(100 + i)); + batch_values.push_back("batch-value-" + std::to_string(100 + i)); + } + std::vector batch_offsets; + const auto write_records = [&](size_t count) { + std::vector record_keys(batch_keys.begin(), + batch_keys.begin() + count); + std::vector record_values(batch_values.begin(), + batch_values.begin() + count); + batch_offsets.assign(count, 0); + std::vector record_sizes(count); + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, blob_file_number, + record_keys, record_values, kNoCompression, batch_offsets, + record_sizes); + }; + ReadOptions lazy_options = read_options; + lazy_options.io_activity = Env::IOActivity::kLazyResolve; + // Positions of the two equal reads in the batch. + for (const auto& pair : + std::vector>{{0, 1}, + {kOldRecords, kOldRecords + 1}, + {0, kOldRecords + 1}, + {5, 12}, + {9, 10}}) { + SCOPED_TRACE(std::to_string(pair.first) + "," + + std::to_string(pair.second)); + // Drop the reader of the previous case, and write the longer file to + // learn the offset of each record. + write_records(kOldRecords); + blob_file_cache.Evict(blob_file_number); + write_records(kOldRecords + 1); + std::vector indexes(kOldRecords + 1); + for (size_t i = 0; i <= kOldRecords; ++i) { + const uint32_t checksum = + crc32c::Value(batch_values[i].data(), batch_values[i].size()); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, blob_file_number, + {.origin_file_number = blob_file_number, + .original_value_offset = batch_offsets[i], + .stored_value_size = batch_values[i].size(), + .compression = kNoCompression, + .stable_checksum = checksum, + .stored_checksum = BlobIndex::ExtendChecksumWithCompression( + checksum, kNoCompression)}); + ASSERT_OK(indexes[i].DecodeFrom(encoded)); + } + // Write the shorter file again and cache a reader of it, then add the new + // record back. The cached reader is now stale: it still has the size of + // the shorter file. + write_records(kOldRecords); + { + Status prime_status; + PinnableSlice prime_value; + IndirectBlobReadRequest prime; + prime.user_key = batch_keys[0]; + prime.blob_index = &indexes[0]; + prime.result = &prime_value; + prime.status = &prime_status; + blob_source.MultiGetIndirectBlobFromOneFile( + read_options, + TestPhysicalFile(blob_file_number, file_size(), + /*is_relocation=*/false, /*directory_digest=*/0), + mutable_cf_options, /*num_requests=*/1, &prime, + /*bytes_read=*/nullptr); + ASSERT_OK(prime_status); + } + write_records(kOldRecords + 1); + constexpr size_t kRequests = kOldRecords + 2; + std::array statuses; + std::array results; + std::array requests; + std::array record_of; + size_t next_old = 0; + for (size_t i = 0; i < kRequests; ++i) { + record_of[i] = + (i == pair.first || i == pair.second) ? kOldRecords : next_old++; + requests[i].user_key = batch_keys[record_of[i]]; + requests[i].blob_index = &indexes[record_of[i]]; + requests[i].result = &results[i]; + requests[i].status = &statuses[i]; + } + const uint64_t lazy_reads = + statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT); + blob_source.MultiGetIndirectBlobFromOneFile( + lazy_options, + TestPhysicalFile(blob_file_number, file_size(), + /*is_relocation=*/false, /*directory_digest=*/0), + mutable_cf_options, kRequests, requests.data(), + /*bytes_read=*/nullptr); + for (size_t i = 0; i < kRequests; ++i) { + ASSERT_OK(statuses[i]); + EXPECT_EQ(results[i], batch_values[record_of[i]]); + } + // kOldRecords distinct reads plus one read of the new record. + EXPECT_EQ(statistics->getTickerCount(BLOB_DB_LAZY_READ_COUNT), + lazy_reads + kOldRecords + 1); + } + // Put back the two-record file for the cases below. + WriteBlobFile(immutable_options, column_family_id, /*has_ttl=*/false, + expiration_range, expiration_range, blob_file_number, keys, + blobs, kNoCompression, offsets, sizes); + blob_file_cache.Evict(blob_file_number); + } + + // A real corruption fails the retry too and is returned. + std::string contents; + const std::string path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + ASSERT_OK(ReadFileToString(env_, path, &contents)); + contents[offsets[1]] ^= 1; + ASSERT_OK(WriteStringToFile(env_, contents, path)); + blob_file_cache.Evict(blob_file_number); + value.Reset(); + Status status = read(1, second_index, file_size(), &value); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_TRUE(value.empty()); + + // When the file is deleted, the retry cannot reopen it. The result is the + // Corruption from the cached reader (the offset is past its file size), with + // the reopen error added to its message. + value.Reset(); + ASSERT_OK(read(0, make_index(0, offsets[0]), contents.size(), &value)); + ASSERT_OK(env_->DeleteFile(path)); + value.Reset(); + status = read(1, make_index(1, contents.size()), contents.size(), &value); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_NE(status.ToString().find("refresh retry failed"), std::string::npos) + << status.ToString(); +} + class BlobSecondaryCacheTest : public DBTestBase { protected: public: diff --git a/db/blob/db_blob_basic_test.cc b/db/blob/db_blob_basic_test.cc index dd418727a8ca..460a37de6a5c 100644 --- a/db/blob/db_blob_basic_test.cc +++ b/db/blob/db_blob_basic_test.cc @@ -6,8 +6,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -15,26 +18,45 @@ #include #include +#include "cache/cache_helpers.h" #include "cache/compressed_secondary_cache.h" +#include "db/blob/blob_file_cache.h" #include "db/blob/blob_file_partition_manager.h" #include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" #include "db/blob/blob_log_sequential_reader.h" +#include "db/blob/blob_reference_census.h" +#include "db/blob/blob_reference_metadata.h" +#include "db/blob/blob_relocation_manifest.h" #include "db/column_family.h" #include "db/db_test_util.h" #include "db/db_with_timestamp_test_util.h" +#include "db/log_writer.h" +#include "db/manifest_ops.h" +#include "db/table_cache.h" +#include "db/version_edit.h" #include "env/composite_env_wrapper.h" #include "file/filename.h" #include "file/random_access_file_reader.h" +#include "file/writable_file_writer.h" +#include "options/cf_options.h" #include "port/stack_trace.h" #include "rocksdb/convenience.h" +#include "rocksdb/experimental.h" #include "rocksdb/file_system.h" #include "rocksdb/lazy_wide_columns.h" #include "rocksdb/trace_reader_writer.h" #include "rocksdb/trace_record.h" +#include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/options_util.h" #include "rocksdb/utilities/replayer.h" +#include "table/adaptive/adaptive_table_factory.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/table_builder.h" #include "test_util/sync_point.h" +#include "trace_replay/block_cache_tracer.h" #include "util/compression.h" #include "util/defer.h" #include "util/file_checksum_helper.h" @@ -43,6 +65,62 @@ namespace ROCKSDB_NAMESPACE { +namespace { + +struct RelocationTestRecord { + uint64_t origin_file_number; + uint64_t original_value_offset; + Slice stored_payload; + CompressionType compression; +}; + +void WriteRelocationFileForDBTest( + const ImmutableOptions& immutable_options, + const MutableCFOptions& mutable_cf_options, + const BlockBasedTableOptions& table_options, uint32_t column_family_id, + uint64_t blob_file_number, const std::vector& records, + BlobRelocationFileMetadata* metadata, uint64_t* file_size) { + assert(metadata != nullptr); + assert(file_size != nullptr); + const std::string file_path = + BlobFileName(immutable_options.cf_paths.front().path, blob_file_number); + std::unique_ptr file; + ASSERT_OK(immutable_options.fs->NewWritableFile(file_path, FileOptions(), + &file, /*dbg=*/nullptr)); + std::unique_ptr writer = + std::make_unique( + std::move(file), file_path, FileOptions(), immutable_options.clock); + + const InternalKeyComparator comparator(BytewiseComparator()); + std::unique_ptr builder; + BlobRelocationTableFileInfo file_info; + file_info.column_family_id = column_family_id; + file_info.column_family_name = kDefaultColumnFamilyName; + file_info.db_id = "test-db"; + file_info.db_session_id = "test-session"; + file_info.file_number = blob_file_number; + ASSERT_OK(NewBlobRelocationTableBuilder( + immutable_options, mutable_cf_options, table_options, ReadOptions(), + WriteOptions(), comparator, file_info, writer.get(), &builder)); + for (const RelocationTestRecord& record : records) { + ASSERT_OK(builder->AddBlobRelocation( + record.origin_file_number, record.original_value_offset, + record.stored_payload, record.compression)); + } + // List each origin file as a served file of the relocation file: an old file + // number that a route may send to this file. + for (const RelocationTestRecord& record : records) { + ASSERT_OK(builder->AddBlobRelocationServedFile(record.origin_file_number)); + } + ASSERT_OK(builder->Finish()); + ASSERT_OK(builder->GetBlobRelocationFileMetadata(metadata)); + ASSERT_OK(writer->Close(IOOptions(), /*dbg=*/nullptr)); + ASSERT_OK(immutable_options.fs->GetFileSize(file_path, IOOptions(), file_size, + /*dbg=*/nullptr)); +} + +} // namespace + class DBBlobBasicTest : public DBTestBase { protected: DBBlobBasicTest() @@ -78,6 +156,98 @@ class DBBlobBasicTest : public DBTestBase { return options; } + // Blob indirection is on. Indirect blob GC stays off, because blob GC and + // auto compaction are disabled. EnableIndirectBlobGC() turns both on. + Options IndirectBlobGCOptions() { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + return options; + } + + // WriteIndirectBlobGCCandidate() writes kIndirectBlobGCKeyCount (4) keys with + // 4 KB values into one blob file, deletes all but the last `live_keys` of + // them and compacts. It returns the blob file number. + static constexpr size_t kIndirectBlobGCKeyCount = 4; + static std::string IndirectBlobGCKey(size_t i) { + return "key" + std::to_string(i); + } + static std::string IndirectBlobGCValue(size_t i) { + return std::string(4096, static_cast('a' + i)); + } + uint64_t WriteIndirectBlobGCCandidate(size_t live_keys) { + for (size_t i = 0; i < kIndirectBlobGCKeyCount; ++i) { + EXPECT_OK(Put(IndirectBlobGCKey(i), IndirectBlobGCValue(i))); + } + EXPECT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + for (size_t i = 0; i + live_keys < kIndirectBlobGCKeyCount; ++i) { + EXPECT_OK(Delete(IndirectBlobGCKey(i))); + } + EXPECT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + EXPECT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + return file_number; + } + + void EnableIndirectBlobGC() { + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + for (int round = 0; round < 3; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + } + + std::set BlobFilesOnDisk() { + std::vector files; + EXPECT_OK(env_->GetChildren(dbname_, &files)); + std::set result; + for (const std::string& name : files) { + uint64_t number = 0; + FileType type; + if (ParseFileName(name, &number, /*info_log_name_prefix=*/"", &type) && + type == kBlobFile) { + result.insert(number); + } + } + return result; + } + + uint64_t RouteTarget(uint64_t file_number) { + return dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetBlobRelocationRouteTarget(file_number); + } + + bool HasBlobFile(uint64_t file_number) { + return dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetBlobFileMetaData(file_number) != nullptr; + } + void AssertBlobReference(const Slice& key, uint64_t blob_file_number) { PinnableSlice value; bool is_blob_index = false; @@ -127,147 +297,9937 @@ class DBBlobBasicTest : public DBTestBase { ASSERT_EQ(TotalTableFiles(), 1); } - Close(); - ASSERT_OK(env_->DeleteFile(BlobFileName(dbname_, *missing_blob_file))); - } + Close(); + ASSERT_OK(env_->DeleteFile(BlobFileName(dbname_, *missing_blob_file))); + } + + void RecoverMissingNonOldestBlob(const Options& options, + bool preserve_older_value, + uint64_t* missing_blob_file) { + ASSERT_NO_FATAL_FAILURE(CreateSstWithMissingBlobFile( + options, preserve_older_value, /*delete_oldest=*/false, + missing_blob_file)); + Options recovery_options = options; + recovery_options.best_efforts_recovery = true; + Reopen(recovery_options); + const std::vector live_blob_files = GetBlobFileNumbers(); + ASSERT_EQ(live_blob_files.size(), 1); + ASSERT_NE(live_blob_files.front(), *missing_blob_file); + ASSERT_EQ(TotalTableFiles(), 1); + ASSERT_NO_FATAL_FAILURE(AssertBlobReference("b", *missing_blob_file)); + ASSERT_NO_FATAL_FAILURE(AssertBlobReference("d", *missing_blob_file)); + } + + std::vector GetDroppedBlobFiles() { + return dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetDroppedBlobFiles(); + } + + static Options RepairOptions(Options options) { + options.best_efforts_blob_db_repair = true; + return options; + } + + // Returns the keys and values a forward and a reverse scan read, which must + // agree. + std::string ScanAll(ReadOptions read_options = ReadOptions()) { + std::string forward; + std::string reverse; + { + std::unique_ptr iter(db_->NewIterator(read_options)); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + EXPECT_TRUE(iter->PrepareValue()); + forward += + iter->key().ToString() + "=" + iter->value().ToString() + ";"; + } + EXPECT_OK(iter->status()); + for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { + EXPECT_TRUE(iter->PrepareValue()); + reverse.insert( + 0, iter->key().ToString() + "=" + iter->value().ToString() + ";"); + } + EXPECT_OK(iter->status()); + } + EXPECT_EQ(forward, reverse); + return forward; + } + + // Runs best-efforts BlobDB repair on a DB whose newer blob file is missing + // (see CreateSstWithMissingBlobFile()) and checks what it recorded. + void RepairMissingNonOldestBlob(const Options& options, + uint64_t* missing_blob_file) { + ASSERT_NO_FATAL_FAILURE(CreateSstWithMissingBlobFile( + options, /*preserve_older_value=*/false, /*delete_oldest=*/false, + missing_blob_file)); + Reopen(RepairOptions(options)); + ASSERT_EQ(GetDroppedBlobFiles(), std::vector{*missing_blob_file}); + ASSERT_EQ(TotalTableFiles(), 1); + } + + std::shared_ptr GetCensusSnapshot( + ColumnFamilyData* cfd) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + return cfd->blob_reference_census()->TEST_GetReadyState( + cfd->current()->GetVersionNumber()); + } + + // The number of SSTs in the current Version of the default CF that have + // exact blob reference metadata (a reference descriptor). + size_t CountExactSsts() { + VersionStorageInfo* const storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + size_t count = 0; + for (int level = 0; level < storage->num_levels(); ++level) { + for (const FileMetaData* file : storage->LevelFiles(level)) { + count += file->blob_reference_descriptor.empty() ? 0 : 1; + } + } + return count; + } + + // Builds a census for the current Version from the reference indexes of its + // SSTs, and checks that the live census has the same generation numbers, SSTs + // and totals. + void ExpectCensusMatchesVersion() { + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + // Pin the current Version (a reference taken under the DB mutex, so that + // a background install cannot free it), then read the census state of that + // version number. If a background install replaced the Version before the + // census caught up with it, try again with the new Version, at most + // kAttempts (10) times. + constexpr int kAttempts = 10; + Version* version = nullptr; + std::shared_ptr census; + for (int attempt = 0; attempt < kAttempts && census == nullptr; ++attempt) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + if (version != nullptr) { + version->Unref(); + } + version = cfd->current(); + version->Ref(); + } + // Read without the DB mutex: the copy takes the census worker mutex. + census = cfd->blob_reference_census()->TEST_GetReadyState( + version->GetVersionNumber()); + } + auto unref = Defer([this, version]() { + InstrumentedMutexLock lock(dbfull()->mutex()); + version->Unref(); + }); + ASSERT_NE(census, nullptr); + std::vector> files; + VersionStorageInfo* const storage = version->storage_info(); + for (int level = 0; level < storage->num_levels(); ++level) { + for (FileMetaData* file : storage->LevelFiles(level)) { + if (file->blob_reference_descriptor.empty()) { + continue; + } + std::shared_ptr summary; + ASSERT_OK(cfd->table_cache()->ReadBlobReferenceSstSummary( + ReadOptions(), FileOptions(), cfd->internal_comparator(), *file, + version->GetMutableCFOptions(), &summary, + /*reservation_manager=*/nullptr)); + files.push_back(std::move(summary)); + } + } + std::shared_ptr expected; + ASSERT_OK(BlobReferenceCensus::BuildState(version->GetVersionNumber(), + files, &expected)); + std::string difference; + EXPECT_TRUE(expected->SameMemberships(*census, &difference)) << difference; + } + + // Options with a block cache that tracks the kBlobReferenceCensus charge. + Options CensusChargeTrackingOptions( + std::shared_ptr< + TargetCacheChargeTrackingCache>* + cache) { + *cache = std::make_shared< + TargetCacheChargeTrackingCache>( + NewLRUCache(64 << 20)); + BlockBasedTableOptions table_options; + table_options.block_cache = *cache; + Options options = GetDefaultOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + return options; + } + + // Counts census rebuild installs and SST reads by the census build. + struct CensusEventCounter { + std::atomic rebuilds{0}; + std::atomic reads{0}; + void Start() { + SyncPoint::GetInstance()->SetCallBack( + "BlobReferenceCensus::Install:Rebuild", + [this](void*) { ++rebuilds; }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:ReadSst", + [this](void*) { ++reads; }); + SyncPoint::GetInstance()->EnableProcessing(); + } + ~CensusEventCounter() { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } + }; + + // Opens the DB with small blob files (2 MB) and flushes kManyGroupsKeyCount + // keys into one SST. The references of the SST form several groups, one per + // blob file. A group spans more than one reference data block, and a group + // can start inside a block that the previous group also uses. `compression` + // is the table compression, which also compresses the reference data blocks. + static constexpr int kManyGroupsKeyCount = 120000; + Options FlushManyBlobReferenceGroups( + CompressionType compression = kNoCompression) { + Options options = GetDefaultOptions(); + options.compression = compression; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_file_size = 2 << 20; + options.disable_auto_compactions = true; + // Keep every reference in one SST, also when it is compacted. + options.target_file_size_base = 1 << 30; + options.create_if_missing = true; + DestroyAndReopen(options); + for (int i = 0; i < kManyGroupsKeyCount; ++i) { + EXPECT_OK(Put(Key(i), std::string(10 + i % 23, 'v'))); + } + EXPECT_OK(Flush()); + EXPECT_OK(BuildExactSstAtLevel1()); + return options; + } + + // The only SST of `cfd`, at any level, or nullptr if there is not exactly + // one. + static const FileMetaData* OnlySstForTest(ColumnFamilyData* cfd) { + const VersionStorageInfo* storage = cfd->current()->storage_info(); + const FileMetaData* result = nullptr; + for (int level = 0; level < storage->num_levels(); ++level) { + for (const FileMetaData* file : storage->LevelFiles(level)) { + if (result != nullptr) { + return nullptr; + } + result = file; + } + } + return result; + } + + // Reads and decodes the reference index of an SST from its file. + BlobReferenceIndex ReadBlobReferenceIndexForTest(const FileMetaData& file) { + BlobReferenceMetadataDescriptor descriptor; + EXPECT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + file.blob_reference_descriptor, &descriptor)); + std::string contents; + EXPECT_OK(ReadFileToString( + env_, MakeTableFileName(dbname_, file.fd.GetNumber()), &contents)); + BlobReferenceIndex index; + const BlockHandle& handle = descriptor.index_handle; + if (handle.offset() + handle.size() > contents.size()) { + ADD_FAILURE() << "Reference index is past the end of the file"; + return index; + } + EXPECT_OK(DecodeBlobReferenceIndex( + Slice(contents.data() + handle.offset(), handle.size()), descriptor, + &index)); + return index; + } + + // Compacts each L0 SST alone into `level`, so SSTs with separate key ranges + // stay separate. When `level` is 1 or higher, each output has exact metadata. + void CompactEachL0FileTo(int level) { + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + for (const auto& file : metadata.levels[0].files) { + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {file.name}, level)); + } + } + + Status BuildExactSstAtLevel1() { + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + compact_options.blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kDisable; + return db_->CompactRange(compact_options, nullptr, nullptr); + } + + // Flush output is always coarse. This helper uses the FlushJob test hook to + // make one flush write exact reference metadata, for tests that need an exact + // SST. + Status FlushInjectedBlobIndexesWithExactMetadata() { + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("FlushJob::WriteLevel0Table:TableBuilderOptions", + [](void* arg) { + static_cast(arg) + ->collect_blob_reference_metadata = true; + }); + sync_point->EnableProcessing(); + const Defer cleanup([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + return Flush(); + } + + // Registers an indirect blob GC job on the census of `cfd` at its current + // Version and ends it at once. Returns the status of + // BlobReferenceCensus::BeginGcJob(): Busy while an indirect blob GC install + // has an unknown MANIFEST result, OK when a GC job could start. + Status TryBeginGcJob(ColumnFamilyData* cfd) { + InstrumentedMutexLock lock(dbfull()->mutex()); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + if (census == nullptr) { + return Status::NotFound("No blob reference census"); + } + uint64_t token = 0; + Status status = + census->BeginGcJob(cfd->current()->GetVersionNumber(), &token); + if (status.ok()) { + census->EndGcJob(token); + } + return status; + } + + // Checks that Get() and MultiGet() find a live key next to a reference cap + // cut inside a range tombstone. + // + // Setup: the cap cuts one compaction to L0 into two exact L0 files with one + // epoch number (an epoch run). The cut is inside the range tombstone + // [Key(4090), Key(4100)), so both files keep the whole tombstone, and the + // left file ends with the exclusive sentinel + // . Key(4095) is the "cut + // key": the first key of the right file. If Get() reads the file that does + // not hold the key first, that file's untruncated tombstone must not end the + // search: the key is live and newer than the tombstone. + // + // How the compaction to L0 is made: + // - universal: num_levels = 1, so CompactRange() writes L0; + // - leveled: CompactFiles() with output level 0 (an intra-L0 compaction). + // L1 and deeper are empty, so the compaction is bottommost and its output + // gets exact reference metadata, which the cap needs. + // + // Each case writes one more key (the "newer key") after the live key. The + // output file that holds the newer key has the larger largest seqno, and L0 + // reads the files of one epoch in order of largest seqno, larger first. So + // the newer key decides which file Get() reads first. + // - live key 4095 (the cut key), newer key 10 on the left: the left file is + // read first, and the lookup is at its sentinel; + // - live key 4092, newer key 4500 on the right: the right file is read + // first, and the key is live in the left file. + // + // Each case runs with 2 L0 files and with 4 L0 files (two newer flushes of + // one key each). With at most 3 L0 files and no other level, FilePicker + // asks every L0 file. With more than 3, it skips files whose user key range + // does not hold the key, but it still asks the left file about the cut key, + // because the left file's largest user key is the cut key. With 4 files, the + // case with live key 4092 skips the right file, so only the cut-key case + // reaches the check. + void CheckPointLookupAcrossRangeTombstoneCut(CompactionStyle style) { + constexpr int kKeys = 5000; + // Keys 0 to 4094 are 4095 references, and one of them (the live key or the + // newer key) has a second version under the snapshot, so the left file + // reaches the cap of 4096 references just before Key(4095). + constexpr int kCut = 4095; + struct Case { + int live_key; + int newer_key; + }; + for (const int extra_l0_files : {0, 2}) { + for (const Case& c : {Case{kCut, 10}, Case{4092, 4500}}) { + SCOPED_TRACE("live_key=" + std::to_string(c.live_key) + + " extra_l0_files=" + std::to_string(extra_l0_files)); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.compaction_style = style; + if (style == kCompactionStyleUniversal) { + options.num_levels = 1; + } + options.target_file_size_base = 1ULL << 30; + options.blob_indirection_max_references_per_sst = 4096; + DestroyAndReopen(options); + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(Key(i), "old" + std::to_string(i))); + } + // Keeps the tombstone and the covered versions in the bottommost + // compaction below. + const Snapshot* const snapshot = db_->GetSnapshot(); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + Key(4090), Key(4100))); + ASSERT_OK(Put(Key(c.live_key), "live")); + ASSERT_OK(Put(Key(c.newer_key), "newer")); + ASSERT_OK(Flush()); + if (style == kCompactionStyleUniversal) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } else { + std::vector flushed; + db_->GetLiveFilesMetaData(&flushed); + ASSERT_EQ(flushed.size(), 1U); + ASSERT_EQ(flushed[0].level, 0); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {flushed[0].name}, + /*output_level=*/0)); + } + + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + { + const std::vector& files = + cfd->current()->storage_info()->LevelFiles(0); + ASSERT_EQ(files.size(), 2U); + ASSERT_EQ(files[0]->epoch_number, files[1]->epoch_number); + // Both outputs have exact reference metadata. + for (const FileMetaData* file : files) { + ASSERT_FALSE(file->blob_reference_descriptor.empty()); + } + } + for (int i = 0; i < extra_l0_files; ++i) { + ASSERT_OK(Put(Key(i), "extra")); + ASSERT_OK(Flush()); + } + std::vector files; + db_->GetLiveFilesMetaData(&files); + ASSERT_EQ(files.size(), 2U + extra_l0_files); + for (const LiveFileMetaData& file : files) { + ASSERT_EQ(file.level, 0); + } + + EXPECT_EQ(Get(Key(c.live_key)), "live"); + // One MultiGet batch with the live key and one covered key on each + // side of the cut. + const std::string live_key = Key(c.live_key); + const std::string covered_left = Key(4091); + const std::string covered_right = Key(4097); + const std::vector keys{covered_left, live_key, covered_right}; + std::vector values; + const std::vector statuses = + db_->MultiGet(ReadOptions(), keys, &values); + ASSERT_EQ(statuses.size(), 3U); + EXPECT_TRUE(statuses[0].IsNotFound()) << statuses[0].ToString(); + EXPECT_OK(statuses[1]); + EXPECT_EQ(values[1], "live"); + EXPECT_TRUE(statuses[2].IsNotFound()) << statuses[2].ToString(); + // Covered keys on both sides of the cut stay deleted. + for (int i = 4090; i < 4100; ++i) { + if (i != c.live_key) { + ASSERT_EQ(Get(Key(i)), "NOT_FOUND") << i; + } + } + // The snapshot still sees the old values. + ASSERT_EQ(Get(Key(4093), snapshot), "old4093"); + ASSERT_EQ(Get(Key(kCut), snapshot), "old" + std::to_string(kCut)); + db_->ReleaseSnapshot(snapshot); + } + } + } + + Status ApplyVersionEdit(VersionEdit* edit, bool new_descriptor_log) { + assert(edit != nullptr); + DBImpl* db_impl = dbfull(); + VersionSet* version_set = db_impl->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + return ApplyVersionEdit(cfd, edit, new_descriptor_log); + } + + Status ApplyVersionEdit(ColumnFamilyData* cfd, VersionEdit* edit, + bool new_descriptor_log) { + assert(cfd != nullptr); + assert(edit != nullptr); + DBImpl* db_impl = dbfull(); + VersionSet* version_set = db_impl->GetVersionSet(); + db_impl->TEST_LockMutex(); + Status status = version_set->LogAndApply( + cfd, ReadOptions(), WriteOptions(), edit, db_impl->TEST_Mutex(), + /*dir_contains_current_file=*/nullptr, new_descriptor_log, + /*column_family_options=*/nullptr, + /*manifest_wcb=*/{}, /*pre_cb=*/{}); + db_impl->TEST_UnlockMutex(); + return status; + } + + // Writes a relocation file with one value (origin 1, offset 11) into the DB + // directory and adds it to the default column family with a VersionEdit. + // `*file` gets what a reader needs to open it. + void InstallOneRelocationFile(const Options& options, + BlobPhysicalFile* file) { + VersionSet* const version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* const cfd = + version_set->GetColumnFamilySet()->GetDefault(); + dbfull()->TEST_LockMutex(); + const uint64_t file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + file_number, + {{/*origin_file_number=*/1, /*original_value_offset=*/11, "payload", + kNoCompression}}, + &metadata, &file_size); + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + file_number, file_size, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), metadata.encoded_directory, + metadata.tail_size)); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + const std::shared_ptr installed = + cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + file_number); + ASSERT_NE(installed, nullptr); + *file = BlobPhysicalFile{ + BlobFileOpenInfo{file_number, installed->GetChecksumValue(), + installed->GetChecksumMethod()}, + installed->GetFileSize(), /*is_relocation=*/true, + installed->GetTailSize(), installed->GetDirectoryDigest()}; + } + + // Coarse pin race tests. A flush takes its blob file number (for example 25) + // early, and installs its coarse L0 SST (frontier 25; see "Coarse SST" in + // db/blob/blob_reference_metadata.h) later. A GC job that selects an input + // numbered above 25 in between sees the coarse pin appear before it installs. + // These helpers make that order with sync points, without sleeps. + // + // SetUpCoarsePinRace() installs two one-shot pauses. Each pause holds a + // background thread without the DB mutex: + // - Flush pause: the first flush that reaches + // "BuildTable:BeforeFinishBuildTable" after *pause_flush becomes true. + // Its blob file number is allocated, and its coarse SST is not + // installed yet. The test waits for it with + // "DBBlobBasicTest::CoarsePinRace:WaitFlushPaused" and lets it go on + // with "DBBlobBasicTest::CoarsePinRace:ResumeFlush". + // - GC pause: the first indirect blob GC job that reaches + // "DBImpl::RunIndirectBlobGC:BeforeOutputWork" after *pause_gc becomes + // true. It has selected its inputs and has not installed. The test + // waits for it with "DBBlobBasicTest::CoarsePinRace:WaitGcPaused" and + // lets it go on with "DBBlobBasicTest::CoarsePinRace:ResumeGc". + // *stale_installs counts the install checks (in the LogAndApply + // precondition) that found a stale input. + void SetUpCoarsePinRace(std::atomic* pause_flush, + std::atomic* pause_gc, + std::atomic* stale_installs) { + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->LoadDependency( + {{"DBBlobBasicTest::CoarsePinRace:FlushPaused", + "DBBlobBasicTest::CoarsePinRace:WaitFlushPaused"}, + {"DBBlobBasicTest::CoarsePinRace:ResumeFlush", + "DBBlobBasicTest::CoarsePinRace:FlushResumed"}, + {"DBBlobBasicTest::CoarsePinRace:GcPaused", + "DBBlobBasicTest::CoarsePinRace:WaitGcPaused"}, + {"DBBlobBasicTest::CoarsePinRace:ResumeGc", + "DBBlobBasicTest::CoarsePinRace:GcResumed"}}); + sync_point->SetCallBack("BuildTable:BeforeFinishBuildTable", + [pause_flush](void* /*arg*/) { + if (pause_flush->exchange(false)) { + TEST_SYNC_POINT( + "DBBlobBasicTest::CoarsePinRace:" + "FlushPaused"); + TEST_SYNC_POINT( + "DBBlobBasicTest::CoarsePinRace:" + "FlushResumed"); + } + }); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:BeforeOutputWork", + [pause_gc](void* /*arg*/) { + if (pause_gc->exchange(false)) { + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:GcPaused"); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:GcResumed"); + } + }); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:InstallPrecondition", + [stale_installs](void* arg) { + if (!*static_cast(arg)) { + stale_installs->fetch_add(1); + } + }); + sync_point->EnableProcessing(); + } + + // Starts a background flush of `value` under `key`, and returns when the + // flush pause of SetUpCoarsePinRace() holds it. The caller set *pause_flush + // to true. + void StartPausedCoarseFlush(const std::string& key, + const std::string& value) { + ASSERT_OK(Put(key, value)); + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(db_->Flush(flush_options)); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:WaitFlushPaused"); + } + + // Compacts every L0 and L1 SST into L1, so a deletion in L0 drops the + // references to the deleted values. CompactFiles() does not flush the + // memtables, so it runs while a flush is paused. CompactFiles() has no + // blob_garbage_collection_policy, so the helper turns compaction-time blob + // GC off for the compaction. Otherwise the compaction may move live values + // into a new blob file, and the test inputs would lose their live values. + // The helper turns blob GC on again afterwards, because indirect blob GC + // relocates live values only when enable_blob_garbage_collection is true. + void CompactL0AndL1IntoL1() { + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_garbage_collection", "false"}})); + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + std::vector inputs; + for (int level = 0; level <= 1; ++level) { + for (const auto& file : metadata.levels[level].files) { + inputs.push_back(file.name); + } + } + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, /*level=*/1)); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_garbage_collection", "true"}})); + } + + bool IsCoarsePinned(uint64_t file_number) { + return dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->IsCoarseIndirectOriginBlobPinned(file_number); + } +}; + +class BlobFileChecksumCapturingFS : public FileSystemWrapper { + public: + explicit BlobFileChecksumCapturingFS(const std::shared_ptr& base) + : FileSystemWrapper(base) {} + + static const char* kClassName() { return "BlobFileChecksumCapturingFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + if (fname.find(".blob") != std::string::npos) { + std::lock_guard lock(mu_); + file_checksum_ = opts.file_checksum; + file_checksum_func_name_ = opts.file_checksum_func_name; + ++capture_count_; + } + return target()->NewRandomAccessFile(fname, opts, result, dbg); + } + + std::string GetFileChecksum() { + std::lock_guard lock(mu_); + return file_checksum_; + } + + std::string GetFileChecksumFuncName() { + std::lock_guard lock(mu_); + return file_checksum_func_name_; + } + + int GetCaptureCount() { + std::lock_guard lock(mu_); + return capture_count_; + } + + private: + std::mutex mu_; + std::string file_checksum_; + std::string file_checksum_func_name_; + int capture_count_ = 0; +}; + +namespace { + +// Keeps every info log line, so that the indirect blob GC signal tests can +// check the LOG lines that a job writes. A WARN or ERROR line starts with +// "[WARN] " or "[ERROR] " (see Logger::Logv()). +class IndirectBlobGcLogCapture : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[4096]; + vsnprintf(buffer, sizeof(buffer), format, ap); + std::lock_guard lock(mutex_); + lines_.emplace_back(buffer); + } + // The number of lines that contain every string of `parts`. + size_t Count(const std::vector& parts) { + std::lock_guard lock(mutex_); + size_t count = 0; + for (const std::string& line : lines_) { + bool all = true; + for (const std::string& part : parts) { + all = all && line.find(part) != std::string::npos; + } + count += all ? 1 : 0; + } + return count; + } + + private: + std::mutex mutex_; + std::vector lines_; +}; + +// Records the path and the creation reason of each blob file that a listener +// sees start. +class BlobCreationReasonRecorder : public EventListener { + public: + void OnBlobFileCreationStarted( + const BlobFileCreationBriefInfo& info) override { + std::lock_guard lock(mutex_); + started_.emplace_back(info.file_path, info.reason); + } + // The paths of the started blob files whose reason is `reason`. + std::vector Paths(BlobFileCreationReason reason) { + std::lock_guard lock(mutex_); + std::vector result; + for (const auto& entry : started_) { + if (entry.second == reason) { + result.push_back(entry.first); + } + } + return result; + } + + private: + std::mutex mutex_; + std::vector> started_; +}; + +// A DB clock that moves only when the test calls Advance(). The idle time +// test uses it, so that two reads of the idle time give the same value and +// the test can make the idle time longer than the stats period without a +// sleep. Only the DB clock is manual: sleeps and timed waits still use the +// real clock. +class ManualIndirectBlobGcClock : public SystemClockWrapper { + public: + explicit ManualIndirectBlobGcClock(const std::shared_ptr& base) + : SystemClockWrapper(base), now_micros_(base->NowMicros()) {} + static const char* kClassName() { return "ManualIndirectBlobGcClock"; } + const char* Name() const override { return kClassName(); } + uint64_t NowMicros() override { return now_micros_.load(); } + void Advance(uint64_t micros) { now_micros_.fetch_add(micros); } + + private: + std::atomic now_micros_; +}; + +// The indirect blob GC tickers and their values at one point in time. +struct IndirectBlobGcTickers { + static constexpr std::array kTickers{ + {INDIRECT_BLOB_GC_JOBS_STARTED, INDIRECT_BLOB_GC_JOBS_INSTALLED, + INDIRECT_BLOB_GC_JOBS_STALE, INDIRECT_BLOB_GC_JOBS_NOTHING_INSTALLED, + INDIRECT_BLOB_GC_JOBS_FAILED, INDIRECT_BLOB_GC_CORRUPT_INPUTS, + INDIRECT_BLOB_GC_KEYS_RELOCATED, INDIRECT_BLOB_GC_BYTES_RELOCATED, + INDIRECT_BLOB_GC_BYTES_READ, INDIRECT_BLOB_GC_BYTES_RECLAIMED, + INDIRECT_BLOB_GC_CENSUS_MISMATCHES, BLOB_DB_GC_FAILURES, + BLOB_DB_GC_NUM_KEYS_RELOCATED}}; + explicit IndirectBlobGcTickers(Statistics* statistics) { + for (Tickers ticker : kTickers) { + values[ticker] = statistics->getTickerCount(ticker); + } + } + std::map values; + // The count of `ticker` since `before`. + uint64_t Since(const IndirectBlobGcTickers& before, Tickers ticker) const { + return values.at(ticker) - before.values.at(ticker); + } +}; + +} // namespace + +// Checks the signals of an indirect blob GC job that installs: the job +// outcome tickers, the job histogram, the relocation byte and key tickers, the +// creation reason that listeners see for the relocation file, the end-of-job +// LOG line, and the idle time: a job start clears it, and it is 0 after a job +// that leaves no file to collect. +TEST_F(DBBlobBasicTest, IndirectBlobGCInstalledJobSignals) { + Options options = IndirectBlobGCOptions(); + options.statistics = CreateDBStatistics(); + auto log = std::make_shared(); + options.info_log = log; + auto recorder = std::make_shared(); + options.listeners.push_back(recorder); + DestroyAndReopen(options); + // Three of the four values are garbage, so GC relocates the live one. + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + Statistics* const statistics = options.statistics.get(); + const IndirectBlobGcTickers before(statistics); + ASSERT_TRUE(recorder->Paths(BlobFileCreationReason::kIndirectBlobGC).empty()); + + // A job that has started clears the idle time of its column family. The + // census noted the candidate before the job (at time `noted_micros`), so the + // idle time start is not 0. During the job it is 0: a census publish does + // not start it while a GC job is registered. + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + dbfull()->TEST_WaitForBlobReferenceCensus(); + const uint64_t noted_micros = cfd->IndirectBlobGcCandidatesSinceMicros(); + ASSERT_NE(noted_micros, 0U); + std::atomic idle_reads{0}; + std::atomic since_during_job{noted_micros}; + std::atomic idle_during_job{1}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:BeforeOutputWork", [&](void*) { + since_during_job.store(cfd->IndirectBlobGcCandidatesSinceMicros()); + uint64_t idle = 1; + EXPECT_TRUE(db_->GetIntProperty( + DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, &idle)); + idle_during_job.store(idle); + ++idle_reads; + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + const uint64_t target = RouteTarget(file_number); + ASSERT_NE(target, kInvalidBlobFileNumber); + + const IndirectBlobGcTickers after(statistics); + const uint64_t started = after.Since(before, INDIRECT_BLOB_GC_JOBS_STARTED); + ASSERT_GE(started, 1U); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_JOBS_INSTALLED), 1U); + // Each started job has exactly one outcome. + ASSERT_EQ(started, + after.Since(before, INDIRECT_BLOB_GC_JOBS_INSTALLED) + + after.Since(before, INDIRECT_BLOB_GC_JOBS_STALE) + + after.Since(before, INDIRECT_BLOB_GC_JOBS_NOTHING_INSTALLED) + + after.Since(before, INDIRECT_BLOB_GC_JOBS_FAILED)); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_FAILED), 0U); + ASSERT_EQ(after.Since(before, BLOB_DB_GC_FAILURES), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_CORRUPT_INPUTS), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_CENSUS_MISMATCHES), 0U); + // The job histogram has one sample per started job. + HistogramData job_micros; + statistics->histogramData(INDIRECT_BLOB_GC_JOB_MICROS, &job_micros); + ASSERT_EQ(job_micros.count, started); + + // One live value was relocated. The base ticker counts the same keys, + // because no compaction-time blob GC ran. + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_KEYS_RELOCATED), 1U); + ASSERT_EQ(after.Since(before, BLOB_DB_GC_NUM_KEYS_RELOCATED), 1U); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_BYTES_RELOCATED), + IndirectBlobGCValue(0).size()); + ASSERT_GT(after.Since(before, INDIRECT_BLOB_GC_BYTES_READ), 0U); + // The input held four 4 KB values; the output holds one. + ASSERT_GT(after.Since(before, INDIRECT_BLOB_GC_BYTES_RECLAIMED), 0U); + + // Listeners see the relocation file with the indirect blob GC reason. + ASSERT_EQ(recorder->Paths(BlobFileCreationReason::kIndirectBlobGC), + std::vector{BlobFileName(dbname_, target)}); + + // The end-of-job line names the outcome and the relocated value count. + ASSERT_EQ(log->Count({"Indirect blob GC job ended: installed", + "1 values relocated"}), + 1U); + + ASSERT_GE(idle_reads.load(), 1); + ASSERT_EQ(since_during_job.load(), 0U); + ASSERT_EQ(idle_during_job.load(), 0U); + // The job left no file that GC can pick (the relocation file has no + // garbage), so the idle time stays 0. + std::map gc_stats; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kIndirectBlobGcStats, &gc_stats)); + ASSERT_EQ(gc_stats["garbage-bytes-collectable"], "0"); + ASSERT_EQ(gc_stats["idle-micros-with-candidates"], "0"); + uint64_t idle = 1; + ASSERT_TRUE(db_->GetIntProperty( + DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, &idle)); + ASSERT_EQ(idle, 0U); +} + +// Checks the signals of stale jobs. A census mismatch (injected once) makes +// the first job stale before it reads any input, and rebuilds the census. A +// failed install check (injected once) makes the second job stale after it +// read its input. A later job installs. The end-of-job line of each stale job +// says that it installed nothing. +TEST_F(DBBlobBasicTest, IndirectBlobGCStaleJobSignals) { + Options options = IndirectBlobGCOptions(); + options.statistics = CreateDBStatistics(); + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + Statistics* const statistics = options.statistics.get(); + const IndirectBlobGcTickers before(statistics); + std::atomic checks{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:CensusTotalsCheck", [&](void* arg) { + if (checks++ == 0) { + *static_cast(arg) = true; + } + }); + std::atomic install_checks{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:InstallPrecondition", [&](void* arg) { + if (install_checks++ == 0) { + *static_cast(arg) = false; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_GE(checks.load(), 2); + ASSERT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + + const IndirectBlobGcTickers after(statistics); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_CENSUS_MISMATCHES), 1U); + ASSERT_GE(install_checks.load(), 2); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_JOBS_STALE), 2U); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_JOBS_INSTALLED), 1U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_FAILED), 0U); + // A stale job is not a failure. + ASSERT_EQ(after.Since(before, BLOB_DB_GC_FAILURES), 0U); + HistogramData job_micros; + statistics->histogramData(INDIRECT_BLOB_GC_JOB_MICROS, &job_micros); + ASSERT_EQ(job_micros.count, + after.Since(before, INDIRECT_BLOB_GC_JOBS_STARTED)); + ASSERT_GE(log->Count({"Indirect blob GC job ended: stale", + "0 inputs read, 0 installed"}), + 1U); + ASSERT_GE(log->Count({"Indirect blob GC job ended: stale", + "1 inputs read, 0 installed"}), + 1U); + ASSERT_GE(log->Count({"Indirect blob GC job ended: installed", + "1 inputs read, 1 installed"}), + 1U); +} + +// Checks the signals of a job that ends without an install and without an +// error: an injected live-set validation failure suppresses the only input. +TEST_F(DBBlobBasicTest, IndirectBlobGCNothingInstalledJobSignals) { + Options options = IndirectBlobGCOptions(); + options.statistics = CreateDBStatistics(); + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + Statistics* const statistics = options.statistics.get(); + const IndirectBlobGcTickers before(statistics); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterLiveSet", [](void* arg) { + *static_cast(arg) = Status::Corruption("injected fault"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + + const IndirectBlobGcTickers after(statistics); + const uint64_t started = after.Since(before, INDIRECT_BLOB_GC_JOBS_STARTED); + ASSERT_GE(started, 1U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_NOTHING_INSTALLED), + started); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_INSTALLED), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_FAILED), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_KEYS_RELOCATED), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_BYTES_RELOCATED), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_BYTES_RECLAIMED), 0U); + HistogramData job_micros; + statistics->histogramData(INDIRECT_BLOB_GC_JOB_MICROS, &job_micros); + ASSERT_EQ(job_micros.count, started); + ASSERT_GE(log->Count({"Indirect blob GC job ended: nothing installed", + "0 inputs read, 0 installed"}), + 1U); +} + +// Checks the signals of a failed job: an injected write error of the output +// fails the job. The job counts in both the indirect blob GC failure ticker +// and the base BLOB_DB_GC_FAILURES ticker, and the error WARN line names +// indirect blob GC. +TEST_F(DBBlobBasicTest, IndirectBlobGCFailedJobSignals) { + Options options = IndirectBlobGCOptions(); + options.statistics = CreateDBStatistics(); + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + Statistics* const statistics = options.statistics.get(); + const IndirectBlobGcTickers before(statistics); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterAddBlobRelocation", [](void* arg) { + *static_cast(arg) = Status::IOError("injected write fault"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + dbfull()->TEST_WaitForCompact().PermitUncheckedError(); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsIOError()); + ASSERT_TRUE(HasBlobFile(file_number)); + + const IndirectBlobGcTickers after(statistics); + const uint64_t failed = after.Since(before, INDIRECT_BLOB_GC_JOBS_FAILED); + ASSERT_GE(failed, 1U); + ASSERT_EQ(after.Since(before, BLOB_DB_GC_FAILURES), failed); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_INSTALLED), 0U); + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_KEYS_RELOCATED), 0U); + HistogramData job_micros; + statistics->histogramData(INDIRECT_BLOB_GC_JOB_MICROS, &job_micros); + ASSERT_EQ(job_micros.count, + after.Since(before, INDIRECT_BLOB_GC_JOBS_STARTED)); + // The job read its input and installed nothing. + ASSERT_GE(log->Count({"Indirect blob GC job ended: failed", + "1 inputs read, 0 installed"}), + 1U); + ASSERT_GE( + log->Count({"[WARN] ", + "Indirect blob GC error: IO error: injected write fault"}), + 1U); + ASSERT_EQ(log->Count({"Compaction error: ", "injected write fault"}), 0U); +} + +// Checks the corrupt input signals: a flipped bit in the reference block of +// one input fails only that input. The ticker counts it and an ERROR line +// names the input and the Status, also when the other inputs install. +TEST_F(DBBlobBasicTest, IndirectBlobGCCorruptInputSignals) { + if (encrypted_env_) { + // test::CorruptFile() rewrites the whole SST, and the encrypted env gives + // the new file a new encryption prefix that the open reader does not use. + ROCKSDB_GTEST_BYPASS("Test corrupts an SST that has an open reader"); + return; + } + Options options = IndirectBlobGCOptions(); + // Keep every reference in one SST. + options.target_file_size_base = 1 << 30; + options.statistics = CreateDBStatistics(); + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + // Origin 0 holds many 100-byte values, so its live group spans several + // reference blocks. Origins 1 and 2 hold four 4 KB values each. Half of + // origin 0 and three quarters of origins 1 and 2 become garbage. + for (int o = 0; o < 3; ++o) { + const int count = o == 0 ? 18000 : 4; + for (int i = 0; i < count; ++i) { + ASSERT_OK( + Put("o" + std::to_string(o) + "-" + Key(i), + std::string(o == 0 ? 100 : 4096, static_cast('a' + o)))); + } + ASSERT_OK(Flush()); + } + std::vector origins; + for (const auto& blob_file : cfd->current()->storage_info()->GetBlobFiles()) { + origins.push_back(blob_file->GetBlobFileNumber()); + } + ASSERT_EQ(origins.size(), 3U); + for (int o = 0; o < 3; ++o) { + const int dead = o == 0 ? 9000 : 3; + for (int i = 0; i < dead; ++i) { + ASSERT_OK(Delete("o" + std::to_string(o) + "-" + Key(i))); + } + } + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const FileMetaData* const sst = OnlySstForTest(cfd); + ASSERT_NE(sst, nullptr); + const BlobReferenceIndex index = ReadBlobReferenceIndexForTest(*sst); + ASSERT_EQ(index.groups.size(), 3U); + ASSERT_EQ(index.groups[0].group.origin_file_number, origins[0]); + ASSERT_GE(index.groups[1].start_block, index.groups[0].start_block + 2); + // Flip one byte in a middle reference block of origin 0. + ASSERT_OK(test::CorruptFile( + env_, MakeTableFileName(dbname_, sst->fd.GetNumber()), + static_cast(index.blocks[index.groups[0].start_block + 1].offset() + + 1), + /*bytes_to_corrupt=*/1, /*verify_checksum=*/false)); + + Statistics* const statistics = options.statistics.get(); + const IndirectBlobGcTickers before(statistics); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_EQ(RouteTarget(origins[0]), kInvalidBlobFileNumber); + ASSERT_NE(RouteTarget(origins[1]), kInvalidBlobFileNumber); + ASSERT_NE(RouteTarget(origins[2]), kInvalidBlobFileNumber); + + const IndirectBlobGcTickers after(statistics); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_CORRUPT_INPUTS), 1U); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_JOBS_INSTALLED), 1U); + ASSERT_GE(after.Since(before, INDIRECT_BLOB_GC_KEYS_RELOCATED), 2U); + // A corrupt input is not a job failure. + ASSERT_EQ(after.Since(before, INDIRECT_BLOB_GC_JOBS_FAILED), 0U); + ASSERT_GE(log->Count({"[ERROR] ", "Indirect blob GC found corrupt input #" + + std::to_string(origins[0]) + + ": Corruption: "}), + 1U); + ASSERT_EQ(log->Count({"Indirect blob GC found corrupt input #" + + std::to_string(origins[1])}), + 0U); +} + +// Checks the idle time signals. Candidates exist, and no job starts because +// automatic compactions are off. The idle time property grows with the DB +// clock and equals the idle-micros-with-candidates map key. DumpStats() writes +// a WARN line only when the idle time is longer than stats_dump_period_sec. +TEST_F(DBBlobBasicTest, IndirectBlobGCIdleTimeSignals) { + auto clock = + std::make_shared(env_->GetSystemClock()); + CompositeEnvWrapper env(env_, clock); + Options options = IndirectBlobGCOptions(); + options.env = &env; + // 60 s is an example period; the test moves the clock past it. + constexpr unsigned int kStatsPeriodSec = 60; + options.stats_dump_period_sec = kStatsPeriodSec; + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + WriteIndirectBlobGCCandidate(/*live_keys=*/1); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + const auto read_idle = [&](uint64_t* property, uint64_t* map_key) { + ASSERT_TRUE(db_->GetIntProperty( + DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, property)); + std::map gc_stats; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kIndirectBlobGcStats, &gc_stats)); + ASSERT_NE(gc_stats["tracked-files"], "0"); + *map_key = std::stoull(gc_stats["idle-micros-with-candidates"]); + }; + const std::vector warn{"[WARN] ", + "Indirect blob GC has had candidates"}; + uint64_t idle = 0; + uint64_t idle_key = 0; + ASSERT_NO_FATAL_FAILURE(read_idle(&idle, &idle_key)); + ASSERT_EQ(idle, idle_key); + // The idle time is shorter than the stats period: no WARN line. + dbfull()->DumpStats(); + ASSERT_EQ(log->Count(warn), 0U); + + // 30 s is half of the period: still no WARN line. + clock->Advance(uint64_t{kStatsPeriodSec / 2} * 1000 * 1000); + uint64_t half_period_idle = 0; + ASSERT_NO_FATAL_FAILURE(read_idle(&half_period_idle, &idle_key)); + ASSERT_EQ(half_period_idle, idle_key); + ASSERT_EQ(half_period_idle - idle, uint64_t{kStatsPeriodSec / 2} * 1000000); + dbfull()->DumpStats(); + ASSERT_EQ(log->Count(warn), 0U); + + // Past the period, DumpStats() warns once per call. + clock->Advance(uint64_t{kStatsPeriodSec} * 1000 * 1000); + uint64_t long_idle = 0; + ASSERT_NO_FATAL_FAILURE(read_idle(&long_idle, &idle_key)); + ASSERT_EQ(long_idle, idle_key); + ASSERT_GT(long_idle, uint64_t{kStatsPeriodSec} * 1000000); + dbfull()->DumpStats(); + ASSERT_EQ(log->Count(warn), 1U); + ASSERT_EQ(log->Count({"[default] Indirect blob GC has had candidates for " + + std::to_string(long_idle / 1000000) + + " s without starting a job"}), + 1U); + Close(); +} + +// Checks that a DB without blob indirection keeps every indirect blob GC +// signal at 0, also when compaction-time blob GC relocates values and the +// clock passes the stats period. +// Checks that a GC job that runs longer than one stats period does not count +// as idle time. The test holds a GC job after it has started, makes the census +// publish a new Version while the job is held (a flush, then a census run), +// and moves the manual DB clock past stats_dump_period_sec. The publish sees +// that a GC job is registered, so it does not start the idle time again: +// DumpStats() writes no idle WARN, and the idle property stays 0. After the +// job is released and installs, no file is left to collect, so the idle time +// stays 0. +TEST_F(DBBlobBasicTest, IndirectBlobGCLongJobIsNotIdle) { + auto clock = + std::make_shared(env_->GetSystemClock()); + CompositeEnvWrapper env(env_, clock); + Options options = IndirectBlobGCOptions(); + options.env = &env; + // 60 s is an example period; the test moves the clock past it. + constexpr unsigned int kStatsPeriodSec = 60; + options.stats_dump_period_sec = kStatsPeriodSec; + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + // Holds the first GC job before its output work, after the job has started + // and registered on the census. + std::atomic hold_job{true}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->LoadDependency( + {{"DBBlobBasicTest::LongJob:Held", "DBBlobBasicTest::LongJob:WaitHeld"}, + {"DBBlobBasicTest::LongJob:Release", + "DBBlobBasicTest::LongJob:Released"}}); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:BeforeOutputWork", [&](void*) { + if (hold_job.exchange(false)) { + TEST_SYNC_POINT("DBBlobBasicTest::LongJob:Held"); + TEST_SYNC_POINT("DBBlobBasicTest::LongJob:Released"); + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + TEST_SYNC_POINT("DBBlobBasicTest::LongJob:WaitHeld"); + + // A flush installs a new Version, and the census publishes it while the job + // is held. The GC input is still a file to collect in that Version. + ASSERT_OK(Put("other", std::string(4096, 'o'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + ASSERT_TRUE(cfd->blob_reference_census()->IsReady( + cfd->current()->GetVersionNumber())); + } + std::map gc_stats; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kIndirectBlobGcStats, &gc_stats)); + ASSERT_NE(gc_stats["garbage-bytes-collectable"], "0"); + + // The job has run for longer than one stats period. + clock->Advance(uint64_t{kStatsPeriodSec + 1} * 1000 * 1000); + const std::vector warn{"[WARN] ", + "Indirect blob GC has had candidates"}; + uint64_t idle = 1; + ASSERT_TRUE(db_->GetIntProperty( + DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, &idle)); + ASSERT_EQ(idle, 0U); + dbfull()->DumpStats(); + ASSERT_EQ(log->Count(warn), 0U); + + TEST_SYNC_POINT("DBBlobBasicTest::LongJob:Release"); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + // The job installed and left no file to collect, so the idle time stays 0 + // also after more time passes. + clock->Advance(uint64_t{kStatsPeriodSec + 1} * 1000 * 1000); + ASSERT_TRUE(db_->GetIntProperty( + DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, &idle)); + ASSERT_EQ(idle, 0U); + dbfull()->DumpStats(); + ASSERT_EQ(log->Count(warn), 0U); + Close(); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCSignalsStayZeroWithoutIndirection) { + Options options = IndirectBlobGCOptions(); + options.enable_blob_indirection = false; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + options.stats_dump_period_sec = 1; + options.statistics = CreateDBStatistics(); + auto log = std::make_shared(); + options.info_log = log; + auto recorder = std::make_shared(); + options.listeners.push_back(recorder); + DestroyAndReopen(options); + WriteIndirectBlobGCCandidate(/*live_keys=*/1); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + // Compaction-time blob GC relocated the live value. + Statistics* const statistics = options.statistics.get(); + ASSERT_GT(statistics->getTickerCount(BLOB_DB_GC_NUM_KEYS_RELOCATED), 0U); + for (Tickers ticker : IndirectBlobGcTickers::kTickers) { + if (ticker != BLOB_DB_GC_NUM_KEYS_RELOCATED) { + ASSERT_EQ(statistics->getTickerCount(ticker), 0U) << ticker; + } + } + HistogramData job_micros; + statistics->histogramData(INDIRECT_BLOB_GC_JOB_MICROS, &job_micros); + ASSERT_EQ(job_micros.count, 0U); + ASSERT_TRUE(recorder->Paths(BlobFileCreationReason::kIndirectBlobGC).empty()); + uint64_t idle = 1; + ASSERT_TRUE(db_->GetIntProperty( + DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, &idle)); + ASSERT_EQ(idle, 0U); + dbfull()->DumpStats(); + ASSERT_EQ(log->Count({"Indirect blob GC"}), 0U); +} + +TEST_F(DBBlobBasicTest, ExactReferenceBudgetKeepsCompleteL0Histories) { + // A user key's history is all its kept versions, including deletions and + // inline versions with no references. Compaction never splits a history. + // a_references is the reference count of the only version of key A. + // b_references gives the reference count of each kept version of key B, + // oldest first. The test sets the cap to 4 with the sync point + // BlockBasedTableBuilder::BlobReferenceLimit (below the option's minimum of + // 4096). B starts a new output only when A already filled the output; `split` + // says that this happens. B's history then stays in one output, also past the + // cap. + struct Scenario { + int a_references; + std::vector b_references; + int deletion; // 0: none, 1: point deletion, 2: range deletion + bool split; + }; + const std::vector scenarios{{4, {3}, 1, true}, + {2, {4, 1}, 0, false}, + {4, {3, 0, 2}, 0, true}, + {4, {4, 3}, 0, true}, + {4, {3}, 2, true}}; + const std::string blob(4096, 'v'); + const std::array names{{"a", "b", "c", "d"}}; + const auto columns_for = [&](int references) { + WideColumns columns; + if (references == 0) { + columns.emplace_back("a", "inline"); + } else { + for (int i = 0; i < references; ++i) { + columns.emplace_back(names[i], blob); + } + } + return columns; + }; + for (size_t scenario_index = 0; scenario_index < scenarios.size(); + ++scenario_index) { + SCOPED_TRACE(scenario_index); + const Scenario& scenario = scenarios[scenario_index]; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 1024; + options.target_file_size_base = 1ULL << 30; + DestroyAndReopen(options); + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "A", + columns_for(scenario.a_references))); + std::vector snapshots; + const Defer release_snapshots([&] { + for (const Snapshot* snapshot : snapshots) { + db_->ReleaseSnapshot(snapshot); + } + }); + uint64_t b_references = 0; + for (int references : scenario.b_references) { + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "B", + columns_for(references))); + snapshots.push_back(db_->GetSnapshot()); + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + b_references += references; + } + if (scenario.deletion != 0) { + if (scenario.deletion == 1) { + ASSERT_OK(Delete("B")); + } else { + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), + "B", "C")); + } + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + } + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + std::vector inputs; + for (const auto& file : metadata.levels[0].files) { + inputs.push_back(file.name); + } + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "BlockBasedTableBuilder::BlobReferenceLimit", + [](void* arg) { *static_cast(arg) = 4; }); + sync_point->EnableProcessing(); + const Defer cleanup([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, 0)); + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + std::multiset counts; + for (const auto& entry : properties) { + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + entry.second->user_collected_properties.at( + kBlobReferenceDescriptorPropertyName), + &descriptor)); + counts.insert(descriptor.reference_count); + } + const uint64_t a_references = scenario.a_references; + const std::multiset expected = + scenario.split ? std::multiset{a_references, b_references} + : std::multiset{a_references + b_references}; + ASSERT_EQ(counts, expected); + for (size_t i = 0; i < snapshots.size(); ++i) { + ReadOptions read; + read.snapshot = snapshots[i]; + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(read, db_->DefaultColumnFamily(), "B", &result)); + ASSERT_EQ(result.columns(), columns_for(scenario.b_references[i])); + } + PinnableWideColumns result; + const Status read_status = + db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), "B", &result); + if (scenario.deletion != 0) { + ASSERT_TRUE(read_status.IsNotFound()); + } else { + ASSERT_OK(read_status); + ASSERT_EQ(result.columns(), columns_for(scenario.b_references.back())); + } + } +} + +// Keeps the table_file_creation lines of the event log. +class TableFileCreationEventLogger : public Logger { + public: + using Logger::Logv; + void Logv(const char* format, va_list ap) override { + char buffer[16384]; + vsnprintf(buffer, sizeof(buffer), format, ap); + if (strstr(buffer, "\"table_file_creation\"") != nullptr) { + std::lock_guard lock(mutex_); + lines_.emplace_back(buffer); + } + } + std::vector lines() { + std::lock_guard lock(mutex_); + return lines_; + } + + private: + std::mutex mutex_; + std::vector lines_; +}; + +TEST_F(DBBlobBasicTest, BlobReferenceOptionCutsCompactionOutputs) { + // 5000 user keys with two kept versions each (10000 references) and a cap of + // 4096 references: three outputs, two cut by the cap, never inside a user + // key's history. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.target_file_size_base = 1ULL << 30; + options.blob_indirection_max_references_per_sst = 4096; + options.statistics = CreateDBStatistics(); + auto logger = std::make_shared(); + options.info_log = logger; + DestroyAndReopen(options); + constexpr int kKeys = 5000; + const auto key_of = [](int i) { + char buffer[16]; + snprintf(buffer, sizeof(buffer), "key%05d", i); + return std::string(buffer); + }; + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(key_of(i), "old" + std::to_string(i))); + } + ASSERT_OK(Flush()); + const Snapshot* const snapshot = db_->GetSnapshot(); + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(key_of(i), "new" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(options.statistics->Reset()); + // One compaction of both L0 files into L1. + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + std::vector inputs; + for (const auto& file : metadata.levels[0].files) { + inputs.push_back(file.name); + } + ASSERT_EQ(inputs.size(), 2U); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, /*output_level=*/1)); + + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_TRUE(metadata.levels[0].files.empty()); + ASSERT_EQ(metadata.levels[1].files.size(), 3U); + // Files are sorted by key; no user key spans two files. + const auto& files = metadata.levels[1].files; + for (size_t i = 1; i < files.size(); ++i) { + ASSERT_LT(files[i - 1].largestkey, files[i].smallestkey); + } + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + uint64_t total = 0; + for (const auto& entry : properties) { + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + entry.second->user_collected_properties.at( + kBlobReferenceDescriptorPropertyName), + &descriptor)); + ASSERT_LE(descriptor.reference_count, 4096U); + ASSERT_EQ(descriptor.reference_count % 2, 0U); + total += descriptor.reference_count; + } + ASSERT_EQ(total, 2U * kKeys); + ASSERT_EQ(options.statistics->getTickerCount( + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS), + 2U); + HistogramData histogram; + options.statistics->histogramData(COMPACTION_OUTPUT_BLOB_REFERENCES, + &histogram); + ASSERT_EQ(histogram.count, 3U); + ASSERT_EQ(histogram.sum, 2U * kKeys); + + // The table_file_creation event of each compaction output has + // "blob_references". The events of the two outputs that the cap closed also + // have "close_reason": "blob_reference_limit". + int creations = 0; + int cut_by_limit = 0; + for (const std::string& line : logger->lines()) { + if (line.find("\"reason\"") != std::string::npos || + line.find("\"blob_references\": ") == std::string::npos) { + continue; + } + if (line.find("\"column_family_name\": \"default\"") == std::string::npos) { + continue; + } + ++creations; + cut_by_limit += line.find("\"close_reason\": \"blob_reference_limit\"") != + std::string::npos; + } + // Two flushes write no blob_references field; three compaction outputs do. + ASSERT_EQ(creations, 3); + ASSERT_EQ(cut_by_limit, 2); + + for (int i = 0; i < kKeys; i += 997) { + ASSERT_EQ(Get(key_of(i)), "new" + std::to_string(i)); + ASSERT_EQ(Get(key_of(i), snapshot), "old" + std::to_string(i)); + } + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBBlobBasicTest, BlobReferenceOptionCutsLevel0Outputs) { + // Universal compaction with one level. The test hook makes both flushed L0 + // files exact. A compaction to L0 that the cap cuts then makes three exact L0 + // outputs with one epoch number. A later compaction of one of them takes all + // three. Reads and the census are still correct after reopen. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options.target_file_size_base = 1ULL << 30; + options.blob_indirection_max_references_per_sst = 4096; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + constexpr int kKeys = 5000; + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(Key(i), "old" + std::to_string(i))); + } + // A compaction to L0 makes exact output only from exact L0 inputs, or when + // the compaction is bottommost. The compactions below are bottommost (there + // is only one level), but the test also makes the flushes exact (see + // FlushInjectedBlobIndexesWithExactMetadata()), so the outputs are exact also + // by the exact-input rule. + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + const Snapshot* const snapshot = db_->GetSnapshot(); + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(Key(i), "new" + std::to_string(i))); + } + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto l0_names = [&]() { + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + std::vector names; + for (const auto& file : metadata.levels[0].files) { + names.push_back(file.name); + } + return names; + }; + std::vector inputs = l0_names(); + ASSERT_EQ(inputs.size(), 2U); + ASSERT_OK(options.statistics->Reset()); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, /*output_level=*/0)); + ASSERT_EQ(options.statistics->getTickerCount( + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS), + 2U); + std::vector outputs = l0_names(); + ASSERT_EQ(outputs.size(), 3U); + { + const auto& files = cfd->current()->storage_info()->LevelFiles(0); + ASSERT_EQ(files.size(), 3U); + for (const FileMetaData* file : files) { + EXPECT_FALSE(file->blob_reference_descriptor.empty()); + EXPECT_EQ(file->epoch_number, files.front()->epoch_number); + } + } + // Compacting one of the outputs takes its epoch siblings too. + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {outputs[1]}, + /*output_level=*/0)); + const std::vector after = l0_names(); + for (const std::string& name : outputs) { + EXPECT_EQ(std::find(after.begin(), after.end(), name), after.end()) << name; + } + ASSERT_EQ(after.size(), 3U); + + // The snapshot belongs to this DB object, so release it before the reopen. + db_->ReleaseSnapshot(snapshot); + Reopen(options); + for (int i = 0; i < kKeys; i += 499) { + ASSERT_EQ(Get(Key(i)), "new" + std::to_string(i)); + } + ExpectCensusMatchesVersion(); +} + +TEST_F(DBBlobBasicTest, CoarseL0TrivialMoveBlockedTicker) { + // A coarse L0 file must be rewritten with exact blob reference metadata when + // it leaves L0, so the leveled L0 trivial move stops before it and ticks + // COMPACTION_COARSE_L0_TRIVIAL_MOVE_BLOCKED. Without blob indirection, L0 + // files are not coarse: the same workload makes trivial moves and the ticker + // stays 0. + for (const bool indirection : {false, true}) { + SCOPED_TRACE(indirection); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + options.enable_blob_files = true; + options.enable_blob_indirection = indirection; + options.min_blob_size = 0; + options.level0_file_num_compaction_trigger = 4; + DestroyAndReopen(options); + // The L0 trivial move needs a non-empty output level, so the first key + // range is compacted out of L0 first. + ASSERT_OK(Put(Key(0), std::string(512, 'v'))); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + std::atomic trivial_moves{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void*) { ++trivial_moves; }); + SyncPoint::GetInstance()->EnableProcessing(); + // Four L0 files with disjoint key ranges after Key(0) reach the compaction + // trigger, and none of them overlaps the output level. + constexpr int kFiles = 4; + constexpr int kKeysPerFile = 10; + for (int file = 0; file < kFiles; ++file) { + for (int i = 0; i < kKeysPerFile; ++i) { + ASSERT_OK(Put(Key(1 + file * kKeysPerFile + i), std::string(512, 'v'))); + } + ASSERT_OK(Flush()); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + const uint64_t blocked = options.statistics->getTickerCount( + COMPACTION_COARSE_L0_TRIVIAL_MOVE_BLOCKED); + if (indirection) { + EXPECT_GT(blocked, 0U); + EXPECT_EQ(trivial_moves.load(), 0); + } else { + EXPECT_EQ(blocked, 0U); + EXPECT_GT(trivial_moves.load(), 0); + } + for (int key = 0; key <= kFiles * kKeysPerFile; ++key) { + ASSERT_EQ(Get(Key(key)), std::string(512, 'v')); + } + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGcStatsCoarseKeys) { + // The keys coarse-sst-count and lowest-coarse-frontier of + // rocksdb.indirect-blob-gc-stats show the coarse SSTs of the current + // Version. A coarse SST records only its oldest indirect-origin blob file + // number (its frontier), and blob files at or above the lowest frontier + // cannot be collected. The keys are: + // - 0 and 0 without blob indirection, + // - the number of coarse flushes and the oldest blob file number after + // plain flushes, + // - 0 and 0 after a full compaction, which writes exact SSTs, and after a + // flush that writes exact metadata. + const auto coarse_keys = [&]() { + std::map values; + EXPECT_TRUE( + db_->GetMapProperty(DB::Properties::kIndirectBlobGcStats, &values)); + EXPECT_EQ(values.count("coarse-sst-count"), 1U); + EXPECT_EQ(values.count("lowest-coarse-frontier"), 1U); + return std::make_pair(values["coarse-sst-count"], + values["lowest-coarse-frontier"]); + }; + const auto zero = std::make_pair(std::string("0"), std::string("0")); + for (const bool indirection : {false, true}) { + SCOPED_TRACE(indirection); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = indirection; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_EQ(coarse_keys(), zero); + ASSERT_OK(Put("a", "value_a")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "value_b")); + ASSERT_OK(Flush()); + if (!indirection) { + ASSERT_EQ(coarse_keys(), zero); + continue; + } + const std::vector blob_files = GetBlobFileNumbers(); + ASSERT_EQ(blob_files.size(), 2U); + // GetBlobFileNumbers() is sorted, so the oldest blob file comes first. + ASSERT_EQ( + coarse_keys(), + std::make_pair(std::string("2"), std::to_string(blob_files.front()))); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(coarse_keys(), zero); + ASSERT_OK(Put("c", "value_c")); + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + ASSERT_EQ(coarse_keys(), zero); + } +} + +TEST_F(DBBlobBasicTest, RoutedReadIsInBlockCacheTrace) { + // A Get of a value that indirect blob GC moved to a relocation file goes + // through a route (BLOB_DB_INDIRECT_READ_ROUTED goes up). While a block + // cache trace runs, the block reads of the relocation file are in the trace + // with the caller kBlobRelocationFileRead, because the DB passes its block + // cache tracer to the relocation file readers. The first Get misses in the + // block cache. A second Get finds the data block in the block cache, and the + // cache hit is traced too. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.statistics = CreateDBStatistics(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + // The table reader traces only block cache lookups. The index block is not + // kept in the block cache (cache_index_and_filter_blocks is false), so the + // records are data block lookups. + const auto set_block_cache = [&options]() { + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(8 << 20); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + }; + set_block_cache(); + DestroyAndReopen(options); + // Four values go to one blob file. Three of them are deleted, so indirect + // blob GC moves the live value of "key3" to a relocation file and adds a + // route from the blob file to it. + const std::string value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), value)); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {origin})); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_NE( + cfd->current()->storage_info()->GetBlobRelocationRouteTarget(origin), + kInvalidBlobFileNumber); + // Reopen with a new, empty block cache, so the traced Get reads the blocks + // of the relocation file. + set_block_cache(); + Reopen(options); + + const std::string trace_path = dbname_ + "/block_cache_trace"; + { + std::unique_ptr trace_writer; + ASSERT_OK( + NewFileTraceWriter(env_, EnvOptions(), trace_path, &trace_writer)); + ASSERT_OK( + db_->StartBlockCacheTrace(TraceOptions(), std::move(trace_writer))); + } + const uint64_t routed_before = + options.statistics->getTickerCount(BLOB_DB_INDIRECT_READ_ROUTED); + ASSERT_EQ(Get("key3"), value); + ASSERT_EQ(options.statistics->getTickerCount(BLOB_DB_INDIRECT_READ_ROUTED), + routed_before + 1); + ASSERT_EQ(Get("key3"), value); + ASSERT_EQ(options.statistics->getTickerCount(BLOB_DB_INDIRECT_READ_ROUTED), + routed_before + 2); + ASSERT_OK(db_->EndBlockCacheTrace()); + + std::unique_ptr trace_reader; + ASSERT_OK(NewFileTraceReader(env_, EnvOptions(), trace_path, &trace_reader)); + BlockCacheTraceReader reader(std::move(trace_reader)); + BlockCacheTraceHeader header; + ASSERT_OK(reader.ReadHeader(&header)); + size_t data_block_misses = 0; + size_t data_block_hits = 0; + BlockCacheTraceRecord record; + while (reader.ReadAccess(&record).ok()) { + if (record.caller == TableReaderCaller::kBlobRelocationFileRead && + record.block_type == TraceType::kBlockTraceDataBlock) { + ++(record.is_cache_hit ? data_block_hits : data_block_misses); + } + } + ASSERT_GT(data_block_misses, 0U); + ASSERT_GT(data_block_hits, 0U); +} + +namespace { +// Counts compactions as they start. `intra_l0` counts the ones that read L0 +// and write L0. +class CompactionBeginCounter : public EventListener { + public: + void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& info) override { + ++compactions; + if (info.base_input_level == 0 && info.output_level == 0) { + ++intra_l0; + } + } + std::atomic compactions{0}; + std::atomic intra_l0{0}; +}; + +// Sets the blob reference cap of every exact SST to `cap` references with the +// sync point BlockBasedTableBuilder::BlobReferenceLimit. The option +// blob_indirection_max_references_per_sst does not accept values below 4096, +// and a small cap keeps the tests small. The caller must disable sync point +// processing and clear the callbacks. +void SetBlobReferenceCapForTest(uint64_t cap) { + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "BlockBasedTableBuilder::BlobReferenceLimit", + [cap](void* arg) { *static_cast(arg) = cap; }); + sync_point->EnableProcessing(); +} +} // namespace + +// Background for the next three tests. With blob indirection, the reference +// cap can cut one exact compaction output to L0 into several L0 files. These +// files have one epoch number and do not overlap, so together they are one +// sorted run (an "exact epoch run"). The L0 compaction score, the L0 +// compaction triggers and the L0 write stall triggers must count such a run +// as one sorted run, not as one run per file. + +TEST_F(DBBlobBasicTest, UniversalOneLevelExactEpochRunSettles) { + // Universal compaction with num_levels = 1 and a compaction trigger of 4. + // Four flushes of 20 new keys each hold 80 blob references. Their full + // compaction is bottommost, so its output is exact, and the cap of 16 + // references cuts it into 5 L0 files with one epoch number. The 5 files are + // one sorted run, so the compaction score drops below 1 and no other + // compaction starts. If each file counted as a sorted run, 5 runs would + // reach the trigger again after every compaction, and the DB would compact + // the same data forever with no writes. The wait has a timeout, so that + // case fails and does not hang. + constexpr int kTrigger = 4; + constexpr uint64_t kCap = 16; + constexpr int kFlushes = 4; + constexpr int kKeysPerFlush = 20; + constexpr int kOutputs = kFlushes * kKeysPerFlush / static_cast(kCap); + auto counter = std::make_shared(); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + options.level0_file_num_compaction_trigger = kTrigger; + options.target_file_size_base = 1ULL << 30; + // The flushes below must not start a compaction. SetOptions() turns + // automatic compactions on after the last flush. + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + options.listeners.push_back(counter); + DestroyAndReopen(options); + for (int flush = 0; flush < kFlushes; ++flush) { + for (int i = 0; i < kKeysPerFlush; ++i) { + const int key = flush * kKeysPerFlush + i; + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + } + ASSERT_EQ(NumTableFilesAtLevel(0), kFlushes); + + SetBlobReferenceCapForTest(kCap); + const Defer cleanup([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(options.statistics->Reset()); + ASSERT_OK(db_->SetOptions({{"disable_auto_compactions", "false"}})); + WaitForCompactOptions wait_options; + wait_options.timeout = std::chrono::seconds(5); + const Status wait_status = dbfull()->WaitForCompact(wait_options); + // Exactly one compaction: the full compaction that makes the run. + EXPECT_EQ(counter->compactions.load(), 1); + ASSERT_OK(wait_status); + ASSERT_EQ(options.statistics->getTickerCount( + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS), + static_cast(kOutputs - 1)); + ASSERT_EQ(NumTableFilesAtLevel(0), kOutputs); + ASSERT_GE(kOutputs, kTrigger); + uint64_t pending = 1; + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kCompactionPending, &pending)); + ASSERT_EQ(pending, 0U); + for (int key = 0; key < kFlushes * kKeysPerFlush; key += 7) { + ASSERT_EQ(Get(Key(key)), "value" + std::to_string(key)); + } +} + +TEST_F(DBBlobBasicTest, LeveledSizeBasedIntraL0ExactEpochRunSettles) { + // Leveled compaction picks a size-based intra-L0 compaction (L0 to L0) when + // L0 has at least max(2, trigger) files and Lbase is more than 20 times + // larger than L0 (PickSizeBasedIntraL0Compaction). + // + // Setup: + // - Lbase holds 2000 inline values of 1000 bytes each, with keys "a..." + // and one more key "zz". + // - L0 holds four flushes of 20 blob values each, with keys "z...". These + // keys are inside the key range of Lbase, so the L0 trivial move to + // Lbase cannot take the L0 files. + // - A compaction to L0 is not bottommost while a lower level has files, so + // the test makes the last flush exact + // (FlushInjectedBlobIndexesWithExactMetadata()). A compaction to L0 with + // an exact input writes exact output. + // + // The cap of 16 references cuts the output of the intra-L0 compaction into + // 5 L0 files with one epoch number. The 5 files are one sorted run, so they + // do not reach the trigger of 4, and no other compaction starts. If each + // file counted, the picker would pick the same run again and again with no + // writes. The wait has a timeout, so that case fails and does not hang. + constexpr int kTrigger = 4; + constexpr uint64_t kCap = 16; + constexpr int kFlushes = 4; + constexpr int kKeysPerFlush = 20; + constexpr int kOutputs = kFlushes * kKeysPerFlush / static_cast(kCap); + constexpr int kBaseKeys = 2000; + constexpr int kBaseValueSize = 1000; + auto counter = std::make_shared(); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + // Lbase values stay inline, so Lbase is large. SetOptions() below sends the + // L0 values to blob files. + options.min_blob_size = 1 << 20; + options.level0_file_num_compaction_trigger = kTrigger; + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + options.listeners.push_back(counter); + DestroyAndReopen(options); + Random rnd(301); + for (int i = 0; i < kBaseKeys; ++i) { + ASSERT_OK(Put("a" + Key(i), rnd.RandomString(kBaseValueSize))); + } + ASSERT_OK(Put("zz", "last")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + ASSERT_OK(db_->SetOptions({{"min_blob_size", "0"}})); + for (int flush = 0; flush < kFlushes; ++flush) { + for (int i = 0; i < kKeysPerFlush; ++i) { + const int key = flush * kKeysPerFlush + i; + ASSERT_OK(Put("z" + Key(key), "value" + std::to_string(key))); + } + if (flush == kFlushes - 1) { + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + } else { + ASSERT_OK(Flush()); + } + } + ASSERT_EQ(NumTableFilesAtLevel(0), kFlushes); + + SetBlobReferenceCapForTest(kCap); + const Defer cleanup([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(options.statistics->Reset()); + const int compactions_before = counter->compactions.load(); + ASSERT_EQ(counter->intra_l0.load(), 0); + ASSERT_OK(db_->SetOptions({{"disable_auto_compactions", "false"}})); + WaitForCompactOptions wait_options; + wait_options.timeout = std::chrono::seconds(5); + const Status wait_status = dbfull()->WaitForCompact(wait_options); + // Exactly one compaction: the intra-L0 compaction that makes the run. + EXPECT_EQ(counter->intra_l0.load(), 1); + EXPECT_EQ(counter->compactions.load() - compactions_before, 1); + ASSERT_OK(wait_status); + ASSERT_EQ(options.statistics->getTickerCount( + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS), + static_cast(kOutputs - 1)); + ASSERT_EQ(NumTableFilesAtLevel(0), kOutputs); + ASSERT_GE(kOutputs, kTrigger); + for (int key = 0; key < kFlushes * kKeysPerFlush; key += 7) { + ASSERT_EQ(Get("z" + Key(key)), "value" + std::to_string(key)); + } +} + +TEST_F(DBBlobBasicTest, WriteStallCountsExactEpochRunAsOneRun) { + // L0 write stall triggers and the L0 part of the pending compaction bytes + // estimate count sorted runs. The compaction trigger is 4, the slowdown + // trigger is 8 and the stop trigger is 12, and L0 holds 10 files. + // - With blob indirection, the 10 files are one exact epoch run: the cap of + // 16 references cuts one compaction to L0 of 160 references into 10 + // files. This is one sorted run, so writes neither slow down nor stop, + // and the estimate has no L0 part. + // - Without blob indirection, 10 flushes make 10 sorted runs, so writes + // slow down and the estimate counts L0. This shows that 10 files do reach + // both triggers. + // Write stall conditions apply only with automatic compactions on. All LOW + // pool threads are busy while SetOptions() turns them on, so no compaction + // can change L0 before the checks. + constexpr uint64_t kCap = 16; + constexpr int kFiles = 10; + constexpr int kKeysPerFile = static_cast(kCap); + for (const bool indirection : {false, true}) { + SCOPED_TRACE(indirection); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = indirection; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.target_file_size_base = 1ULL << 30; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 8; + options.level0_stop_writes_trigger = 12; + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + if (indirection) { + for (int key = 0; key < kFiles * kKeysPerFile; ++key) { + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + // No lower level holds these keys, so the compaction to L0 is + // bottommost and its output is exact, which the cap needs. + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(metadata.levels[0].files.size(), 1U); + SetBlobReferenceCapForTest(kCap); + const Status compact_status = db_->CompactFiles( + CompactionOptions(), {metadata.levels[0].files[0].name}, + /*output_level=*/0); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(compact_status); + ASSERT_EQ(options.statistics->getTickerCount( + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS), + static_cast(kFiles - 1)); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const std::vector& files = + cfd->current()->storage_info()->LevelFiles(0); + ASSERT_EQ(files.size(), static_cast(kFiles)); + for (const FileMetaData* file : files) { + ASSERT_EQ(file->epoch_number, files.front()->epoch_number); + } + } else { + for (int file = 0; file < kFiles; ++file) { + for (int i = 0; i < kKeysPerFile; ++i) { + const int key = file * kKeysPerFile + i; + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + } + } + // NumTableFilesAtLevel() reads rocksdb.num-files-at-level0, which counts + // files, not sorted runs. + ASSERT_EQ(NumTableFilesAtLevel(0), kFiles); + + const int low_threads = env_->GetBackgroundThreads(Env::Priority::LOW); + std::vector sleeping_tasks(low_threads); + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &task, + Env::Priority::LOW); + task.WaitUntilSleeping(); + } + const Defer wake([&sleeping_tasks] { + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + }); + ASSERT_OK(db_->SetOptions({{"disable_auto_compactions", "false"}})); + uint64_t delayed_write_rate = 0; + uint64_t write_stopped = 0; + ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kActualDelayedWriteRate, + &delayed_write_rate)); + ASSERT_TRUE( + db_->GetIntProperty(DB::Properties::kIsWriteStopped, &write_stopped)); + // The pending compaction bytes estimate counts L0 only when L0 reaches the + // compaction trigger of 4 sorted runs (or max_bytes_for_level_base, which + // these small files do not reach). No other level holds data, so the + // estimate is 0 when L0 does not count. + uint64_t pending_bytes = 0; + ASSERT_TRUE(db_->GetIntProperty( + DB::Properties::kEstimatePendingCompactionBytes, &pending_bytes)); + EXPECT_EQ(write_stopped, 0U); + if (indirection) { + EXPECT_EQ(delayed_write_rate, 0U); + EXPECT_EQ(pending_bytes, 0U); + } else { + EXPECT_GT(delayed_write_rate, 0U); + EXPECT_GT(pending_bytes, 0U); + } + } +} + +namespace { +// Records the write stall flags of FlushJobInfo that OnFlushBegin and +// OnFlushCompleted receive. Each entry is the pair (triggered_writes_slowdown, +// triggered_writes_stop). +class FlushStallFlagsRecorder : public EventListener { + public: + void OnFlushBegin(DB* /*db*/, const FlushJobInfo& info) override { + std::lock_guard lock(mutex); + begin.emplace_back(info.triggered_writes_slowdown, + info.triggered_writes_stop); + } + void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override { + std::lock_guard lock(mutex); + completed.emplace_back(info.triggered_writes_slowdown, + info.triggered_writes_stop); + } + std::mutex mutex; + std::vector> begin; + std::vector> completed; +}; +} // namespace + +TEST_F(DBBlobBasicTest, FlushInfoStallFlagsCountExactEpochRunAsOneRun) { + // FlushJobInfo::triggered_writes_slowdown and triggered_writes_stop must use + // the same L0 count as the write stall decision: sorted runs, where an exact + // epoch run counts once. The setup is the one of + // WriteStallCountsExactEpochRunAsOneRun: the slowdown trigger is 8, the stop + // trigger is 12, and L0 holds 10 files. Then one more flush runs, and a + // listener records the flags. OnFlushBegin sees L0 before the new file is + // added (10 files), and OnFlushCompleted sees it after (11 files). + // - With blob indirection, the 10 files are one exact epoch run, so L0 holds + // 1 sorted run, and then 2. Both are below the slowdown trigger, so all + // flags are false. + // - Without blob indirection, the 10 files are 10 sorted runs, and then 11. + // Both reach the slowdown trigger but not the stop trigger. + // Automatic compactions stay off, so no compaction changes L0 and writes do + // not stall. + constexpr uint64_t kCap = 16; + constexpr int kFiles = 10; + constexpr int kKeysPerFile = static_cast(kCap); + for (const bool indirection : {false, true}) { + SCOPED_TRACE(indirection); + auto recorder = std::make_shared(); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = indirection; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.target_file_size_base = 1ULL << 30; + options.level0_file_num_compaction_trigger = 4; + options.level0_slowdown_writes_trigger = 8; + options.level0_stop_writes_trigger = 12; + options.disable_auto_compactions = true; + options.listeners.push_back(recorder); + DestroyAndReopen(options); + if (indirection) { + for (int key = 0; key < kFiles * kKeysPerFile; ++key) { + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + // No lower level holds these keys, so the compaction to L0 is + // bottommost and its output is exact, which the cap needs. + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(metadata.levels[0].files.size(), 1U); + SetBlobReferenceCapForTest(kCap); + const Status compact_status = db_->CompactFiles( + CompactionOptions(), {metadata.levels[0].files[0].name}, + /*output_level=*/0); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(compact_status); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const std::vector& files = + cfd->current()->storage_info()->LevelFiles(0); + ASSERT_EQ(files.size(), static_cast(kFiles)); + for (const FileMetaData* file : files) { + ASSERT_EQ(file->epoch_number, files.front()->epoch_number); + } + } else { + for (int file = 0; file < kFiles; ++file) { + for (int i = 0; i < kKeysPerFile; ++i) { + const int key = file * kKeysPerFile + i; + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + } + } + ASSERT_EQ(NumTableFilesAtLevel(0), kFiles); + + { + std::lock_guard lock(recorder->mutex); + recorder->begin.clear(); + recorder->completed.clear(); + } + ASSERT_OK(Put("zz", "last")); + ASSERT_OK(Flush()); + // Flush() can return before the flush job calls OnFlushCompleted, so wait + // until the background work is done. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_EQ(NumTableFilesAtLevel(0), kFiles + 1); + + std::lock_guard lock(recorder->mutex); + ASSERT_EQ(recorder->begin.size(), 1U); + ASSERT_EQ(recorder->completed.size(), 1U); + const bool expect_slowdown = !indirection; + EXPECT_EQ(recorder->begin[0].first, expect_slowdown); + EXPECT_FALSE(recorder->begin[0].second); + EXPECT_EQ(recorder->completed[0].first, expect_slowdown); + EXPECT_FALSE(recorder->completed[0].second); + } +} + +TEST_F(DBBlobBasicTest, PointLookupAcrossRangeTombstoneCutInLevel0Run) { + CheckPointLookupAcrossRangeTombstoneCut(kCompactionStyleUniversal); +} + +TEST_F(DBBlobBasicTest, PointLookupAcrossRangeTombstoneCutInLeveledIntraL0) { + CheckPointLookupAcrossRangeTombstoneCut(kCompactionStyleLevel); +} + +TEST_F(DBBlobBasicTest, BlobReferenceOptionCutsEachPlacementTarget) { + // Per-key placement, turned on by a sync point: versions newer than the + // snapshot go to the proximal level (L5), and older versions go to the last + // level (L6). Each of the two output levels counts its own references against + // the cap, and no user key spans two files of one level. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 7; + options.last_level_temperature = Temperature::kCold; + options.target_file_size_base = 1ULL << 30; + options.blob_indirection_max_references_per_sst = 4096; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + constexpr int kKeys = 5000; + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(Key(i), "old" + std::to_string(i))); + } + ASSERT_OK(Flush()); + const Snapshot* const snapshot = db_->GetSnapshot(); + const SequenceNumber split = snapshot->GetSequenceNumber(); + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(Put(Key(i), "new" + std::to_string(i))); + } + ASSERT_OK(Flush()); + SyncPoint::GetInstance()->SetCallBack( + "Compaction::SupportsPerKeyPlacement:Enabled", + [](void* arg) { *static_cast(arg) = true; }); + SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::PrepareTimes():preclude_last_level_min_seqno", + [&](void* arg) { *static_cast(arg) = split; }); + SyncPoint::GetInstance()->EnableProcessing(); + const Defer cleanup([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(options.statistics->Reset()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_TRUE(metadata.levels[0].files.empty()); + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + for (int level : {5, 6}) { + SCOPED_TRACE(level); + const auto& files = metadata.levels[level].files; + // Each level gets 5000 references, so the cap cuts it once: one file has + // 4096 references and the other has the rest. + ASSERT_EQ(files.size(), 2U); + ASSERT_LT(files[0].largestkey, files[1].smallestkey); + std::multiset counts; + for (const auto& file : files) { + const auto found = properties.find(file.db_path + file.name); + ASSERT_NE(found, properties.end()) << file.name; + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + found->second->user_collected_properties.at( + kBlobReferenceDescriptorPropertyName), + &descriptor)); + counts.insert(descriptor.reference_count); + } + ASSERT_EQ(counts, (std::multiset{4096, kKeys - 4096})); + } + ASSERT_EQ(options.statistics->getTickerCount( + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS), + 2U); + for (int i = 0; i < kKeys; i += 499) { + ASSERT_EQ(Get(Key(i)), "new" + std::to_string(i)); + ASSERT_EQ(Get(Key(i), snapshot), "old" + std::to_string(i)); + } + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBBlobBasicTest, BlobReferenceOptionRange) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.blob_indirection_max_references_per_sst = 4095; + ASSERT_TRUE(TryReopen(options).IsInvalidArgument()); + options.blob_indirection_max_references_per_sst = + kMaxBlobIndirectionMaxReferencesPerSst + 1; + ASSERT_TRUE(TryReopen(options).IsInvalidArgument()); + options.blob_indirection_max_references_per_sst = + kMaxBlobIndirectionMaxReferencesPerSst; + Reopen(options); + ASSERT_TRUE( + db_->SetOptions({{"blob_indirection_max_references_per_sst", "4095"}}) + .IsInvalidArgument()); + ASSERT_OK( + db_->SetOptions({{"blob_indirection_max_references_per_sst", "4096"}})); + ASSERT_EQ(dbfull()->GetOptions().blob_indirection_max_references_per_sst, + 4096U); +} + +TEST_F(DBBlobBasicTest, BlobMetadataAmbiguityResolvesAfterManifestRecovery) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.max_bgerror_resume_count = 0; + CreateAndReopenWithCF({"blob"}, options); + const std::string value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put(1, "key" + std::to_string(i), value)); + } + ASSERT_OK(Flush(1)); + ColumnFamilyData* const cfd = + static_cast_with_check(handles_[1])->cfd(); + const uint64_t origin = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete(1, "key" + std::to_string(i))); + } + ASSERT_OK(Flush(1)); + CompactRangeOptions exact_options; + exact_options.change_level = true; + exact_options.target_level = 1; + exact_options.bottommost_level_compaction = BottommostLevelCompaction::kForce; + exact_options.blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kDisable; + ASSERT_OK(db_->CompactRange(exact_options, handles_[1], nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + std::atomic failures{0}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) { + if (failures.fetch_add(1) == 0) { + auto* status = static_cast(arg); + *status = IOStatus::IOError("Injected retryable blob MANIFEST error"); + status->SetRetryable(true); + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_TRUE(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {origin}).IsIOError()); + ASSERT_EQ(failures.load(), 1); + // The MANIFEST write result is unknown, so no indirect blob GC job can start + // until MANIFEST recovery. + ASSERT_TRUE(TryBeginGcJob(cfd).IsBusy()); + // TEST_RunIndirectBlobGCBatch does not report errors through + // BackgroundCompaction. So the test sets the same retryable MANIFEST error as + // the background error, and then calls Resume(). + const IOStatus io_status = dbfull()->GetVersionSet()->io_status(); + ASSERT_TRUE(io_status.IsIOError()); + dbfull()->TEST_SetBGError(io_status, BackgroundErrorReason::kManifestWrite); + sync_point->ClearCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest"); + ASSERT_OK(db_->Resume()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(TryBeginGcJob(cfd)); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_EQ(Get(1, "key3"), value); + ASSERT_OK(Put(1, "after-recovery", value)); +} +TEST_F(DBBlobBasicTest, IndirectBlobGCRunsAfterAmbiguityResolves) { + // A background indirect blob GC install fails its MANIFEST write, so the + // outcome of its metadata commit is unknown until recovery. A GC attempt in + // that state (BeginGcJob returns Busy) ends with no error and no install. + // After Resume() recovers the MANIFEST, GC runs again by itself: no new write + // and no manual compaction is needed. + Options options = IndirectBlobGCOptions(); + options.max_bgerror_resume_count = 0; + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + std::atomic failures{0}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) { + if (failures.fetch_add(1) == 0) { + auto* status = static_cast(arg); + *status = IOStatus::IOError("Injected retryable blob MANIFEST error"); + status->SetRetryable(true); + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + dbfull()->TEST_WaitForCompact().PermitUncheckedError(); + ASSERT_EQ(failures.load(), 1); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsIOError()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_TRUE(TryBeginGcJob(cfd).IsBusy()); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + + // While the outcome is unknown, a GC attempt ends quietly. + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {file_number})); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + + sync_point->ClearCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest"); + ASSERT_OK(db_->Resume()); + for (int round = 0; round < 3; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + ASSERT_FALSE(HasBlobFile(file_number)); + ASSERT_EQ(Get(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1)), + IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); +} + +TEST_F(DBBlobBasicTest, PrefixRepairRejectsValidatedRelocationFile) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("legacy", "value")); + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + const ImmutableOptions immutable_options = cfd->ioptions(); + const MutableCFOptions mutable_options = cfd->GetLatestMutableCFOptions(); + Close(); + const std::string payload = "stored"; + BlobRelocationFileMetadata metadata; + uint64_t size = 0; + constexpr uint64_t number = 1000000; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, BlockBasedTableOptions(), 0, number, + {{1, 42, payload, kNoCompression}}, &metadata, &size); + const std::string path = BlobFileName(dbname_, number); + std::string before; + ASSERT_OK(ReadFileToString(env_, path, &before)); + ASSERT_TRUE(RepairDB(dbname_, options).IsNotSupported()); + std::string after; + ASSERT_OK(ReadFileToString(env_, path, &after)); + ASSERT_EQ(before, after); +} +TEST_F(DBBlobBasicTest, BlobReferenceCensusShutdownAfterDispatch) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + Reopen(options); + + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + Close(); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->LoadDependency( + {{"DBImpl::BGWorkBlobReferenceCensus:BeforeBuildReady", + "DBBlobBasicTest::CensusShutdownWorkerDispatched"}, + {"DBBlobBasicTest::ReleaseCensusShutdownWorker", + "DBImpl::BGWorkBlobReferenceCensus:BeforeBuildWait"}, + {"DBImpl::BGWorkBlobReferenceCensus:Done", + "DBBlobBasicTest::CensusShutdownWorkerDone"}}); + sync_point->EnableProcessing(); + bool released = false; + const Defer cleanup([&] { + if (!released) { + TEST_SYNC_POINT("DBBlobBasicTest::ReleaseCensusShutdownWorker"); + } + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Reopen(options); + TEST_SYNC_POINT("DBBlobBasicTest::CensusShutdownWorkerDispatched"); + // The worker has taken a live CF from the queue, but shutdown starts before + // the build sets up its state, so the build returns at once. In + // ASSERT_STATUS_CHECKED builds, the default OK status that this path does not + // use must still be marked as checked. + dbfull()->CancelAllBackgroundWork(/*wait=*/false); + TEST_SYNC_POINT("DBBlobBasicTest::ReleaseCensusShutdownWorker"); + released = true; + TEST_SYNC_POINT("DBBlobBasicTest::CensusShutdownWorkerDone"); + Close(); +} +TEST_F(DBBlobBasicTest, FifoRejectsOwnStickyFenceAfterPolicyDisable) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.max_open_files = -1; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + DestroyAndReopen(options); + ASSERT_OK(Put("feature", "value")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_indirection", "false"}})); + Close(); + options.enable_blob_indirection = false; + options.compaction_style = kCompactionStyleFIFO; + ASSERT_TRUE(TryReopen(options).IsNotSupported()); + options.compaction_style = kCompactionStyleLevel; + Reopen(options); + ASSERT_EQ(Get("feature"), "value"); +} + +TEST_F(DBBlobBasicTest, FifoRejectsOnlyOwnColumnFamilyFence) { + Options legacy = GetDefaultOptions(); + legacy.create_if_missing = true; + legacy.max_open_files = -1; + legacy.compaction_style = kCompactionStyleFIFO; + legacy.num_levels = 1; + DestroyAndReopen(legacy); + ColumnFamilyOptions feature(GetDefaultOptions()); + feature.enable_blob_files = true; + feature.enable_blob_indirection = true; + feature.min_blob_size = 0; + ColumnFamilyHandle* handle = nullptr; + ASSERT_OK(db_->CreateColumnFamily(feature, "feature", &handle)); + ASSERT_OK(db_->Put(WriteOptions(), handle, "key", "value")); + ASSERT_OK(db_->Flush(FlushOptions(), handle)); + delete handle; + Close(); + std::vector handles; + std::unique_ptr database; + ASSERT_OK(DB::Open(DBOptions(legacy), dbname_, + {{kDefaultColumnFamilyName, ColumnFamilyOptions(legacy)}, + {"feature", feature}}, + &handles, &database)); + std::string value; + ASSERT_OK(database->Get(ReadOptions(), handles[1], "key", &value)); + ASSERT_EQ(value, "value"); + for (ColumnFamilyHandle* opened : handles) { + delete opened; + } + database.reset(); +} + +TEST_F(DBBlobBasicTest, FifoRejectsFeatureButPreservesLegacyBlobDb) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.max_open_files = -1; + options.compaction_style = kCompactionStyleFIFO; + options.num_levels = 1; + options.enable_blob_files = true; + options.min_blob_size = 0; + DestroyAndReopen(options); + ASSERT_OK(Put("legacy", "legacy_value")); + ASSERT_OK(Flush()); + ASSERT_EQ(Get("legacy"), "legacy_value"); + ASSERT_TRUE(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_indirection", "true"}}) + .IsNotSupported()); + ColumnFamilyOptions feature(options); + feature.enable_blob_indirection = true; + ColumnFamilyHandle* handle = nullptr; + ASSERT_TRUE(db_->CreateColumnFamily(feature, "unsupported", &handle) + .IsNotSupported()); + ASSERT_EQ(handle, nullptr); + Close(); + options.enable_blob_indirection = true; + ASSERT_TRUE(TryReopen(options).IsNotSupported()); + std::unique_ptr readonly; + ASSERT_TRUE( + DB::OpenForReadOnly(options, dbname_, &readonly).IsNotSupported()); + ASSERT_EQ(readonly, nullptr); + // The multi-CF read-only open and the secondary open check each CF too. + std::vector handles; + ASSERT_TRUE(DB::OpenForReadOnly( + DBOptions(options), dbname_, + {ColumnFamilyDescriptor(kDefaultColumnFamilyName, + ColumnFamilyOptions(options))}, + &handles, &readonly) + .IsNotSupported()); + ASSERT_EQ(readonly, nullptr); + ASSERT_TRUE(handles.empty()); + std::unique_ptr secondary; + ASSERT_TRUE(DB::OpenAsSecondary(options, dbname_, + test::PerThreadDBPath(env_, "fifo_secondary"), + &secondary) + .IsNotSupported()); + ASSERT_EQ(secondary, nullptr); + options.enable_blob_indirection = false; + Reopen(options); + ASSERT_EQ(Get("legacy"), "legacy_value"); +} + +TEST_F(DBBlobBasicTest, FencedInstallsKeepTheDbMutex) { + // Versions and builders are destroyed under the DB mutex. A JobContext frees + // the relocation state of a destroyed Version later, without the mutex. + // LogAndApply releases the mutex to free relocation state only when the batch + // staged relocation file or route changes. A later fenced flush or a + // SetOptions stages none. So it holds the mutex for its whole install: it + // installs its Version, removes the flushed memtables (flush only), and + // installs the SuperVersion without releasing the mutex. + for (bool atomic_flush : {false, true}) { + SCOPED_TRACE(atomic_flush); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.atomic_flush = atomic_flush; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + DestroyAndReopen(options); + CreateAndReopenWithCF({"one"}, options); + ASSERT_OK(Put(0, "key", "default-value")); + ASSERT_OK(Put(1, "key", "one-value")); + int unlocked = 0; + for (const char* sync_point : {"VersionSet::ProcessManifestWrites:" + "CandidateRelocationStateFreedUnlocked", + "VersionSet::ProcessManifestWrites:" + "BuilderRelocationEditsFreedUnlocked"}) { + SyncPoint::GetInstance()->SetCallBack(sync_point, + [&](void*) { ++unlocked; }); + } + SyncPoint::GetInstance()->EnableProcessing(); + const Status status = atomic_flush + ? db_->Flush(FlushOptions(), handles_) + : db_->Flush(FlushOptions(), handles_[0]); + ASSERT_OK(status); + ASSERT_TRUE(dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->HasBlobIndirectionFormat()); + ASSERT_OK(db_->SetOptions(handles_[0], {{"blob_file_size", "2097152"}})); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + EXPECT_EQ(unlocked, 0); + EXPECT_EQ(Get(0, "key"), "default-value"); + EXPECT_EQ(Get(1, "key"), "one-value"); + } +} + +TEST_F(DBBlobBasicTest, ReplacedVersionRelocationStateIsFreedUnlocked) { + // Each flush of a fenced CF changes its blob files, so the new Version gets + // its own relocation state. The replaced Version is destroyed under the DB + // mutex. When it is the last owner of its relocation state, it hands the + // state to a JobContext, which frees it without the mutex. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + DestroyAndReopen(options); + ASSERT_OK(Put("key0", "value0")); + ASSERT_OK(Flush()); + std::atomic deferred{0}; + std::atomic freed{0}; + SyncPoint::GetInstance()->SetCallBack( + "Version::~Version:DeferRelocationState", + [&](void*) { deferred.fetch_add(1); }); + SyncPoint::GetInstance()->SetCallBack( + "JobContext::Clean:FreeBlobRelocationStates", [&](void*) { + // The DB mutex is free here: another thread can take it. + std::thread locker([&]() { + dbfull()->TEST_LockMutex(); + dbfull()->TEST_UnlockMutex(); + }); + locker.join(); + freed.fetch_add(1); + }); + SyncPoint::GetInstance()->EnableProcessing(); + for (int i = 1; i <= 3; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), "value" + std::to_string(i))); + ASSERT_OK(Flush()); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + // The census can also own a replaced Version's relocation state, through its + // change log or its worker. Then the census frees that state later, without + // the DB mutex, and the Version does not hand it to a JobContext. So the test + // only checks that a state handed to a JobContext is freed. + if (deferred.load() > 0) { + EXPECT_GE(freed.load(), 1); + } + // FindObsoleteFiles() moved every deferred state into a JobContext, so none + // is left in the VersionSet. + dbfull()->TEST_LockMutex(); + std::vector> left; + dbfull()->GetVersionSet()->TakeObsoleteBlobRelocationStates(&left); + dbfull()->TEST_UnlockMutex(); + EXPECT_TRUE(left.empty()); + for (int i = 0; i <= 3; ++i) { + EXPECT_EQ(Get("key" + std::to_string(i)), "value" + std::to_string(i)); + } +} + +TEST_F(DBBlobBasicTest, DeferredRelocationStateOutlivesDroppedColumnFamily) { + // CF "x" deletes relocation file R, so the Version that held R dies. Its + // relocation file metadata is freed under the DB mutex (the file's deleter + // uses the CF), and only its file table is deferred. A flush of the default + // CF takes the table into its JobContext; "x" is dropped and destroyed + // before the JobContext frees it. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + DestroyAndReopen(options); + CreateAndReopenWithCF({"x"}, options); + const std::string key = "key"; + const std::string expected = "x-value"; + ASSERT_OK(Put(1, key, expected)); + ASSERT_OK(Flush(1)); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = + static_cast(handles_[1])->cfd(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{origin_file_number, original_value_offset, expected, kNoCompression}}, + &metadata, &relocation_file_size); + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + ASSERT_OK(ApplyVersionEdit(cfd, &install, /*new_descriptor_log=*/false)); + VersionEdit deletion; + deletion.SetColumnFamily(cfd->GetID()); + deletion.DeleteBlobRelocationFile(relocation_file_number); + ASSERT_OK(ApplyVersionEdit(cfd, &deletion, /*new_descriptor_log=*/false)); + + port::Mutex sync_mutex; + port::CondVar sync_cv(&sync_mutex); + bool found = false; + bool dropped = false; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCallFlush:FilesFound", [&](void*) { + MutexLock lock(&sync_mutex); + found = true; + sync_cv.SignalAll(); + while (!dropped) { + sync_cv.Wait(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put(0, "default-key", "default-value")); + FlushOptions flush_options; + flush_options.wait = false; + ASSERT_OK(db_->Flush(flush_options, handles_[0])); + { + MutexLock lock(&sync_mutex); + while (!found) { + sync_cv.Wait(); + } + } + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[1])); + handles_.resize(1); + { + MutexLock lock(&sync_mutex); + dropped = true; + sync_cv.SignalAll(); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[0])); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_EQ(Get(0, "default-key"), "default-value"); +} + +TEST_F(DBBlobBasicTest, IndirectMultiGetUsesContiguousThirtyTwoRequestBatch) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_file_size = 1 << 20; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + constexpr size_t count = 32; + std::array key_strings; + std::array keys; + const std::string expected(1024, 'v'); + for (size_t i = 0; i < count; ++i) { + key_strings[i] = Key(static_cast(i)); + keys[i] = key_strings[i]; + ASSERT_OK(Put(keys[i], expected)); + } + ASSERT_OK(Flush()); + ReadOptions read_options; + read_options.fill_cache = false; + std::array values; + std::array statuses; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), count, keys.data(), + values.data(), statuses.data(), /*sorted_input=*/true); + for (size_t i = 0; i < count; ++i) { + ASSERT_OK(statuses[i]); + EXPECT_EQ(values[i], expected); + } +} + +TEST_F(DBBlobBasicTest, WalRecoveryPublishesFirstIndirectionFence) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = false; + options.min_blob_size = 0; + options.avoid_flush_during_shutdown = true; + options.avoid_flush_during_recovery = false; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + const std::string value(4096, 'w'); + ASSERT_OK(Put("wal-only", value)); + ASSERT_OK(db_->FlushWAL(true)); + Close(); + + options.enable_blob_indirection = true; + Reopen(options); + ASSERT_EQ(Get("wal-only"), value); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_TRUE(storage->HasBlobIndirectionFormat()); + ASSERT_FALSE(storage->GetBlobFiles().empty()); + ASSERT_TRUE(storage->GetBlobFiles().front()->IsIndirectOrigin()); + } + options.enable_blob_indirection = false; + Reopen(options); + ASSERT_EQ(Get("wal-only"), value); +} + +TEST_F(DBBlobBasicTest, GetBlob) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + + constexpr char key[] = "key"; + constexpr char blob_value[] = "blob_value"; + + ASSERT_OK(Put(key, blob_value)); + + ASSERT_OK(Flush()); + + ASSERT_EQ(Get(key), blob_value); + + // Try again with no I/O allowed. The table and the necessary blocks should + // already be in their respective caches; however, the blob itself can only be + // read from the blob file, so the read should return Incomplete. + ReadOptions read_options; + read_options.read_tier = kBlockCacheTier; + + PinnableSlice result; + ASSERT_TRUE(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result) + .IsIncomplete()); +} + +TEST_F(DBBlobBasicTest, BlobIndirectionWritesExactSstMetadata) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + + Reopen(options); + ASSERT_OK(Put("key1", "blob_value_1")); + ASSERT_OK(Put("key2", "blob_value_2")); + ASSERT_OK(Flush()); + + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_TRUE(storage->HasBlobIndirectionFormat()); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + ASSERT_TRUE(storage->GetBlobFiles().front()->IsIndirectOrigin()); + + // The flush writes unflattened references (generation == origin), so the + // normal blob read path finds each value in its origin blob file. + ASSERT_EQ(Get("key1"), "blob_value_1"); + ASSERT_EQ(Get("key2"), "blob_value_2"); + + const FileMetaData* const coarse = storage->LevelFiles(0).front(); + ASSERT_TRUE(coarse->blob_reference_descriptor.empty()); + ASSERT_EQ(coarse->oldest_coarse_indirect_origin_file_number, + storage->GetBlobFiles().front()->GetBlobFileNumber()); + ASSERT_OK(BuildExactSstAtLevel1()); + + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1); + const auto& user_properties = + properties.begin()->second->user_collected_properties; + const auto descriptor_it = + user_properties.find(kBlobReferenceDescriptorPropertyName); + ASSERT_NE(descriptor_it, user_properties.end()); + + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(descriptor_it->second, + &descriptor)); + EXPECT_FALSE(descriptor.IsEmpty()); + EXPECT_EQ(descriptor.reference_count, 2); + EXPECT_EQ(descriptor.stored_value_bytes, + strlen("blob_value_1") + strlen("blob_value_2")); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusRebuildsAndTracksInstalledSsts) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + + Reopen(options); + ASSERT_OK(Put("key1", "blob_value_1")); + ASSERT_OK(Put("key2", "blob_value_2")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->SstFileNumbers().size(), 1); + + // The census is built in the background after open. Reopen now that the + // format fence exists, so the initial build reads the recovered SST. + Reopen(options); + dbfull()->TEST_WaitForBlobReferenceCensus(); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->SstFileNumbers().size(), 1); + ASSERT_EQ(census->generation_count(), 1); + + ASSERT_OK(Put("key3", "blob_value_3")); + ASSERT_OK(Flush()); + census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->SstFileNumbers().size(), 1); + ASSERT_OK(BuildExactSstAtLevel1()); + census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->SstFileNumbers().size(), 1); + ASSERT_EQ(census->generation_count(), 2); + + ASSERT_OK(Delete("key1")); + ASSERT_OK(Delete("key2")); + ASSERT_OK(Delete("key3")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->generation_count(), 0); +} + +TEST_F(DBBlobBasicTest, SetOptionsOnFencedColumnFamilyPublishesAtomically) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + Reopen(options); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + auto* const cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + ASSERT_TRUE(cfd->current()->storage_info()->HasBlobIndirectionFormat()); + ASSERT_FALSE(cfd->GetLatestMutableCFOptions().disable_auto_compactions); + + // With the format fence, the options publication also prepares a census + // install. A reader that takes the DB mutex during SetOptions must see the + // options, Version and SuperVersion all old or all new. + std::unique_ptr reader; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WakeUpAndNotDone", [&](void* arg) { + if (reader != nullptr) { + return; + } + auto* mu = static_cast(arg); + reader = std::make_unique([mu, cfd]() { + InstrumentedMutexLock l(mu); + const bool disabled = + cfd->GetLatestMutableCFOptions().disable_auto_compactions; + ASSERT_EQ( + disabled, + cfd->current()->GetMutableCFOptions().disable_auto_compactions); + ASSERT_EQ(disabled, + cfd->GetCurrentMutableCFOptions().disable_auto_compactions); + }); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(db_->SetDBOptions({{"max_open_files", "100"}})); + ASSERT_OK(db_->SetOptions({{"disable_auto_compactions", "true"}})); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_NE(reader, nullptr); + reader->join(); + ASSERT_TRUE(cfd->GetLatestMutableCFOptions().disable_auto_compactions); + ASSERT_TRUE(cfd->current()->GetMutableCFOptions().disable_auto_compactions); + ASSERT_TRUE(cfd->GetCurrentMutableCFOptions().disable_auto_compactions); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCRelocatesExactLiveSet) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + ASSERT_OK(Put("older", std::string(4096, 'o'))); + ASSERT_OK(Flush()); + const std::array keys{{"key0", "key1", "key2", "key3"}}; + const std::array values{ + {std::string(4096, 'a'), std::string(4096, 'b'), std::string(4096, 'c'), + std::string(4096, 'd')}}; + for (size_t i = 0; i < keys.size(); ++i) { + ASSERT_OK(Put(keys[i], values[i])); + } + ASSERT_OK(Flush()); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobFiles().size(), 2U); + const uint64_t older_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + ASSERT_LT(older_file_number, origin_file_number); + + for (size_t i = 0; i + 1 < keys.size(); ++i) { + ASSERT_OK(Delete(keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + std::shared_ptr source = + cfd->current()->storage_info()->GetBlobFileMetaData(origin_file_number); + ASSERT_NE(source, nullptr); + ASSERT_GE(source->GetGarbageBlobBytes(), + source->GetTotalBlobBytes() - source->GetGarbageBlobBytes()); + source.reset(); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + const uint64_t route_target = + storage->GetBlobRelocationRouteTarget(origin_file_number); + ASSERT_NE(route_target, kInvalidBlobFileNumber); + ASSERT_NE(storage->GetBlobRelocationFileMetaData(route_target), nullptr); + ASSERT_NE(storage->GetBlobFileMetaData(older_file_number), nullptr); + ASSERT_EQ(storage->GetBlobFileMetaData(origin_file_number), nullptr); + for (size_t i = 0; i + 1 < keys.size(); ++i) { + std::string ignored; + ASSERT_TRUE(db_->Get(ReadOptions(), keys[i], &ignored).IsNotFound()); + } + ASSERT_EQ(Get(keys.back()), values.back()); + ASSERT_EQ(Get("older"), std::string(4096, 'o')); + + // In MANIFEST replay, the old record that added the input comes before the + // blob file retirement record of the input. Reopen must accept that the input + // file is gone from disk, while the older normal blob file is still live. + const std::string source_path = + BlobFileName(cfd->ioptions().cf_paths.front().path, origin_file_number); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_TRUE(options.env->FileExists(source_path).IsNotFound()); + Reopen(options); + ASSERT_EQ(Get(keys.back()), values.back()); + ASSERT_EQ(Get("older"), std::string(4096, 'o')); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(older_file_number), nullptr); + ASSERT_EQ(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_NE(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + + // Compaction garbage accounting (the per-blob-file garbage counters) still + // uses the value's origin file number. Indirect blob GC has done the blob + // file retirement of the origin file, so the compaction must ignore that + // garbage update instead of failing. + ASSERT_OK(Delete(keys.back())); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + std::string ignored; + ASSERT_TRUE(db_->Get(ReadOptions(), keys.back(), &ignored).IsNotFound()); + ASSERT_OK(Put("post_gc", "still writable")); + ASSERT_OK(Flush()); + + Reopen(options); + ASSERT_EQ(Get("post_gc"), "still writable"); +} + +namespace { +// Sends every compaction back to local execution. With a separate remote +// compaction pool, background compactions still start on remote coordinator +// threads. +class LocalOnlyCompactionService : public CompactionService { + public: + const char* Name() const override { return "LocalOnlyCompactionService"; } + CompactionServiceScheduleResponse Schedule( + const CompactionServiceJobInfo& /*info*/, + const std::string& /*compaction_service_input*/) override { + return CompactionServiceScheduleResponse( + CompactionServiceJobStatus::kUseLocal); + } + CompactionServiceJobStatus Wait(const std::string& /*scheduled_job_id*/, + std::string* /*result*/) override { + return CompactionServiceJobStatus::kUseLocal; + } +}; +} // namespace + +TEST_F(DBBlobBasicTest, IndirectBlobGCChargesItsOwnCacheRole) { + const auto role_bytes = [](Cache* cache, CacheEntryRole role) { + size_t bytes = 0; + cache->ApplyToAllEntries( + [&](const Slice& /*key*/, Cache::ObjectPtr /*value*/, size_t charge, + const Cache::CacheItemHelper* helper) { + if (helper != nullptr && helper->role == role) { + bytes += charge; + } + }, + Cache::ApplyToAllEntriesOptions()); + return bytes; + }; + for (const bool charged : {true, false}) { + SCOPED_TRACE(charged); + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(64 << 20); + if (!charged) { + table_options.cache_usage_options + .options_overrides[CacheEntryRole::kIndirectBlobGcWorkingMemory] = { + CacheEntryRoleOptions::Decision::kDisabled}; + } + Cache* const cache = table_options.block_cache.get(); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + DestroyAndReopen(options); + ASSERT_OK(Put("older", std::string(4096, 'o'))); + ASSERT_OK(Flush()); + const std::array keys{{"key0", "key1", "key2", "key3"}}; + for (const std::string& key : keys) { + ASSERT_OK(Put(key, std::string(4096, key.back()))); + } + ASSERT_OK(Flush()); + for (size_t i = 0; i + 1 < keys.size(); ++i) { + ASSERT_OK(Delete(keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // When charging is on, indirect blob GC charges its working memory to the + // block cache under CacheEntryRole::kIndirectBlobGcWorkingMemory while the + // live set exists. When that role is disabled, GC charges nothing. + std::atomic gc_bytes{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterLiveSet", [&](void* /*arg*/) { + gc_bytes = std::max( + gc_bytes.load(), + role_bytes(cache, CacheEntryRole::kIndirectBlobGcWorkingMemory)); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobRelocationFiles().size(), + 1U); + if (charged) { + ASSERT_GT(gc_bytes.load(), 0U); + } else { + ASSERT_EQ(gc_bytes.load(), 0U); + } + // The charge ends with the job. The block cache entry stats + // (DB::Properties::kBlockCacheEntryStats) have a separate entry for + // CacheEntryRole::kIndirectBlobGcWorkingMemory. + ASSERT_EQ(role_bytes(cache, CacheEntryRole::kIndirectBlobGcWorkingMemory), + 0U); + std::map values; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values)); + ASSERT_EQ(values.count(BlockCacheEntryStatsMapKeys::UsedBytes( + CacheEntryRole::kIndirectBlobGcWorkingMemory)), + 1U); + ASSERT_EQ(Get(keys.back()), std::string(4096, keys.back().back())); + ASSERT_EQ(Get("older"), std::string(4096, 'o')); + } +} + +TEST_F(DBBlobBasicTest, RelocationFileOpenIOErrorIsNotRemembered) { + // A short-lived open failure of a relocation file (IOError, or TimedOut from + // one reader's deadline) fails only the read that hit it. The next read opens + // the file again. + class OpenFailureFS : public FileSystemWrapper { + public: + explicit OpenFailureFS(const std::shared_ptr& target) + : FileSystemWrapper(target) {} + const char* Name() const override { return "OpenFailureFS"; } + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& opts, + std::unique_ptr* result, + IODebugContext* dbg) override { + if (fname.size() >= fail_suffix.size() && + fname.compare(fname.size() - fail_suffix.size(), fail_suffix.size(), + fail_suffix) == 0 && + fail_next.exchange(false)) { + return fail_with; + } + return target()->NewRandomAccessFile(fname, opts, result, dbg); + } + std::string fail_suffix; + IOStatus fail_with; + std::atomic fail_next{false}; + }; + const auto fs = std::make_shared(env_->GetFileSystem()); + CompositeEnvWrapper fault_env(env_, fs); + Options options = GetDefaultOptions(); + options.env = &fault_env; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + Reopen(options); + const std::string value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), value)); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {origin})); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t relocation_file = + cfd->current()->storage_info()->GetBlobRelocationRouteTarget(origin); + ASSERT_NE(relocation_file, kInvalidBlobFileNumber); + + fs->fail_suffix = BlobFileName("", relocation_file); + for (const IOStatus& injected : + {IOStatus::IOError("Injected relocation file open failure"), + IOStatus::TimedOut("Injected relocation file open timeout")}) { + SCOPED_TRACE(injected.ToString()); + fs->fail_with = injected; + // Drop the cached reader, so the read opens the file again. + // BlobFileCache::Evict() is for normal blob files only. + TableCache::Evict(dbfull()->TEST_table_cache(), relocation_file); + fs->fail_next.store(true); + std::string result; + const Status status = db_->Get(ReadOptions(), "key3", &result); + EXPECT_EQ(status.code(), injected.code()) << status.ToString(); + ASSERT_FALSE(fs->fail_next.load()); + ASSERT_EQ(Get("key3"), value); + } + Close(); +} + +TEST_F(DBBlobBasicTest, SequentialL0FilesDoNotStall) { + // Sequential inserts make L0 files with disjoint key ranges. Run twice: + // - Coarse L0 files (plain Flush()) are never trivially moved, so the level + // picker must make a normal L0 pick each time instead of picking nothing. + // - Exact L0 files (made exact by the test hook) are trivially moved. + // In both runs L0 stays below the compaction trigger. + for (const bool exact : {false, true}) { + SCOPED_TRACE(exact); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.level0_file_num_compaction_trigger = 4; + DestroyAndReopen(options); + std::atomic trivial_moves{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void*) { ++trivial_moves; }); + SyncPoint::GetInstance()->EnableProcessing(); + constexpr int kFiles = 12; + constexpr int kKeysPerFile = 20; + for (int file = 0; file < kFiles; ++file) { + for (int i = 0; i < kKeysPerFile; ++i) { + ASSERT_OK(Put(Key(file * kKeysPerFile + i), std::string(512, 'v'))); + } + if (exact) { + // The helper turns sync point processing off and clears all callbacks + // when it returns, so register the callback again. + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCompaction:TrivialMove", + [&](void*) { ++trivial_moves; }); + SyncPoint::GetInstance()->EnableProcessing(); + } else { + ASSERT_OK(Flush()); + } + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + EXPECT_LT(NumTableFilesAtLevel(0), + options.level0_file_num_compaction_trigger); + if (exact) { + EXPECT_GT(trivial_moves.load(), 0); + } else { + EXPECT_EQ(trivial_moves.load(), 0); + } + for (int key = 0; key < kFiles * kKeysPerFile; ++key) { + ASSERT_EQ(Get(Key(key)), std::string(512, 'v')); + } + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCTotalsCheckRebuildsCensus) { + // Indirect blob GC compares the SSTs it read with the census record. An + // injected mismatch aborts the job and rebuilds the census; the retried job + // then relocates the live value. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + DestroyAndReopen(options); + ASSERT_OK(Put("older", std::string(4096, 'o'))); + ASSERT_OK(Flush()); + const std::array keys{{"key0", "key1", "key2", "key3"}}; + for (const std::string& key : keys) { + ASSERT_OK(Put(key, std::string(4096, key.back()))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + for (size_t i = 0; i + 1 < keys.size(); ++i) { + ASSERT_OK(Delete(keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::atomic checks{0}; + std::atomic rebuilds{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:CensusTotalsCheck", [&](void* arg) { + if (checks++ == 0) { + *static_cast(arg) = true; + } + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:ReadSst", + [&](void* /*arg*/) { ++rebuilds; }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + for (int round = 0; round < 3; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + EXPECT_GE(checks.load(), 2); + // The census was rebuilt by reading the SSTs again. + EXPECT_GT(rebuilds.load(), 0); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + EXPECT_NE(cfd->current()->storage_info()->GetBlobRelocationRouteTarget( + origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get(keys.back()), std::string(4096, keys.back().back())); + ASSERT_EQ(Get("older"), std::string(4096, 'o')); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCSuppressesMismatchRebuildCannotClear) { + // A mismatch that stays after the census rebuild suppresses that input until + // its census bytes change, instead of rebuilding at every Version. An + // unrelated Version does not retry the input. A change to its references + // does. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + DestroyAndReopen(options); + const std::array keys{{"key0", "key1", "key2", "key3"}}; + for (const std::string& key : keys) { + ASSERT_OK(Put(key, std::string(4096, key.back()))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + for (size_t i = 0; i + 1 < keys.size(); ++i) { + ASSERT_OK(Delete(keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::atomic inject{true}; + std::atomic checks{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:CensusTotalsCheck", [&](void* arg) { + ++checks; + if (inject.load()) { + *static_cast(arg) = true; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + for (int round = 0; round < 5; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + // The first mismatch rebuilt the census. The second mismatch suppressed the + // input. No more GC attempts followed. + EXPECT_EQ(cfd->blob_reference_census()->gc_mismatch_suppress_count(), 1U); + EXPECT_EQ(checks.load(), 2); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + EXPECT_EQ(cfd->current()->storage_info()->GetBlobRelocationRouteTarget( + origin_file_number), + kInvalidBlobFileNumber); + + // An unrelated Version keeps the input suppressed. + inject = false; + ASSERT_OK(Put("other", "v")); + ASSERT_OK(Flush()); + for (int round = 0; round < 5; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + EXPECT_EQ(checks.load(), 2); + + // Dropping the last live value changes the input's census bytes. GC runs + // again, finds no live value, and removes the input by blob file retirement, + // with no output. + ASSERT_OK(Delete(keys.back())); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + for (int round = 0; round < 5; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + EXPECT_GT(checks.load(), 2); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + EXPECT_EQ( + cfd->current()->storage_info()->GetBlobFileMetaData(origin_file_number), + nullptr); + EXPECT_EQ(cfd->current()->storage_info()->GetBlobRelocationRouteTarget( + origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get(keys.back()), "NOT_FOUND"); + ASSERT_EQ(Get("other"), "v"); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCMatchClearsRememberedMismatch) { + // The census remembers the file of its last mismatch report. A second + // mismatch of the same file, with no match in between, means the rebuild did + // not help, so GC suppresses the file. When a GC job checks the file and the + // census matches, GC forgets the mismatch. Then a later mismatch of the file + // is a new problem: it must request a census rebuild, not suppress the file. + // + // The job checks (counted from 1) do this: + // 1. An injected mismatch. The census is rebuilt. + // 2. A match. The test stops this job before it decides any input (it sets + // plan_end to 0), so the file stays a candidate. + // 3. An injected mismatch. It must request a census rebuild again. + // 4. A match. The job relocates the live value. + DestroyAndReopen(IndirectBlobGCOptions()); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + + std::atomic checks{0}; + std::atomic rebuild_reads{0}; + std::atomic rebuild_reads_at_check3{-1}; + std::atomic stop_job_after_match{false}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:CensusTotalsCheck", [&](void* arg) { + const int check = ++checks; + bool* const census_mismatch = static_cast(arg); + if (check == 1 || check == 3) { + *census_mismatch = true; + } + if (check == 2) { + EXPECT_FALSE(*census_mismatch); + stop_job_after_match = true; + } + if (check == 3) { + rebuild_reads_at_check3 = rebuild_reads.load(); + } + }); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:PlanEnd", [&](void* arg) { + if (stop_job_after_match.exchange(false)) { + *static_cast(arg) = 0; + } + }); + sync_point->SetCallBack("DBImpl::BuildBlobReferenceCensus:ReadSst", + [&](void* /*arg*/) { ++rebuild_reads; }); + sync_point->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + // The mismatch at check 3 was not suppressed: it rebuilt the census (the + // build read the SSTs again), and a later job relocated the live value. + EXPECT_EQ(cfd->blob_reference_census()->gc_mismatch_suppress_count(), 0U); + ASSERT_GE(checks.load(), 4); + EXPECT_GT(rebuild_reads.load(), rebuild_reads_at_check3.load()); + EXPECT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + const size_t live_key = kIndirectBlobGCKeyCount - 1; + ASSERT_EQ(Get(IndirectBlobGCKey(live_key)), IndirectBlobGCValue(live_key)); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCKeepsSnapshotVersions) { + // A value hidden by a newer write but visible to a snapshot is live: indirect + // blob GC relocates it, and the snapshot still reads it. + const Options options = IndirectBlobGCOptions(); + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + const std::string key = IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1); + const std::string old_value = + IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1); + const Snapshot* const snapshot = db_->GetSnapshot(); + ASSERT_OK(Put(key, "new value")); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + ASSERT_FALSE(HasBlobFile(file_number)); + ReadOptions snapshot_read; + snapshot_read.snapshot = snapshot; + std::string value; + ASSERT_OK(db_->Get(snapshot_read, key, &value)); + ASSERT_EQ(value, old_value); + ASSERT_EQ(Get(key), "new value"); + db_->ReleaseSnapshot(snapshot); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCRetiresFileWithNoLiveValue) { + // The real indirect blob GC path removes a file with no live value by blob + // file retirement: no output, no route. + const Options options = IndirectBlobGCOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("other", std::string(4096, 'o'))); + ASSERT_OK(Flush()); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/0); + ASSERT_TRUE(HasBlobFile(file_number)); + const std::set before = BlobFilesOnDisk(); + + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_FALSE(HasBlobFile(file_number)); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + dbfull()->TEST_DeleteObsoleteFiles(); + std::set expected = before; + expected.erase(file_number); + ASSERT_EQ(BlobFilesOnDisk(), expected); + ASSERT_EQ(Get("other"), std::string(4096, 'o')); + Reopen(options); + ASSERT_FALSE(HasBlobFile(file_number)); + ASSERT_EQ(Get("other"), std::string(4096, 'o')); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCRetriesStaleInputAtQueueHead) { + // The install check in the MANIFEST queue fails once (injected), as if the + // input changed while the install waited. The job deletes its output and is + // retried, and the retry installs a new output. + const Options options = IndirectBlobGCOptions(); + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + const std::set before = BlobFilesOnDisk(); + std::atomic attempts{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:InstallPrecondition", [&](void* arg) { + if (attempts.fetch_add(1) == 0) { + *static_cast(arg) = false; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + // The first install check failed; a later attempt installed. + ASSERT_GE(attempts.load(), 2); + const uint64_t target = RouteTarget(file_number); + ASSERT_NE(target, kInvalidBlobFileNumber); + ASSERT_OK(dbfull()->TEST_GetBGError()); + // Only the second output is left: the first was deleted. + dbfull()->TEST_DeleteObsoleteFiles(); + std::set expected = before; + expected.erase(file_number); + expected.insert(target); + ASSERT_EQ(BlobFilesOnDisk(), expected); + ASSERT_EQ(Get(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1)), + IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCCompactionInstallsAfterGC) { + // A compaction starts before GC and installs after GC installs. Its outputs + // still use the input as the generation, and reads follow the route. + Options options = IndirectBlobGCOptions(); + // GC needs a second compaction slot while the compaction waits. RocksDB runs + // more than one compaction only under compaction pressure, so the test holds + // a compaction pressure token. + options.max_background_compactions = 2; + options.env->SetBackgroundThreads(3, Env::Priority::LOW); + DestroyAndReopen(options); + std::unique_ptr pressure = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + const std::string key = IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1); + // A second SST over the same range, so the compaction rewrites the SST that + // holds the reference. + ASSERT_OK(Put("key9", "small")); + ASSERT_OK(Flush()); + + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BackgroundCompaction:NonTrivial:BeforeRun", + "DBBlobBasicTest::EnableGC"}, + {"DBImpl::RunIndirectBlobGC:Installed", + "DBImpl::BackgroundCompaction:NonTrivial:AfterRun"}}); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + port::Thread compaction([&]() { + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + // Keep the references: compaction-time blob GC would move the value to a + // new blob file instead. + compact_options.blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kDisable; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + }); + TEST_SYNC_POINT("DBBlobBasicTest::EnableGC"); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + compaction.join(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + pressure.reset(); + + const uint64_t target = RouteTarget(file_number); + ASSERT_NE(target, kInvalidBlobFileNumber); + ASSERT_FALSE(HasBlobFile(file_number)); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_EQ(Get(key), IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); + ASSERT_EQ(Get("key9"), "small"); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + Reopen(options); + ASSERT_EQ(Get(key), IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCInstallsAfterReferenceDropped) { + // The input has two live values. A compaction drops the reference to one of + // them after indirect blob GC has chosen its live set (the list of values + // that it will copy) and before GC copies them. GC copies the live set it + // chose, so the output holds both values, and the dropped one is garbage in + // the output. + // + // GC does not follow that compaction: it installs on its first try, with + // the routes of its census snapshot. The census job applies the + // compaction's change, so the census then counts only the reference that + // is left. + // + // Both values are 4 KB, so the output is then 50% garbage. GC skips a file + // only when less than half of its bytes are garbage, so a later GC job may + // relocate the output again. Sync point callbacks record two facts about + // the first install: how many GC jobs had started, and how many entries the + // output holds. The test checks the census and the values at the end. + Options options = IndirectBlobGCOptions(); + // The compaction that drops the reference needs a second compaction slot + // while GC runs. RocksDB runs more than one compaction only under compaction + // pressure, so the test holds a compaction pressure token. + options.max_background_compactions = 2; + options.env->SetBackgroundThreads(3, Env::Priority::LOW); + DestroyAndReopen(options); + std::unique_ptr pressure = + dbfull()->TEST_write_controler().GetCompactionPressureToken(); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/2); + const std::string dropped_key = + IndirectBlobGCKey(kIndirectBlobGCKeyCount - 2); + const std::string kept_key = IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::RunIndirectBlobGC:AfterLiveSet", + "DBBlobBasicTest::DropReference"}, + {"DBBlobBasicTest::ReferenceDropped", + "DBImpl::RunIndirectBlobGC:BeforeOutputWork"}}); + int gc_jobs = 0; + int jobs_at_first_install = 0; + uint64_t first_target = kInvalidBlobFileNumber; + uint64_t first_output_entries = 0; + // Only GC jobs run these callbacks, and only one GC job runs per column + // family at a time. The test thread reads the values only after all + // background work ends. + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterLiveSet", [&](void*) { ++gc_jobs; }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterMetadataInstall", [&](void* arg) { + if (jobs_at_first_install != 0) { + return; + } + jobs_at_first_install = gc_jobs; + VersionStorageInfo* const storage = + static_cast(arg)->current()->storage_info(); + first_target = storage->GetBlobRelocationRouteTarget(file_number); + const std::shared_ptr output = + storage->GetBlobRelocationFileMetaData(first_target); + first_output_entries = + output != nullptr ? output->GetTotalEntries() : 0; + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + TEST_SYNC_POINT("DBBlobBasicTest::DropReference"); + ASSERT_OK(Delete(dropped_key)); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.exclusive_manual_compaction = false; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + const std::string begin = IndirectBlobGCKey(0); + const Slice begin_slice(begin); + const Slice end_slice(kept_key); + ASSERT_OK(db_->CompactRange(compact_options, &begin_slice, &end_slice)); + TEST_SYNC_POINT("DBBlobBasicTest::ReferenceDropped"); + for (int round = 0; round < 3; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + pressure.reset(); + SyncPoint::GetInstance()->DisableProcessing(); + + ASSERT_OK(dbfull()->TEST_GetBGError()); + // The first GC job installed, so it was not retried with a new live set. Its + // output holds both values. + ASSERT_EQ(jobs_at_first_install, 1); + ASSERT_NE(first_target, kInvalidBlobFileNumber); + ASSERT_EQ(first_output_entries, 2U); + ASSERT_FALSE(HasBlobFile(file_number)); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + const BlobCensusGenerationTotals* const totals = + census->FindTotals(file_number); + ASSERT_NE(totals, nullptr); + ASSERT_EQ(totals->reference_count, 1U); + ASSERT_EQ(Get(kept_key), IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); + ASSERT_EQ(Get(dropped_key), "NOT_FOUND"); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCManifestFailureKeepsOutputUntilReopen) { + // The indirect blob GC install fails to write the MANIFEST. The output is + // quarantined, not deleted, and the input stays. This is also the state after + // a crash between the output rename and the install: on reopen the output, + // which the MANIFEST does not know, is removed and the input is kept. + Options options = IndirectBlobGCOptions(); + options.max_bgerror_resume_count = 0; + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + const std::set before = BlobFilesOnDisk(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:InstallPrecondition", + [&](void*) { env_->manifest_write_error_.store(true); }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + dbfull()->TEST_WaitForCompact().PermitUncheckedError(); + ASSERT_NOK(dbfull()->TEST_GetBGError()); + SyncPoint::GetInstance()->DisableProcessing(); + env_->manifest_write_error_.store(false); + + ASSERT_TRUE(HasBlobFile(file_number)); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + const std::set after_failure = BlobFilesOnDisk(); + std::vector outputs; + std::set_difference(after_failure.begin(), after_failure.end(), + before.begin(), before.end(), + std::back_inserter(outputs)); + ASSERT_EQ(outputs.size(), 1U); + + Close(); + Reopen(options); + ASSERT_TRUE(HasBlobFile(file_number)); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + ASSERT_EQ(BlobFilesOnDisk().count(outputs[0]), 0U); + ASSERT_EQ(Get(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1)), + IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); +} + +namespace { +class BlobFileEventRecorder : public EventListener { + public: + // Tests check only some of the recorded statuses. A vector resize moves the + // statuses, so they are marked checked here, not when they are recorded. + ~BlobFileEventRecorder() override { + for (const auto& file : created_) { + file.second.PermitUncheckedError(); + } + } + void OnBlobFileCreationStarted( + const BlobFileCreationBriefInfo& /*info*/) override { + ++started_; + } + int started() const { return started_.load(); } + void OnBlobFileCreated(const BlobFileCreationInfo& info) override { + std::lock_guard lock(mutex_); + created_.emplace_back(info.file_path, info.status); + } + void OnBlobFileDeleted(const BlobFileDeletionInfo& info) override { + std::lock_guard lock(mutex_); + deleted_.push_back(info.file_path); + } + // Tests check only some of the returned statuses. + std::vector> created() { + std::lock_guard lock(mutex_); + std::vector> result = created_; + for (const auto& file : result) { + file.second.PermitUncheckedError(); + } + return result; + } + std::vector deleted() { + std::lock_guard lock(mutex_); + return deleted_; + } + + private: + std::atomic started_{0}; + std::mutex mutex_; + std::vector> created_; + std::vector deleted_; +}; +} // namespace + +TEST_F(DBBlobBasicTest, IndirectBlobGCRelocatesValueOverReadBatch) { + // A live value larger than one indirect blob GC read batch + // (kIndirectBlobGCReadBatchBytes, 1 MB) is streamed in + // kBlobRelocationStreamChunkSize (256 KB) chunks, so GC relocates the file + // that holds it. + const Options options = IndirectBlobGCOptions(); + DestroyAndReopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("dead" + std::to_string(i), std::string(4 << 20, 'd'))); + } + ASSERT_OK(Put("large", std::string(9 << 20, 'l'))); + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("dead" + std::to_string(i))); + } + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_FALSE(HasBlobFile(file_number)); + ASSERT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + ASSERT_EQ(Get("large"), std::string(9 << 20, 'l')); + Reopen(options); + ASSERT_EQ(Get("large"), std::string(9 << 20, 'l')); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCFailedOutputSendsNoDeleteEvent) { + // An output that fails before its rename never had its final name: + // listeners see a failed creation and no deletion of that name. + Options options = IndirectBlobGCOptions(); + auto recorder = std::make_shared(); + options.listeners.push_back(recorder); + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + const size_t created_before = recorder->created().size(); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterAddBlobRelocation", [](void* arg) { + *static_cast(arg) = Status::IOError("injected write fault"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + dbfull()->TEST_WaitForCompact().PermitUncheckedError(); + ASSERT_TRUE(dbfull()->TEST_GetBGError().IsIOError()); + + const std::vector> created = + recorder->created(); + ASSERT_EQ(created.size(), created_before + 1); + const std::string& output_path = created.back().first; + ASSERT_TRUE(created.back().second.IsIOError()); + for (const std::string& deleted : recorder->deleted()) { + ASSERT_NE(deleted, output_path); + } + ASSERT_TRUE(env_->FileExists(output_path).IsNotFound()); + ASSERT_TRUE(HasBlobFile(file_number)); + ASSERT_EQ(Get(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1)), + IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCTempOutputSurvivesFullPurge) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + const std::array keys{{"key0", "key1", "key2", "key3"}}; + for (const std::string& key : keys) { + ASSERT_OK(Put(key, std::string(4096, key.back()))); + } + ASSERT_OK(Flush()); + for (size_t i = 0; i + 1 < keys.size(); ++i) { + ASSERT_OK(Delete(keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobFiles().size(), 1U); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + + // A full obsolete-file scan runs while the output of indirect blob GC is + // still a temp file. + std::atomic purges{0}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:BeforeRenameOutputs", + [&](void*) { + if (purges.fetch_add(1) == 0) { + dbfull()->TEST_DeleteObsoleteFiles(); + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GE(purges.load(), 1); + + ASSERT_OK(dbfull()->TEST_GetBGError()); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get(keys.back()), std::string(4096, keys.back().back())); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCRunsAsLocalWorkWithSeparateRemotePool) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.compaction_service = std::make_shared(); + options.use_separate_remote_compaction_pool = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + Reopen(options); + + ASSERT_OK(Put("older", std::string(4096, 'o'))); + ASSERT_OK(Flush()); + const std::array keys{{"key0", "key1", "key2", "key3"}}; + for (const std::string& key : keys) { + ASSERT_OK(Put(key, std::string(4096, key.back()))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobFiles().size(), 2U); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + for (size_t i = 0; i + 1 < keys.size(); ++i) { + ASSERT_OK(Delete(keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::mutex threads_mutex; + std::set local_work_threads; + std::atomic gc_output_phases{0}; + std::atomic gc_output_phases_in_local_work{0}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("DBImpl::BGWorkLocalCompaction:Start", [&](void*) { + std::lock_guard lock(threads_mutex); + local_work_threads.insert(std::this_thread::get_id()); + }); + sync_point->SetCallBack("DBImpl::BGWorkLocalCompaction:Finished", [&](void*) { + std::lock_guard lock(threads_mutex); + local_work_threads.erase(std::this_thread::get_id()); + }); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:AfterLiveSet", [&](void*) { + ++gc_output_phases; + std::lock_guard lock(threads_mutex); + if (local_work_threads.count(std::this_thread::get_id()) != 0) { + ++gc_output_phases_in_local_work; + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([&] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // Indirect blob GC data work runs in the LOW pool under a local compaction + // credit, not on the remote coordinator thread. + ASSERT_GT(gc_output_phases.load(), 0); + ASSERT_EQ(gc_output_phases_in_local_work.load(), gc_output_phases.load()); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(Get(keys.back()), std::string(4096, keys.back().back())); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCChangesDuringCopy) { + // Indirect blob GC does not follow the Version changes after its census + // snapshot. This test makes changes while GC copies its first output, in two + // modes: + // - kDropDuringCopy: the column family is dropped. The job installs + // nothing. + // - kInstallsDuringCopy: two flushes install two other Versions. GC still + // installs its edit, and every value stays readable. + // In both modes the job never sets a background error, and writes continue. + enum Mode { kDropDuringCopy, kInstallsDuringCopy }; + for (int mode = kDropDuringCopy; mode <= kInstallsDuringCopy; ++mode) { + SCOPED_TRACE(mode); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + DestroyAndReopen(options); + CreateAndReopenWithCF({"gc"}, options); + const std::string live_value(4096, 'v'); + // Two input blob files, each with one live value, so the job writes two + // outputs. + for (const char* prefix : {"first", "second"}) { + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put(1, prefix + std::to_string(i), + i == 3 ? live_value : std::string(4096, 'd'))); + } + ASSERT_OK(Flush(1)); + } + for (const char* prefix : {"first", "second"}) { + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete(1, prefix + std::to_string(i))); + } + } + ASSERT_OK(Flush(1)); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, + nullptr)); + + std::atomic copies{0}; + std::atomic installs{0}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterAddBlobRelocation", [&](void*) { + if (copies.fetch_add(1) != 0) { + return; + } + if (mode == kDropDuringCopy) { + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + return; + } + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Put(1, "other" + std::to_string(i), "value")); + ASSERT_OK(Flush(1)); + } + }); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:Installed", + [&](void*) { installs.fetch_add(1); }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + // blob_file_size (6000) makes the indirect blob GC target output size 5400 + // bytes. Each input holds one live 4 KB value, and its estimated output + // size (which includes a 64 KB fixed overhead) is larger than the target. + // So each input gets its own output. + ASSERT_OK(db_->SetOptions(handles_[1], + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}, + {"blob_file_size", "6000"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(copies.load(), 0); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_OK(Put("after", "write")); + ASSERT_EQ(Get("after"), "write"); + if (mode == kDropDuringCopy) { + // The job ended at its install check. + ASSERT_EQ(installs.load(), 0); + } else { + // One job copied both live values and installed once. + ASSERT_EQ(installs.load(), 1); + ASSERT_EQ(copies.load(), 2); + ASSERT_EQ(Get(1, "first3"), live_value); + ASSERT_EQ(Get(1, "second3"), live_value); + ASSERT_EQ(Get(1, "other1"), "value"); + } + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + Close(); + } +} + +TEST_F(DBBlobBasicTest, + IndirectBlobGCColumnFamilyDropAfterSnapshotEndsWithoutError) { + // The CF is dropped right after indirect blob GC takes its census snapshot + // (at the SnapshotSelected sync point). The job then ends with no background + // error, and writes continue. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.paranoid_checks = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"gc"}, options); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put(1, "key" + std::to_string(i), std::string(4096, 'd'))); + } + ASSERT_OK(Flush(1)); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete(1, "key" + std::to_string(i))); + } + ASSERT_OK(Flush(1)); + ASSERT_OK( + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr)); + + std::atomic dropped{false}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:SnapshotSelected", + [&](void*) { + if (!dropped.exchange(true)) { + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(handles_[1], + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_TRUE(dropped.load()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_OK(Put("after", "write")); + ASSERT_EQ(Get("after"), "write"); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCWithMmapReads) { + // Indirect blob GC opens normal blob-file inputs for direct reads, except + // when mmap reads are on, because the two cannot be combined. GC still works + // with mmap reads. + if (encrypted_env_) { + ROCKSDB_GTEST_BYPASS("Test requires mmap reads"); + return; + } + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.allow_mmap_reads = true; + Reopen(options); + const std::string live_value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), + i == 3 ? live_value : std::string(4096, 'd'))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin = + cfd->current()->storage_info()->GetBlobFiles()[0]->GetBlobFileNumber(); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_NE( + cfd->current()->storage_info()->GetBlobRelocationRouteTarget(origin), + kInvalidBlobFileNumber); + ASSERT_EQ(Get("key3"), live_value); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCPacksOriginsAndReclaimsRelocation) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + const std::array first_keys{ + {"first0", "first1", "first2", "first3"}}; + const std::array second_keys{ + {"second0", "second1", "second2", "second3"}}; + const std::string first_live_value(4096, 'a'); + const std::string second_live_value(4096, 'b'); + for (size_t i = 0; i < first_keys.size(); ++i) { + ASSERT_OK(Put(first_keys[i], + i + 1 == first_keys.size() + ? first_live_value + : std::string(4096, static_cast('c' + i)))); + } + ASSERT_OK(Flush()); + for (size_t i = 0; i < second_keys.size(); ++i) { + ASSERT_OK(Put(second_keys[i], + i + 1 == second_keys.size() + ? second_live_value + : std::string(4096, static_cast('g' + i)))); + } + ASSERT_OK(Flush()); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobFiles().size(), 2U); + const uint64_t first_origin = + cfd->current()->storage_info()->GetBlobFiles()[0]->GetBlobFileNumber(); + const uint64_t second_origin = + cfd->current()->storage_info()->GetBlobFiles()[1]->GetBlobFileNumber(); + for (size_t i = 0; i + 1 < first_keys.size(); ++i) { + ASSERT_OK(Delete(first_keys[i])); + ASSERT_OK(Delete(second_keys[i])); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + std::atomic unlocked_apply_count{0}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("VersionSet::LogAndApplyHelper:ApplyUnlocked", + [&](void*) { unlocked_apply_count.fetch_add(1); }); + sync_point->EnableProcessing(); + auto cleanup_sync_point = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GT(unlocked_apply_count.load(), 0U); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + uint64_t first_route = storage->GetBlobRelocationRouteTarget(first_origin); + uint64_t second_route = storage->GetBlobRelocationRouteTarget(second_origin); + ASSERT_NE(first_route, kInvalidBlobFileNumber); + ASSERT_NE(second_route, kInvalidBlobFileNumber); + const uint64_t packed_file_number = first_route; + ASSERT_EQ(second_route, packed_file_number); + std::shared_ptr packed_file = + storage->GetBlobRelocationFileMetaData(packed_file_number); + ASSERT_NE(packed_file, nullptr); + // One output holds both inputs: one live value of each origin. + ASSERT_EQ(packed_file->GetTotalEntries(), 2U); + ASSERT_EQ(storage->GetBlobFileMetaData(first_origin), nullptr); + ASSERT_EQ(storage->GetBlobFileMetaData(second_origin), nullptr); + ASSERT_EQ(Get(first_keys.back()), first_live_value); + ASSERT_EQ(Get(second_keys.back()), second_live_value); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "true"}, + {"enable_blob_garbage_collection", "false"}})); + ASSERT_OK(Delete(first_keys.back())); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobRelocationRouteTarget(second_origin), + kInvalidBlobFileNumber); + const std::shared_ptr before_regc_census = + GetCensusSnapshot(cfd); + ASSERT_NE(before_regc_census, nullptr); + ASSERT_EQ(before_regc_census->GetSstCount(first_origin), 0U); + ASSERT_GT(before_regc_census->GetSstCount(second_origin), 0U); + ASSERT_EQ(Get(second_keys.back()), second_live_value); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}, + {"blob_file_size", "0"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(first_origin), + kInvalidBlobFileNumber); + second_route = storage->GetBlobRelocationRouteTarget(second_origin); + ASSERT_NE(second_route, kInvalidBlobFileNumber); + ASSERT_NE(second_route, packed_file_number); + ASSERT_EQ(storage->GetBlobRelocationFileMetaData(packed_file_number), + nullptr); + // Refresh this thread's cached SuperVersion after indirect blob GC before + // checking that no reader still holds the relocation file that GC deleted + // (kBlobRelocationFileDeletion). + ASSERT_EQ(Get(second_keys.back()), second_live_value); + ASSERT_EQ(packed_file.use_count(), 1U); + std::string ignored; + ASSERT_TRUE( + db_->Get(ReadOptions(), first_keys.back(), &ignored).IsNotFound()); + + const std::string packed_path = + BlobFileName(cfd->ioptions().cf_paths.front().path, packed_file_number); + packed_file.reset(); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_OK(dbfull()->TEST_WaitForPurge()); + ASSERT_TRUE(options.env->FileExists(packed_path).IsNotFound()); + Reopen(options); + ASSERT_EQ(Get(second_keys.back()), second_live_value); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCRelocationInputIsNotPinned) { + // With pin_blob_relocation_location_blocks on, indirect blob GC still opens a + // relocation input without keeping its location blocks: GC is about to + // reclaim the file. The test checks the cached reader of the input while GC + // runs. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + Reopen(options); + const std::string value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), value)); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + // GC moves the two live values of the origin into a relocation file. + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {origin})); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t relocation_file = + cfd->current()->storage_info()->GetBlobRelocationRouteTarget(origin); + ASSERT_NE(relocation_file, kInvalidBlobFileNumber); + + // One of the two relocated values becomes garbage, so the relocation file + // is a GC input. No read opens it before GC does. + ASSERT_OK(Delete("key2")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + ASSERT_OK(db_->SetOptions({{"pin_blob_relocation_location_blocks", "true"}})); + // SetOptions() installs a new Version. GC starts only when the census is + // ready at the current Version, so wait for the census job to catch up. + dbfull()->TEST_WaitForBlobReferenceCensus(); + TableCache::Evict(dbfull()->TEST_table_cache(), relocation_file); + + int checks = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::RunIndirectBlobGC:InputsOpened", [&](void*) { + Cache* const cache = dbfull()->TEST_table_cache(); + uint64_t number = relocation_file; + Cache::Handle* const handle = cache->Lookup(GetSliceForKey(&number)); + ASSERT_NE(handle, nullptr); + const auto* reader = static_cast(cache->Value(handle)); + ASSERT_NE(reader->relocation(), nullptr); + EXPECT_EQ(reader->relocation()->pinned_location_block_bytes(), 0U); + cache->Release(handle); + ++checks; + }); + SyncPoint::GetInstance()->EnableProcessing(); + const Status status = + dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {relocation_file}); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(status); + EXPECT_EQ(checks, 1); + ASSERT_EQ(Get("key3"), value); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCRelocationInputHasNoGenerationCap) { + // Indirect blob GC collects a relocation file together with the generation + // numbers of every old file routed to it, with no cap on how many there are. + // Here 300 old blob files route to one relocation file R. After two of every + // three values die, GC collects R: the live generation numbers route to the + // new output, and the routes of the dead generation numbers are removed. + Options options = IndirectBlobGCOptions(); + // Each value goes to its own blob file. + options.blob_file_size = 1; + DestroyAndReopen(options); + constexpr size_t kGenerationCount = 300; + std::vector keys; + std::vector values; + for (size_t i = 0; i < kGenerationCount; ++i) { + char key[16]; + snprintf(key, sizeof(key), "k%03zu", i); + keys.emplace_back(key); + values.push_back(std::string(100, static_cast('a' + i % 26)) + + std::to_string(i)); + ASSERT_OK(Put(keys.back(), values.back())); + } + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + + VersionSet* const version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + std::vector generations; + for (const auto& file : cfd->current()->storage_info()->GetBlobFiles()) { + generations.push_back(file->GetBlobFileNumber()); + } + // The flush writes the values in key order, and blob_file_size = 1 closes + // each blob file after one value. So the sorted generations[i] is the blob + // file of keys[i]. + ASSERT_EQ(generations.size(), kGenerationCount); + std::sort(generations.begin(), generations.end()); + + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + std::vector records; + for (size_t i = 0; i < kGenerationCount; ++i) { + const uint64_t value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + keys[i].size(); + records.push_back( + {generations[i], value_offset, values[i], kNoCompression}); + } + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, records, &metadata, &relocation_file_size); + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + for (uint64_t generation : generations) { + install.SetBlobRelocationRoute( + BlobRelocationRoute(generation, relocation_file_number)); + install.RetireBlobFile(generation); + } + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + // LogAndApply alone installs no SuperVersion; reopen to read the new Version. + Reopen(options); + for (size_t i = 0; i < kGenerationCount; ++i) { + ASSERT_EQ(Get(keys[i]), values[i]); + } + + // Keep one value in three. + for (size_t i = 0; i < kGenerationCount; ++i) { + if (i % 3 != 0) { + ASSERT_OK(Delete(keys[i])); + } + } + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + compact_options.blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kDisable; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobRelocationFileMetaData(relocation_file_number), + nullptr); + const uint64_t output = RouteTarget(generations[0]); + ASSERT_NE(output, kInvalidBlobFileNumber); + ASSERT_NE(output, relocation_file_number); + for (size_t i = 0; i < kGenerationCount; ++i) { + SCOPED_TRACE(i); + if (i % 3 == 0) { + ASSERT_EQ(RouteTarget(generations[i]), output); + ASSERT_EQ(Get(keys[i]), values[i]); + } else { + ASSERT_EQ(RouteTarget(generations[i]), kInvalidBlobFileNumber); + } + } + Reopen(options); + for (size_t i = 0; i < kGenerationCount; i += 3) { + ASSERT_EQ(Get(keys[i]), values[i]); + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCReclaimsEmptyRelocationFile) { + // A relocation file with no live value left is removed together with its + // routes, and indirect blob GC writes no output for it. + const Options options = IndirectBlobGCOptions(); + DestroyAndReopen(options); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + const uint64_t relocation_file_number = RouteTarget(file_number); + ASSERT_NE(relocation_file_number, kInvalidBlobFileNumber); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "true"}, + {"enable_blob_garbage_collection", "false"}})); + ASSERT_OK(Delete(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1))); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + compact_options.blob_garbage_collection_policy = + BlobGarbageCollectionPolicy::kDisable; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + dbfull()->TEST_DeleteObsoleteFiles(); + const std::set before = BlobFilesOnDisk(); + ASSERT_EQ(before.count(relocation_file_number), 1U); + + ASSERT_NO_FATAL_FAILURE(EnableIndirectBlobGC()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + relocation_file_number), + nullptr); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_OK(dbfull()->TEST_WaitForPurge()); + std::set expected = before; + expected.erase(relocation_file_number); + ASSERT_EQ(BlobFilesOnDisk(), expected); + Reopen(options); + ASSERT_EQ(Get(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1)), "NOT_FOUND"); +} + +TEST_F(DBBlobBasicTest, BlobReferenceReadChecksSharedOutputRoom) { + // Indirect blob GC reads all generation numbers of one input into one live + // set, so several requests of a batch read share one output. Each request is + // checked against the room left after the earlier requests for the same + // output. The output has room for one reference, so the second request gets + // Aborted (as for a census mismatch), not Corruption, and adds nothing. + Options options = IndirectBlobGCOptions(); + // Each value goes to its own blob file, so each generation number has one + // reference. + options.blob_file_size = 1; + DestroyAndReopen(options); + ASSERT_OK(Put("k0", "value0")); + ASSERT_OK(Put("k1", "value1")); + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + std::vector generations; + for (const auto& file : storage->GetBlobFiles()) { + generations.push_back(file->GetBlobFileNumber()); + } + ASSERT_EQ(generations.size(), 2U); + std::sort(generations.begin(), generations.end()); + ASSERT_EQ(storage->LevelFiles(0).size(), 1U); + const FileMetaData& sst = *storage->LevelFiles(0)[0]; + + std::array keys; + std::vector requests(2); + std::vector output; + output.reserve(1); + for (size_t i = 0; i < 2; ++i) { + keys[i].encoding_class = BlobReferenceEncodingClass::kIndirectUnflattened; + keys[i].generation_file_number = generations[i]; + keys[i].origin_file_number = generations[i]; + requests[i].group = &keys[i]; + requests[i].output = &output; + } + ASSERT_EQ(output.capacity(), 1U); + ASSERT_OK(cfd->table_cache()->ReadBlobReferences( + ReadOptions(), FileOptions(), cfd->internal_comparator(), sst, + cfd->GetLatestMutableCFOptions(), &requests)); + ASSERT_OK(requests[0].status); + ASSERT_TRUE(requests[1].status.IsAborted()) << requests[1].status.ToString(); + ASSERT_EQ(output.size(), 1U); + ASSERT_EQ(output[0].generation_file_number, generations[0]); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCCorruptionIsCandidateLocal) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + ASSERT_OK(Put("dead1", std::string(4096, 'a'))); + ASSERT_OK(Put("dead2", std::string(4096, 'b'))); + ASSERT_OK(Put("dead3", std::string(4096, 'c'))); + ASSERT_OK(Put("live", std::string(4096, 'd'))); + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + + ASSERT_OK(Delete("dead1")); + ASSERT_OK(Delete("dead2")); + ASSERT_OK(Delete("dead3")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + SyncPoint* sync_point = SyncPoint::GetInstance(); + std::atomic attempts{0}; + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:AfterLiveSet", + [&](void* argument) { + ++attempts; + *static_cast(argument) = + Status::Corruption("injected candidate fault"); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get("live"), std::string(4096, 'd')); + ASSERT_EQ(attempts.load(), 1); + ASSERT_OK(Put("after", "still writable")); + ASSERT_OK(Flush()); + ASSERT_EQ(Get("after"), "still writable"); + // The census keeps the candidate suppressed across the new Version. + for (int round = 0; round < 3; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + ASSERT_EQ(attempts.load(), 1); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCOutputCorruptionIsBackgroundError) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + ASSERT_OK(Put("dead1", std::string(4096, 'a'))); + ASSERT_OK(Put("dead2", std::string(4096, 'b'))); + ASSERT_OK(Put("dead3", std::string(4096, 'c'))); + ASSERT_OK(Put("live", std::string(4096, 'd'))); + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + + ASSERT_OK(Delete("dead1")); + ASSERT_OK(Delete("dead2")); + ASSERT_OK(Delete("dead3")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:AfterOutputWork", + [](void* argument) { + *static_cast(argument) = + Status::Corruption("injected output fault"); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + const Status wait_status = dbfull()->TEST_WaitForCompact(); + ASSERT_TRUE(wait_status.IsCorruption()) << wait_status.ToString(); + const Status background_error = dbfull()->TEST_GetBGError(); + ASSERT_TRUE(background_error.IsCorruption()) << background_error.ToString(); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get("live"), std::string(4096, 'd')); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCMixedReadErrorIsBackgroundError) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + for (int i = 0; i < 5; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), + std::string(4096, static_cast('a' + i)))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "BlobFileReader::MultiGetStoredBlobsForRelocation:TamperWithResult", + [](void* argument) { + auto* statuses = static_cast*>(argument); + ASSERT_GE(statuses->size(), 2U); + (*statuses)[0] = + Status::Corruption("injected candidate-local read fault"); + (*statuses)[1] = Status::IOError("injected external read fault"); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + const Status wait_status = dbfull()->TEST_WaitForCompact(); + ASSERT_TRUE(wait_status.IsIOError()) << wait_status.ToString(); + const Status background_error = dbfull()->TEST_GetBGError(); + ASSERT_TRUE(background_error.IsIOError()) << background_error.ToString(); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); +} + +TEST_F(DBBlobBasicTest, + IndirectBlobGCOutputCorruptionPrecedesSourceCorruption) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + for (int i = 0; i < 5; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), + std::string(4096, static_cast('a' + i)))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "BlobFileReader::MultiGetStoredBlobsForRelocation:TamperWithResult", + [](void* argument) { + auto* statuses = static_cast*>(argument); + ASSERT_GE(statuses->size(), 2U); + (*statuses)[1] = Status::Corruption("injected source-record fault"); + }); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:AfterAddBlobRelocation", [](void* argument) { + *static_cast(argument) = + Status::Corruption("injected output-format fault"); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + const Status wait_status = dbfull()->TEST_WaitForCompact(); + ASSERT_TRUE(wait_status.IsCorruption()) << wait_status.ToString(); + const Status background_error = dbfull()->TEST_GetBGError(); + ASSERT_TRUE(background_error.IsCorruption()) << background_error.ToString(); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusCatchesDeletionDuringInitialScan) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + + Reopen(options); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + uint64_t starting_file_number = 0; + for (int level = 1; level < options.num_levels; ++level) { + const auto& files = cfd->current()->storage_info()->LevelFiles(level); + if (!files.empty()) { + starting_file_number = files.front()->fd.GetNumber(); + break; + } + } + ASSERT_NE(starting_file_number, 0); + + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->LoadDependency( + {{"DBBlobBasicTest::ReleaseCensusScan", + "DBImpl::BuildBlobReferenceCensus:BeforeReadFile"}}); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { sync_point->DisableProcessing(); }); + + Reopen(options); + ASSERT_OK(DeleteFilesInRange(db_.get(), db_->DefaultColumnFamily(), + /*begin=*/nullptr, /*end=*/nullptr)); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetFileMetaDataByNumber( + starting_file_number), + nullptr); + TEST_SYNC_POINT("DBBlobBasicTest::ReleaseCensusScan"); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + const std::vector files = census->SstFileNumbers(); + ASSERT_EQ(std::count(files.begin(), files.end(), starting_file_number), 0); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusReplaysLegacySnapshotMarker) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + + Reopen(options); + ASSERT_OK(Put("legacy", "legacy_blob_value")); + ASSERT_OK(Flush()); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_indirection", "true"}})); + ASSERT_OK(Put("feature", "feature_blob_value")); + ASSERT_OK(Flush()); + + VersionEdit rotate; + rotate.SetColumnFamily(/*column_family_id=*/0); + ASSERT_OK(ApplyVersionEdit(&rotate, /*new_descriptor_log=*/true)); + + Close(); + options.enable_blob_indirection = true; + Reopen(options); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->SstFileNumbers().size(), 0); + ASSERT_EQ(census->generation_count(), 0); + ASSERT_EQ(Get("legacy"), "legacy_blob_value"); + ASSERT_EQ(Get("feature"), "feature_blob_value"); + ASSERT_OK(BuildExactSstAtLevel1()); + const auto exact_census = GetCensusSnapshot(cfd); + ASSERT_NE(exact_census, nullptr); + ASSERT_EQ(exact_census->SstFileNumbers().size(), 1); + // Legacy direct references do not add generation numbers to the census. + ASSERT_EQ(exact_census->generation_count(), 1); +} + +TEST_F(DBBlobBasicTest, + BlobReferenceCensusFailureDoesNotAbortFlushAndRebuilds) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + Reopen(options); + + // Flush output is coarse, so both flushes use the test hook to write exact + // metadata. The census rebuild then has two SSTs to read. + ASSERT_OK(Put("first", "first_blob_value")); + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(GetCensusSnapshot(cfd), nullptr); + + SyncPoint* sync_point = SyncPoint::GetInstance(); + std::atomic injected{0}; + sync_point->SetCallBack( + "VersionSet::ProcessManifestWrites:PrepareCensusInstall", + [&](void* argument) { + ++injected; + *static_cast(argument) = + Status::IOError("injected census-only read failure"); + }); + sync_point->SetCallBack("FlushJob::WriteLevel0Table:TableBuilderOptions", + [](void* arg) { + static_cast(arg) + ->collect_blob_reference_metadata = true; + }); + // A rebuild reads the reference index of every SST again. + std::atomic rebuild_reads{0}; + sync_point->SetCallBack("DBImpl::BuildBlobReferenceCensus:ReadSst", + [&](void*) { ++rebuild_reads; }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + // The census install of the second flush fails to prepare. The flush still + // commits, and the census is rebuilt for the new Version. + ASSERT_OK(Put("second", "second_blob_value")); + ASSERT_OK(Flush()); + ASSERT_GE(injected.load(), 1); + ASSERT_EQ(Get("first"), "first_blob_value"); + ASSERT_EQ(Get("second"), "second_blob_value"); + dbfull()->TEST_WaitForBlobReferenceCensus(); + // The rebuild read both SSTs. + ASSERT_GE(rebuild_reads.load(), 2); + sync_point->DisableProcessing(); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusAtReopenMatchesBuildTime) { + // After the flush, the census uses the summary from the table builder. After + // reopen, the census reads the groups from the reference index of the SST. + // Both must give the same census. + const Options options = FlushManyBlobReferenceGroups(); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const FileMetaData* const sst = OnlySstForTest(cfd); + ASSERT_NE(sst, nullptr); + const uint64_t file_number = sst->fd.GetNumber(); + const auto built = GetCensusSnapshot(cfd); + ASSERT_NE(built, nullptr); + ASSERT_GE(built->generation_count(), 3U); + ASSERT_EQ(built->SstFileNumbers(), std::vector{file_number}); + // Check the layout: there are more data blocks than groups, and at least one + // group starts inside a block (run_offset > 0). + const BlobReferenceIndex index = ReadBlobReferenceIndexForTest(*sst); + ASSERT_GT(index.blocks.size(), index.groups.size()); + ASSERT_TRUE(std::any_of(index.groups.begin(), index.groups.end(), + [](const BlobReferenceIndexEntry& entry) { + return entry.run_offset > 0; + })); + + Reopen(options); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto loaded = GetCensusSnapshot(cfd); + ASSERT_NE(loaded, nullptr); + std::string difference; + EXPECT_TRUE(built->SameMemberships(*loaded, &difference)) << difference; +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusReusesTableCacheReader) { + // With max_open_files=-1, the DB open loads the table reader of every SST + // before the census builds. The census then reads the reference index + // through that reader and opens no new reader, so it does not read the table + // tail of the SST again. + Options options = FlushManyBlobReferenceGroups(); + options.max_open_files = -1; + std::atomic fresh_opens{0}; + std::atomic cache_hits{0}; + SyncPoint::GetInstance()->SetCallBack( + "TableCache::FindTable:FreshTableReader", + [&](void*) { fresh_opens.fetch_add(1); }); + SyncPoint::GetInstance()->SetCallBack( + "TableCache::GetBlobReferenceReader:Lookup", [&](void* argument) { + if (static_cast(argument)->ok()) { + cache_hits.fetch_add(1); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(GetCensusSnapshot(cfd), nullptr); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + // The debug self-check after a publish (DBImpl::CheckBlobReferenceCensus()) + // may read the SST once more, through the same table cache reader. + EXPECT_EQ(fresh_opens.load(), 0); + EXPECT_GE(cache_hits.load(), 1); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusMissDoesNotFillTableCache) { + // With a limited table cache, the reader of an SST may be missing from the + // table cache. The census then opens an ephemeral reader and does not insert + // it, so a census build never evicts the reader of another SST. + // max_open_files=20 gives a table cache capacity of 10 (the DB keeps 10 file + // handles for other files). The open then loads at most capacity / 4 = 2 + // readers, so most of the L1 SSTs are not in the table cache when the census + // builds. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.target_file_size_base = 16 << 10; + options.max_open_files = 20; + options.create_if_missing = true; + DestroyAndReopen(options); + for (int i = 0; i < 8000; ++i) { + ASSERT_OK(Put(Key(i), std::string(100, 'v'))); + } + ASSERT_OK(Flush()); + // Compact into several exact SSTs at L1. + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_GE(NumTableFilesAtLevel(1), 4); + std::atomic fresh_opens{0}; + SyncPoint::GetInstance()->SetCallBack( + "TableCache::FindTable:FreshTableReader", + [&](void*) { fresh_opens.fetch_add(1); }); + // The census build at reopen waits after it starts until the test has read + // the table cache usage. + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BuildBlobReferenceCensus:NewBuildReady", + "DBBlobBasicTest::MissBuildStarted"}, + {"DBBlobBasicTest::MissUsageRead", + "DBImpl::BuildBlobReferenceCensus:NewBuildWait"}}); + SyncPoint::GetInstance()->EnableProcessing(); + Reopen(options); + TEST_SYNC_POINT("DBBlobBasicTest::MissBuildStarted"); + const size_t usage_before_build = dbfull()->TEST_table_cache()->GetUsage(); + TEST_SYNC_POINT("DBBlobBasicTest::MissUsageRead"); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(GetCensusSnapshot(cfd), nullptr); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + EXPECT_GE(fresh_opens.load(), 2); + EXPECT_EQ(dbfull()->TEST_table_cache()->GetUsage(), usage_before_build); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusAppliesCompactionRemovedGroups) { + // A compaction that does not remove every SST reads the summaries of its + // inputs and passes them to the census in its edit. So the census applies a + // delta and is not rebuilt. The job charges the summaries to + // kBlobReferenceCensus while it holds them. + std::shared_ptr< + TargetCacheChargeTrackingCache> + cache; + Options options = CensusChargeTrackingOptions(&cache); + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + // Flush output is coarse. Compacting each L0 file to L1 rewrites it with + // exact metadata. + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(CompactEachL0FileTo(1)); + ASSERT_OK(Put("c", "blob_c")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(CompactEachL0FileTo(1)); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + std::vector inputs; + for (const auto& file : metadata.levels[1].files) { + if (file.smallestkey != "a") { + inputs.push_back(file.name); + } + } + ASSERT_EQ(inputs.size(), 2U); + const size_t charge_before = cache->GetCacheCharge(); + size_t charge_during = 0; + SyncPoint::GetInstance()->SetCallBack("CompactionJob::Run():End", [&](void*) { + charge_during = cache->GetCacheCharge(); + }); + CensusEventCounter counter; + counter.Start(); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, /*output_level=*/2)); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_EQ(counter.rebuilds.load(), 0); + EXPECT_EQ(counter.reads.load(), 0); + // The summaries are charged from the read at the start of the job until the + // census has applied the install. + EXPECT_GT(charge_during, charge_before); + EXPECT_EQ(cache->GetCacheCharge(), charge_before); + ASSERT_EQ(Get("a"), "blob_a"); + ASSERT_EQ(Get("b"), "blob_b"); + ASSERT_EQ(Get("c"), "blob_c"); +} + +TEST_F(DBBlobBasicTest, CoarseL0FilesStayInL0) { + // A coarse L0 file pins every generation number at or above its frontier, so + // it must be rewritten with exact metadata when it leaves L0. PromoteL0 + // returns NotSupported. CompactRange rewrites the file instead of moving it, + // also under universal compaction with allow_trivial_move. + for (const bool universal : {false, true}) { + SCOPED_TRACE(universal); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + if (universal) { + options.compaction_style = kCompactionStyleUniversal; + options.compaction_options_universal.allow_trivial_move = true; + options.num_levels = 4; + } + DestroyAndReopen(options); + ASSERT_OK(Put("a", "value_a")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("b", "value_b")); + ASSERT_OK(Flush()); + const auto files_by_level = [&]() { + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + return metadata; + }; + ASSERT_EQ(files_by_level().levels[0].files.size(), 2U); + if (!universal) { + const Status status = + experimental::PromoteL0(db_.get(), db_->DefaultColumnFamily(), 1); + ASSERT_TRUE(status.IsNotSupported()) << status.ToString(); + ASSERT_EQ(files_by_level().levels[0].files.size(), 2U); + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + const VersionStorageInfo* const storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + size_t files = 0; + for (int level = 1; level < storage->num_levels(); ++level) { + for (const FileMetaData* file : storage->LevelFiles(level)) { + EXPECT_EQ(file->oldest_coarse_indirect_origin_file_number, 0U); + EXPECT_FALSE(file->blob_reference_descriptor.empty()); + ++files; + } + } + EXPECT_GT(files, 0U); + ASSERT_EQ(Get("a"), "value_a"); + ASSERT_EQ(Get("b"), "value_b"); + } +} + +namespace { +// Turns on blob indirection with universal compaction and one level, so every +// compaction writes to L0. Auto compaction and blob GC are off, so the test +// decides when compactions run. Every value goes to a blob file +// (min_blob_size = 0). +Options UniversalOneLevelIndirectionOptions(Options options) { + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.compaction_style = kCompactionStyleUniversal; + options.num_levels = 1; + return options; +} + +// Runs a full (bottommost) compaction with a reference cap of `cap` references +// per output SST. The option blob_indirection_max_references_per_sst has a +// minimum of 4096, so the cap is set with the sync point +// BlockBasedTableBuilder::BlobReferenceLimit, which lets a test cut outputs +// after a few keys. +Status FullCompactionWithReferenceCap(DB* db, uint64_t cap) { + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "BlockBasedTableBuilder::BlobReferenceLimit", + [cap](void* arg) { *static_cast(arg) = cap; }); + sync_point->EnableProcessing(); + const Defer cleanup([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + CompactRangeOptions full; + full.bottommost_level_compaction = BottommostLevelCompaction::kForce; + return db->CompactRange(full, nullptr, nullptr); +} +} // namespace + +TEST_F(DBBlobBasicTest, + UniversalOneLevelDeleteTriggeredCompactionTakesWholeExactRun) { + // Background: with universal compaction and one level, a compaction with the + // reference cap can write an exact epoch run, which is several exact L0 + // files with one epoch number. Their key ranges do not overlap, but their L0 + // order (newest largest sequence number first) can differ from their key + // order. The run is one sorted run. With num_levels = 1, the + // delete-triggered pick never starts at the last (oldest) sorted run. It + // starts at the first sorted run with a file that CompactOnDeletionCollector + // marked, and takes every older sorted run. If it took only part of a run, + // its output would get the run's epoch number and could overlap a run file + // that it left out. The Version would then reject the install and set a + // background error. + // + // The test builds a run of three files above one older L0 file X (keys + // 20..23), so the run is not the last sorted run. In L0 order the run files + // are: file N (keys 4..7), file M (keys 8..11, with deletions of keys 9 and + // 10, so it is marked) and file O (keys 0..3). A compaction of only M and O + // would keep keys 0..3, 8 and 11, so its output would cover keys 0..11 and + // overlap N. The test checks that the delete-triggered compaction takes all + // three run files and X, sets no background error, and leaves no two L0 + // files of one epoch with overlapping key ranges. + Options options = UniversalOneLevelIndirectionOptions(GetDefaultOptions()); + // The collector marks an SST when a window of 4 entries has 2 deletions. + // Only file M (and the flushed SST that the run comes from) has deletions. + options.table_properties_collector_factories.emplace_back( + NewCompactOnDeletionCollectorFactory(/*sliding_window_size=*/4, + /*deletion_trigger=*/2)); + // Size-ratio and size-amplification picks need at least 8 sorted runs, and + // the DB has 2, so only the delete-triggered pick runs. + options.level0_file_num_compaction_trigger = 8; + DestroyAndReopen(options); + // File X is the oldest L0 file. + for (int key = 20; key < 24; ++key) { + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + const uint64_t file_x = [this]() { + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + return metadata.levels[0].files[0].file_number; + }(); + // Keys 0..3 (file O) and 8..11 (file M) are written first. Keys 9 and 10 are + // then deleted, while a snapshot keeps the old values, so the compaction + // below keeps both the values and the deletions. Keys 4..7 (file N) are + // written last. The largest sequence numbers then order the files N, M, O. + for (const int key : {0, 1, 2, 3, 8, 9, 10, 11}) { + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + const Snapshot* snapshot = db_->GetSnapshot(); + for (const int key : {9, 10}) { + ASSERT_OK(Delete(Key(key))); + } + for (int key = 4; key < 8; ++key) { + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + // The compaction of only the new SST to L0 is not bottommost, because file X + // is older. A compaction to L0 writes exact output when an input is exact, + // so this flush writes exact metadata. + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + std::string flushed; + { + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(metadata.levels[0].files.size(), 2U); + for (const auto& file : metadata.levels[0].files) { + if (file.file_number != file_x) { + flushed = file.name; + } + } + } + // Each run file has 4 references (a deletion has none), so a cap of 4 cuts + // the output at keys 4 and 8. + SetBlobReferenceCapForTest(/*cap=*/4); + Status status = + db_->CompactFiles(CompactionOptions(), {flushed}, /*output_level=*/0); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + db_->ReleaseSnapshot(snapshot); + ASSERT_OK(status); + + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto storage = [cfd]() { return cfd->current()->storage_info(); }; + std::vector run; + { + const std::vector& files = storage()->LevelFiles(0); + ASSERT_EQ(files.size(), 4U); + for (size_t i = 0; i < 3; ++i) { + ASSERT_FALSE(files[i]->blob_reference_descriptor.empty()); + ASSERT_EQ(files[i]->epoch_number, files.front()->epoch_number); + run.push_back(files[i]->fd.GetNumber()); + } + ASSERT_EQ(files[0]->smallest.user_key(), Key(4)); + ASSERT_EQ(files[1]->smallest.user_key(), Key(8)); + ASSERT_EQ(files[2]->smallest.user_key(), Key(0)); + ASSERT_FALSE(files[0]->marked_for_compaction); + ASSERT_TRUE(files[1]->marked_for_compaction); + ASSERT_FALSE(files[2]->marked_for_compaction); + ASSERT_EQ(files[3]->fd.GetNumber(), file_x); + ASSERT_LT(files[3]->epoch_number, files[0]->epoch_number); + ASSERT_FALSE(files[3]->marked_for_compaction); + } + + // Turning on auto compaction schedules the delete-triggered compaction. + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}})); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_OK(dbfull()->TEST_GetBGError()); + const std::vector& files = storage()->LevelFiles(0); + ASSERT_FALSE(files.empty()); + for (const FileMetaData* file : files) { + EXPECT_EQ(std::find(run.begin(), run.end(), file->fd.GetNumber()), + run.end()) + << file->fd.GetNumber(); + // The pick takes every sorted run older than the run, so X is gone too. + EXPECT_NE(file->fd.GetNumber(), file_x); + } + const Comparator* const ucmp = BytewiseComparator(); + for (size_t i = 0; i < files.size(); ++i) { + for (size_t j = i + 1; j < files.size(); ++j) { + if (files[i]->epoch_number != files[j]->epoch_number) { + continue; + } + EXPECT_TRUE(ucmp->Compare(files[i]->largest.user_key(), + files[j]->smallest.user_key()) < 0 || + ucmp->Compare(files[j]->largest.user_key(), + files[i]->smallest.user_key()) < 0) + << files[i]->fd.GetNumber() << " " << files[j]->fd.GetNumber(); + } + } + for (int key = 0; key < 24; ++key) { + if (key >= 12 && key < 20) { + continue; + } + const bool deleted = key == 9 || key == 10; + ASSERT_EQ(Get(Key(key)), + deleted ? "NOT_FOUND" : "value" + std::to_string(key)); + } + ExpectCensusMatchesVersion(); +} + +TEST_F(DBBlobBasicTest, UniversalOneLevelFullCompactionWritesExactOutput) { + // Background: a flush writes a coarse SST. A coarse SST records only its + // oldest indirect-origin blob file number (its frontier). The Version pins + // every indirect-origin blob file at or above the oldest frontier, so + // indirect blob GC cannot collect those files. With universal compaction and + // num_levels = 1, no compaction writes to L1 or deeper. So a bottommost + // compaction must write exact output, or the data stays coarse for ever. + // + // The test flushes three coarse SSTs with three blob files and runs a full + // compaction. It checks that every output SST is exact, that the Version has + // no coarse frontier, and that no blob file is pinned. + DestroyAndReopen(UniversalOneLevelIndirectionOptions(GetDefaultOptions())); + constexpr int kFlushes = 3; + constexpr int kKeysPerFlush = 4; + for (int flush = 0; flush < kFlushes; ++flush) { + for (int i = 0; i < kKeysPerFlush; ++i) { + const int key = flush * kKeysPerFlush + i; + ASSERT_OK(Put(Key(key), "value" + std::to_string(key))); + } + ASSERT_OK(Flush()); + } + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto storage = [cfd]() { return cfd->current()->storage_info(); }; + ASSERT_EQ(storage()->LevelFiles(0).size(), static_cast(kFlushes)); + ASSERT_EQ(CountExactSsts(), 0U); + const std::vector blob_files = GetBlobFileNumbers(); + ASSERT_EQ(blob_files.size(), static_cast(kFlushes)); + // GetBlobFileNumbers() is sorted, so the oldest blob file comes first. + ASSERT_EQ(storage()->OldestCoarseIndirectOriginFileNumber(), + blob_files.front()); + for (uint64_t blob_file : blob_files) { + ASSERT_TRUE(storage()->IsCoarseIndirectOriginBlobPinned(blob_file)) + << blob_file; + } + + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + const std::vector& files = storage()->LevelFiles(0); + ASSERT_FALSE(files.empty()); + for (const FileMetaData* file : files) { + EXPECT_FALSE(file->blob_reference_descriptor.empty()); + EXPECT_EQ(file->oldest_coarse_indirect_origin_file_number, + kInvalidBlobFileNumber); + } + ASSERT_EQ(CountExactSsts(), files.size()); + ASSERT_EQ(storage()->OldestCoarseIndirectOriginFileNumber(), + kInvalidBlobFileNumber); + // Every value is still live, so the compaction keeps all three blob files. + ASSERT_EQ(GetBlobFileNumbers(), blob_files); + for (uint64_t blob_file : blob_files) { + ASSERT_FALSE(storage()->IsCoarseIndirectOriginBlobPinned(blob_file)) + << blob_file; + } + for (int key = 0; key < kFlushes * kKeysPerFlush; ++key) { + ASSERT_EQ(Get(Key(key)), "value" + std::to_string(key)); + } + ExpectCensusMatchesVersion(); +} + +TEST_F(DBBlobBasicTest, UniversalOneLevelPartialCompactionKeepsNewerL0Coarse) { + // Universal compaction with one level: an exact epoch run (several exact L0 + // files with one epoch number, written by one compaction) sits at the bottom + // of L0, and newer coarse flushes sit above it. The test checks three + // things: + // 1. A compaction of only the newer coarse files is not bottommost (their + // keys also exist in the older run) and has no exact input, so its + // output stays coarse. The exact run is not touched. + // 2. A compaction that names one file of the exact run takes all files of + // the run (VersionStorageInfo::ExpandExactL0Inputs()), so no run is + // split. + // 3. A later full compaction makes every SST exact again and removes the + // coarse frontier. + DestroyAndReopen(UniversalOneLevelIndirectionOptions(GetDefaultOptions())); + constexpr int kKeys = 12; + for (int key = 0; key < kKeys; ++key) { + ASSERT_OK(Put(Key(key), "old" + std::to_string(key))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto storage = [cfd]() { return cfd->current()->storage_info(); }; + const auto l0_numbers = [&]() { + std::vector numbers; + for (const FileMetaData* file : storage()->LevelFiles(0)) { + numbers.push_back(file->fd.GetNumber()); + } + return numbers; + }; + const auto table_names = [](const std::vector& numbers) { + std::vector names; + for (uint64_t number : numbers) { + names.push_back(MakeTableFileName(number)); + } + return names; + }; + CompactRangeOptions full; + full.bottommost_level_compaction = BottommostLevelCompaction::kForce; + + // The full compaction runs with a reference cap of 4. The 12 keys have one + // reference each, so the cap cuts the output into three exact L0 files with + // one epoch number. + ASSERT_OK(FullCompactionWithReferenceCap(db_.get(), /*cap=*/4)); + const std::vector run = l0_numbers(); + ASSERT_EQ(run.size(), 3U); + for (const FileMetaData* file : storage()->LevelFiles(0)) { + ASSERT_FALSE(file->blob_reference_descriptor.empty()); + ASSERT_EQ(file->epoch_number, + storage()->LevelFiles(0).front()->epoch_number); + } + ASSERT_EQ(storage()->OldestCoarseIndirectOriginFileNumber(), + kInvalidBlobFileNumber); + const uint64_t run_blob_file = GetBlobFileNumbers().back(); + + // Two coarse flushes overwrite keys at both ends and in the middle of the + // run's key range. + for (const int key : {0, 5, 11}) { + ASSERT_OK(Put(Key(key), "mid" + std::to_string(key))); + } + ASSERT_OK(Flush()); + for (const int key : {0, 6, 11}) { + ASSERT_OK(Put(Key(key), "new" + std::to_string(key))); + } + ASSERT_OK(Flush()); + std::vector l0 = l0_numbers(); + ASSERT_EQ(l0.size(), 5U); + // L0 is newest first: the two flushes, then the run. + const std::vector newer(l0.begin(), l0.begin() + 2); + ASSERT_EQ(std::vector(l0.begin() + 2, l0.end()), run); + ASSERT_EQ(CountExactSsts(), run.size()); + const uint64_t frontier = storage()->OldestCoarseIndirectOriginFileNumber(); + ASSERT_GT(frontier, run_blob_file); + ASSERT_FALSE(storage()->IsCoarseIndirectOriginBlobPinned(run_blob_file)); + + // Step 1: compact only the two newer coarse files. + ASSERT_OK(db_->CompactFiles(CompactionOptions(), table_names(newer), + /*output_level=*/0)); + l0 = l0_numbers(); + ASSERT_EQ(l0.size(), 4U); + ASSERT_EQ(std::vector(l0.begin() + 1, l0.end()), run); + const FileMetaData* const coarse = storage()->LevelFiles(0).front(); + ASSERT_TRUE(coarse->blob_reference_descriptor.empty()); + ASSERT_EQ(coarse->oldest_coarse_indirect_origin_file_number, frontier); + ASSERT_EQ(CountExactSsts(), run.size()); + ASSERT_EQ(storage()->OldestCoarseIndirectOriginFileNumber(), frontier); + ASSERT_TRUE(storage()->IsCoarseIndirectOriginBlobPinned(frontier)); + ASSERT_FALSE(storage()->IsCoarseIndirectOriginBlobPinned(run_blob_file)); + const uint64_t coarse_number = coarse->fd.GetNumber(); + + // Step 2: name only the middle file of the run. The compaction takes all + // three run files and none of the newer coarse file. Its output is + // bottommost (nothing older is left in L0), so it is exact. + ASSERT_OK(db_->CompactFiles(CompactionOptions(), table_names({run[1]}), + /*output_level=*/0)); + l0 = l0_numbers(); + ASSERT_EQ(l0.front(), coarse_number); + for (uint64_t number : run) { + EXPECT_EQ(std::find(l0.begin(), l0.end(), number), l0.end()) << number; + } + ASSERT_EQ(CountExactSsts(), l0.size() - 1); + ASSERT_EQ(storage()->OldestCoarseIndirectOriginFileNumber(), frontier); + + // Step 3: a full compaction makes every SST exact again. + ASSERT_OK(db_->CompactRange(full, nullptr, nullptr)); + ASSERT_FALSE(storage()->LevelFiles(0).empty()); + ASSERT_EQ(CountExactSsts(), storage()->LevelFiles(0).size()); + ASSERT_EQ(storage()->OldestCoarseIndirectOriginFileNumber(), + kInvalidBlobFileNumber); + for (uint64_t blob_file : GetBlobFileNumbers()) { + ASSERT_FALSE(storage()->IsCoarseIndirectOriginBlobPinned(blob_file)) + << blob_file; + } + for (int key = 0; key < kKeys; ++key) { + std::string expected = "old" + std::to_string(key); + if (key == 0 || key == 6 || key == 11) { + expected = "new" + std::to_string(key); + } else if (key == 5) { + expected = "mid" + std::to_string(key); + } + ASSERT_EQ(Get(Key(key)), expected); + } + ExpectCensusMatchesVersion(); +} + +TEST_F(DBBlobBasicTest, UniversalOneLevelFullCompactionLetsIndirectBlobGcRun) { + // Universal compaction with one level: flushes are coarse, and a coarse SST + // pins its blob file, so indirect blob GC skips it. The test makes one blob + // file 3/4 garbage and checks that GC skips it while only coarse SSTs exist. + // A full compaction writes exact output (it is bottommost), which removes the + // pin. GC then collects the blob file and the live value is still readable. + Options options = + UniversalOneLevelIndirectionOptions(IndirectBlobGCOptions()); + // Indirect blob GC relocates live values only when blob GC is on. Auto + // compaction stays off, so GC runs only when the test calls it. + options.enable_blob_garbage_collection = true; + DestroyAndReopen(options); + for (size_t i = 0; i < kIndirectBlobGCKeyCount; ++i) { + ASSERT_OK(Put(IndirectBlobGCKey(i), IndirectBlobGCValue(i))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const std::vector blob_files = GetBlobFileNumbers(); + ASSERT_EQ(blob_files.size(), 1U); + const uint64_t blob_file = blob_files.front(); + // Delete all keys but the last one, so 3 of the 4 values are garbage. + for (size_t i = 0; i + 1 < kIndirectBlobGCKeyCount; ++i) { + ASSERT_OK(Delete(IndirectBlobGCKey(i))); + } + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_TRUE(IsCoarsePinned(blob_file)); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {blob_file})); + ASSERT_TRUE(HasBlobFile(blob_file)); + ASSERT_EQ(RouteTarget(blob_file), kInvalidBlobFileNumber); + + // The full compaction runs with compaction-time blob GC off, so the values + // stay in their blob file and only indirect blob GC can move them. + CompactRangeOptions full; + full.bottommost_level_compaction = BottommostLevelCompaction::kForce; + full.blob_garbage_collection_policy = BlobGarbageCollectionPolicy::kDisable; + ASSERT_OK(db_->CompactRange(full, nullptr, nullptr)); + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->LevelFiles(0).size(), 1U); + ASSERT_FALSE( + storage->LevelFiles(0).front()->blob_reference_descriptor.empty()); + ASSERT_EQ(GetBlobFileNumbers(), blob_files); + ASSERT_FALSE(IsCoarsePinned(blob_file)); + + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {blob_file})); + ASSERT_FALSE(HasBlobFile(blob_file)); + ASSERT_NE(RouteTarget(blob_file), kInvalidBlobFileNumber); + const size_t live = kIndirectBlobGCKeyCount - 1; + ASSERT_EQ(Get(IndirectBlobGCKey(live)), IndirectBlobGCValue(live)); + for (size_t i = 0; i < live; ++i) { + ASSERT_EQ(Get(IndirectBlobGCKey(i)), "NOT_FOUND"); + } + ExpectCensusMatchesVersion(); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusSkipsTrivialMove) { + // A trivial move deletes and re-adds the same SST in one edit, so the census + // has no change to apply and does not rebuild. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(CompactEachL0FileTo(/*level=*/1)); + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(CompactEachL0FileTo(/*level=*/1)); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + std::atomic changed_files{0}; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CensusRemovalWithoutGroups", + [&](void*) { ++changed_files; }); + CensusEventCounter counter; + counter.Start(); + MoveFilesToLevel(/*level=*/2); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_EQ(counter.rebuilds.load(), 0); + EXPECT_EQ(changed_files.load(), 0); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusRebuildsForSstWithoutSummary) { + // If a new exact SST has no summary at install (a bug, injected here), the + // commit still succeeds. The census is rebuilt from the SSTs on disk and is + // not ready until the rebuild ends. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + ASSERT_EQ(census->missing_summary_rebuild_count(), 0U); + + CensusEventCounter counter; + // Make the flush exact with the test hook (see + // FlushInjectedBlobIndexesWithExactMetadata()). + SyncPoint::GetInstance()->SetCallBack( + "FlushJob::WriteLevel0Table:TableBuilderOptions", [](void* arg) { + static_cast(arg) + ->collect_blob_reference_metadata = true; + }); + SyncPoint::GetInstance()->SetCallBack( + "BuildTable:BlobReferenceSummary", [](void* arg) { + static_cast(arg)->blob_reference_summary.reset(); + }); + // Debug builds assert on the missing summary. This callback turns the assert + // off, so the test checks the release behavior. + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CensusAdditionWithoutSummary", + [](void* arg) { *static_cast(arg) = true; }); + SyncPoint::GetInstance()->LoadDependency( + {{"DBBlobBasicTest::CensusNotReadyChecked", + "DBImpl::BGWorkBlobReferenceCensus:BeforeBuildWait"}}); + counter.Start(); + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + EXPECT_EQ(census->missing_summary_rebuild_count(), 1U); + EXPECT_EQ(counter.rebuilds.load(), 1); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_FALSE(census->IsReady(cfd->current()->GetVersionNumber())); + } + TEST_SYNC_POINT("DBBlobBasicTest::CensusNotReadyChecked"); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_TRUE(census->IsReady(cfd->current()->GetVersionNumber())); + } + ASSERT_EQ(Get("a"), "blob_a"); + ASSERT_EQ(Get("b"), "blob_b"); +} + +TEST_F(DBBlobBasicTest, FlushThreadAppliesCensusWithoutCensusJob) { + // Every LOW-pool thread is busy, so the census job stays in its run queue + // and does not run. After its install, the flush thread applies the flush's + // census delta and publishes the census for the new Version (see "Apply + // after an install" in db/blob/blob_reference_census.h). + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + // The first flush thread can still be in its own apply step after Flush() + // returns. Wait for it, so the sync point below comes from the second flush. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + + const int low_threads = env_->GetBackgroundThreads(Env::Priority::LOW); + std::vector sleeping_tasks(low_threads); + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &task, + Env::Priority::LOW); + task.WaitUntilSleeping(); + } + auto wake = Defer([&sleeping_tasks] { + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + }); + // The sync point is reached at the end of every apply step. + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::ApplyBlobReferenceCensusAfterInstall:Done", + "DBBlobBasicTest::BeforeInlineApplyCheck"}}); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT("DBBlobBasicTest::BeforeInlineApplyCheck"); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_TRUE(census->IsReady(cfd->current()->GetVersionNumber())); + } + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + sleeping_tasks.clear(); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_EQ(Get("b"), "blob_b"); +} + +TEST_F(DBBlobBasicTest, FlushReturnsAfterSuperVersionWithCensusCleanup) { + // Background: a Flush(wait=true) caller waits on bg_cv_ until the flushed + // memtables are gone from the immutable list. The flush job passes a + // callback to its MANIFEST write (the writer callback). The writer callback + // removes the flushed memtables. After the MANIFEST write returns, the + // flush job installs its new SuperVersion while it still holds the DB + // mutex. So a Flush() caller that sees the memtables removed also sees the + // new SuperVersion, as long as the DB mutex is not released between the + // writer callback and the SuperVersion install. + // + // A flush of a CF with blob indirection also installs census state. After + // VersionSet::ProcessManifestWrites() installs the new Versions, it frees + // the old census state while the DB mutex is released (the census cleanup, + // sync point "VersionSet::ProcessManifestWrites:CleanupUnlocked"). + // + // Ordering rule: the writer callback runs only after the census cleanup + // takes the DB mutex again. Another thread can signal bg_cv_ while the + // census cleanup runs. If the writer callback ran before the census + // cleanup, that signal would let Flush() return while the SuperVersion + // still has the old Version without the new L0 file. + // + // Test steps (the checked flush is the second flush, of key "b"): + // 1. In the census cleanup of the checked flush, the test reads the number + // of immutable memtables of the CF and then wakes the Flush() caller. + // 2. If that number is 0, the rule is broken. Then the flush thread waits + // until Flush() has returned and the test thread has read the + // SuperVersion. So Flush() returns while the SuperVersion has only 1 L0 + // file, and both checks at the end fail. + // 3. After Flush() returns, the test checks that the memtable was still in + // the list during the census cleanup and that the SuperVersion has the + // new L0 file. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + // Flush once first and wait for all background work, so that only the + // checked flush below writes to the MANIFEST while the callbacks are set. + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + + // The flush thread sets this right before its MANIFEST write, so the next + // cleanup is the one of the checked flush. + std::atomic flush_commit_started{false}; + std::atomic cleanup_count{0}; + // The number of immutable memtables of the CF during the cleanup of the + // checked flush, or -1 if that cleanup did not run. + std::atomic memtables_during_cleanup{-1}; + std::promise flush_returned; + std::shared_future flush_returned_future = + flush_returned.get_future().share(); + + SyncPoint::GetInstance()->SetCallBack( + "MemTableList::TryInstallMemtableFlushResults:AfterComputeMinWalToKeep", + [&](void*) { flush_commit_started.store(true); }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CleanupUnlocked", [&](void*) { + cleanup_count.fetch_add(1); + if (!flush_commit_started.exchange(false)) { + return; + } + int memtables = 0; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + memtables = cfd->imm()->NumNotFlushed(); + } + memtables_during_cleanup.store(memtables); + // Signal bg_cv_ in the way any other thread can, to wake the waiting + // Flush() caller. + dbfull()->TEST_SignalAllBgCv(); + if (memtables == 0) { + flush_returned_future.wait(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + // Releases a waiting flush thread on every exit path. It is declared after + // `cleanup`, so it runs first. This order matters because `cleanup` waits + // for running sync point callbacks, and a callback that waits on + // `flush_returned_future` would block `cleanup` forever. + bool flush_returned_set = false; + auto set_flush_returned = [&]() { + if (!flush_returned_set) { + flush_returned_set = true; + flush_returned.set_value(); + } + }; + auto release_flush_thread = Defer(set_flush_returned); + + ASSERT_OK(Put("b", "blob_b")); + const Status flush_status = Flush(); + int super_version_l0_files = 0; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + super_version_l0_files = + cfd->GetSuperVersion()->current->storage_info()->NumLevelFiles(0); + } + set_flush_returned(); + ASSERT_OK(flush_status); + ASSERT_GT(cleanup_count.load(), 0); + EXPECT_EQ(memtables_during_cleanup.load(), 1); + EXPECT_EQ(super_version_l0_files, 2); + ASSERT_EQ(Get("b"), "blob_b"); +} + +TEST_F(DBBlobBasicTest, CensusInlineFailureStopsLaterInlineApplies) { + // Every LOW-pool thread is busy, so only flush threads apply census deltas. + // 1. The first flush's apply fails (the test injects Corruption after the + // delta is applied). The thread stores the failure in + // BlobReferenceCensusBuildContext::failure. + // 2. The second flush's apply sees the stored failure and applies nothing, + // so the failed delta is not applied again and the census stays not + // ready. + // 3. When the LOW threads are free, the census job fails the build with the + // stored failure and builds the census again. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + + const int low_threads = env_->GetBackgroundThreads(Env::Priority::LOW); + std::vector sleeping_tasks(low_threads); + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &task, + Env::Priority::LOW); + task.WaitUntilSleeping(); + } + auto wake = Defer([&sleeping_tasks] { + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + }); + std::mutex mutex; + std::condition_variable cv; + int applies_done = 0; + std::atomic apply_calls{0}; + std::atomic new_builds{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::ApplyBlobReferenceCensusDeltas:ApplyStatus", + [&](void* argument) { + if (apply_calls.fetch_add(1) == 0) { + *static_cast(argument) = Status::Corruption("injected"); + } + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::ApplyBlobReferenceCensusAfterInstall:Done", [&](void*) { + std::lock_guard lock(mutex); + ++applies_done; + cv.notify_all(); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:NewBuildReady", + [&](void*) { new_builds.fetch_add(1); }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkBlobReferenceCensus:FailureRetryDelay", + [](void* argument) { *static_cast(argument) = 0; }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + const auto wait_for_applies = [&](int count) { + std::unique_lock lock(mutex); + return cv.wait_for(lock, std::chrono::seconds(30), + [&] { return applies_done >= count; }); + }; + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + ASSERT_TRUE(wait_for_applies(1)); + ASSERT_OK(Put("c", "blob_c")); + ASSERT_OK(Flush()); + ASSERT_TRUE(wait_for_applies(2)); + EXPECT_EQ(apply_calls.load(), 1); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_FALSE(census->IsReady(cfd->current()->GetVersionNumber())); + } + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + sleeping_tasks.clear(); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_GE(new_builds.load(), 1); + ASSERT_EQ(Get("c"), "blob_c"); +} + +TEST_F(DBBlobBasicTest, CensusJobFailureStopsInlineApplyBeforeFailBuild) { + // A census job run that fails releases worker_run_mutex() before it calls + // FailBuild(). The failed delta is still queued in the build context. A + // flush that ends in between must apply nothing: + // 1. An indirect blob GC job pauses the worker, so the first flush's apply + // step gets TryAgain and leaves its delta queued. + // 2. The GC job ends. The census job applies the delta in its catch-up, + // and the test injects a failure there. The run stops after it releases + // the run lock and before FailBuild(). + // 3. A second flush ends in that window. Its apply step sees the stored + // failure and applies nothing. + // 4. The run goes on, fails the build, and the census is built again. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + uint64_t gc_token = 0; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + ASSERT_OK( + census->BeginGcJob(cfd->current()->GetVersionNumber(), &gc_token)); + } + std::mutex mutex; + std::condition_variable cv; + int applies_done = 0; + std::atomic inject{false}; + std::atomic injected{false}; + std::atomic inline_applies{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:CatchUpApplyStatus", + [&](void* argument) { + Status* const status = static_cast(argument); + if (status->ok() && inject.load() && !injected.exchange(true)) { + *status = Status::Corruption("injected"); + } + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::ApplyBlobReferenceCensusDeltas:ApplyStatus", + [&](void*) { inline_applies.fetch_add(1); }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::ApplyBlobReferenceCensusAfterInstall:Done", [&](void*) { + std::lock_guard lock(mutex); + ++applies_done; + cv.notify_all(); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkBlobReferenceCensus:FailureRetryDelay", + [](void* argument) { *static_cast(argument) = 0; }); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkBlobReferenceCensus:FailedRunReady", + "DBBlobBasicTest::JobRunFailed"}, + {"DBBlobBasicTest::SecondFlushApplied", + "DBImpl::BGWorkBlobReferenceCensus:FailedRunWait"}}); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + // If an assertion below returns early, still let the blocked census run go + // on, so that the DB can close. + bool released = false; + auto release_run = Defer([&released] { + if (!released) { + TEST_SYNC_POINT("DBBlobBasicTest::SecondFlushApplied"); + } + }); + const auto wait_for_applies = [&](int count) { + std::unique_lock lock(mutex); + return cv.wait_for(lock, std::chrono::seconds(30), + [&] { return applies_done >= count; }); + }; + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + ASSERT_TRUE(wait_for_applies(1)); + inject.store(true); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + census->EndGcJob(gc_token); + } + // The periodic task runs the census job again (the GC pause put it in the + // retry list). + TEST_SYNC_POINT("DBBlobBasicTest::JobRunFailed"); + EXPECT_TRUE(injected.load()); + const int inline_applies_before = inline_applies.load(); + ASSERT_OK(Put("c", "blob_c")); + ASSERT_OK(Flush()); + ASSERT_TRUE(wait_for_applies(2)); + EXPECT_EQ(inline_applies.load(), inline_applies_before); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_FALSE(census->IsReady(cfd->current()->GetVersionNumber())); + } + released = true; + TEST_SYNC_POINT("DBBlobBasicTest::SecondFlushApplied"); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_EQ(Get("c"), "blob_c"); +} + +TEST_F(DBBlobBasicTest, CensusApplyAfterInstallVisitsOnlyJobColumnFamilies) { + // Every LOW-pool thread is busy, so the census job does not run. An indirect + // blob GC job on column family "one" keeps its census paused, so a flush of + // "one" leaves it queued for the census job. A later flush of the default + // column family then visits only the default column family in its apply step, + // not "one". + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"one"}, options); + ASSERT_OK(Put(0, "a", "blob_a")); + ASSERT_OK(Flush(0)); + ASSERT_OK(Put(1, "a", "blob_a")); + ASSERT_OK(Flush(1)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const one = + static_cast_with_check(handles_[1])->cfd(); + BlobReferenceCensus* const one_census = one->blob_reference_census(); + uint64_t gc_token = 0; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + ASSERT_OK( + one_census->BeginGcJob(one->current()->GetVersionNumber(), &gc_token)); + } + + const int low_threads = env_->GetBackgroundThreads(Env::Priority::LOW); + std::vector sleeping_tasks(low_threads); + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &task, + Env::Priority::LOW); + task.WaitUntilSleeping(); + } + auto wake = Defer([&sleeping_tasks] { + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + }); + std::mutex mutex; + std::condition_variable cv; + int applies_done = 0; + std::vector visited; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::ApplyBlobReferenceCensusDeltas:Start", [&](void* argument) { + std::lock_guard lock(mutex); + visited.push_back(static_cast(argument)->GetID()); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::ApplyBlobReferenceCensusAfterInstall:Done", [&](void*) { + std::lock_guard lock(mutex); + ++applies_done; + cv.notify_all(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + const auto wait_for_applies = [&](int count) { + std::unique_lock lock(mutex); + return cv.wait_for(lock, std::chrono::seconds(30), + [&] { return applies_done >= count; }); + }; + ASSERT_OK(Put(1, "b", "blob_b")); + ASSERT_OK(Flush(1)); + ASSERT_TRUE(wait_for_applies(1)); + { + std::lock_guard lock(mutex); + EXPECT_EQ(visited, std::vector{one->GetID()}); + visited.clear(); + } + ASSERT_OK(Put(0, "b", "blob_b")); + ASSERT_OK(Flush(0)); + ASSERT_TRUE(wait_for_applies(2)); + { + std::lock_guard lock(mutex); + EXPECT_EQ(visited, std::vector{0}); + } + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_FALSE(one_census->IsReady(one->current()->GetVersionNumber())); + one_census->EndGcJob(gc_token); + } + for (test::SleepingBackgroundTask& task : sleeping_tasks) { + task.WakeUp(); + task.WaitUntilDone(); + } + sleeping_tasks.clear(); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(Get(1, "b"), "blob_b"); +} + +TEST_F(DBBlobBasicTest, CensusApplyAfterInstallWaitsForGcSnapshot) { + // A registered indirect blob GC job pauses the worker until it copies its + // snapshot. The flush thread's apply step then gets TryAgain and leaves the + // delta in the build context for the census job, so the census is not ready. + // After the GC job ends, the census job applies the delta and publishes. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + uint64_t gc_token = 0; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + ASSERT_OK( + census->BeginGcJob(cfd->current()->GetVersionNumber(), &gc_token)); + } + ASSERT_OK(Put("b", "blob_b")); + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_FALSE(census->IsReady(cfd->current()->GetVersionNumber())); + const std::shared_ptr context = + census->GetBuildContext(); + ASSERT_NE(context, nullptr); + EXPECT_EQ(context->pending_deltas.size(), 1U); + census->EndGcJob(gc_token); + } + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_EQ(Get("b"), "blob_b"); +} + +TEST_F(DBBlobBasicTest, CensusRestartBuildOverflowBacksOffThenFinishes) { + // The change log limit is lowered to 1 byte (restart limit 4 bytes), so every + // queued delta after the first overflows it. + // 1. Three flushes while the test holds the worker run lock: the census + // cursor overflows, and the census job restarts the build with the + // restart limit. + // 2. The restarted build stops right after its build cursor opened. Three + // more flushes overflow its restart limit. The build fails, the job + // bumps BLOB_DB_CENSUS_RESTART_OVERFLOW_COUNT, and it retries after a + // backoff (the test sets the backoff to 0). + // 3. The test stops lowering the limit. The retried build, which still has + // the restart limit, publishes. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.statistics = CreateDBStatistics(); + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + + // The callbacks run on background threads and share this state. It lives + // on the heap, so it stays valid until the last callback returns. + struct State { + std::atomic tiny_limit{false}; + std::atomic backoffs{0}; + }; + auto state = std::make_shared(); + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("BlobReferenceCensus::AppendChange:Limit", + [state](void* arg) { + if (state->tiny_limit.load()) { + *static_cast(arg) = 1; + } + }); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:FailureRetryDelay", + [state](void* arg) { + state->backoffs.fetch_add(1); + *static_cast(arg) = 0; + }); + // The first new build after EnableProcessing() is the restarted build: it + // waits after its build cursor opened until the test has made its flushes. + sync_point->LoadDependency( + {{"DBImpl::BuildBlobReferenceCensus:NewBuildReady", + "DBBlobBasicTest::RestartBuildStopped"}, + {"DBBlobBasicTest::RestartFlushesDone", + "DBImpl::BuildBlobReferenceCensus:NewBuildWait"}}); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + + // Step 1. + state->tiny_limit.store(true); + { + std::lock_guard run_lock(census->worker_run_mutex()); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("k" + std::to_string(i), "blob_k")); + ASSERT_OK(Flush()); + } + } + // Step 2. + TEST_SYNC_POINT("DBBlobBasicTest::RestartBuildStopped"); + ASSERT_EQ(census->overflow_restart_count(), 1U); + for (int i = 3; i < 6; ++i) { + ASSERT_OK(Put("k" + std::to_string(i), "blob_k")); + ASSERT_OK(Flush()); + } + state->tiny_limit.store(false); + TEST_SYNC_POINT("DBBlobBasicTest::RestartFlushesDone"); + // Step 3. + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_EQ( + options.statistics->getTickerCount(BLOB_DB_CENSUS_RESTART_OVERFLOW_COUNT), + 1U); + EXPECT_GE(state->backoffs.load(), 1); + EXPECT_EQ(census->overflow_restart_count(), 1U); + ASSERT_EQ(Get("k5"), "blob_k"); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusRebuildsAfterFileDeletion) { + // DeleteFilesInRange removes SSTs without passing their summaries, so the + // census is rebuilt and then matches the Version. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(CompactEachL0FileTo(/*level=*/1)); + ASSERT_OK(Put("m", "blob_m")); + ASSERT_OK(Flush()); + ASSERT_NO_FATAL_FAILURE(CompactEachL0FileTo(/*level=*/1)); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + CensusEventCounter counter; + counter.Start(); + const Slice begin("a"); + const Slice end("b"); + ASSERT_OK( + DeleteFilesInRange(db_.get(), db_->DefaultColumnFamily(), &begin, &end)); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_EQ(counter.rebuilds.load(), 1); + ASSERT_EQ(Get("a"), "NOT_FOUND"); + ASSERT_EQ(Get("m"), "blob_m"); +} + +TEST_F(DBBlobBasicTest, CensusScanKeepsReadSstWhileGcJobPauses) { + // An indirect blob GC job registers while the census is ready. Then + // DeleteFilesInRange starts a rebuild before the job copies its snapshot, so + // the job still pauses the worker during the rebuild's scan of the SSTs: + // 1. The scan reads the first SST. A flush then queues a delta. + // 2. The catch-up after the read gets TryAgain. The scan keeps the summary + // it read as a seed and stops; the build is incomplete. + // 3. After the GC job ends, the census job runs again. The scan adds the + // SST from the seed without reading it again. + // So every SST is read once, and the census then matches the Version. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.target_file_size_base = 16 << 10; + DestroyAndReopen(options); + for (int i = 0; i < 8000; ++i) { + ASSERT_OK(Put(Key(i), std::string(100, 'v'))); + } + ASSERT_OK(Flush()); + // Compact into several exact SSTs at L1. + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_GE(NumTableFilesAtLevel(1), 3); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + std::string first_smallest; + std::string first_largest; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + const FileMetaData* const first = + cfd->current()->storage_info()->LevelFiles(1).front(); + first_smallest = first->smallest.user_key().ToString(); + first_largest = first->largest.user_key().ToString(); + } + uint64_t gc_token = 0; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + ASSERT_OK( + census->BeginGcJob(cfd->current()->GetVersionNumber(), &gc_token)); + } + std::atomic sst_reads{0}; + std::atomic sst_adds{0}; + std::mutex mutex; + std::condition_variable cv; + int job_runs_done = 0; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BGWorkBlobReferenceCensus:Done", [&](void*) { + std::lock_guard lock(mutex); + ++job_runs_done; + cv.notify_all(); + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:ReadSst", + [&](void*) { sst_reads.fetch_add(1); }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:AfterReadFile", + [&](void*) { sst_adds.fetch_add(1); }); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BuildBlobReferenceCensus:ReadSst", + "DBBlobBasicTest::FirstSstRead"}, + {"DBBlobBasicTest::FlushQueued", + "DBImpl::BuildBlobReferenceCensus:AfterReadFile"}}); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + // If an assertion below returns early, still let the blocked census run go + // on, so that the DB can close. + bool flush_queued = false; + auto release_run = Defer([&flush_queued] { + if (!flush_queued) { + TEST_SYNC_POINT("DBBlobBasicTest::FlushQueued"); + } + }); + const Slice begin(first_smallest); + const Slice end(first_largest); + ASSERT_OK( + DeleteFilesInRange(db_.get(), db_->DefaultColumnFamily(), &begin, &end)); + const int starting_ssts = NumTableFilesAtLevel(1); + TEST_SYNC_POINT("DBBlobBasicTest::FirstSstRead"); + ASSERT_OK(Put("z", "blob_z")); + ASSERT_OK(Flush()); + int runs_before = 0; + { + std::lock_guard lock(mutex); + runs_before = job_runs_done; + } + flush_queued = true; + TEST_SYNC_POINT("DBBlobBasicTest::FlushQueued"); + // The blocked run gets TryAgain after the read and ends. Later runs only get + // TryAgain until the GC job ends, so the test ends it now. + bool run_ended = false; + { + std::unique_lock lock(mutex); + run_ended = cv.wait_for(lock, std::chrono::seconds(30), + [&] { return job_runs_done > runs_before; }); + } + { + InstrumentedMutexLock lock(dbfull()->mutex()); + EXPECT_FALSE(census->IsReady(cfd->current()->GetVersionNumber())); + census->EndGcJob(gc_token); + } + ASSERT_TRUE(run_ended); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_EQ(sst_reads.load(), starting_ssts); + // The first SST passes AfterReadFile twice: once after its read, and once + // when the scan adds it from the seed. + EXPECT_EQ(sst_adds.load(), starting_ssts + 1); + ASSERT_EQ(Get("z"), "blob_z"); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusOverflowRestartsBuildOnce) { + // The test sets the change log limit to 1 byte. While the initial build + // waits, flush installs overflow its change log. The build restarts once with + // no limit and then completes, instead of restarting after every overflow. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.env->SetBackgroundThreads(2, Env::Priority::LOW); + DestroyAndReopen(options); + ASSERT_OK(Put("a", "blob_a")); + // Write an exact SST, so the census build at reopen has a file to read and + // waits before reading it. + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + Close(); + + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("BlobReferenceCensus::AppendChange:Limit", + [](void* arg) { *static_cast(arg) = 1; }); + // Make the flushes exact with the test hook (see + // FlushInjectedBlobIndexesWithExactMetadata()), so that each flush adds + // census changes. + sync_point->SetCallBack("FlushJob::WriteLevel0Table:TableBuilderOptions", + [](void* arg) { + static_cast(arg) + ->collect_blob_reference_metadata = true; + }); + sync_point->LoadDependency( + {{"DBBlobBasicTest::FlushedDuringBuild", + "DBImpl::BuildBlobReferenceCensus:BeforeReadFile"}}); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + sync_point->ClearTrace(); + }); + Reopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("k" + std::to_string(i), "blob_value")); + ASSERT_OK(Flush()); + } + TEST_SYNC_POINT("DBBlobBasicTest::FlushedDuringBuild"); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + EXPECT_EQ(cfd->blob_reference_census()->overflow_restart_count(), 1U); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusSelfCheckRunsAfterPublish) { + // After a publish, debug builds can compare the census with a census built + // from the SSTs of the Version (DBImpl::CheckBlobReferenceCensus()). This + // test forces the check after every publish. The check must pass after + // flushes, a partial compaction and a full compaction. + std::atomic checks{0}; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::CheckBlobReferenceCensus:Force", [&](void* arg) { + *static_cast(arg) = true; + ++checks; + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("a", "blob_a" + std::to_string(i))); + ASSERT_OK(Put("k" + std::to_string(i), "blob_value")); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + } + MoveFilesToLevel(/*level=*/1); + ASSERT_OK(Put("z", "blob_z")); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_GT(checks.load(), 0); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusFullCompactionInstallsByDelta) { + // A full compaction (every SST is an input) reads the summaries of its inputs + // at job start, like any compaction. Its install applies a census delta: the + // census is not rebuilt and reads no SST. + std::shared_ptr< + TargetCacheChargeTrackingCache> + cache; + Options options = CensusChargeTrackingOptions(&cache); + DestroyAndReopen(options); + // Every SST covers the whole key range, so the compaction is not a series + // of trivial moves. A first compaction writes exact SSTs below L0, so the + // full compaction below has exact inputs even when flushes write coarse + // SSTs. + for (int i = 0; i < 6; ++i) { + ASSERT_OK(Put("key0", "blob_value" + std::to_string(i))); + ASSERT_OK(Put("key3", "blob_value" + std::to_string(i))); + ASSERT_OK(Put("key" + std::to_string(i), "blob_value" + std::to_string(i))); + ASSERT_OK(Flush()); + if (i == 3) { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + } + } + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + const size_t exact_inputs = CountExactSsts(); + ASSERT_GT(exact_inputs, 0U); + std::atomic input_reads{0}; + std::atomic removals_without_groups{0}; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CensusRemovalWithoutGroups", + [&](void*) { ++removals_without_groups; }); + SyncPoint::GetInstance()->SetCallBack( + "CompactionJob::ReadInputBlobReferenceSummaries:Status", [&](void* arg) { + ASSERT_OK(*static_cast(arg)); + ++input_reads; + }); + CensusEventCounter counter; + counter.Start(); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + // The job reads the summary of each exact input once. + EXPECT_EQ(static_cast(input_reads.load()), exact_inputs); + EXPECT_EQ(removals_without_groups.load(), 0); + EXPECT_EQ(counter.rebuilds.load(), 0); + EXPECT_EQ(counter.reads.load(), 0); + ASSERT_EQ(Get("key0"), "blob_value5"); + ASSERT_EQ(Get("key1"), "blob_value1"); + ASSERT_EQ(Get("key3"), "blob_value5"); + ASSERT_EQ(Get("key4"), "blob_value4"); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusFullCompactionWithConcurrentFlush) { + // A flush installs an SST while a full compaction runs. The compaction read + // the summaries of its inputs at job start, so its install applies a census + // delta, and the census is not rebuilt. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + // Make an exact SST in L1 and a coarse flushed SST in L0 that overlaps it. + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Put("key0", "blob_value" + std::to_string(i))); + ASSERT_OK(Put("key3", "blob_value" + std::to_string(i))); + ASSERT_OK(Flush()); + if (i == 1) { + ASSERT_OK(BuildExactSstAtLevel1()); + } + } + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + ASSERT_GT(CountExactSsts(), 0U); + std::atomic removals_without_groups{0}; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CensusRemovalWithoutGroups", + [&](void*) { ++removals_without_groups; }); + SyncPoint::GetInstance()->LoadDependency( + {{"CompactionJob::Run():Start", + "BlobReferenceCensusFullCompactionWithConcurrentFlush:Flush"}, + {"BlobReferenceCensusFullCompactionWithConcurrentFlush:Flushed", + "CompactionJob::Run():End"}}); + CensusEventCounter counter; + counter.Start(); + port::Thread compaction([this]() { + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + }); + TEST_SYNC_POINT("BlobReferenceCensusFullCompactionWithConcurrentFlush:Flush"); + ASSERT_OK(Put("key5", "blob_flushed")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT( + "BlobReferenceCensusFullCompactionWithConcurrentFlush:Flushed"); + compaction.join(); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_NO_FATAL_FAILURE(ExpectCensusMatchesVersion()); + EXPECT_EQ(counter.rebuilds.load(), 0); + EXPECT_EQ(removals_without_groups.load(), 0); + ASSERT_EQ(Get("key0"), "blob_value2"); + ASSERT_EQ(Get("key3"), "blob_value2"); + ASSERT_EQ(Get("key5"), "blob_flushed"); +} + +TEST_F(DBBlobBasicTest, ReadBlobReferencesFailureKeepsEarlierOutput) { + // Indirect blob GC reads every SST of one input into one output, one call per + // SST. When the read of a later SST fails on the first block of its group, + // the references that the earlier call appended stay in the output. + if (encrypted_env_) { + // test::CorruptFile() rewrites the whole SST. The encrypted env gives the + // new file a new random encryption prefix, but the open table reader keeps + // the old one. That reader then decrypts the whole file to garbage, so the + // read does not see just one flipped bit. + ROCKSDB_GTEST_BYPASS("Test corrupts an SST that has an open reader"); + return; + } + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + // Flush output is always coarse, so the test hook makes these flushes exact. + ASSERT_OK(Put("a", "value_a")); + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ASSERT_OK(Put("b", "value_b")); + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const std::vector& files = + cfd->current()->storage_info()->LevelFiles(0); + ASSERT_EQ(files.size(), 2U); + const FileMetaData& first = *files[0]; + const FileMetaData& second = *files[1]; + const BlobReferenceIndex first_index = ReadBlobReferenceIndexForTest(first); + const BlobReferenceIndex second_index = ReadBlobReferenceIndexForTest(second); + ASSERT_EQ(first_index.groups.size(), 1U); + ASSERT_EQ(second_index.groups.size(), 1U); + const BlobReferenceGroup first_group = first_index.groups[0].group; + const BlobReferenceGroup second_group = second_index.groups[0].group; + + std::vector output; + output.reserve(static_cast(first_group.reference_count + + second_group.reference_count)); + const auto read = [&](const FileMetaData& file, + const BlobReferenceGroup& group, Status* status) { + std::vector requests(1); + requests[0].group = &group; + requests[0].output = &output; + ASSERT_OK(cfd->table_cache()->ReadBlobReferences( + ReadOptions(), FileOptions(), cfd->internal_comparator(), file, + cfd->GetLatestMutableCFOptions(), &requests)); + *status = requests[0].status; + }; + Status status; + read(first, first_group, &status); + ASSERT_OK(status); + ASSERT_EQ(output.size(), first_group.reference_count); + + // A flipped bit in the first (and only) reference data block of the second + // SST makes its read fail before it appends anything. + ASSERT_OK( + test::CorruptFile(env_, MakeTableFileName(dbname_, second.fd.GetNumber()), + static_cast(second_index.blocks[0].offset() + 1), + /*bytes_to_corrupt=*/1, /*verify_checksum=*/false)); + read(second, second_group, &status); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_EQ(output.size(), first_group.reference_count); + ASSERT_EQ(output[0].generation_file_number, + first_group.generation_file_number); +} + +TEST_F(DBBlobBasicTest, ReadBlobReferencesBatchReadsEachBlockOnce) { + // Small blob files first, so several small groups share a reference data + // block, then one large blob file, so one group spans several blocks. One + // compaction puts every reference into one SST. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_file_size = 32 << 10; + options.disable_auto_compactions = true; + options.target_file_size_base = 1 << 30; + DestroyAndReopen(options); + for (int i = 0; i < 10000; ++i) { + ASSERT_OK(Put(Key(i), std::string(10 + i % 23, 'v'))); + } + ASSERT_OK(Flush()); + ASSERT_OK(dbfull()->SetOptions({{"blob_file_size", "8388608"}})); + for (int i = 10000; i < 40000; ++i) { + ASSERT_OK(Put(Key(i), std::string(10 + i % 23, 'v'))); + } + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const FileMetaData* const sst = OnlySstForTest(cfd); + ASSERT_NE(sst, nullptr); + const FileMetaData& file = *sst; + const BlobReferenceIndex index = ReadBlobReferenceIndexForTest(file); + std::vector groups; + for (const BlobReferenceIndexEntry& entry : index.groups) { + groups.push_back(entry.group); + } + bool shared_block = false; + size_t spanning_group = groups.size(); + for (size_t g = 0; g + 1 < index.groups.size(); ++g) { + shared_block |= + index.groups[g].start_block == index.groups[g + 1].start_block; + if (index.groups[g + 1].start_block >= index.groups[g].start_block + 3) { + spanning_group = g; + } + } + if (index.blocks.size() >= index.groups.back().start_block + 3) { + spanning_group = index.groups.size() - 1; + } + ASSERT_TRUE(shared_block); + ASSERT_LT(spanning_group, groups.size()); + + // Reads `selected` groups in one batch; each output is reserved first. + const auto read_batch = + [&](const std::vector& selected, + std::vector>* outputs, + std::vector* requests) { + outputs->assign(selected.size(), {}); + requests->assign(selected.size(), {}); + for (size_t i = 0; i < selected.size(); ++i) { + (*outputs)[i].reserve( + static_cast(selected[i]->reference_count)); + (*requests)[i].group = selected[i]; + (*requests)[i].output = &(*outputs)[i]; + } + return cfd->table_cache()->ReadBlobReferences( + ReadOptions(), FileOptions(), cfd->internal_comparator(), file, + cfd->GetLatestMutableCFOptions(), requests); + }; + std::vector all; + for (const BlobReferenceGroup& group : groups) { + all.push_back(&group); + } + + // Count the MultiRead calls, their read requests and their bytes. + size_t multi_reads = 0; + size_t read_requests = 0; + uint64_t bytes_read = 0; + uint64_t largest_call = 0; + // A batch budget of the largest on-disk block, so the reads split into + // several batches. + uint64_t test_batch_bytes = 0; + for (const BlockHandle& handle : index.blocks) { + test_batch_bytes = std::max( + test_batch_bytes, BlockBasedTable::BlockSizeWithTrailer(handle)); + } + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::ReadBlobReferenceDataBlocks:BatchBytes", + [&](void* arg) { + *static_cast(arg) = static_cast(test_batch_bytes); + }); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::ReadBlobReferenceDataBlocks:MultiRead", [&](void* arg) { + const auto* requests = static_cast*>(arg); + uint64_t call_bytes = 0; + for (const FSReadRequest& request : *requests) { + call_bytes += request.len; + } + ++multi_reads; + read_requests += requests->size(); + bytes_read += call_bytes; + largest_call = std::max(largest_call, call_bytes); + }); + SyncPoint::GetInstance()->EnableProcessing(); + std::vector> batch_outputs; + std::vector requests; + ASSERT_OK(read_batch(all, &batch_outputs, &requests)); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + for (const BlobReferenceGroupRead& request : requests) { + ASSERT_OK(request.status); + } + // Every block is needed and read exactly once. The blocks are adjacent in the + // file, so each MultiRead call is one coalesced request, and the calls split + // at the batch budget. + ASSERT_GT(index.blocks.size(), 2U); + uint64_t all_block_bytes = 0; + for (const BlockHandle& handle : index.blocks) { + all_block_bytes += BlockBasedTable::BlockSizeWithTrailer(handle); + } + EXPECT_EQ(bytes_read, all_block_bytes); + EXPECT_EQ(read_requests, multi_reads); + EXPECT_LE(largest_call, test_batch_bytes); + EXPECT_GE(multi_reads, + (all_block_bytes + test_batch_bytes - 1) / test_batch_bytes); + EXPECT_GE(multi_reads, 2U); + + // The batch gives the same references as reading each group alone. + uint64_t total = 0; + for (size_t g = 0; g < groups.size(); ++g) { + std::vector> single; + std::vector single_request; + ASSERT_OK(read_batch({&groups[g]}, &single, &single_request)); + ASSERT_OK(single_request[0].status); + ASSERT_EQ(single[0].size(), groups[g].reference_count); + // The read reports the index totals of the group. + EXPECT_EQ(single_request[0].found_reference_count, + groups[g].reference_count); + EXPECT_EQ(single_request[0].found_stored_value_bytes, + groups[g].stored_value_bytes); + EXPECT_EQ(single_request[0].found_record_bytes, groups[g].record_bytes); + ASSERT_EQ(batch_outputs[g].size(), single[0].size()); + for (size_t i = 0; i < single[0].size(); ++i) { + EXPECT_EQ(batch_outputs[g][i].original_value_offset, + single[0][i].original_value_offset); + EXPECT_EQ(batch_outputs[g][i].stable_checksum, + single[0][i].stable_checksum); + } + total += single[0].size(); + } + EXPECT_EQ(total, 40000U); + + // Unsorted or repeated requests are rejected. + ASSERT_GE(all.size(), 2U); + EXPECT_TRUE(read_batch({all[1], all[0]}, &batch_outputs, &requests) + .IsInvalidArgument()); + EXPECT_TRUE(read_batch({all[0], all[0]}, &batch_outputs, &requests) + .IsInvalidArgument()); + + // A group the SST does not have (NotFound), or one larger than its output + // (Aborted), fails only its own request and leaves its output empty. + BlobReferenceGroup missing = groups.back(); + missing.generation_file_number += 1000; + missing.origin_file_number += 1000; + BlobReferenceGroup too_large = groups[1]; + --too_large.reference_count; + // Fresh outputs, so each has exactly the reserved capacity. + std::vector> fresh_outputs; + ASSERT_OK(read_batch({&groups[0], &too_large, &groups[2], &missing}, + &fresh_outputs, &requests)); + EXPECT_OK(requests[0].status); + EXPECT_TRUE(requests[1].status.IsAborted()); + EXPECT_TRUE(fresh_outputs[1].empty()); + EXPECT_OK(requests[2].status); + EXPECT_TRUE(requests[3].status.IsNotFound()); + EXPECT_TRUE(fresh_outputs[3].empty()); + + // A flipped bit in a middle block of the spanning group fails only that + // group's request. The encrypted env cannot run this part: see the comment + // in ReadBlobReferencesFailureKeepsEarlierOutput. + if (encrypted_env_) { + return; + } + const size_t middle_block = index.groups[spanning_group].start_block + 1; + ASSERT_OK(test::CorruptFile( + env_, MakeTableFileName(dbname_, file.fd.GetNumber()), + static_cast(index.blocks[middle_block].offset() + 1), + /*bytes_to_corrupt=*/1, /*verify_checksum=*/false)); + ASSERT_OK(read_batch(all, &batch_outputs, &requests)); + for (size_t g = 0; g < groups.size(); ++g) { + if (g == spanning_group) { + EXPECT_TRUE(requests[g].status.IsCorruption()); + EXPECT_TRUE(batch_outputs[g].empty()); + } else { + EXPECT_OK(requests[g].status) << g; + EXPECT_EQ(batch_outputs[g].size(), groups[g].reference_count) << g; + } + } +} + +namespace { + +// Writes `origins` blob files with `per_origin` values of `value_size` bytes +// each, and deletes all but the last `live` values of each file. One full +// compaction then writes every live reference into one SST (the caller sets a +// large target_file_size_base). Fills `origin_numbers` with the blob file +// numbers in file-number order, and `live_keys` with the keys of the live +// values. +void WriteGarbageHeavyOrigins(DBBlobBasicTest* test, DB* db, size_t origins, + size_t per_origin, size_t live, size_t value_size, + std::vector* origin_numbers, + std::vector* live_keys) { + ColumnFamilyData* cfd = static_cast_with_check(db) + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault(); + for (size_t o = 0; o < origins; ++o) { + for (size_t i = 0; i < per_origin; ++i) { + const std::string key = + "o" + std::to_string(o) + "-" + DBTestBase::Key(static_cast(i)); + ASSERT_OK(db->Put(WriteOptions(), key, + std::string(value_size, static_cast('a' + o)))); + if (i + live >= per_origin) { + live_keys->push_back(key); + } + } + ASSERT_OK(db->Flush(FlushOptions())); + } + for (const auto& blob_file : cfd->current()->storage_info()->GetBlobFiles()) { + origin_numbers->push_back(blob_file->GetBlobFileNumber()); + } + for (size_t o = 0; o < origins; ++o) { + for (size_t i = 0; i + live < per_origin; ++i) { + ASSERT_OK( + db->Delete(WriteOptions(), "o" + std::to_string(o) + "-" + + DBTestBase::Key(static_cast(i)))); + } + } + ASSERT_OK(db->Flush(FlushOptions())); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db->CompactRange(compact_options, nullptr, nullptr)); + (void)test; +} + +Options SstMajorGCOptions(Options options) { + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.target_file_size_base = 1 << 30; + options.create_if_missing = true; + return options; +} + +} // namespace + +TEST_F(DBBlobBasicTest, IndirectBlobGCReadsSharedSstOnce) { + // Four inputs whose live groups share one SST and one reference block. The + // job reads that SST once, with all four groups in one batch, and closes its + // reader before going on. + Options options = SstMajorGCOptions(GetDefaultOptions()); + DestroyAndReopen(options); + std::vector origins; + std::vector live_keys; + WriteGarbageHeavyOrigins(this, db_.get(), 4, 4, 1, 4096, &origins, + &live_keys); + ASSERT_EQ(origins.size(), 4U); + + std::atomic largest_batch{0}; + std::atomic open_reader_charge{0}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:SstBatchRead", [&](void* arg) { + const auto* report = static_cast*>(arg); + largest_batch.store(std::max(largest_batch.load(), report->first)); + open_reader_charge.fetch_add(report->second); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + EXPECT_EQ(largest_batch.load(), 4U); + EXPECT_EQ(open_reader_charge.load(), 0U); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + for (uint64_t origin : origins) { + EXPECT_NE( + cfd->current()->storage_info()->GetBlobRelocationRouteTarget(origin), + kInvalidBlobFileNumber) + << origin; + } + for (size_t i = 0; i < live_keys.size(); ++i) { + EXPECT_EQ(Get(live_keys[i]), std::string(4096, static_cast('a' + i))); + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCCorruptGroupFailsOnlyItsInput) { + // The first origin's live group spans several reference blocks; a flipped bit + // in a middle block fails only that input. The other inputs, whose groups are + // in other blocks of the same SST, are still relocated. + if (encrypted_env_) { + // test::CorruptFile() rewrites the whole SST. The encrypted env gives the + // new file a new random encryption prefix, but the open table reader keeps + // the old one. That reader then decrypts the whole file to garbage, so GC + // does not see just one flipped bit. + ROCKSDB_GTEST_BYPASS("Test corrupts an SST that has an open reader"); + return; + } + Options options = SstMajorGCOptions(GetDefaultOptions()); + DestroyAndReopen(options); + std::vector origins; + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + for (int o = 0; o < 3; ++o) { + const int count = o == 0 ? 18000 : 4; + for (int i = 0; i < count; ++i) { + // Origin 0 holds many 100-byte values, so its live group spans several + // reference blocks. Origins 1 and 2 hold four 4 KB values each. + ASSERT_OK( + Put("o" + std::to_string(o) + "-" + Key(i), + std::string(o == 0 ? 100 : 4096, static_cast('a' + o)))); + } + ASSERT_OK(Flush()); + } + for (const auto& blob_file : cfd->current()->storage_info()->GetBlobFiles()) { + origins.push_back(blob_file->GetBlobFileNumber()); + } + ASSERT_EQ(origins.size(), 3U); + for (int o = 0; o < 3; ++o) { + const int dead = o == 0 ? 9000 : 3; + for (int i = 0; i < dead; ++i) { + ASSERT_OK(Delete("o" + std::to_string(o) + "-" + Key(i))); + } + } + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const FileMetaData* const sst = OnlySstForTest(cfd); + ASSERT_NE(sst, nullptr); + const BlobReferenceIndex index = ReadBlobReferenceIndexForTest(*sst); + ASSERT_EQ(index.groups.size(), 3U); + ASSERT_EQ(index.groups[0].group.origin_file_number, origins[0]); + ASSERT_GE(index.groups[1].start_block, index.groups[0].start_block + 2); + ASSERT_OK(test::CorruptFile( + env_, MakeTableFileName(dbname_, sst->fd.GetNumber()), + static_cast(index.blocks[index.groups[0].start_block + 1].offset() + + 1), + /*bytes_to_corrupt=*/1, /*verify_checksum=*/false)); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + EXPECT_EQ(storage->GetBlobRelocationRouteTarget(origins[0]), + kInvalidBlobFileNumber); + EXPECT_NE(storage->GetBlobFileMetaData(origins[0]), nullptr); + for (size_t o = 1; o < origins.size(); ++o) { + EXPECT_NE(storage->GetBlobRelocationRouteTarget(origins[o]), + kInvalidBlobFileNumber) + << o; + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCMemoryLimitDefersOrSuppresses) { + // If the live-set reservation fails for a later input, that input and every + // input after it wait for a later job. If it fails for the first input, + // indirect blob GC suppresses only that input and still plans the other + // inputs. Neither case reports a census mismatch. + for (const int failing_reservation : {2, 1}) { + SCOPED_TRACE(failing_reservation); + Options options = SstMajorGCOptions(GetDefaultOptions()); + DestroyAndReopen(options); + std::vector origins; + std::vector live_keys; + WriteGarbageHeavyOrigins(this, db_.get(), 4, 4, 1, 4096, &origins, + &live_keys); + + std::atomic reservations{0}; + std::atomic first_plan_end{-1}; + std::atomic census_mismatches{0}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:ReserveLiveSet", [&](void* arg) { + if (++reservations == failing_reservation) { + *static_cast(arg) = + Status::MemoryLimit("injected live set limit"); + } + }); + // A deferred or suppressed input reads no groups, so GC must not check its + // census totals. The callback counts mismatches; the test expects none. + sync_point->SetCallBack("DBImpl::RunIndirectBlobGC:CensusTotalsCheck", + [&](void* arg) { + if (*static_cast(arg)) { + ++census_mismatches; + } + }); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:PlanEnd", [&](void* arg) { + int expected = -1; + first_plan_end.compare_exchange_strong( + expected, static_cast(*static_cast(arg))); + }); + sync_point->EnableProcessing(); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + ASSERT_GE(reservations.load(), failing_reservation); + EXPECT_EQ(census_mismatches.load(), 0); + if (failing_reservation == 2) { + // The inputs after the first input are deferred. + EXPECT_EQ(first_plan_end.load(), 1); + } else { + // The first input is suppressed and the later ones are planned. + EXPECT_GT(first_plan_end.load(), 1); + } + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCSnapshotMemoryLimitBacksOff) { + // A census snapshot that does not fit the memory budget is transient: its + // candidates back off, and the job is not retried every few milliseconds. + Options options = SstMajorGCOptions(GetDefaultOptions()); + DestroyAndReopen(options); + std::vector origins; + std::vector live_keys; + WriteGarbageHeavyOrigins(this, db_.get(), 4, 4, 1, 4096, &origins, + &live_keys); + std::vector live_values; + for (const std::string& key : live_keys) { + live_values.push_back(Get(key)); + } + + std::atomic snapshots{0}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:SnapshotSelected", [&](void* arg) { + ++snapshots; + *static_cast(arg) = Status::MemoryLimit("injected"); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + // Without the back-off this wait would not end: the retried pick would return + // the same candidates again. + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GE(snapshots.load(), 1); + ASSERT_LE(snapshots.load(), 2); + ASSERT_OK(dbfull()->TEST_GetBGError()); + for (size_t i = 0; i < live_keys.size(); ++i) { + ASSERT_EQ(Get(live_keys[i]), live_values[i]); + } +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCWorkspaceChargeFailureSuppressesOnce) { + // Indirect blob GC charges its I/O workspace to the block cache before it + // reads any input. When that charge fails, SelectInputs() suppresses every + // input once, with a transient suppression. The check for the block-based + // table format that follows must not suppress the inputs a second time. + // A second suppression would count as a second failure and double the + // first retry delay from 1 s to 2 s (see + // BlobGcCandidateHeap::SuppressTransient()). + // + // This block cache fails the first insert of the GC working memory role + // after the job has copied its census snapshot. The snapshot charge uses the + // same role, so the cache is armed only after the snapshot. The workspace + // charge is then the first insert of that role. + class FailWorkspaceChargeCache : public CacheWrapper { + public: + explicit FailWorkspaceChargeCache(std::shared_ptr target) + : CacheWrapper(std::move(target)) {} + const char* Name() const override { return "FailWorkspaceChargeCache"; } + using Cache::Insert; + Status Insert(const Slice& key, ObjectPtr value, + const CacheItemHelper* helper, size_t charge, + Handle** handle = nullptr, Priority priority = Priority::LOW, + const Slice& compressed_value = Slice(), + CompressionType type = kNoCompression) override { + if (helper != nullptr && + helper->role == CacheEntryRole::kIndirectBlobGcWorkingMemory && + armed.exchange(false)) { + ++failed_inserts; + return Status::MemoryLimit("injected GC workspace charge failure"); + } + return CacheWrapper::Insert(key, value, helper, charge, handle, priority, + compressed_value, type); + } + std::atomic armed{false}; + std::atomic failed_inserts{0}; + }; + auto cache = + std::make_shared(NewLRUCache(64 << 20)); + BlockBasedTableOptions table_options; + table_options.block_cache = cache; + auto clock = + std::make_shared(env_->GetSystemClock()); + CompositeEnvWrapper env(env_, clock); + Options options = IndirectBlobGCOptions(); + options.env = &env; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + auto log = std::make_shared(); + options.info_log = log; + DestroyAndReopen(options); + // The DB uses the local `env`, so close it before `env` goes away, also when + // an assertion ends the test early. + auto close_db = Defer([this] { Close(); }); + const uint64_t file_number = WriteIndirectBlobGCCandidate(/*live_keys=*/1); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + // `jobs` counts the GC jobs that reach input selection (one census snapshot + // per job). Only the first job gets the charge failure. + std::atomic jobs{0}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImpl::RunIndirectBlobGC:SnapshotSelected", [&](void* arg) { + if (static_cast(arg)->ok() && ++jobs == 1) { + cache->armed = true; + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + // Runs the census and the GC jobs until no more work is queued. + const auto run_gc = [&]() { + for (int round = 0; round < 3; ++round) { + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } + }; + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + ASSERT_NO_FATAL_FAILURE(run_gc()); + + // The first job failed the charge, and suppressed its input once, for the + // workspace reason only. The DB clock does not move, so no retry is due yet. + ASSERT_EQ(cache->failed_inserts.load(), 1); + ASSERT_EQ(jobs.load(), 1); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + const std::string skipped = + "Indirect blob GC skipped file #" + std::to_string(file_number) + ": "; + ASSERT_EQ(log->Count({skipped + "blob GC I/O workspace cache pressure"}), 1U); + ASSERT_EQ(log->Count({skipped + "block-based table format is required"}), 0U); + ASSERT_EQ(log->Count({"Indirect blob GC skipped file #"}), 1U); + + // Just before 1 s, the input is still suppressed: no new job starts. + constexpr uint64_t kFirstRetryMicros = 1000 * 1000; + clock->Advance(kFirstRetryMicros - 1000); + ASSERT_NO_FATAL_FAILURE(run_gc()); + ASSERT_EQ(jobs.load(), 1); + ASSERT_EQ(RouteTarget(file_number), kInvalidBlobFileNumber); + + // At 1 s after the failure, the census brings the input back, and a new job + // collects it. A double suppression would keep it out until 2 s. + clock->Advance(1000); + ASSERT_NO_FATAL_FAILURE(run_gc()); + ASSERT_EQ(jobs.load(), 2); + ASSERT_NE(RouteTarget(file_number), kInvalidBlobFileNumber); + ASSERT_EQ(cache->failed_inserts.load(), 1); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_EQ(Get(IndirectBlobGCKey(kIndirectBlobGCKeyCount - 1)), + IndirectBlobGCValue(kIndirectBlobGCKeyCount - 1)); +} + +namespace { + +// Opens a new table reader for `file`, outside the shared table cache. +Status OpenFreshTableReaderForTest(ColumnFamilyData* cfd, + const FileMetaData& file, + std::unique_ptr* reader) { + TableCacheOpenOptions open_options; + open_options.open_ephemeral_table_reader = true; + open_options.avoid_shared_metadata_cache = true; + TableCache::TypedHandle* handle = nullptr; + TableReader* raw_reader = nullptr; + return cfd->table_cache()->FindTable( + ReadOptions(), FileOptions(), cfd->internal_comparator(), file, &handle, + cfd->GetLatestMutableCFOptions(), &raw_reader, /*no_io=*/false, + /*file_read_hist=*/nullptr, /*skip_filters=*/true, /*level=*/-1, + /*prefetch_index_and_filter_in_cache=*/false, + /*max_file_size_for_l0_meta_pin=*/0, file.temperature, + /*pin_table_handle=*/false, /*file_open_metadata=*/nullptr, reader, + open_options); +} + +} // namespace + +TEST_F(DBBlobBasicTest, BlobReferenceReadReportsDecodedIndexSize) { + // Indirect blob GC charges the decoded reference index of each SST at its + // real size, which ReadBlobReferences() reports through `after_index_read`. + // With blob_file_size = 1, each value gets its own blob file, so the SST has + // one group per value. Each BlobReferenceIndexEntry is 56 bytes, so 1300 + // groups need about 71 KB (1300 x 56 bytes) of decoded index, which is more + // than 64 KB. So a fixed estimate, such as the 64 KB that GC charges for the + // table reader itself, would be too small. + constexpr int kGroups = 1300; + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_file_size = 1; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + for (int i = 0; i < kGroups; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), "value" + std::to_string(i))); + } + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const FileMetaData* const sst = OnlySstForTest(cfd); + ASSERT_NE(sst, nullptr); + const BlobReferenceIndex index = ReadBlobReferenceIndexForTest(*sst); + ASSERT_EQ(index.groups.size(), static_cast(kGroups)); + const BlobReferenceGroup group = index.groups.back().group; + std::unique_ptr reader; + ASSERT_OK(OpenFreshTableReaderForTest(cfd, *sst, &reader)); + + // The callback records the reported size and charges it to the cache, as + // indirect blob GC does. After a successful read, the lambda checks that the + // cache holds at least the reported size. + const auto read = [&](const std::shared_ptr& cache, + size_t* reported_bytes, + std::vector* references) { + CacheReservationManagerImpl + charge(cache); + references->reserve(1); + std::vector requests(1); + requests[0].group = &group; + requests[0].output = references; + const Status status = reader->ReadBlobReferences( + ReadOptions(), sst->blob_reference_descriptor, + [&](size_t index_bytes) { + *reported_bytes = index_bytes; + return charge.UpdateCacheReservation(index_bytes); + }, + &requests); + if (status.ok()) { + EXPECT_OK(requests[0].status); + EXPECT_GE(cache->GetPinnedUsage(), *reported_bytes); + } + return status; + }; + size_t reported_bytes = 0; + std::vector references; + // A 16 MB strict cache has room for the charge, so the read succeeds. + ASSERT_OK(read(NewLRUCache(16 << 20, /*num_shard_bits=*/0, + /*strict_capacity_limit=*/true), + &reported_bytes, &references)); + ASSERT_GE(reported_bytes, kGroups * sizeof(BlobReferenceIndexEntry)); + ASSERT_GT(reported_bytes, size_t{64} << 10); + ASSERT_EQ(references.size(), 1U); + + // A strict 32 KB cache fails the charge. CacheReservationManager reserves + // memory in dummy entries of kSizeDummyEntry (256 KB), so this cache cannot + // hold even one entry for the index. The read returns that status before it + // reads any data block, and the output stays empty. + reported_bytes = 0; + references.clear(); + const Status status = read(NewLRUCache(32 << 10, /*num_shard_bits=*/0, + /*strict_capacity_limit=*/true), + &reported_bytes, &references); + ASSERT_TRUE(status.IsMemoryLimit()) << status.ToString(); + ASSERT_GT(reported_bytes, size_t{64} << 10); + ASSERT_TRUE(references.empty()); +} + +TEST_F(DBBlobBasicTest, BlobReferenceBlockChecksumMismatchIsCorruption) { + // A flipped bit in a group's start block fails the block checksum. + // ReadBlobReferences fails the request with Corruption. + // ReadBlobReferenceGroups (the census read, which reads only the index block) + // still succeeds. A flipped bit in the index block makes + // ReadBlobReferenceGroups fail too. + FlushManyBlobReferenceGroups(); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const FileMetaData* const sst = OnlySstForTest(cfd); + ASSERT_NE(sst, nullptr); + const FileMetaData& file = *sst; + ASSERT_NE(GetCensusSnapshot(cfd), nullptr); + const BlobReferenceIndex index = ReadBlobReferenceIndexForTest(file); + ASSERT_GE(index.groups.size(), 2U); + const BlobReferenceIndexEntry& entry = index.groups[1]; + const BlobReferenceGroup& group = entry.group; + const std::string file_name = MakeTableFileName(dbname_, file.fd.GetNumber()); + ASSERT_OK(test::CorruptFile( + env_, file_name, + static_cast(index.blocks[entry.start_block].offset() + 1), + /*bytes_to_corrupt=*/1, /*verify_checksum=*/false)); + + std::unique_ptr reader; + ASSERT_OK(OpenFreshTableReaderForTest(cfd, file, &reader)); + std::vector groups; + ASSERT_OK(reader->ReadBlobReferenceGroups( + ReadOptions(), file.blob_reference_descriptor, + /*before_allocation=*/nullptr, &groups)); + EXPECT_EQ(groups.size(), index.groups.size()); + std::vector references; + references.reserve(static_cast(group.reference_count)); + std::vector requests(1); + requests[0].group = &group; + requests[0].output = &references; + ASSERT_OK( + reader->ReadBlobReferences(ReadOptions(), file.blob_reference_descriptor, + /*after_index_read=*/nullptr, &requests)); + ASSERT_TRUE(requests[0].status.IsCorruption()); + ASSERT_TRUE(references.empty()); + + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + file.blob_reference_descriptor, &descriptor)); + ASSERT_OK(test::CorruptFile( + env_, file_name, static_cast(descriptor.index_handle.offset() + 1), + /*bytes_to_corrupt=*/1, /*verify_checksum=*/false)); + reader.reset(); + ASSERT_OK(OpenFreshTableReaderForTest(cfd, file, &reader)); + ASSERT_TRUE(reader + ->ReadBlobReferenceGroups( + ReadOptions(), file.blob_reference_descriptor, + /*before_allocation=*/nullptr, &groups) + .IsCorruption()); +} + +TEST_F(DBBlobBasicTest, BlobReferenceMetadataSurvivesTrivialMove) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto& l1_files = cfd->current()->storage_info()->LevelFiles(1); + ASSERT_EQ(l1_files.size(), 1); + const uint64_t file_number = l1_files.front()->fd.GetNumber(); + ASSERT_FALSE(l1_files.front()->blob_reference_descriptor.empty()); + + MoveFilesToLevel(/*level=*/2); + const auto& l2_files = cfd->current()->storage_info()->LevelFiles(2); + ASSERT_EQ(l2_files.size(), 1); + ASSERT_EQ(l2_files.front()->fd.GetNumber(), file_number); + ASSERT_FALSE(l2_files.front()->blob_reference_descriptor.empty()); + + Reopen(options); + dbfull()->TEST_WaitForBlobReferenceCensus(); + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + const std::vector files = census->SstFileNumbers(); + ASSERT_EQ(std::count(files.begin(), files.end(), file_number), 1); + ASSERT_EQ(Get("key"), "blob_value"); +} + +TEST_F(DBBlobBasicTest, BlobReferenceMetadataSurvivesWalRecoveryFlush) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.avoid_flush_during_recovery = false; + Reopen(options); + + ASSERT_OK(Put("flushed", "flushed_blob_value")); + ASSERT_OK(Flush()); + ASSERT_OK(Put("recovered", "recovered_blob_value")); + + Close(); + Reopen(options); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(Get("flushed"), "flushed_blob_value"); + ASSERT_EQ(Get("recovered"), "recovered_blob_value"); + + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + size_t file_count = 0; + for (int level = 0; level < cfd->NumberLevels(); ++level) { + for (const FileMetaData* file : + cfd->current()->storage_info()->LevelFiles(level)) { + ++file_count; + ASSERT_TRUE(file->blob_reference_descriptor.empty()); + ASSERT_NE(file->oldest_coarse_indirect_origin_file_number, + kInvalidBlobFileNumber); + ASSERT_FALSE(file->blob_reference_legacy_metadata); + } + } + ASSERT_GE(file_count, 2); + const auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->SstFileNumbers().size(), 0U); + ASSERT_OK(BuildExactSstAtLevel1()); + const auto exact_census = GetCensusSnapshot(cfd); + ASSERT_NE(exact_census, nullptr); + ASSERT_EQ(exact_census->SstFileNumbers().size(), 1U); +} + +TEST_F(DBBlobBasicTest, BlobIndirectionReadsCompressedUnflattenedIndexes) { + if (!CompressionTypeSupported(kZSTD)) { + ROCKSDB_GTEST_SKIP("ZSTD is required for this test"); + return; + } + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_compression_type = kZSTD; + options.blob_cache = NewLRUCache(1 << 20); + + Reopen(options); + const std::array expected{std::string(4096, 'a'), + std::string(4096, 'b')}; + ASSERT_OK(Put("key1", expected[0])); + ASSERT_OK(Put("key2", expected[1])); + ASSERT_OK(Flush()); + + const std::array keys{{"key2", "key1"}}; + std::array values; + std::array statuses; + ReadOptions read_options; + read_options.fill_cache = false; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_OK(statuses[0]); + ASSERT_OK(statuses[1]); + EXPECT_EQ(values[0], expected[1]); + EXPECT_EQ(values[1], expected[0]); + + PinnableSlice value; + read_options.fill_cache = true; + ASSERT_OK(db_->Get(read_options, db_->DefaultColumnFamily(), "key1", &value)); + EXPECT_EQ(value, expected[0]); + value.Reset(); + read_options.fill_cache = false; + read_options.read_tier = kBlockCacheTier; + ASSERT_OK(db_->Get(read_options, db_->DefaultColumnFamily(), "key1", &value)); + EXPECT_EQ(value, expected[0]); +} + +TEST_F(DBBlobBasicTest, FlattenedReferenceReadChecksOriginInReader) { + // A flattened reference has generation = a relocation file, so the read goes + // to that file directly, with no route. The Version does not search the + // file's directory. The relocation file reader seeks (origin, original + // offset) and checks that the group it finds has the same origin, so a + // reference to an origin that the file does not hold fails with Corruption. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + + Reopen(options); + const std::string key = "key"; + const std::string expected = "relocated-value"; + ASSERT_OK(Put(key, expected)); + ASSERT_OK(Flush()); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + const uint64_t origin_file_number = + storage->GetBlobFiles().front()->GetBlobFileNumber(); + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + const std::vector records{ + {origin_file_number, original_value_offset, expected, kNoCompression}}; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, records, &metadata, &relocation_file_size); + + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + + const auto put_flattened = [&](const std::string& flattened_key, + uint64_t flattened_origin) { + std::string blob_index; + const uint32_t stable_checksum = + crc32c::Value(expected.data(), expected.size()); + BlobIndex::EncodeIndirectBlob( + &blob_index, relocation_file_number, + {.origin_file_number = flattened_origin, + .original_value_offset = original_value_offset, + .stored_value_size = expected.size(), + .compression = kNoCompression, + .stable_checksum = stable_checksum, + .stored_checksum = BlobIndex::ExtendChecksumWithCompression( + stable_checksum, kNoCompression)}); + WriteBatch batch; + ASSERT_OK( + WriteBatchInternal::PutBlobIndex(&batch, 0, flattened_key, blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + }; + put_flattened("flattened", origin_file_number); + // A flattened generation is above its origin, so use a lower origin that the + // relocation file does not hold. + ASSERT_GT(origin_file_number, 1U); + put_flattened("outside", origin_file_number - 1); + // A coarse SST cannot hold a flattened reference, so the flush writes exact + // metadata (see FlushInjectedBlobIndexesWithExactMetadata()). + ASSERT_OK(FlushInjectedBlobIndexesWithExactMetadata()); + ASSERT_EQ(Get("flattened"), expected); + PinnableSlice result; + const Status status = + db_->Get(ReadOptions(), db_->DefaultColumnFamily(), "outside", &result); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); +} + +TEST_F(DBBlobBasicTest, CensusStartsWhenFenceIsWrittenAfterOpen) { + // A new DB has no format fence at open, so open schedules no census. The + // first flush writes the fence, and installing its SuperVersion starts the + // census. The test does not call TEST_WaitForBlobReferenceCensus(), which + // would schedule the census by itself. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.create_if_missing = true; + DestroyAndReopen(options); + ASSERT_FALSE(dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->HasBlobIndirectionFormat()); + SyncPoint::GetInstance()->LoadDependency( + {{"DBImpl::BGWorkBlobReferenceCensus:Start", + "DBBlobBasicTest::CensusStartsWhenFenceIsWrittenAfterOpen:Wait"}}); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + TEST_SYNC_POINT( + "DBBlobBasicTest::CensusStartsWhenFenceIsWrittenAfterOpen:Wait"); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_TRUE(dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->HasBlobIndirectionFormat()); + ASSERT_EQ(Get("key"), "value"); +} + +TEST_F(DBBlobBasicTest, BlobRelocationLossAndFingerprintSurviveReopen) { + // The test routes a new, unused file number (routed_generation), which stands + // for an old blob file, to a relocation file (relocation_file_number). The + // fingerprint of the relocation state must be the same after a reopen and + // after a MANIFEST snapshot. Then a loss record (MarkBlobPhysicalFileLost()) + // marks the relocation file lost, so a read that follows the route returns + // NotFound. The lost mark and the new fingerprint must also stay the same + // after a reopen and after a snapshot. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + dbfull()->TEST_LockMutex(); + const uint64_t routed_generation = version_set->NewFileNumber(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + const std::string expected = "relocated-value"; + const uint64_t original_value_offset = 100; + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{routed_generation, original_value_offset, expected, kNoCompression}}, + &metadata, &relocation_file_size); + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(routed_generation, relocation_file_number)); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + { + std::string blob_index; + const uint32_t stable_checksum = + crc32c::Value(expected.data(), expected.size()); + BlobIndex::EncodeIndirectBlob( + &blob_index, routed_generation, + {.origin_file_number = routed_generation, + .original_value_offset = original_value_offset, + .stored_value_size = expected.size(), + .compression = kNoCompression, + .stable_checksum = stable_checksum, + .stored_checksum = BlobIndex::ExtendChecksumWithCompression( + stable_checksum, kNoCompression)}); + WriteBatch batch; + ASSERT_OK( + WriteBatchInternal::PutBlobIndex(&batch, 0, "routed", blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + ASSERT_EQ(Get("routed"), expected); + + const auto fingerprint = [&]() { + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + return std::make_pair(storage->GetBlobRelocationStateFingerprintHigh(), + storage->GetBlobRelocationStateFingerprintLow()); + }; + const auto reopen_both_ways = [&]() { + Close(); + Reopen(options); + const auto replayed = fingerprint(); + VersionEdit snapshot; + snapshot.SetColumnFamily(/*column_family_id=*/0); + EXPECT_OK(ApplyVersionEdit(&snapshot, /*new_descriptor_log=*/true)); + Close(); + Reopen(options); + EXPECT_EQ(fingerprint(), replayed); + return replayed; + }; + const auto before_loss = fingerprint(); + ASSERT_EQ(reopen_both_ways(), before_loss); + ASSERT_EQ(Get("routed"), expected); + + VersionEdit loss; + loss.SetColumnFamily(0); + loss.MarkBlobPhysicalFileLost(relocation_file_number); + ASSERT_OK(ApplyVersionEdit(&loss, /*new_descriptor_log=*/false)); + const auto after_loss = fingerprint(); + ASSERT_NE(after_loss, before_loss); + const auto check_lost = [&]() { + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_TRUE(storage->HasLostBlobRelocationFiles()); + ASSERT_TRUE(storage->GetBlobRelocationFileMetaData(relocation_file_number) + ->IsMarkedLost()); + PinnableSlice value; + const Status status = + db_->Get(ReadOptions(), db_->DefaultColumnFamily(), "routed", &value); + ASSERT_TRUE(status.IsNotFound()) << status.ToString(); + ASSERT_EQ(Get("key"), "value"); + }; + // LogAndApply alone installs no SuperVersion; reopen to read the new + // Version. + ASSERT_EQ(reopen_both_ways(), after_loss); + check_lost(); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCKeepsRelocationPinnedByCoarseSst) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + + Reopen(options); + const std::string key = "key"; + const std::string value = "value-routed-from-a-coarse-sst"; + ASSERT_OK(Put(key, value)); + // Flush output is coarse: the exact census does not index it. + ASSERT_OK(Flush()); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + const uint64_t origin_file_number = + storage->GetBlobFiles().front()->GetBlobFileNumber(); + ASSERT_TRUE(storage->IsCoarseIndirectOriginBlobPinned(origin_file_number)); + const uint64_t value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{origin_file_number, value_offset, value, kNoCompression}}, &metadata, + &relocation_file_size); + + // Route the coarse-pinned indirect-origin file to the relocation file. + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(origin_file_number, relocation_file_number)); + install.RetireBlobFile(origin_file_number); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + + // The coarse SST still needs the route, so indirect blob GC must not treat + // the routed relocation file as empty and delete it + // (kBlobRelocationFileDeletion). + storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobRelocationFileMetaData(relocation_file_number), + nullptr); + ASSERT_NE(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get(key), value); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCCollectsRelocationInputAboveCoarseFlush) { + // A coarse SST pins indirect-origin blob files only. A relocation input is + // never pinned, even when its number is at or above the frontier of a coarse + // SST. This test makes that case with a real flush: + // 1. A flush takes its blob file number (for example 25) and pauses. + // 2. GC job 1 collects normal file A (below 25) into relocation file R + // (for example 30, above 25). + // 3. A compaction drops one of the two values in R, so R is half garbage. + // 4. GC job 2 selects R and pauses before it installs. + // 5. The flush installs its coarse SST (frontier 25). R is now at or above + // the frontier. + // 6. GC job 2 installs: R is collected, with no stale job and no assert. + Options options = IndirectBlobGCOptions(); + // Indirect blob GC relocates live values only when blob GC is on. Auto + // compaction stays off, so GC runs only when the test calls it. + options.enable_blob_garbage_collection = true; + DestroyAndReopen(options); + for (size_t i = 0; i < kIndirectBlobGCKeyCount; ++i) { + ASSERT_OK(Put(IndirectBlobGCKey(i), IndirectBlobGCValue(i))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + // Keys 2 and 3 stay live in A. Key 2 is deleted by an L0 SST that the test + // compacts after GC job 1. + ASSERT_OK(Delete(IndirectBlobGCKey(0))); + ASSERT_OK(Delete(IndirectBlobGCKey(1))); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + ASSERT_OK(Delete(IndirectBlobGCKey(2))); + ASSERT_OK(Flush()); + + std::atomic pause_flush{true}; + std::atomic pause_gc{false}; + std::atomic stale_installs{0}; + SetUpCoarsePinRace(&pause_flush, &pause_gc, &stale_installs); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + const std::string flush_key = "flush-key"; + const std::string flush_value(4096, 'f'); + ASSERT_NO_FATAL_FAILURE(StartPausedCoarseFlush(flush_key, flush_value)); + + // GC job 1 allocates R after the flush took its blob file number, so R is + // above it. + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {origin})); + const uint64_t relocation_file = RouteTarget(origin); + ASSERT_NE(relocation_file, kInvalidBlobFileNumber); + ASSERT_NO_FATAL_FAILURE(CompactL0AndL1IntoL1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_FALSE(IsCoarsePinned(relocation_file)); + + pause_gc = true; + Status gc_status; + port::Thread gc_thread([&]() { + gc_status = dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {relocation_file}); + }); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:WaitGcPaused"); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:ResumeFlush"); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + // The coarse SST of the flush is installed, and R is at or above its + // frontier. + ASSERT_NE( + cfd->current()->storage_info()->OldestCoarseIndirectOriginFileNumber(), + kInvalidBlobFileNumber); + ASSERT_TRUE(IsCoarsePinned(relocation_file)); + ASSERT_FALSE(IsCoarsePinned(origin)); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:ResumeGc"); + gc_thread.join(); + + ASSERT_OK(gc_status); + ASSERT_EQ(stale_installs.load(), 0); + ASSERT_OK(dbfull()->TEST_GetBGError()); + // R is collected: its live value moved to a new relocation file, and the + // route of A now points to that file. + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobRelocationFileMetaData(relocation_file), nullptr); + const uint64_t new_target = RouteTarget(origin); + ASSERT_NE(new_target, kInvalidBlobFileNumber); + ASSERT_NE(new_target, relocation_file); + ASSERT_NE(storage->GetBlobRelocationFileMetaData(new_target), nullptr); + const size_t live_key = kIndirectBlobGCKeyCount - 1; + ASSERT_EQ(Get(IndirectBlobGCKey(live_key)), IndirectBlobGCValue(live_key)); + ASSERT_EQ(Get(IndirectBlobGCKey(2)), "NOT_FOUND"); + ASSERT_EQ(Get(flush_key), flush_value); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Reopen(options); + ASSERT_EQ(Get(IndirectBlobGCKey(live_key)), IndirectBlobGCValue(live_key)); + ASSERT_EQ(Get(flush_key), flush_value); +} + +TEST_F(DBBlobBasicTest, IndirectBlobGCNormalInputPinnedByCoarseFlushIsStale) { + // A normal input that becomes coarse pinned while a GC job runs makes the + // job stale: the install check returns TryAgain and nothing is installed. + // The coarse SST may reference the input, so GC must not move it. It is a + // normal race, not a broken invariant, so a debug build does not assert. + // 1. A flush takes its blob file number (for example 25) and pauses. + // 2. An L0 to L0 compaction with compaction-time blob GC moves 4 values + // into a new normal blob file N (for example 28, above 25). Its output + // is a coarse L0 SST, because compaction-time blob GC rewrites an + // indirect value only into coarse L0 output. An exact L1 SST below the + // key range, written first, makes this compaction not bottommost, + // because a bottommost compaction writes exact output. + // 3. A second compaction, without blob GC, writes the L0 and L1 SSTs as + // one exact L1 SST and drops 2 of the 4 values, so N is half garbage + // and no coarse SST is left. + // 4. GC job 1 selects N and pauses before it installs. + // 5. The flush installs its coarse SST (frontier 25), which pins N. + // 6. GC job 1 is stale and installs nothing. + // 7. When the coarse SST is rewritten as an exact SST, the pin goes away + // and GC job 2 collects N. + Options options = IndirectBlobGCOptions(); + // Indirect blob GC relocates live values only when blob GC is on. Auto + // compaction stays off, so GC runs only when the test calls it. + options.enable_blob_garbage_collection = true; + DestroyAndReopen(options); + // The L1 SST covers the key range of the L0 SSTs: "key" sorts before "key0", + // and "key9" sorts after "key3". + ASSERT_OK(Put("key", "low")); + ASSERT_OK(Put("key9", "high")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + for (size_t i = 0; i < kIndirectBlobGCKeyCount; ++i) { + ASSERT_OK(Put(IndirectBlobGCKey(i), IndirectBlobGCValue(i))); + } + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t flushed_blob_file = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + std::string value_sst; + { + ColumnFamilyMetaData metadata; + db_->GetColumnFamilyMetaData(&metadata); + ASSERT_EQ(metadata.levels[0].files.size(), 1U); + value_sst = metadata.levels[0].files.front().name; + } + ASSERT_OK(Delete(IndirectBlobGCKey(0))); + ASSERT_OK(Delete(IndirectBlobGCKey(1))); + ASSERT_OK(Flush()); + + std::atomic pause_flush{true}; + std::atomic pause_gc{false}; + std::atomic stale_installs{0}; + SetUpCoarsePinRace(&pause_flush, &pause_gc, &stale_installs); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + const std::string flush_key = "flush-key"; + const std::string flush_value(4096, 'f'); + ASSERT_NO_FATAL_FAILURE(StartPausedCoarseFlush(flush_key, flush_value)); + + // Compaction-time blob GC with age cutoff 1.0 (every blob file is old + // enough) moves every value of the first L0 SST into a new blob file N, + // which is numbered above the flush's blob file. + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"blob_garbage_collection_age_cutoff", "1.0"}})); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {value_sst}, /*level=*/0)); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + const uint64_t input = storage->GetBlobFiles().back()->GetBlobFileNumber(); + ASSERT_GT(input, flushed_blob_file); + ASSERT_TRUE(storage->GetBlobFileMetaData(input)->IsIndirectOrigin()); + // Drop keys 0 and 1, so half of N is garbage. + ASSERT_NO_FATAL_FAILURE(CompactL0AndL1IntoL1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_FALSE(IsCoarsePinned(input)); + + pause_gc = true; + Status gc_status; + port::Thread gc_thread([&]() { + gc_status = dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {input}); + }); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:WaitGcPaused"); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:ResumeFlush"); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_TRUE(IsCoarsePinned(input)); + TEST_SYNC_POINT("DBBlobBasicTest::CoarsePinRace:ResumeGc"); + gc_thread.join(); + + // The job is stale (TryAgain) and installs nothing: N keeps its values and + // has no route. + ASSERT_EQ(gc_status.code(), Status::Code::kTryAgain) << gc_status.ToString(); + ASSERT_EQ(stale_installs.load(), 1); + ASSERT_OK(dbfull()->TEST_GetBGError()); + ASSERT_TRUE(HasBlobFile(input)); + ASSERT_EQ(RouteTarget(input), kInvalidBlobFileNumber); + for (size_t i = 2; i < kIndirectBlobGCKeyCount; ++i) { + ASSERT_EQ(Get(IndirectBlobGCKey(i)), IndirectBlobGCValue(i)); + } + ASSERT_EQ(Get(IndirectBlobGCKey(0)), "NOT_FOUND"); + ASSERT_EQ(Get(flush_key), flush_value); + + // Rewriting the coarse L0 SST as an exact SST removes the pin, and a later + // job collects N. + ASSERT_OK(BuildExactSstAtLevel1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_FALSE(IsCoarsePinned(input)); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {input})); + ASSERT_EQ(stale_installs.load(), 1); + ASSERT_FALSE(HasBlobFile(input)); + ASSERT_NE(RouteTarget(input), kInvalidBlobFileNumber); + for (size_t i = 2; i < kIndirectBlobGCKeyCount; ++i) { + ASSERT_EQ(Get(IndirectBlobGCKey(i)), IndirectBlobGCValue(i)); + } + ASSERT_EQ(Get(flush_key), flush_value); +} + +TEST_F(DBBlobBasicTest, BlobRelocationRouteSurvivesReplayAndManifestSnapshot) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + + Reopen(options); + const std::string key = "key"; + const std::string expected = "relocated-value"; + ASSERT_OK(Put(key, expected)); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + const uint64_t origin_file_number = + storage->GetBlobFiles().front()->GetBlobFileNumber(); + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + + // An old blob file G that indirect blob GC already collected into relocation + // file R: a reference to G (generation == origin == G, so flush output can + // hold it) is read through route[G]. + dbfull()->TEST_LockMutex(); + const uint64_t routed_generation = version_set->NewFileNumber(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + const std::vector records{ + {origin_file_number, original_value_offset, expected, kNoCompression}, + {routed_generation, original_value_offset, expected, kNoCompression}}; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, records, &metadata, &relocation_file_size); + + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(routed_generation, relocation_file_number)); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + { + std::string blob_index; + const uint32_t stable_checksum = + crc32c::Value(expected.data(), expected.size()); + BlobIndex::EncodeIndirectBlob( + &blob_index, routed_generation, + {.origin_file_number = routed_generation, + .original_value_offset = original_value_offset, + .stored_value_size = expected.size(), + .compression = kNoCompression, + .stable_checksum = stable_checksum, + .stored_checksum = BlobIndex::ExtendChecksumWithCompression( + stable_checksum, kNoCompression)}); + WriteBatch batch; + ASSERT_OK( + WriteBatchInternal::PutBlobIndex(&batch, 0, "routed", blob_index)); + ASSERT_OK(db_->Write(WriteOptions(), &batch)); + ASSERT_OK(Flush()); + } + const std::string key_through_route = "routed"; + + // Reopen once from incremental records, then rewrite the MANIFEST and reopen + // again from the complete state snapshot. + Close(); + Reopen(options); + ASSERT_EQ(Get(key_through_route), expected); + VersionEdit snapshot; + snapshot.SetColumnFamily(/*column_family_id=*/0); + ASSERT_OK(ApplyVersionEdit(&snapshot, /*new_descriptor_log=*/true)); + Close(); + Reopen(options); + + // A checkpoint copies the relocation file together with the MANIFEST that + // names it. After the origin file is deleted from the checkpoint, the + // checkpoint still reads the value through the route. + const std::string checkpoint_path = dbname_ + "_relocation_checkpoint"; + { + Checkpoint* raw_checkpoint = nullptr; + ASSERT_OK(Checkpoint::Create(db_.get(), &raw_checkpoint)); + std::unique_ptr checkpoint(raw_checkpoint); + ASSERT_OK(checkpoint->CreateCheckpoint(checkpoint_path)); + } + ASSERT_OK( + env_->FileExists(BlobFileName(checkpoint_path, relocation_file_number))); + ASSERT_OK( + env_->DeleteFile(BlobFileName(checkpoint_path, origin_file_number))); + { + std::unique_ptr checkpoint_db; + ASSERT_OK(DB::Open(options, checkpoint_path, &checkpoint_db)); + std::string value; + ASSERT_OK(checkpoint_db->Get(ReadOptions(), key_through_route, &value)); + ASSERT_EQ(value, expected); + } + ASSERT_OK(DestroyDB(checkpoint_path, options)); + + // Reads must use the route, not the origin file. Delete the indirect-origin + // file after recovery, then check that Get, MultiGet and an iterator all + // still return the value. + ASSERT_OK(env_->DeleteFile(BlobFileName(dbname_, origin_file_number))); + ASSERT_EQ(Get(key_through_route), expected); + + const std::array keys{{key_through_route}}; + std::array values; + std::array statuses; + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), keys.size(), + keys.data(), values.data(), statuses.data()); + ASSERT_OK(statuses.front()); + ASSERT_EQ(values.front(), expected); + + std::unique_ptr iterator(db_->NewIterator(ReadOptions())); + iterator->Seek(key_through_route); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ(iterator->key(), key_through_route); + ASSERT_EQ(iterator->value(), expected); + ASSERT_OK(iterator->status()); +} + +TEST_F(DBBlobBasicTest, BlobFileRetirementSurvivesReplayAndManifestSnapshot) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + + Reopen(options); + const std::string key = "key"; + const std::string expected = "relocated-value"; + ASSERT_OK(Put(key, expected)); + ASSERT_OK(Flush()); + // Write a second blob file whose only value is then deleted. After a + // compaction that rewrites every SST, no SST uses that file. The file still + // stays in the Version, because an indirect-origin blob file stays until a + // blob file retirement record removes it. + ASSERT_OK(Put("doomed", "doomed-value")); + ASSERT_OK(Flush()); + ASSERT_OK(Delete("doomed")); + ASSERT_OK(Flush()); + { + CompactRangeOptions rewrite_all; + rewrite_all.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(rewrite_all, nullptr, nullptr)); + } + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobFiles().size(), 2U); + const uint64_t origin_file_number = + storage->GetBlobFiles().front()->GetBlobFileNumber(); + const uint64_t empty_file_number = + storage->GetBlobFiles().back()->GetBlobFileNumber(); + ASSERT_TRUE(storage->GetBlobFiles().front()->IsIndirectOrigin()); + ASSERT_TRUE(storage->GetBlobFiles().back()->IsIndirectOrigin()); + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + const std::vector records{ + {origin_file_number, original_value_offset, expected, kNoCompression}}; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, records, &metadata, &relocation_file_size); + + // The test writes by hand an edit with the same form as an indirect blob GC + // install edit, in one record batch. It adds relocation file + // relocation_file_number, routes origin_file_number to it, and records the + // blob file retirement of origin_file_number. It also records the blob file + // retirement of empty_file_number with no route, because no SST uses that + // file. + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(origin_file_number, relocation_file_number)); + install.RetireBlobFile(origin_file_number); + install.RetireBlobFile(empty_file_number); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + // After its blob file retirement, this MANIFEST rejects adding the collected + // file back. + { + VersionEdit readd; + readd.SetColumnFamily(cfd->GetID()); + readd.AddBlobFile(origin_file_number, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + ASSERT_TRUE( + ApplyVersionEdit(&readd, /*new_descriptor_log=*/false).IsCorruption()); + } + + const auto check = [&]() { + const VersionStorageInfo* current = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_EQ(current->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(current->GetBlobFileMetaData(empty_file_number), nullptr); + ASSERT_EQ(current->GetBlobRelocationRouteTarget(origin_file_number), + relocation_file_number); + ASSERT_EQ(current->GetBlobRelocationRouteTarget(empty_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(Get(key), expected); + }; + check(); + // Reopen from incremental records, then from a MANIFEST snapshot, without + // the collected files (the DB may have deleted them already). + Close(); + for (uint64_t file_number : {origin_file_number, empty_file_number}) { + const Status deleted = env_->DeleteFile(BlobFileName(dbname_, file_number)); + ASSERT_TRUE(deleted.ok() || deleted.IsNotFound()) << deleted.ToString(); + } + Reopen(options); + check(); + VersionEdit snapshot; + snapshot.SetColumnFamily(/*column_family_id=*/0); + ASSERT_OK(ApplyVersionEdit(&snapshot, /*new_descriptor_log=*/true)); + Close(); + Reopen(options); + check(); + + // SSTs still hold the reference with the collected file's number as its + // generation number. A compaction that drops the reference finds a route for + // that number in its input Version. So it writes no garbage record + // (BlobFileGarbage) for the collected file, and the compaction succeeds. + ASSERT_OK(Put(key, "overwritten")); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_EQ(Get(key), "overwritten"); +} + +TEST_F(DBBlobBasicTest, CompactionPickedBeforeRetirementInstallsAfterIt) { + // A compaction is picked while blob file F is live, and it drops a reference + // to F. Before the compaction installs, an indirect blob GC install edit + // records the blob file retirement of F and routes F to relocation file R. + // The compaction's garbage for F is ignored, and its install succeeds. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + const std::string key = "key"; + const std::string expected = "relocated-value"; + ASSERT_OK(Put(key, expected)); + ASSERT_OK(Put("other", "other-value")); + ASSERT_OK(Flush()); + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + const uint64_t other_value_offset = original_value_offset + expected.size() + + BlobLogRecord::kHeaderSize + + std::string("other").size(); + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{origin_file_number, original_value_offset, expected, kNoCompression}, + {origin_file_number, other_value_offset, "other-value", kNoCompression}}, + &metadata, &relocation_file_size); + + // The compaction drops the old value of "key". + ASSERT_OK(Put(key, "overwritten")); + ASSERT_OK(Flush()); + port::Mutex sync_mutex; + port::CondVar sync_cv(&sync_mutex); + bool picked = false; + bool installed = false; + SyncPoint::GetInstance()->SetCallBack("CompactionJob::Run():Start", + [&](void*) { + MutexLock lock(&sync_mutex); + picked = true; + sync_cv.SignalAll(); + while (!installed) { + sync_cv.Wait(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + // Force a rewrite, so the compaction runs and drops the old value. + std::thread compaction([&]() { + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + }); + { + MutexLock lock(&sync_mutex); + while (!picked) { + sync_cv.Wait(); + } + } + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(origin_file_number, relocation_file_number)); + install.RetireBlobFile(origin_file_number); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + { + MutexLock lock(&sync_mutex); + installed = true; + sync_cv.SignalAll(); + } + compaction.join(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_EQ(Get(key), "overwritten"); + ASSERT_EQ(Get("other"), "other-value"); + ASSERT_EQ( + cfd->current()->storage_info()->GetBlobFileMetaData(origin_file_number), + nullptr); +} + +TEST_F(DBBlobBasicTest, BestEffortsRecoveryRollsBackRetirement) { + // An indirect blob GC install edit records the blob file retirement of blob + // file F and routes F to relocation file R. When R is missing at a + // best-efforts open, recovery drops the whole edit: F is live again and reads + // use it. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + const std::string key = "key"; + const std::string expected = "value-in-origin"; + ASSERT_OK(Put(key, expected)); + ASSERT_OK(Flush()); + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const uint64_t origin_file_number = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{origin_file_number, original_value_offset, expected, kNoCompression}}, + &metadata, &relocation_file_size); + // Keep a copy of F: the DB may delete it once its blob file retirement is + // installed. + const std::string origin_path = BlobFileName(dbname_, origin_file_number); + const std::string origin_copy = origin_path + ".copy"; + CopyFile(origin_path, origin_copy); + + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(origin_file_number, relocation_file_number)); + install.RetireBlobFile(origin_file_number); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + Close(); + + const Status deleted = env_->DeleteFile(origin_path); + ASSERT_TRUE(deleted.ok() || deleted.IsNotFound()) << deleted.ToString(); + ASSERT_OK(env_->RenameFile(origin_copy, origin_path)); + ASSERT_OK(env_->DeleteFile(BlobFileName(dbname_, relocation_file_number))); + options.best_efforts_recovery = true; + Reopen(options); + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(origin_file_number), nullptr); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(origin_file_number), + kInvalidBlobFileNumber); + ASSERT_EQ(storage->GetBlobRelocationFileMetaData(relocation_file_number), + nullptr); + ASSERT_EQ(Get(key), expected); +} + +TEST_F(DBBlobBasicTest, PinnedOldVersionReadsReplacedBlobMetadata) { + // The file table of a Version does not own its blob file entries; the + // Version's blob file vector does. An iterator pins an old Version; later + // Versions replace the metadata of its blob file (new links) and the + // Versions in between are freed. The pinned Version still reads through + // its own entries (checked by the sanitizers). + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + ASSERT_OK(Put("a", "old-a")); + ASSERT_OK(Put("b", "value-b")); + ASSERT_OK(Flush()); + const uint64_t blob_file_number = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + // The iterator does not see "old-a", so a compaction can drop it and add + // garbage to the blob file. + ASSERT_OK(Put("a", "new-a")); + std::unique_ptr iterator(db_->NewIterator(ReadOptions())); + // Keep only a copy of the links: holding the metadata here would keep it + // alive. + const BlobFileMetaData::LinkedSsts pinned_links = + dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetBlobFileMetaData(blob_file_number) + ->GetLinkedSsts(); + // Each forced compaction rewrites every SST, so each new Version links the + // blob file from a new SST. + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + ASSERT_OK(Put("c" + std::to_string(i), "value-c")); + } + const std::shared_ptr current_metadata = + dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetBlobFileMetaData(blob_file_number); + // Compaction rewrote the SSTs that link the blob file, so later Versions + // have new metadata for it. + ASSERT_NE(current_metadata, nullptr); + ASSERT_NE(current_metadata->GetLinkedSsts(), pinned_links); + + iterator->Seek("b"); + ASSERT_TRUE(iterator->Valid()); + ASSERT_EQ(iterator->key(), "b"); + ASSERT_EQ(iterator->value(), "value-b"); + ASSERT_OK(iterator->status()); +} + +TEST_F(DBBlobBasicTest, + BestEffortsRecoveryMissingRelocationFileUsesPriorVersion) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + + Reopen(options); + const std::string key = "key"; + const std::string expected = "original-value-remains-readable"; + ASSERT_OK(Put(key, expected)); + ASSERT_OK(Flush()); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + const uint64_t origin_file_number = + storage->GetBlobFiles().front()->GetBlobFileNumber(); + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + + dbfull()->TEST_LockMutex(); + const uint64_t routed_generation = version_set->NewFileNumber(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{origin_file_number, original_value_offset, expected, kNoCompression}}, + &metadata, &relocation_file_size); + + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size)); + install.SetBlobRelocationRoute( + BlobRelocationRoute(routed_generation, relocation_file_number)); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + Close(); + + ASSERT_OK(env_->DeleteFile(BlobFileName(dbname_, relocation_file_number))); + options.best_efforts_recovery = true; + Reopen(options); + ASSERT_EQ(Get(key), expected); + + storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_EQ(storage->GetBlobRelocationFileMetaData(relocation_file_number), + nullptr); + ASSERT_EQ(storage->GetBlobRelocationRouteTarget(routed_generation), + kInvalidBlobFileNumber); +} + +TEST_F(DBBlobBasicTest, DroppedRelocationFileIsPurgedAndEvicted) { + // A relocation file that no Version owns is deleted by PurgeObsoleteFiles(), + // which also evicts its cached reader. The file has no blob log footer, so + // the purge must recognize it by its prefix. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + Reopen(options); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobPhysicalFile file; + ASSERT_NO_FATAL_FAILURE(InstallOneRelocationFile(options, &file)); + const uint64_t file_number = file.open_info.file_number; + { + CacheHandleGuard reader; + ASSERT_OK(cfd->blob_file_cache()->GetRelocationFileReader( + ReadOptions(), file, cfd->GetLatestMutableCFOptions(), &reader)); + } + const auto reader_is_cached = [&]() { + Cache::Handle* const handle = + dbfull()->TEST_table_cache()->Lookup(GetSliceForKey(&file_number)); + if (handle == nullptr) { + return false; + } + dbfull()->TEST_table_cache()->Release(handle); + return true; + }; + ASSERT_TRUE(reader_is_cached()); + + VersionEdit deletion; + deletion.SetColumnFamily(cfd->GetID()); + deletion.DeleteBlobRelocationFile(file_number); + ASSERT_OK(ApplyVersionEdit(&deletion, /*new_descriptor_log=*/false)); + dbfull()->TEST_DeleteObsoleteFiles(); + EXPECT_FALSE(reader_is_cached()); + EXPECT_TRUE( + env_->FileExists(BlobFileName(dbname_, file_number)).IsNotFound()); +} + +TEST_F(DBBlobBasicTest, RelocationFileListedByOneColumnFamilyIsNotPurged) { + // The obsolete file purge is DB-wide: it keeps a .blob file while any column + // family's Version lists the file as a blob file or as a relocation file + // (Version::RemoveLiveFiles()). This test checks that a relocation file of + // column family "one" stays on disk and still serves reads when column + // family "two" puts the same file number on the obsolete list. + // + // Setup: "one" writes a value to blob file B. The test writes relocation + // file R with that value and installs it in "one" like indirect blob GC + // does: add R, route B to R, and retire B. Reads in "one" now go to R. + // + // Checks: + // 1. "two" adds R and then deletes R. This destroys the relocation file + // metadata of "two", so R goes on the obsolete file list. The purge + // after the next flush checks each listed file against all Versions + // (VersionSet::RemoveLiveFiles()) and keeps R. + // 2. VersionSet::RemoveLiveFiles() drops a candidate for R, the case of a + // column family that never listed R. + // 3. A full-scan purge (DBImpl::TEST_DeleteObsoleteFiles() and the purge at + // open) uses the live file list (VersionSet::AddLiveFiles()) and keeps R + // too. + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + DestroyAndReopen(options); + CreateAndReopenWithCF({"one", "two"}, options); + const std::string key = "key"; + const std::string expected = "relocated-value"; + ASSERT_OK(Put(1, key, expected)); + ASSERT_OK(Flush(1)); + // A flush of "two" sets the format fence of "two". + ASSERT_OK(Put(2, "other-key", "other-value")); + ASSERT_OK(Flush(2)); + + VersionSet* const version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* const one_cfd = + static_cast(handles_[1])->cfd(); + ColumnFamilyData* const two_cfd = + static_cast(handles_[2])->cfd(); + ASSERT_EQ(one_cfd->current()->storage_info()->GetBlobFiles().size(), 1U); + const uint64_t origin_file_number = one_cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + const uint64_t original_value_offset = + BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key.size(); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, one_cfd->GetID(), + relocation_file_number, + {{origin_file_number, original_value_offset, expected, kNoCompression}}, + &metadata, &relocation_file_size); + const BlobRelocationFileAddition addition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size); + + VersionEdit install; + install.SetColumnFamily(one_cfd->GetID()); + install.AddBlobRelocationFile(addition); + install.SetBlobRelocationRoute( + BlobRelocationRoute(origin_file_number, relocation_file_number)); + install.RetireBlobFile(origin_file_number); + ASSERT_OK(ApplyVersionEdit(one_cfd, &install, /*new_descriptor_log=*/false)); + ASSERT_EQ(Get(1, key), expected); + + const std::string relocation_path = + BlobFileName(dbname_, relocation_file_number); + + // Check 1: "two" lists R, then deletes R. + { + VersionEdit two_addition; + two_addition.SetColumnFamily(two_cfd->GetID()); + two_addition.AddBlobRelocationFile(addition); + ASSERT_OK(ApplyVersionEdit(two_cfd, &two_addition, + /*new_descriptor_log=*/false)); + VersionEdit two_deletion; + two_deletion.SetColumnFamily(two_cfd->GetID()); + two_deletion.DeleteBlobRelocationFile(relocation_file_number); + ASSERT_OK(ApplyVersionEdit(two_cfd, &two_deletion, + /*new_descriptor_log=*/false)); + } + ASSERT_EQ(two_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + relocation_file_number), + nullptr); + // The flush job runs the purge that is not a full scan, and + // TEST_WaitForBackgroundWork() waits until that purge is done. + ASSERT_OK(Put(0, "flush-key", "flush-value")); + ASSERT_OK(Flush(0)); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + ASSERT_OK(env_->FileExists(relocation_path)); + ASSERT_NE(one_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + relocation_file_number), + nullptr); + ASSERT_EQ(Get(1, key), expected); + + // Check 2: a candidate for R from a column family that never listed R. + { + std::vector sst_candidates; + std::vector blob_candidates; + blob_candidates.emplace_back(relocation_file_number, dbname_); + dbfull()->TEST_LockMutex(); + version_set->RemoveLiveFiles(sst_candidates, blob_candidates); + dbfull()->TEST_UnlockMutex(); + ASSERT_TRUE(blob_candidates.empty()); + } + + // Check 3: the full-scan purges keep R, and after the reopen the value reads + // from R with a new reader. + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_OK(env_->FileExists(relocation_path)); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "one", "two"}, options); + ASSERT_OK(env_->FileExists(relocation_path)); + ASSERT_EQ(Get(1, key), expected); +} + +TEST_F(DBBlobBasicTest, KeptObsoleteRelocationFileReaderIsEvicted) { + // The purge keeps an obsolete relocation file when it cannot read the file's + // prefix (here an injected I/O error). The file has then left the obsolete + // list, but its cached reader is still evicted. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + Reopen(options); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobPhysicalFile file; + ASSERT_NO_FATAL_FAILURE(InstallOneRelocationFile(options, &file)); + const uint64_t file_number = file.open_info.file_number; + { + CacheHandleGuard reader; + ASSERT_OK(cfd->blob_file_cache()->GetRelocationFileReader( + ReadOptions(), file, cfd->GetLatestMutableCFOptions(), &reader)); + } + const auto reader_is_cached = [&]() { + Cache::Handle* const handle = + dbfull()->TEST_table_cache()->Lookup(GetSliceForKey(&file_number)); + if (handle == nullptr) { + return false; + } + dbfull()->TEST_table_cache()->Release(handle); + return true; + }; + ASSERT_TRUE(reader_is_cached()); + + SyncPoint::GetInstance()->SetCallBack( + "ShouldKeepFooterlessBlobFile:PrefixRead", [](void* arg) { + *static_cast(arg) = IOStatus::IOError("injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + VersionEdit deletion; + deletion.SetColumnFamily(cfd->GetID()); + deletion.DeleteBlobRelocationFile(file_number); + ASSERT_OK(ApplyVersionEdit(&deletion, /*new_descriptor_log=*/false)); + dbfull()->TEST_DeleteObsoleteFiles(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + EXPECT_OK(env_->FileExists(BlobFileName(dbname_, file_number))); + EXPECT_FALSE(reader_is_cached()); +} + +TEST_F(DBBlobBasicTest, RelocationPinDoesNotBlockVersionRelease) { + // A relocation file open with pin_blob_relocation_location_blocks holds the + // stripe mutex of its file in the blob file cache while it reads the + // location blocks. The release of a Version that drops the same file runs + // the file's deleter under the DB mutex. The deleter must not wait for that + // stripe mutex: here the open waits inside the pin until the release is + // done, so a deleter that waited would never finish. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + Reopen(options); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobPhysicalFile file; + ASSERT_NO_FATAL_FAILURE(InstallOneRelocationFile(options, &file)); + const uint64_t relocation_file_number = file.open_info.file_number; + std::shared_ptr installed = + cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + relocation_file_number); + ASSERT_NE(installed, nullptr); + // The deleter runs when the last owner of the metadata goes away, so the + // test keeps only a weak pointer. + const std::weak_ptr weak_installed = installed; + installed.reset(); + + std::promise pin_started; + std::promise release_done; + std::shared_future release_done_future = release_done.get_future(); + SyncPoint::GetInstance()->SetCallBack( + "BlobRelocationFileReader::Open:PinLocationBlocks", [&](void*) { + pin_started.set_value(); + release_done_future.wait(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + MutableCFOptions pin_options(options); + pin_options.pin_blob_relocation_location_blocks = true; + Status open_status; + std::thread opener([&]() { + CacheHandleGuard reader; + open_status = cfd->blob_file_cache()->GetRelocationFileReader( + ReadOptions(), file, pin_options, &reader); + }); + pin_started.get_future().wait(); + + // Drop the file from the current Version. No SuperVersion holds the Version + // that added the file, so LogAndApply() releases it under the DB mutex when + // it replaces it. + VersionEdit deletion; + deletion.SetColumnFamily(cfd->GetID()); + deletion.DeleteBlobRelocationFile(relocation_file_number); + const Status deletion_status = + ApplyVersionEdit(&deletion, /*new_descriptor_log=*/false); + release_done.set_value(); + opener.join(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_OK(deletion_status); + ASSERT_OK(open_status); + EXPECT_TRUE(weak_installed.expired()); +} + +TEST_F(DBBlobBasicTest, BlobRelocationDeletionIsElidedFromManifestSnapshot) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + Reopen(options); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* cfd = version_set->GetColumnFamilySet()->GetDefault(); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + Options relocation_options = options; + relocation_options.cf_paths.clear(); + relocation_options.cf_paths.emplace_back(dbname_, /*target_size=*/0); + BlockBasedTableOptions table_options; + relocation_options.table_factory.reset( + NewBlockBasedTableFactory(table_options)); + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + BlobRelocationFileMetadata metadata; + uint64_t relocation_file_size = 0; + WriteRelocationFileForDBTest( + immutable_options, mutable_options, table_options, cfd->GetID(), + relocation_file_number, + {{/*origin_file_number=*/1, /*original_value_offset=*/11, "payload", + kNoCompression}}, + &metadata, &relocation_file_size); + + const BlobRelocationFileAddition addition( + relocation_file_number, relocation_file_size, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + metadata.encoded_directory, metadata.tail_size); + VersionEdit install; + install.SetColumnFamily(cfd->GetID()); + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(addition); + ASSERT_OK(ApplyVersionEdit(&install, /*new_descriptor_log=*/false)); + + VersionEdit retirement; + retirement.SetColumnFamily(cfd->GetID()); + retirement.DeleteBlobRelocationFile(relocation_file_number); + ASSERT_OK(ApplyVersionEdit(&retirement, /*new_descriptor_log=*/false)); + // Each LogAndApply builds on the current Version, which no longer lists the + // file. So a repeated deletion in a later LogAndApply names no live file. + const auto expect_no_live_file = [&]() { + const Status status = + ApplyVersionEdit(&retirement, /*new_descriptor_log=*/false); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << status.ToString(); + }; + expect_no_live_file(); + + VersionEdit rotate; + rotate.SetColumnFamily(cfd->GetID()); + ASSERT_OK(ApplyVersionEdit(&rotate, /*new_descriptor_log=*/true)); + Close(); + Reopen(options); + + cfd = dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + relocation_file_number), + nullptr); + expect_no_live_file(); +} + +TEST_F(DBBlobBasicTest, BlobIndirectionPrepopulatesOnlyStableCacheKey) { + Options options = GetDefaultOptions(); + std::shared_ptr blob_cache = NewLRUCache(1 << 20); + std::shared_ptr block_cache = NewLRUCache(1 << 20); + options.blob_cache = blob_cache; + BlockBasedTableOptions table_options; + table_options.block_cache = block_cache; + table_options.cache_index_and_filter_blocks = false; + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlobCache, + {/* charged = */ CacheEntryRoleOptions::Decision::kEnabled}}); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly; + options.statistics = CreateDBStatistics(); + + Reopen(options); + const std::string expected = "prepopulated-stable-value"; + ASSERT_OK(Put("key", expected)); + const size_t block_cache_usage_before_flush = block_cache->GetUsage(); + ASSERT_OK(Flush()); + ASSERT_EQ(options.statistics->getTickerCount(BLOB_DB_CACHE_ADD), 1U); + EXPECT_GT(block_cache->GetUsage(), block_cache_usage_before_flush); + + std::atomic blob_file_opens{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::OpenFile:NewRandomAccessFile", + [&blob_file_opens](void* /*arg*/) { ++blob_file_opens; }); + SyncPoint::GetInstance()->EnableProcessing(); + PinnableSlice value; + const Status status = + db_->Get(ReadOptions(), db_->DefaultColumnFamily(), "key", &value); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); - void RecoverMissingNonOldestBlob(const Options& options, - bool preserve_older_value, - uint64_t* missing_blob_file) { - ASSERT_NO_FATAL_FAILURE(CreateSstWithMissingBlobFile( - options, preserve_older_value, /*delete_oldest=*/false, - missing_blob_file)); - Options recovery_options = options; - recovery_options.best_efforts_recovery = true; - Reopen(recovery_options); - const std::vector live_blob_files = GetBlobFileNumbers(); - ASSERT_EQ(live_blob_files.size(), 1); - ASSERT_NE(live_blob_files.front(), *missing_blob_file); - ASSERT_EQ(TotalTableFiles(), 1); - ASSERT_NO_FATAL_FAILURE(AssertBlobReference("b", *missing_blob_file)); - ASSERT_NO_FATAL_FAILURE(AssertBlobReference("d", *missing_blob_file)); - } + ASSERT_OK(status); + EXPECT_EQ(value, expected); + EXPECT_EQ(blob_file_opens.load(), 0); + EXPECT_EQ(options.statistics->getTickerCount(BLOB_DB_CACHE_MISS), 0U); + EXPECT_EQ(options.statistics->getTickerCount(BLOB_DB_CACHE_HIT), 1U); +} - std::vector GetDroppedBlobFiles() { - return dbfull() - ->GetVersionSet() - ->GetColumnFamilySet() - ->GetDefault() - ->current() - ->storage_info() - ->GetDroppedBlobFiles(); - } +TEST_F(DBBlobBasicTest, BlobIndirectionCacheHitNeedsNoRouteLookup) { + // A read of an indirect value looks up the stable blob cache before the + // route and file lookup (ResolveBlobPhysicalFile()). A miss resolves the + // physical file once; a hit, from Get or MultiGet, resolves nothing. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_cache = NewLRUCache(1 << 20); + Reopen(options); + ASSERT_OK(Put("key", "stable-value")); + ASSERT_OK(Flush()); + std::atomic resolves{0}; + SyncPoint::GetInstance()->SetCallBack("Version::ResolveBlobPhysicalFile", + [&](void*) { ++resolves; }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + ASSERT_EQ(Get("key"), "stable-value"); + EXPECT_EQ(resolves.load(), 1); + resolves = 0; + ASSERT_EQ(Get("key"), "stable-value"); + EXPECT_EQ(resolves.load(), 0); + const Slice key("key"); + PinnableSlice value; + Status status; + db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), 1, &key, &value, + &status); + ASSERT_OK(status); + EXPECT_EQ(value, "stable-value"); + EXPECT_EQ(resolves.load(), 0); +} - static Options RepairOptions(Options options) { - options.best_efforts_blob_db_repair = true; - return options; +TEST_F(DBBlobBasicTest, BlobIndirectionMultiGetReadsOnlyCacheMisses) { + // MultiGet looks up every indirect value in the stable blob cache first. A + // hit is filled at once; only the misses are grouped by file and read. Each + // key is looked up in the cache once. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.blob_cache = NewLRUCache(1 << 20); + options.statistics = CreateDBStatistics(); + Reopen(options); + constexpr int kKeys = 8; + std::vector keys; + std::vector expected; + for (int i = 0; i < kKeys; ++i) { + keys.push_back("key" + std::to_string(i)); + expected.push_back("stable-value-" + std::to_string(i)); + ASSERT_OK(Put(keys.back(), expected.back())); } - - // Returns the keys and values a forward and a reverse scan read, which must - // agree. - std::string ScanAll(ReadOptions read_options = ReadOptions()) { - std::string forward; - std::string reverse; - { - std::unique_ptr iter(db_->NewIterator(read_options)); - for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - EXPECT_TRUE(iter->PrepareValue()); - forward += - iter->key().ToString() + "=" + iter->value().ToString() + ";"; - } - EXPECT_OK(iter->status()); - for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { - EXPECT_TRUE(iter->PrepareValue()); - reverse.insert( - 0, iter->key().ToString() + "=" + iter->value().ToString() + ";"); - } - EXPECT_OK(iter->status()); - } - EXPECT_EQ(forward, reverse); - return forward; + ASSERT_OK(Flush()); + // Cache the even keys. + for (int i = 0; i < kKeys; i += 2) { + ASSERT_EQ(Get(keys[i]), expected[i]); } - // Runs best-efforts BlobDB repair on a DB whose newer blob file is missing - // (see CreateSstWithMissingBlobFile()) and checks what it recorded. - void RepairMissingNonOldestBlob(const Options& options, - uint64_t* missing_blob_file) { - ASSERT_NO_FATAL_FAILURE(CreateSstWithMissingBlobFile( - options, /*preserve_older_value=*/false, /*delete_oldest=*/false, - missing_blob_file)); - Reopen(RepairOptions(options)); - ASSERT_EQ(GetDroppedBlobFiles(), std::vector{*missing_blob_file}); - ASSERT_EQ(TotalTableFiles(), 1); + Statistics* const statistics = options.statistics.get(); + const uint64_t hits = statistics->getTickerCount(BLOB_DB_CACHE_HIT); + const uint64_t misses = statistics->getTickerCount(BLOB_DB_CACHE_MISS); + std::atomic file_reads{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlobFileReader::MultiGetBlob:Requests", [&](void* arg) { + file_reads += static_cast*>(arg)->size(); + }); + SyncPoint::GetInstance()->EnableProcessing(); + std::vector key_slices(keys.begin(), keys.end()); + std::vector values(kKeys); + std::vector statuses(kKeys); + ReadOptions read_options; + read_options.fill_cache = false; + db_->MultiGet(read_options, db_->DefaultColumnFamily(), kKeys, + key_slices.data(), values.data(), statuses.data()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + for (int i = 0; i < kKeys; ++i) { + ASSERT_OK(statuses[i]) << i; + EXPECT_EQ(values[i], expected[i]) << i; } -}; + EXPECT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_HIT), hits + kKeys / 2); + EXPECT_EQ(statistics->getTickerCount(BLOB_DB_CACHE_MISS), misses + kKeys / 2); + EXPECT_EQ(file_reads.load(), static_cast(kKeys / 2)); +} -class BlobFileChecksumCapturingFS : public FileSystemWrapper { - public: - explicit BlobFileChecksumCapturingFS(const std::shared_ptr& base) - : FileSystemWrapper(base) {} +TEST_F(DBBlobBasicTest, BlobIndirectionWritesCompleteEmptySstMetadata) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 1024; - static const char* kClassName() { return "BlobFileChecksumCapturingFS"; } - const char* Name() const override { return kClassName(); } + Reopen(options); + ASSERT_OK(Put("key", "inline")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + ASSERT_TRUE(dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->HasBlobIndirectionFormat()); - IOStatus NewRandomAccessFile(const std::string& fname, - const FileOptions& opts, - std::unique_ptr* result, - IODebugContext* dbg) override { - if (fname.find(".blob") != std::string::npos) { - std::lock_guard lock(mu_); - file_checksum_ = opts.file_checksum; - file_checksum_func_name_ = opts.file_checksum_func_name; - ++capture_count_; - } - return target()->NewRandomAccessFile(fname, opts, result, dbg); - } + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1); + const auto& user_properties = + properties.begin()->second->user_collected_properties; + const auto descriptor_it = + user_properties.find(kBlobReferenceDescriptorPropertyName); + ASSERT_NE(descriptor_it, user_properties.end()); + + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(descriptor_it->second, + &descriptor)); + EXPECT_TRUE(descriptor.IsEmpty()); +} - std::string GetFileChecksum() { - std::lock_guard lock(mu_); - return file_checksum_; - } +TEST_F(DBBlobBasicTest, BlobIndirectionIsDisabledByDefault) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; - std::string GetFileChecksumFuncName() { - std::lock_guard lock(mu_); - return file_checksum_func_name_; - } + Reopen(options); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + ASSERT_EQ(Get("key"), "blob_value"); + + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_FALSE(storage->HasBlobIndirectionFormat()); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + ASSERT_FALSE(storage->GetBlobFiles().front()->IsIndirectOrigin()); - int GetCaptureCount() { - std::lock_guard lock(mu_); - return capture_count_; - } + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1); + const auto& user_properties = + properties.begin()->second->user_collected_properties; + EXPECT_EQ(user_properties.count(kBlobReferenceDescriptorPropertyName), 0); +} - private: - std::mutex mu_; - std::string file_checksum_; - std::string file_checksum_func_name_; - int capture_count_ = 0; -}; +TEST_F(DBBlobBasicTest, BlobIndirectionCanBeEnabledDynamically) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; -TEST_F(DBBlobBasicTest, GetBlob) { + Reopen(options); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_indirection", "true"}})); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1); + const auto& user_properties = + properties.begin()->second->user_collected_properties; + EXPECT_EQ(user_properties.count(kBlobReferenceDescriptorPropertyName), 0); + EXPECT_EQ(user_properties.count(kBlobReferenceClassificationPropertyName), 1); + + // Reopen with the option off (only SetOptions() turned it on). The per-CF + // format fence is permanent, so it stays set, and the blob file stays an + // indirect origin. + Close(); + Reopen(options); + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_TRUE(storage->HasBlobIndirectionFormat()); + ASSERT_EQ(storage->GetBlobFiles().size(), 1U); + ASSERT_TRUE(storage->GetBlobFiles().front()->IsIndirectOrigin()); + ASSERT_EQ(Get("key"), "blob_value"); +} + +TEST_F(DBBlobBasicTest, BlobIndirectionFormatFenceSurvivesActivatedCfDrop) { Options options = GetDefaultOptions(); options.enable_blob_files = true; options.min_blob_size = 0; + Reopen(options); + ColumnFamilyHandle* activated = nullptr; + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(options), "activated", + &activated)); + ASSERT_NE(activated, nullptr); + ASSERT_OK(db_->SetOptions(activated, {{"enable_blob_indirection", "true"}})); + ASSERT_OK(db_->Put(WriteOptions(), activated, "key", "blob_value")); + ASSERT_OK(db_->Flush(FlushOptions(), activated)); + ASSERT_OK(db_->DropColumnFamily(activated)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(activated)); + + VersionEdit rotate; + rotate.SetColumnFamily(/*column_family_id=*/0); + ASSERT_OK(ApplyVersionEdit(&rotate, /*new_descriptor_log=*/true)); + Close(); Reopen(options); - constexpr char key[] = "key"; - constexpr char blob_value[] = "blob_value"; + const VersionStorageInfo* storage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + ASSERT_FALSE(storage->HasBlobIndirectionFormat()); + ASSERT_EQ(dbfull()->GetVersionSet()->TEST_GetDbBlobIndirectionFormatVersion(), + kBlobIndirectionManifestFormatVersion); +} - ASSERT_OK(Put(key, blob_value)); +TEST_F(DBBlobBasicTest, + BlobIndirectionSnapshotDoesNotActivateUnrelatedColumnFamily) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + Reopen(options); + ASSERT_OK(Put("legacy", "legacy_blob_value")); ASSERT_OK(Flush()); - ASSERT_EQ(Get(key), blob_value); + ColumnFamilyHandle* activated = nullptr; + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(options), "activated", + &activated)); + ASSERT_OK(db_->SetOptions(activated, {{"enable_blob_indirection", "true"}})); + ASSERT_OK( + db_->Put(WriteOptions(), activated, "feature", "feature_blob_value")); + ASSERT_OK(db_->Flush(FlushOptions(), activated)); + + VersionEdit rotate; + rotate.SetColumnFamily(/*column_family_id=*/0); + ASSERT_OK(ApplyVersionEdit(&rotate, /*new_descriptor_log=*/true)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(activated)); + ASSERT_OK(TryReopenWithColumnFamilies({"default", "activated"}, options)); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* default_cfd = + version_set->GetColumnFamilySet()->GetDefault(); + ColumnFamilyData* activated_cfd = + version_set->GetColumnFamilySet()->GetColumnFamily("activated"); + ASSERT_NE(activated_cfd, nullptr); + ASSERT_FALSE( + default_cfd->current()->storage_info()->HasBlobIndirectionFormat()); + ASSERT_TRUE( + activated_cfd->current()->storage_info()->HasBlobIndirectionFormat()); + ASSERT_EQ(version_set->TEST_GetDbBlobIndirectionFormatVersion(), + kBlobIndirectionManifestFormatVersion); + ASSERT_EQ(Get("legacy"), "legacy_blob_value"); + std::string feature_value; + ASSERT_OK(db_->Get(ReadOptions(), handles_[1], "feature", &feature_value)); + ASSERT_EQ(feature_value, "feature_blob_value"); +} - // Try again with no I/O allowed. The table and the necessary blocks should - // already be in their respective caches; however, the blob itself can only be - // read from the blob file, so the read should return Incomplete. - ReadOptions read_options; - read_options.read_tier = kBlockCacheTier; +TEST_F(DBBlobBasicTest, BlobRelocationRecordsApplyToOwnColumnFamily) { + // VersionBuilder applies relocation records to the state of the edit's own + // column family. Column family "first" adds a relocation file. A deletion of + // that file by column family "second" names no live file of "second", so + // LogAndApply rejects it. An addition of the same file number by "second" is + // accepted: no check spans column families. A valid writer never does this, + // because new file numbers come from one DB-wide counter. The obsolete file + // purge keeps such a file while any column family lists it + // (RelocationFileListedByOneColumnFamilyIsNotPurged). + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + Reopen(options); - PinnableSlice result; - ASSERT_TRUE(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result) - .IsIncomplete()); + ColumnFamilyHandle* first_handle = nullptr; + ColumnFamilyHandle* second_handle = nullptr; + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(options), "first", + &first_handle)); + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(options), "second", + &second_handle)); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* first_cfd = + version_set->GetColumnFamilySet()->GetColumnFamily(first_handle->GetID()); + ColumnFamilyData* second_cfd = + version_set->GetColumnFamilySet()->GetColumnFamily( + second_handle->GetID()); + ASSERT_NE(first_cfd, nullptr); + ASSERT_NE(second_cfd, nullptr); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/100, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/100)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + + VersionEdit first_addition; + first_addition.SetColumnFamily(first_cfd->GetID()); + first_addition.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + first_addition.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, /*file_size=*/4096, + /*checksum_method=*/"crc32c", /*checksum_value=*/"checksum", + encoded_directory, /*tail_size=*/1)); + ASSERT_OK(ApplyVersionEdit(first_cfd, &first_addition, + /*new_descriptor_log=*/false)); + + VersionEdit other_cf_deletion; + other_cf_deletion.SetColumnFamily(second_cfd->GetID()); + other_cf_deletion.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + other_cf_deletion.DeleteBlobRelocationFile(relocation_file_number); + const Status deletion_status = ApplyVersionEdit( + second_cfd, &other_cf_deletion, /*new_descriptor_log=*/false); + ASSERT_TRUE(deletion_status.IsCorruption()) << deletion_status.ToString(); + ASSERT_NE(deletion_status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << deletion_status.ToString(); + + VersionEdit same_number_addition; + same_number_addition.SetColumnFamily(second_cfd->GetID()); + same_number_addition.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + same_number_addition.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, /*file_size=*/4096, + /*checksum_method=*/"crc32c", /*checksum_value=*/"checksum", + encoded_directory, /*tail_size=*/1)); + ASSERT_OK(ApplyVersionEdit(second_cfd, &same_number_addition, + /*new_descriptor_log=*/false)); + for (ColumnFamilyData* cfd : {first_cfd, second_cfd}) { + ASSERT_NE(cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + relocation_file_number), + nullptr); + } + + ASSERT_OK(db_->DropColumnFamily(first_handle)); + ASSERT_OK(db_->DropColumnFamily(second_handle)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(first_handle)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(second_handle)); +} + +TEST_F(DBBlobBasicTest, + BlobRelocationDeletionOfAnotherColumnFamilyIsRejectedDuringRecovery) { + // A MANIFEST record of column family "second" deletes a relocation file that + // only column family "first" lists. Replay applies the record to the state + // of "second", which has no such live file, so the reopen fails. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + Reopen(options); + + ColumnFamilyHandle* first_handle = nullptr; + ColumnFamilyHandle* second_handle = nullptr; + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(options), "first", + &first_handle)); + ASSERT_OK(db_->CreateColumnFamily(ColumnFamilyOptions(options), "second", + &second_handle)); + + VersionSet* version_set = dbfull()->GetVersionSet(); + ColumnFamilyData* first_cfd = + version_set->GetColumnFamilySet()->GetColumnFamily(first_handle->GetID()); + ColumnFamilyData* second_cfd = + version_set->GetColumnFamilySet()->GetColumnFamily( + second_handle->GetID()); + ASSERT_NE(first_cfd, nullptr); + ASSERT_NE(second_cfd, nullptr); + const uint32_t second_cf_id = second_handle->GetID(); + dbfull()->TEST_LockMutex(); + const uint64_t relocation_file_number = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/100, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/100)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + + VersionEdit first_addition; + first_addition.SetColumnFamily(first_cfd->GetID()); + first_addition.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + first_addition.AddBlobRelocationFile(BlobRelocationFileAddition( + relocation_file_number, /*file_size=*/4096, + /*checksum_method=*/"crc32c", /*checksum_value=*/"checksum", + encoded_directory, /*tail_size=*/1)); + ASSERT_OK(ApplyVersionEdit(first_cfd, &first_addition, + /*new_descriptor_log=*/false)); + + // Set the per-CF format fence on the second CF before adding the bad record. + // Recovery must then fail on the per-CF deletion check, not on a missing + // per-CF fence. + VersionEdit second_fence; + second_fence.SetColumnFamily(second_cfd->GetID()); + second_fence.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + ASSERT_OK(ApplyVersionEdit(second_cfd, &second_fence, + /*new_descriptor_log=*/false)); + + ASSERT_OK(db_->DestroyColumnFamilyHandle(first_handle)); + ASSERT_OK(db_->DestroyColumnFamilyHandle(second_handle)); + Close(); + + std::string manifest_path; + uint64_t manifest_file_number = 0; + const auto& fs = env_->GetFileSystem(); + ASSERT_OK(GetCurrentManifestPath(dbname_, fs.get(), /*is_retry=*/false, + &manifest_path, &manifest_file_number)); + ASSERT_GT(manifest_file_number, 0U); + uint64_t manifest_size = 0; + ASSERT_OK(fs->GetFileSize(manifest_path, IOOptions(), &manifest_size, + /*dbg=*/nullptr)); + std::unique_ptr manifest_file; + ASSERT_OK(fs->ReopenWritableFile(manifest_path, FileOptions(), &manifest_file, + /*dbg=*/nullptr)); + std::unique_ptr manifest_writer(new WritableFileWriter( + std::move(manifest_file), manifest_path, FileOptions())); + log::Writer log_writer(std::move(manifest_writer), /*log_number=*/0, + /*recycle_log_files=*/false, + /*manual_flush=*/false, kNoCompression, + /*track_and_verify_wals=*/false, + manifest_size % log::kBlockSize); + VersionEdit conflicting_tombstone; + conflicting_tombstone.SetColumnFamily(second_cf_id); + conflicting_tombstone.DeleteBlobRelocationFile(relocation_file_number); + std::string record; + ASSERT_TRUE(conflicting_tombstone.EncodeTo(&record)); + ASSERT_OK(log_writer.AddRecord(WriteOptions(), record)); + ASSERT_OK(log_writer.Close(WriteOptions())); + + const Status status = + TryReopenWithColumnFamilies({"default", "first", "second"}, options); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << status.ToString(); +} + +TEST_F(DBBlobBasicTest, BlobIndirectionTracksAllWideColumnBlobReferences) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 4; + + Reopen(options); + const WideColumns columns{{kDefaultWideColumnName, "default_blob"}, + {"inline", "x"}, + {"secondary", "secondary_blob"}}; + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "key", + columns)); + ASSERT_OK(Flush()); + + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), "key", + &result)); + ASSERT_EQ(result.columns(), columns); + ASSERT_OK(BuildExactSstAtLevel1()); + + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1); + const auto& user_properties = + properties.begin()->second->user_collected_properties; + const auto descriptor_it = + user_properties.find(kBlobReferenceDescriptorPropertyName); + ASSERT_NE(descriptor_it, user_properties.end()); + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(descriptor_it->second, + &descriptor)); + EXPECT_EQ(descriptor.reference_count, 2); + EXPECT_EQ(descriptor.stored_value_bytes, + strlen("default_blob") + strlen("secondary_blob")); +} + +TEST_F(DBBlobBasicTest, BlobIndirectionRejectsUnsupportedOptionCombinations) { + Options options = GetDefaultOptions(); + options.enable_blob_indirection = true; + ASSERT_TRUE(TryReopen(options).IsInvalidArgument()); + + options.enable_blob_files = true; + options.enable_blob_direct_write = true; + options.allow_concurrent_memtable_write = false; + ASSERT_TRUE(TryReopen(options).IsNotSupported()); + + options.enable_blob_direct_write = false; + options.table_factory.reset(NewPlainTableFactory()); + ASSERT_TRUE(TryReopen(options).IsNotSupported()); +} + +TEST_F(DBBlobBasicTest, + BlobIndirectionRejectsDynamicEnableForUnsupportedTableFactory) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.table_factory.reset(NewPlainTableFactory()); + options.prefix_extractor.reset(NewNoopTransform()); + Reopen(options); + + ASSERT_TRUE(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_indirection", "true"}}) + .IsNotSupported()); +} + +TEST_F(DBBlobBasicTest, + BlobIndirectionFailsExactCompactionWhenMetadataCannotBeCharged) { + LRUCacheOptions cache_options; + cache_options.capacity = 1; + cache_options.num_shard_bits = 0; + cache_options.strict_capacity_limit = true; + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(cache_options); + + Options options = GetDefaultOptions(); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + + Reopen(options); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + const Status status = BuildExactSstAtLevel1(); + EXPECT_TRUE(status.IsMemoryLimit()) << status.ToString(); +} + +TEST_F(DBBlobBasicTest, BlobReferenceCensusChargesItsOwnCacheRole) { + for (const bool charged : {true, false}) { + SCOPED_TRACE(charged); + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(64 << 20); + if (!charged) { + table_options.cache_usage_options + .options_overrides[CacheEntryRole::kBlobReferenceCensus] = { + CacheEntryRoleOptions::Decision::kDisabled}; + } + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + for (int file = 0; file < 4; ++file) { + for (int key = 0; key < 16; ++key) { + ASSERT_OK(Put("key" + std::to_string(file * 16 + key), + std::string(1024, 'v'))); + } + ASSERT_OK(Flush()); + } + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(GetCensusSnapshot(cfd), nullptr); + + // The census has its own line in the block cache entry stats. + std::map values; + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats, &values)); + const std::string census_key = BlockCacheEntryStatsMapKeys::UsedBytes( + CacheEntryRole::kBlobReferenceCensus); + ASSERT_EQ(values.count(census_key), 1); + const uint64_t census_bytes = std::stoull(values[census_key]); + if (charged) { + ASSERT_GT(census_bytes, 0); + } else { + ASSERT_EQ(census_bytes, 0); + } + ASSERT_EQ(Get("key0"), std::string(1024, 'v')); + } } TEST_F(DBBlobBasicTest, BlobFileChecksumInFileOptions) { @@ -3835,6 +13795,55 @@ TEST_F(DBBlobWithTimestampTest, HistoryCompactionPreservesHealthyBlobKey) { ASSERT_EQ(value, "healthy"); } +TEST_F(DBBlobWithTimestampTest, ExactReferenceBudgetGroupsAllTimestamps) { + Options options = GetDefaultOptions(); + options.create_if_missing = true; + options.enable_blob_files = options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + options.target_file_size_base = 1ULL << 30; + TestComparator comparator(Timestamp(0, 0).size()); + options.comparator = &comparator; + DestroyAndReopen(options); + const std::string value(4096, 'v'); + // The test sets the cap to 2 with the sync point + // BlockBasedTableBuilder::BlobReferenceLimit. Key A has two timestamped + // versions, so it fills the first output. B starts a new output and keeps all + // three of its timestamps there, past the cap. + ASSERT_OK(db_->Put(WriteOptions(), "A", Timestamp(1, 0), value)); + ASSERT_OK(db_->Put(WriteOptions(), "A", Timestamp(2, 0), value)); + for (int i = 1; i <= 3; ++i) { + ASSERT_OK(db_->Put(WriteOptions(), "B", Timestamp(i, 0), value)); + } + ASSERT_OK(Flush()); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("BlockBasedTableBuilder::BlobReferenceLimit", + [](void* arg) { *static_cast(arg) = 2; }); + sync_point->EnableProcessing(); + const Defer cleanup([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + CompactRangeOptions compact; + compact.change_level = true; + compact.target_level = 1; + compact.bottommost_level_compaction = BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact, nullptr, nullptr)); + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 2U); + for (int i = 1; i <= 3; ++i) { + const std::string timestamp = Timestamp(i, 0); + const Slice read_timestamp(timestamp); + ReadOptions read; + read.timestamp = &read_timestamp; + std::string result; + ASSERT_OK(db_->Get(read, "B", &result)); + ASSERT_EQ(result, value); + } +} + TEST_F(DBBlobWithTimestampTest, GetBlob) { Options options = GetDefaultOptions(); options.create_if_missing = true; @@ -5252,6 +15261,1082 @@ TEST_F(DBBlobBasicTest, GetApproximateSizesConservesMixedMetadataBudget) { ASSERT_GT(distribution_change, physical_blob_size / 100); } +TEST_F(DBBlobBasicTest, + ExactReferenceBudgetCutsEachOutputWithoutSplittingEntities) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + + const WideColumns columns{ + {"a", "value_a"}, {"b", "value_b"}, {"c", "value_c"}}; + for (int i = 0; i < 3; ++i) { + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), + "key" + std::to_string(i), columns)); + } + ASSERT_OK(Flush()); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + // Each entity fills the output, so the next key starts a new one. + sync_point->SetCallBack("BlockBasedTableBuilder::BlobReferenceLimit", + [](void* arg) { *static_cast(arg) = 3; }); + sync_point->EnableProcessing(); + const Defer cleanup([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(BuildExactSstAtLevel1()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto& files = cfd->current()->storage_info()->LevelFiles(1); + ASSERT_EQ(files.size(), 3U); + for (const FileMetaData* file : files) { + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + file->blob_reference_descriptor, &descriptor)); + ASSERT_EQ(descriptor.reference_count, 3U); + ASSERT_EQ(file->num_entries, 1U); + } + for (int i = 0; i < 3; ++i) { + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), + "key" + std::to_string(i), &result)); + ASSERT_EQ(result.columns(), columns); + } +} +TEST_F(DBBlobBasicTest, ExactReferenceBudgetKeepsOneOversizedEntity) { + // The limit is soft: one entity with more references than the limit stays + // whole in one output. + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + const WideColumns columns{ + {"a", "value_a"}, {"b", "value_b"}, {"c", "value_c"}}; + ASSERT_OK(db_->PutEntity(WriteOptions(), db_->DefaultColumnFamily(), "key", + columns)); + ASSERT_OK(Flush()); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("BlockBasedTableBuilder::BlobReferenceLimit", + [](void* arg) { *static_cast(arg) = 2; }); + sync_point->EnableProcessing(); + const Defer cleanup([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(BuildExactSstAtLevel1()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto& files = cfd->current()->storage_info()->LevelFiles(1); + ASSERT_EQ(files.size(), 1U); + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom( + files.front()->blob_reference_descriptor, &descriptor)); + ASSERT_EQ(descriptor.reference_count, 3U); + PinnableWideColumns result; + ASSERT_OK(db_->GetEntity(ReadOptions(), db_->DefaultColumnFamily(), "key", + &result)); + ASSERT_EQ(result.columns(), columns); +} + +TEST_F(DBBlobBasicTest, CoarseIntraL0CompactionUsesAgeBasedBlobGc) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = true; + options.blob_garbage_collection_age_cutoff = 1.0; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + // A bottommost compaction writes exact output. An exact L1 SST with the same + // key makes the L0 to L0 compaction below not bottommost, so its output is + // coarse. + ASSERT_OK(Put("key", "low")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + // The blob file of the L0 SST is the newest one. + const uint64_t original_blob = cfd->current() + ->storage_info() + ->GetBlobFiles() + .back() + ->GetBlobFileNumber(); + const FileMetaData* const sst = + cfd->current()->storage_info()->LevelFiles(0).front(); + const std::string input = TableFileName( + cfd->ioptions().cf_paths, sst->fd.GetNumber(), sst->fd.GetPathId()); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), {input}, 0)); + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + ASSERT_EQ(storage->LevelFiles(0).size(), 1U); + ASSERT_TRUE( + storage->LevelFiles(0).front()->blob_reference_descriptor.empty()); + ASSERT_GT( + storage->LevelFiles(0).front()->oldest_coarse_indirect_origin_file_number, + original_blob); + ASSERT_TRUE(storage->GetBlobRelocationFiles().empty()); + ASSERT_EQ(Get("key"), "blob_value"); +} + +TEST_F(DBBlobBasicTest, RelocatingOldestRootPreservesUnlinkedIndirectOrigins) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + Reopen(options); + + const std::string value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("old" + std::to_string(i), value)); + } + ASSERT_OK(Flush()); + ASSERT_OK(Put("new", value)); + ASSERT_OK(Flush()); + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const auto& roots = cfd->current()->storage_info()->GetBlobFiles(); + ASSERT_EQ(roots.size(), 2U); + const uint64_t oldest = roots.front()->GetBlobFileNumber(); + const uint64_t newer = roots.back()->GetBlobFileNumber(); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("old" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + // Both generations now occur in one exact SST, but the legacy linkage + // records only the oldest one. Relocating it must not drop the newer root. + const VersionStorageInfo* storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobFileMetaData(newer), nullptr); + ASSERT_TRUE(storage->GetBlobFileMetaData(newer)->GetLinkedSsts().empty()); + const auto census = GetCensusSnapshot(cfd); + ASSERT_NE(census, nullptr); + ASSERT_EQ(census->GetSstCount(oldest), 1U); + ASSERT_EQ(census->GetSstCount(newer), 1U); + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"disable_auto_compactions", "false"}, + {"enable_blob_garbage_collection", "true"}})); + // SetOptions installs a new Version; indirect blob GC is scheduled only after + // the census has caught up with it and published its candidates. + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + storage = cfd->current()->storage_info(); + ASSERT_NE(storage->GetBlobRelocationRouteTarget(oldest), + kInvalidBlobFileNumber); + ASSERT_EQ(storage->GetBlobFileMetaData(oldest), nullptr); + ASSERT_NE(storage->GetBlobFileMetaData(newer), nullptr); + ASSERT_EQ(Get("old3"), value); + ASSERT_EQ(Get("new"), value); + + options.enable_blob_garbage_collection = true; + options.disable_auto_compactions = false; + Reopen(options); + ASSERT_EQ(Get("old3"), value); + ASSERT_EQ(Get("new"), value); +} + +TEST_F(DBBlobBasicTest, RelocationPropertiesIncludePhysicalOverheadAndGarbage) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = true; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + Reopen(options); + const std::string value(4096, 'v'); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), value)); + } + ASSERT_OK(Flush()); + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const uint64_t origin = cfd->current() + ->storage_info() + ->GetBlobFiles() + .front() + ->GetBlobFileNumber(); + ASSERT_OK(Delete("key0")); + ASSERT_OK(Delete("key1")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_OK(dbfull()->TEST_RunIndirectBlobGCBatch(cfd, {origin})); + dbfull()->TEST_WaitForBlobReferenceCensus(); + const uint64_t physical = + cfd->current()->storage_info()->GetBlobRelocationRouteTarget(origin); + ASSERT_NE(physical, kInvalidBlobFileNumber); + auto relocation = + cfd->current()->storage_info()->GetBlobRelocationFileMetaData(physical); + ASSERT_NE(relocation, nullptr); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobStats().total_file_size, + relocation->GetFileSize()); + ASSERT_OK(Delete("key2")); + ASSERT_OK(Flush()); + ASSERT_OK(BuildExactSstAtLevel1()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + const VersionStorageInfo* const storage = cfd->current()->storage_info(); + const auto stats = storage->GetBlobStats(); + ASSERT_EQ(stats.total_file_size, relocation->GetFileSize()); + ASSERT_EQ(stats.total_garbage_size, + value.size() + kSimpleGen2BlobTrailerSize); + uint64_t table_bytes = 0; + for (int level = 0; level < storage->num_levels(); ++level) { + for (const FileMetaData* file : storage->LevelFiles(level)) { + table_bytes += file->fd.GetFileSize(); + } + } + ASSERT_EQ(storage->EstimateLiveDataSize(), + table_bytes + stats.total_file_size - stats.total_garbage_size); + ASSERT_GT(stats.total_file_size - stats.total_garbage_size, + value.size() + kSimpleGen2BlobTrailerSize); + // Version-owned metadata must be released before its VersionSet is closed. + relocation.reset(); + Reopen(options); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ColumnFamilyData* const reopened_cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(reopened_cfd->current() + ->storage_info() + ->GetBlobStats() + .total_garbage_size, + stats.total_garbage_size); + ASSERT_EQ(Get("key3"), value); +} + +TEST_F(DBBlobBasicTest, CloseWaitsForCensusCallbackCleanup) { + Options options = GetDefaultOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + Reopen(options); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + std::mutex gate_mutex; + std::condition_variable gate_cv; + bool callback_entered = false; + bool close_waiting = false; + bool release_callback = false; + std::atomic closed{false}; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Done", [&](void*) { + std::unique_lock lock(gate_mutex); + callback_entered = true; + gate_cv.notify_all(); + gate_cv.wait(lock, [&] { return release_callback; }); + }); + sync_point->SetCallBack("DBImpl::~DBImpl:WaitJob", [&](void*) { + std::lock_guard lock(gate_mutex); + close_waiting = true; + gate_cv.notify_all(); + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([sync_point] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + ASSERT_OK(Put("key", "blob_value")); + ASSERT_OK(Flush()); + { + std::unique_lock lock(gate_mutex); + gate_cv.wait(lock, [&] { return callback_entered; }); + } + port::Thread closer([&] { + Close(); + closed.store(true); + }); + { + std::unique_lock lock(gate_mutex); + gate_cv.wait(lock, [&] { return close_waiting; }); + EXPECT_FALSE(closed.load()); + release_callback = true; + gate_cv.notify_all(); + } + closer.join(); + EXPECT_TRUE(closed.load()); +} + +namespace { + +// The keys that "rocksdb.indirect-blob-gc-stats" reports from the census diff +// on. Later diffs add more keys, so the tests check that these keys exist, not +// that the map has exactly these keys. +constexpr const char* kCensusSignalKeys[] = {"census-state", + "census-unready-micros", + "census-consecutive-failures", + "census-builds", + "census-bytes", + "route-count", + "relocation-file-count", + "relocation-file-bytes", + "tracked-files", + "garbage-bytes-collectable", + "garbage-bytes-suppressed", + "coarse-pinned-files", + "coarse-pinned-bytes", + "coarse-pinned-garbage-bytes", + "relocation-garbage-bytes"}; + +// The census tests read the census LOG lines (INFO when a build is ready, WARN +// from DumpStats when the census stays unready). This logger keeps every line +// with its level, so a test can count the lines that contain a text. +class CensusSignalLog : public Logger { + public: + CensusSignalLog() : Logger(InfoLogLevel::DEBUG_LEVEL) {} + + using Logger::Logv; + void Logv(const InfoLogLevel log_level, const char* format, + va_list ap) override { + char buffer[4096]; + vsnprintf(buffer, sizeof(buffer), format, ap); + std::lock_guard lock(mutex_); + lines_.emplace_back(log_level, buffer); + } + + // Returns the lines at `log_level` that contain `text`. + std::vector Find(InfoLogLevel log_level, + const std::string& text) const { + std::vector found; + std::lock_guard lock(mutex_); + for (const auto& line : lines_) { + if (line.first == log_level && + line.second.find(text) != std::string::npos) { + found.push_back(line.second); + } + } + return found; + } + + private: + mutable std::mutex mutex_; + std::vector> lines_; +}; + +// The DB clock of the census tests: the real clock plus an offset that the +// test adds. A test moves DB time forward by minutes without a sleep, and +// timers and condition variables still work in real time. +class CensusSignalClock : public SystemClockWrapper { + public: + explicit CensusSignalClock(const std::shared_ptr& target) + : SystemClockWrapper(target) {} + + static const char* kClassName() { return "CensusSignalClock"; } + const char* Name() const override { return kClassName(); } + uint64_t NowMicros() override { + return target_->NowMicros() + offset_micros_.load(); + } + uint64_t NowNanos() override { + return target_->NowNanos() + offset_micros_.load() * 1000; + } + void Advance(uint64_t micros) { offset_micros_.fetch_add(micros); } + + private: + std::atomic offset_micros_{0}; +}; + +// The 10 minutes after which DumpStats warns about an unready census +// (kCensusUnreadyWarnMicros in DBImpl::DumpStats()). +constexpr uint64_t kCensusUnreadyWarnMicros = 10ULL * 60 * 1000 * 1000; +constexpr char kCensusReadyLine[] = "Blob reference census is ready at Version"; +constexpr char kCensusUnreadyLine[] = + "Blob reference census has been unready for"; + +// Options for the census tests: blob indirection with every value in a blob +// file, and no automatic compaction or blob GC, so the tests decide when the +// census changes. +Options CensusSignalOptions(Options options) { + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + // The tests call DumpStats() directly, so the periodic dump does not add + // lines. + options.stats_dump_period_sec = 0; + return options; +} + +std::map GetCensusSignals(DB* db) { + std::map stats; + EXPECT_TRUE(db->GetMapProperty(DB::Properties::kIndirectBlobGcStats, &stats)); + for (const char* key : kCensusSignalKeys) { + EXPECT_EQ(stats.count(key), 1U) << key; + } + return stats; +} + +// Returns the number of a numeric key of "rocksdb.indirect-blob-gc-stats". +uint64_t GetCensusSignal(DB* db, const std::string& key) { + const std::map stats = GetCensusSignals(db); + const auto it = stats.find(key); + EXPECT_NE(it, stats.end()) << key; + return it == stats.end() ? 0 : std::stoull(it->second); +} + +std::string GetCensusState(DB* db) { + return GetCensusSignals(db)["census-state"]; +} + +uint64_t GetCensusUnreadyProperty(DB* db) { + uint64_t value = 0; + EXPECT_TRUE(db->GetIntProperty( + DB::Properties::kBlobReferenceCensusUnreadyMicros, &value)); + return value; +} + +// Reads the number that follows `prefix` in `line` (0 when it is not there). +uint64_t NumberAfter(const std::string& line, const std::string& prefix) { + const size_t position = line.find(prefix); + if (position == std::string::npos) { + return 0; + } + return std::strtoull(line.c_str() + position + prefix.size(), nullptr, 10); +} + +} // namespace + +TEST_F(DBBlobBasicTest, CensusSignalsStayOffWithoutBlobIndirection) { + // A DB with blob files but without blob indirection has no census. Every + // census signal must stay at its "off" value: census-state is "none", every + // number is 0, the census LOG lines never appear, and rocksdb.blob-stats has + // the base RocksDB text without a relocation file count. + auto log = std::make_shared(); + auto clock = std::make_shared(env_->GetSystemClock()); + std::unique_ptr clock_env(new CompositeEnvWrapper(env_, clock)); + auto cleanup = Defer([this] { Close(); }); + Options options = CensusSignalOptions(GetDefaultOptions()); + options.enable_blob_indirection = false; + options.env = clock_env.get(); + options.info_log = log; + Reopen(options); + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), std::string(1024, 'v'))); + } + ASSERT_OK(Flush()); + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + + const std::map stats = GetCensusSignals(db_.get()); + for (const auto& entry : stats) { + EXPECT_EQ(entry.second, entry.first == "census-state" ? "none" : "0") + << entry.first; + } + ASSERT_EQ(GetCensusUnreadyProperty(db_.get()), 0U); + + // Even after more than 10 minutes, DumpStats has no census to warn about. + clock->Advance(kCensusUnreadyWarnMicros + 60ULL * 1000 * 1000); + dbfull()->DumpStats(); + EXPECT_TRUE(log->Find(InfoLogLevel::WARN_LEVEL, kCensusUnreadyLine).empty()); + EXPECT_TRUE(log->Find(InfoLogLevel::INFO_LEVEL, kCensusReadyLine).empty()); + + std::string blob_stats; + ASSERT_TRUE(db_->GetProperty(DB::Properties::kBlobStats, &blob_stats)); + EXPECT_EQ(blob_stats.rfind("Number of blob files: 1\n", 0), 0U) << blob_stats; + EXPECT_EQ(blob_stats.find("relocation files"), std::string::npos) + << blob_stats; +} + +TEST_F(DBBlobBasicTest, CensusSignalsTrackBuildAndGarbage) { + // After a census build, the map reports a ready census with memory and + // tracked files, and the LOG has the "ready" line. After deletes and a + // compaction, the census publishes collectable garbage. + auto log = std::make_shared(); + Options options = CensusSignalOptions(GetDefaultOptions()); + options.info_log = log; + Reopen(options); + // The census starts when the first flush writes the format fence. + ASSERT_EQ(GetCensusState(db_.get()), "none"); + + for (int i = 0; i < 4; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), std::string(4096, 'v'))); + } + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-unready-micros"), 0U); + EXPECT_EQ(GetCensusUnreadyProperty(db_.get()), 0U); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-consecutive-failures"), 0U); + EXPECT_GE(GetCensusSignal(db_.get(), "census-builds"), 1U); + EXPECT_GT(GetCensusSignal(db_.get(), "census-bytes"), 0U); + // Every value is still live, so no garbage is reported yet. + EXPECT_EQ(GetCensusSignal(db_.get(), "garbage-bytes-collectable"), 0U); + EXPECT_EQ(GetCensusSignal(db_.get(), "route-count"), 0U); + EXPECT_EQ(GetCensusSignal(db_.get(), "relocation-file-count"), 0U); + EXPECT_EQ(GetCensusSignal(db_.get(), "relocation-file-bytes"), 0U); + EXPECT_FALSE(log->Find(InfoLogLevel::INFO_LEVEL, kCensusReadyLine).empty()); + + // Three of the four values become garbage in their blob file. + for (int i = 0; i < 3; ++i) { + ASSERT_OK(Delete("key" + std::to_string(i))); + } + ASSERT_OK(Flush()); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + EXPECT_GT(GetCensusSignal(db_.get(), "tracked-files"), 0U); + EXPECT_GE(GetCensusSignal(db_.get(), "garbage-bytes-collectable"), + 3U * 4096U); + EXPECT_EQ(GetCensusSignal(db_.get(), "garbage-bytes-suppressed"), 0U); + ASSERT_EQ(Get("key3"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, CensusSignalsReportHeldBuild) { + // While a census build is held, the census is "building": its unready time + // grows, both unready signals agree, and change log entries add to + // census-bytes. After 10 minutes DumpStats warns. When the build finishes, + // the census is ready, the change log is freed, and the "ready" LOG line + // reports how long the build took. + auto log = std::make_shared(); + auto clock = std::make_shared(env_->GetSystemClock()); + std::unique_ptr clock_env(new CompositeEnvWrapper(env_, clock)); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + bool released = false; + auto cleanup = Defer([&] { + if (!released) { + TEST_SYNC_POINT("DBBlobBasicTest::CensusSignalsReleaseBuild"); + } + Close(); + sync_point->DisableProcessing(); + sync_point->LoadDependency({}); + }); + Options options = CensusSignalOptions(GetDefaultOptions()); + options.env = clock_env.get(); + options.info_log = log; + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + Close(); + + // Hold the build that starts at open after its build cursor opened. The + // build has started, so the census has an unready start time, and the cursor + // keeps the change log entries of later installs. + sync_point->LoadDependency( + {{"DBImpl::BuildBlobReferenceCensus:NewBuildReady", + "DBBlobBasicTest::CensusSignalsBuildHeld"}, + {"DBBlobBasicTest::CensusSignalsReleaseBuild", + "DBImpl::BuildBlobReferenceCensus:NewBuildWait"}}); + sync_point->EnableProcessing(); + Reopen(options); + TEST_SYNC_POINT("DBBlobBasicTest::CensusSignalsBuildHeld"); + ASSERT_EQ(GetCensusState(db_.get()), "building"); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-consecutive-failures"), 0U); + + // The unready time is a duration. The int property and the map key read the + // same value, so the property is between two reads of the key. + clock->Advance(1000 * 1000); + const uint64_t unready_before = + GetCensusSignal(db_.get(), "census-unready-micros"); + const uint64_t unready_property = GetCensusUnreadyProperty(db_.get()); + const uint64_t unready_after = + GetCensusSignal(db_.get(), "census-unready-micros"); + EXPECT_GE(unready_before, 1000U * 1000U); + EXPECT_LE(unready_before, unready_property); + EXPECT_LE(unready_property, unready_after); + + // Each flush installs a new Version while the build cursor holds the change + // log, so the log grows and census-bytes goes up. + uint64_t bytes = GetCensusSignal(db_.get(), "census-bytes"); + EXPECT_GT(bytes, 0U); + for (int i = 0; i < 2; ++i) { + ASSERT_OK(Put("held" + std::to_string(i), std::string(4096, 'h'))); + ASSERT_OK(Flush()); + const uint64_t grown = GetCensusSignal(db_.get(), "census-bytes"); + EXPECT_GT(grown, bytes); + bytes = grown; + } + ASSERT_EQ(GetCensusState(db_.get()), "building"); + + // Below 10 minutes DumpStats does not warn. Above, it warns once per call. + dbfull()->DumpStats(); + EXPECT_TRUE(log->Find(InfoLogLevel::WARN_LEVEL, kCensusUnreadyLine).empty()); + clock->Advance(kCensusUnreadyWarnMicros); + dbfull()->DumpStats(); + const std::vector warnings = + log->Find(InfoLogLevel::WARN_LEVEL, kCensusUnreadyLine); + ASSERT_EQ(warnings.size(), 1U); + EXPECT_GT(NumberAfter(warnings[0], kCensusUnreadyLine), + kCensusUnreadyWarnMicros / 1000000); + EXPECT_NE(warnings[0].find("(state: building, failed builds in a row: 0)"), + std::string::npos) + << warnings[0]; + + const size_t ready_lines_before = + log->Find(InfoLogLevel::INFO_LEVEL, kCensusReadyLine).size(); + TEST_SYNC_POINT("DBBlobBasicTest::CensusSignalsReleaseBuild"); + released = true; + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-unready-micros"), 0U); + EXPECT_EQ(GetCensusUnreadyProperty(db_.get()), 0U); + // The publish frees the change log entries that the build cursor held. + EXPECT_LT(GetCensusSignal(db_.get(), "census-bytes"), bytes); + const std::vector ready_lines = + log->Find(InfoLogLevel::INFO_LEVEL, kCensusReadyLine); + ASSERT_GT(ready_lines.size(), ready_lines_before); + // The held build took more than 10 minutes of DB time. + EXPECT_GT(NumberAfter(ready_lines[ready_lines_before], " after "), + kCensusUnreadyWarnMicros / 1000); + // A ready census gives no more warnings. + dbfull()->DumpStats(); + EXPECT_EQ(log->Find(InfoLogLevel::WARN_LEVEL, kCensusUnreadyLine).size(), 1U); + ASSERT_EQ(Get("held1"), std::string(4096, 'h')); +} + +TEST_F(DBBlobBasicTest, CensusSignalsReportFailedBuild) { + // A census build that fails makes the census "failed" and counts the failure. + // After 10 minutes DumpStats warns with the failed state. When the next build + // succeeds, the census is ready, the failure count is 0, and census-builds + // went up. + auto log = std::make_shared(); + auto clock = std::make_shared(env_->GetSystemClock()); + std::unique_ptr clock_env(new CompositeEnvWrapper(env_, clock)); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + std::mutex mutex; + std::condition_variable cv; + int finished_jobs = 0; + std::atomic inject_failure{true}; + auto cleanup = Defer([&] { + Close(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Options options = CensusSignalOptions(GetDefaultOptions()); + options.env = clock_env.get(); + options.info_log = log; + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + // A forced compaction writes an SST with exact reference metadata, which + // the census build at open reads (a flush can write a coarse SST, which the + // build does not read). + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + Close(); + + sync_point->SetCallBack( + "DBImpl::BuildBlobReferenceCensus:ReadFileStatus", [&](void* arg) { + if (inject_failure.load()) { + *static_cast(arg) = Status::IOError("injected failure"); + } + }); + // A long retry delay (one day) keeps the periodic retry from running a + // second build while the test checks the failed state. + sync_point->SetCallBack( + "DBImpl::BGWorkBlobReferenceCensus:FailureRetryDelay", [](void* arg) { + *static_cast(arg) = 24ULL * 3600 * 1000 * 1000; + }); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Done", [&](void*) { + std::lock_guard lock(mutex); + ++finished_jobs; + cv.notify_all(); + }); + sync_point->EnableProcessing(); + Reopen(options); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(30), + [&] { return finished_jobs >= 1; })); + } + ASSERT_EQ(GetCensusState(db_.get()), "failed"); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-consecutive-failures"), 1U); + const uint64_t failed_builds = GetCensusSignal(db_.get(), "census-builds"); + EXPECT_GE(failed_builds, 1U); + clock->Advance(1000 * 1000); + EXPECT_GE(GetCensusSignal(db_.get(), "census-unready-micros"), 1000U * 1000U); + EXPECT_GE(GetCensusUnreadyProperty(db_.get()), 1000U * 1000U); + + dbfull()->DumpStats(); + EXPECT_TRUE(log->Find(InfoLogLevel::WARN_LEVEL, kCensusUnreadyLine).empty()); + clock->Advance(kCensusUnreadyWarnMicros); + dbfull()->DumpStats(); + const std::vector warnings = + log->Find(InfoLogLevel::WARN_LEVEL, kCensusUnreadyLine); + ASSERT_EQ(warnings.size(), 1U); + EXPECT_NE(warnings[0].find("(state: failed, failed builds in a row: 1)"), + std::string::npos) + << warnings[0]; + + // Move past the retry delay. The wait moves the census to the run queue, and + // the new build succeeds. + inject_failure.store(false); + clock->Advance(2 * 24ULL * 3600 * 1000 * 1000); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-consecutive-failures"), 0U); + EXPECT_GT(GetCensusSignal(db_.get(), "census-builds"), failed_builds); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-unready-micros"), 0U); + EXPECT_EQ(GetCensusUnreadyProperty(db_.get()), 0U); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, CensusSignalsCountRelocationFilesAndRoutes) { + // The relocation file and route keys of the map, and the relocation file + // count in rocksdb.blob-stats, follow the relocation state of the current + // Version. Without relocation files, rocksdb.blob-stats has the base RocksDB + // text; with them, it adds "(+N relocation files)". + Options options = CensusSignalOptions(GetDefaultOptions()); + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + std::string blob_stats; + ASSERT_TRUE(db_->GetProperty(DB::Properties::kBlobStats, &blob_stats)); + EXPECT_EQ(blob_stats.rfind("Number of blob files: 1\n", 0), 0U) << blob_stats; + EXPECT_EQ(blob_stats.find("relocation files"), std::string::npos) + << blob_stats; + + BlobPhysicalFile relocation_file; + ASSERT_NO_FATAL_FAILURE(InstallOneRelocationFile(options, &relocation_file)); + EXPECT_EQ(GetCensusSignal(db_.get(), "relocation-file-count"), 1U); + EXPECT_GT(GetCensusSignal(db_.get(), "relocation-file-bytes"), 0U); + EXPECT_EQ(GetCensusSignal(db_.get(), "route-count"), 0U); + blob_stats.clear(); + ASSERT_TRUE(db_->GetProperty(DB::Properties::kBlobStats, &blob_stats)); + const size_t relocation_files = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info() + ->GetBlobRelocationFiles() + .size(); + ASSERT_EQ(relocation_files, 1U); + EXPECT_EQ(blob_stats.rfind("Number of blob files: 1 (+" + + std::to_string(relocation_files) + + " relocation files)\n", + 0), + 0U) + << blob_stats; + + // A route from an old file number (not a live file) to the relocation file. + VersionSet* const version_set = dbfull()->GetVersionSet(); + dbfull()->TEST_LockMutex(); + const uint64_t routed_generation = version_set->NewFileNumber(); + dbfull()->TEST_UnlockMutex(); + VersionEdit route; + route.SetColumnFamily(0); + route.SetBlobRelocationRoute(BlobRelocationRoute( + routed_generation, relocation_file.open_info.file_number)); + ASSERT_OK(ApplyVersionEdit(&route, /*new_descriptor_log=*/false)); + EXPECT_EQ(GetCensusSignal(db_.get(), "route-count"), 1U); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, PauseBackgroundWorkWaitsForRunningCensusJob) { + // The census job can write the MANIFEST, so PauseBackgroundWork() must wait + // for a running census job and return only after the job ends, without a + // deadlock. + SyncPoint* const sync_point = SyncPoint::GetInstance(); + std::mutex mutex; + std::condition_variable cv; + bool pause_waiting = false; + std::atomic finished_jobs{0}; + bool released = false; + std::unique_ptr pauser; + auto cleanup = Defer([&] { + if (!released) { + TEST_SYNC_POINT("DBBlobBasicTest::PauseReleaseCensusJob"); + } + if (pauser != nullptr) { + pauser->join(); + } + Close(); + sync_point->DisableProcessing(); + sync_point->LoadDependency({}); + sync_point->ClearAllCallBacks(); + }); + Options options = CensusSignalOptions(GetDefaultOptions()); + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + Close(); + + // Hold the census job that starts at open before it builds. + sync_point->LoadDependency( + {{"DBImpl::BGWorkBlobReferenceCensus:BeforeBuildReady", + "DBBlobBasicTest::PauseCensusJobHeld"}, + {"DBBlobBasicTest::PauseReleaseCensusJob", + "DBImpl::BGWorkBlobReferenceCensus:BeforeBuildWait"}}); + sync_point->SetCallBack("DBImpl::PauseBackgroundWork:Wait", [&](void*) { + std::lock_guard lock(mutex); + pause_waiting = true; + cv.notify_all(); + }); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Done", + [&](void*) { finished_jobs.fetch_add(1); }); + sync_point->EnableProcessing(); + Reopen(options); + TEST_SYNC_POINT("DBBlobBasicTest::PauseCensusJobHeld"); + + std::atomic pause_returned{false}; + Status pause_status; + int finished_at_return = -1; + pauser.reset(new port::Thread([&] { + pause_status = dbfull()->PauseBackgroundWork(); + finished_at_return = finished_jobs.load(); + pause_returned.store(true); + })); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(30), + [&] { return pause_waiting; })); + } + // The census job is still held, so the pause still waits. + EXPECT_FALSE(pause_returned.load()); + TEST_SYNC_POINT("DBBlobBasicTest::PauseReleaseCensusJob"); + released = true; + pauser->join(); + pauser.reset(); + ASSERT_OK(pause_status); + EXPECT_GE(finished_at_return, 1); + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, ContinueBackgroundWorkSchedulesPausedCensusJob) { + // While background work is paused, a Version install that leaves the census + // behind does not start the census job. ContinueBackgroundWork() schedules + // the job again by itself, and the census becomes ready. + SyncPoint* const sync_point = SyncPoint::GetInstance(); + std::mutex mutex; + std::condition_variable cv; + int started_jobs = 0; + int finished_jobs = 0; + bool paused = false; + auto cleanup = Defer([&] { + if (paused) { + EXPECT_OK(dbfull()->ContinueBackgroundWork()); + } + Close(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Options options = CensusSignalOptions(GetDefaultOptions()); + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Start", + [&](void*) { + std::lock_guard lock(mutex); + ++started_jobs; + }); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Done", [&](void*) { + std::lock_guard lock(mutex); + ++finished_jobs; + cv.notify_all(); + }); + sync_point->EnableProcessing(); + ASSERT_OK(dbfull()->PauseBackgroundWork()); + paused = true; + // A flush or compaction cannot run while paused, so install a new Version + // directly. The census must catch up with it in its job. + VersionEdit edit; + edit.SetColumnFamily(0); + ASSERT_OK(ApplyVersionEdit(&edit, /*new_descriptor_log=*/false)); + ASSERT_EQ(GetCensusState(db_.get()), "building"); + { + std::lock_guard lock(mutex); + EXPECT_EQ(started_jobs, 0); + } + + // Do not call TEST_WaitForBlobReferenceCensus() before the job ends: it + // schedules the job itself. Wait for the job that ContinueBackgroundWork() + // schedules. + paused = false; + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(30), + [&] { return finished_jobs >= 1; })); + EXPECT_GE(started_jobs, 1); + } + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, PausedCensusJobDoesNotStartWhenRequested) { + // While background work is paused, a request to schedule the census job + // (what a Version install makes) keeps the column family in the queue and + // does not start the job. After ContinueBackgroundWork() the census becomes + // ready. + SyncPoint* const sync_point = SyncPoint::GetInstance(); + std::atomic started_jobs{0}; + bool paused = false; + auto cleanup = Defer([&] { + if (paused) { + EXPECT_OK(dbfull()->ContinueBackgroundWork()); + } + Close(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Options options = CensusSignalOptions(GetDefaultOptions()); + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Start", + [&](void*) { started_jobs.fetch_add(1); }); + sync_point->EnableProcessing(); + ASSERT_OK(dbfull()->PauseBackgroundWork()); + paused = true; + VersionEdit edit; + edit.SetColumnFamily(0); + ASSERT_OK(ApplyVersionEdit(&edit, /*new_descriptor_log=*/false)); + ASSERT_EQ(GetCensusState(db_.get()), "building"); + // The test wait asks to schedule the census job of every column family and + // then waits for a scheduled job. While paused, no job is scheduled, so the + // wait returns at once and the census stays behind. + dbfull()->TEST_WaitForBlobReferenceCensus(); + EXPECT_EQ(started_jobs.load(), 0); + ASSERT_EQ(GetCensusState(db_.get()), "building"); + + paused = false; + ASSERT_OK(dbfull()->ContinueBackgroundWork()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + EXPECT_GE(started_jobs.load(), 1); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, CensusUnreadyTimeStartsWhenCensusIsCreated) { + // A Version install that adds the format fence creates the census of the + // column family before it writes the MANIFEST. When that MANIFEST write + // fails, no census install follows, and the census job is not scheduled. + // The census then exists without a build. Its unready time must still count + // from its creation, so census-unready-micros and the int property do not + // report 0 while the census is unready. + auto clock = std::make_shared(env_->GetSystemClock()); + std::unique_ptr clock_env(new CompositeEnvWrapper(env_, clock)); + SyncPoint* const sync_point = SyncPoint::GetInstance(); + std::atomic fail_manifest{false}; + bool released = false; + auto cleanup = Defer([&] { + if (!released) { + TEST_SYNC_POINT("DBBlobBasicTest::CensusCreatedReleaseJob"); + } + Close(); + sync_point->DisableProcessing(); + sync_point->LoadDependency({}); + sync_point->ClearAllCallBacks(); + }); + Options options = CensusSignalOptions(GetDefaultOptions()); + options.env = clock_env.get(); + options.create_if_missing = true; + // The test resumes by hand, so an automatic resume cannot install a Version + // (and start the unready time) before the checks. + options.max_bgerror_resume_count = 0; + DestroyAndReopen(options); + ASSERT_EQ(GetCensusState(db_.get()), "none"); + + // If a census job starts, hold it before it starts the build (before + // BeginBuild()), so only the census creation can set the unready time. + sync_point->LoadDependency( + {{"DBBlobBasicTest::CensusCreatedReleaseJob", + "DBImpl::BGWorkBlobReferenceCensus:BeforeBuildWait"}}); + sync_point->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) { + if (fail_manifest.load()) { + // A retryable error lets Resume() recover the DB below. + IOStatus error = IOStatus::IOError("injected failure"); + error.SetRetryable(true); + *static_cast(arg) = error; + } + }); + sync_point->EnableProcessing(); + // The first flush writes the format fence. Its MANIFEST write fails. + ASSERT_OK(Put("key", std::string(4096, 'v'))); + fail_manifest.store(true); + ASSERT_NOK(Flush()); + fail_manifest.store(false); + ASSERT_EQ(GetCensusState(db_.get()), "building"); + clock->Advance(1000 * 1000); + EXPECT_GE(GetCensusSignal(db_.get(), "census-unready-micros"), 1000U * 1000U); + EXPECT_GE(GetCensusUnreadyProperty(db_.get()), 1000U * 1000U); + + TEST_SYNC_POINT("DBBlobBasicTest::CensusCreatedReleaseJob"); + released = true; + ASSERT_OK(dbfull()->Resume()); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(GetCensusState(db_.get()), "ready"); + EXPECT_EQ(GetCensusSignal(db_.get(), "census-unready-micros"), 0U); + EXPECT_EQ(GetCensusUnreadyProperty(db_.get()), 0U); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(DBBlobBasicTest, EveryPropertyHasAStringForm) { + // DB::GetProperty() asserts that each property has a string, int or DBImpl + // string handler, and db_stress reads every property with it. A property + // with only a map handler would crash a debug build there. + for (const auto& [name, info] : InternalStats::ppt_name_to_info) { + SCOPED_TRACE(name); + EXPECT_TRUE(info.handle_string != nullptr || info.handle_int != nullptr || + info.handle_string_dbimpl != nullptr); + } + // The blob indirection properties work in every form that they support, + // with the feature off and on. + for (const bool indirection : {false, true}) { + SCOPED_TRACE(indirection ? "indirection on" : "indirection off"); + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = indirection; + options.min_blob_size = 0; + DestroyAndReopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + dbfull()->TEST_WaitForBlobReferenceCensus(); + + std::string value; + std::map map_value; + ASSERT_TRUE(db_->GetProperty(DB::Properties::kIndirectBlobGcStats, &value)); + ASSERT_TRUE( + db_->GetMapProperty(DB::Properties::kIndirectBlobGcStats, &map_value)); + EXPECT_EQ(map_value["census-state"], indirection ? "ready" : "none"); + // The string form has one "key=value" line per map key. + std::string expected; + for (const auto& [key, key_value] : map_value) { + expected += key + "=" + key_value + "\n"; + } + EXPECT_EQ(value, expected); + + for (const std::string& name : + {DB::Properties::kBlobIndirectionFormatVersion, + DB::Properties::kBlobReferenceCensusUnreadyMicros}) { + SCOPED_TRACE(name); + uint64_t int_value = 0; + ASSERT_TRUE(db_->GetProperty(name, &value)); + ASSERT_TRUE(db_->GetIntProperty(name, &int_value)); + EXPECT_EQ(value, std::to_string(int_value)); + } + } +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/blob/db_blob_index_test.cc b/db/blob/db_blob_index_test.cc index f5fbdf9acf7a..0297beb09845 100644 --- a/db/blob/db_blob_index_test.cc +++ b/db/blob/db_blob_index_test.cc @@ -7,16 +7,24 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include #include #include #include +#include +#include #include #include #include +#include "cache/cache_reservation_manager.h" #include "db/arena_wrapped_db_iter.h" #include "db/blob/blob_fetcher.h" +#include "db/blob/blob_gen2_format.h" #include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_reference_census.h" +#include "db/blob/blob_reference_metadata.h" #include "db/column_family.h" #include "db/db_iter.h" #include "db/db_test_util.h" @@ -24,11 +32,20 @@ #include "db/wide/wide_column_test_util.h" #include "db/write_batch_internal.h" #include "file/filename.h" +#include "file/writable_file_writer.h" #include "port/port.h" #include "port/stack_trace.h" #include "rocksdb/perf_context.h" #include "rocksdb/sst_file_writer.h" #include "rocksdb/table.h" +#include "table/block_based/block_based_table_reader.h" +#include "table/table_builder.h" +#include "test_util/mock_time_env.h" +#include "test_util/testutil.h" +#include "util/coding.h" +#include "util/crc32c.h" +#include "util/defer.h" +#include "util/random.h" #include "util/string_util.h" #include "utilities/merge_operators.h" @@ -43,23 +60,3902 @@ void CorruptPinnedBlobIndexOnCleanup(void* arg1, void* /*arg2*/) { (*blob_index)[0] = static_cast(BlobIndex::Type::kUnknown); } +class RejectingReservationManager : public CacheReservationManager { + public: + explicit RejectingReservationManager(size_t limit) : limit_(limit) {} + + class Handle : public CacheReservationHandle { + public: + explicit Handle(RejectingReservationManager* manager) : manager_(manager) {} + ~Handle() override { + manager_->UpdateCacheReservation(memory_used_, false) + .PermitUncheckedError(); + } + Status Resize(size_t memory_used) override { + Status status = manager_->UpdateCacheReservation( + memory_used > memory_used_ ? memory_used - memory_used_ + : memory_used_ - memory_used, + memory_used > memory_used_); + memory_used_ = memory_used; + return status; + } + + private: + RejectingReservationManager* manager_; + size_t memory_used_ = 0; + }; + + Status UpdateCacheReservation(size_t new_memory_used) override { + memory_used_ = new_memory_used; + if (on_update) { + on_update(new_memory_used); + } + if (new_memory_used > limit_) { + return Status::MemoryLimit("test reservation limit"); + } + return Status::OK(); + } + + Status UpdateCacheReservation(size_t memory_used_delta, + bool increase) override { + return UpdateCacheReservation(increase ? memory_used_ + memory_used_delta + : memory_used_ - memory_used_delta); + } + + Status TryIncreaseCacheReservation(size_t memory_used_delta) override { + const size_t old_memory_used = memory_used_; + Status status = UpdateCacheReservation(memory_used_delta, true); + if (!status.ok()) { + memory_used_ = old_memory_used; + } + return status; + } + + Status MakeCacheReservation( + size_t incremental_memory_used, + std::unique_ptr* handle) override { + handle->reset(new Handle(this)); + return (*handle)->Resize(incremental_memory_used); + } + + size_t GetTotalReservedCacheSize() override { return memory_used_; } + size_t GetTotalMemoryUsed() override { return memory_used_; } + + // Called with the new total on every charge update. + std::function on_update; + + private: + size_t limit_; + size_t memory_used_ = 0; +}; + +constexpr uint32_t kTestColumnFamilyId = 0; +constexpr uint64_t kTestSstFileNumber = 1; + +BlobIndex DecodeBlobIndexForTest(const std::string& encoded) { + BlobIndex blob_index; + EXPECT_OK(blob_index.DecodeFrom(encoded)); + return blob_index; +} + +// The metadata that a collector writes in a test: the data blocks, the index +// block and the descriptor. +struct MetadataForTest { + std::vector blocks; + std::string index; + BlobReferenceMetadataDescriptor descriptor; +}; + +// A block writer that places each block after the previous one, starting at +// offset 1000. Like a table, it leaves BlockBasedTable::kBlockTrailerSize (5 +// bytes) after each block for the block trailer. +BlobReferenceMetadataCollector::BlockWriter BlockWriterForTest( + MetadataForTest* metadata) { + return [metadata, next_offset = uint64_t{1000}]( + const Slice& block, bool is_index_block, + BlockHandle* handle) mutable -> Status { + if (is_index_block) { + metadata->index = block.ToString(); + } else { + metadata->blocks.emplace_back(block.data(), block.size()); + } + *handle = BlockHandle(next_offset, block.size()); + next_offset += block.size() + 5; + return Status::OK(); + }; +} + +Status FinishForTest( + BlobReferenceMetadataCollector* collector, MetadataForTest* metadata, + std::shared_ptr* summary = nullptr) { + return collector->Finish(BlockWriterForTest(metadata), &metadata->descriptor, + summary); +} + +// Returns the index groups that are not kDirect. BlobReferenceSstSummary holds +// the same groups. +std::vector IndirectGroupsForTest( + const BlobReferenceIndex& index) { + std::vector groups; + for (const BlobReferenceIndexEntry& entry : index.groups) { + if (entry.group.encoding_class != BlobReferenceEncodingClass::kDirect) { + groups.push_back(entry.group); + } + } + return groups; +} + +BlobReferenceBlockReader BlockReaderForTest(const MetadataForTest& metadata) { + return [&metadata](size_t block, Slice* contents) -> Status { + if (block >= metadata.blocks.size()) { + return Status::Corruption("Block ordinal out of range"); + } + *contents = metadata.blocks[block]; + return Status::OK(); + }; +} + +// Decodes the index and walks every block. +Status DecodeAllForTest(const MetadataForTest& metadata, + BlobReferenceIndex* index, + std::vector* references) { + Status status = + DecodeBlobReferenceIndex(metadata.index, metadata.descriptor, index); + if (!status.ok()) { + return status; + } + if (index->blocks.size() != metadata.blocks.size()) { + return Status::Corruption("Index and block count differ"); + } + return ForEachBlobReference(*index, BlockReaderForTest(metadata), + [&](const BlobReference& reference) { + references->push_back(reference); + return Status::OK(); + }); +} + +// Reads one group through the index. +Status ReadGroupForTest(const MetadataForTest& metadata, + const BlobReferenceIndex& index, size_t group, + std::vector* references) { + return ForEachBlobReferenceInGroup(index, group, BlockReaderForTest(metadata), + [&](const BlobReference& reference) { + references->push_back(reference); + return Status::OK(); + }); +} + +// Builds metadata bytes field by field. +struct BytesForTest { + std::string bytes; + BytesForTest& Byte(uint8_t value) { + bytes.push_back(static_cast(value)); + return *this; + } + BytesForTest& Varint(uint64_t value) { + PutVarint64(&bytes, value); + return *this; + } + BytesForTest& Fixed32(uint32_t value) { + PutFixed32(&bytes, value); + return *this; + } + BytesForTest& Append(const std::string& more) { + bytes += more; + return *this; + } +}; + +auto ReferenceFieldsForTest(const BlobReference& reference) { + return std::make_tuple( + reference.encoding_class, reference.generation_file_number, + reference.origin_file_number, reference.original_value_offset, + reference.stored_value_size, reference.record_bytes, + reference.compression, reference.stable_checksum, + reference.stored_checksum); +} + +BlobIndex UnflattenedForTest(uint64_t origin, uint64_t offset, + uint64_t stored_size) { + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob( + &encoded, {.origin_file_number = origin, + .original_value_offset = offset, + .stored_value_size = stored_size, + .compression = kZSTD, + .stable_checksum = static_cast(offset), + .stored_checksum = static_cast(offset + 1)}); + return DecodeBlobIndexForTest(encoded); +} + +BlobIndex LegacyForTest(uint64_t file_number, uint64_t offset) { + std::string encoded; + BlobIndex::EncodeBlob(&encoded, file_number, offset, /*size=*/100, + kNoCompression); + return DecodeBlobIndexForTest(encoded); +} + } // namespace -TEST(BlobIndexTest, SameFileBlobIndex) { +TEST(BlobIndexTest, SameFileBlobIndex) { + std::string encoded; + BlobIndex::EncodeBlob(&encoded, kCurrentFileBlobIndexFileNumber, + /*offset=*/123, /*size=*/456, kNoCompression); + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + EXPECT_TRUE(blob_index.IsSameFile()); + EXPECT_EQ(blob_index.file_number(), kCurrentFileBlobIndexFileNumber); + EXPECT_NE(blob_index.DebugString(false).find("file:same"), std::string::npos); +} + +TEST(BlobIndexTest, EmptyBlobIndexIsCorruption) { + BlobIndex blob_index; + ASSERT_EQ(blob_index.DecodeFrom(Slice()).code(), Status::Code::kCorruption); +} + +TEST(BlobIndexTest, IndirectBlobIndexRoundTrip) { + constexpr uint64_t kGeneration = 301; + constexpr uint64_t kOrigin = 101; + constexpr uint64_t kOffset = 123456; + constexpr uint64_t kSize = 789; + constexpr uint32_t kChecksum = 0x89abcdefU; + constexpr uint32_t kStoredChecksum = 0x01234567U; + + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, kGeneration, + {.origin_file_number = kOrigin, + .original_value_offset = kOffset, + .stored_value_size = kSize, + .compression = kZSTD, + .stable_checksum = kChecksum, + .stored_checksum = kStoredChecksum}); + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + EXPECT_TRUE(blob_index.IsIndirect()); + EXPECT_FALSE(blob_index.IsInlined()); + EXPECT_FALSE(blob_index.IsSameFile()); + EXPECT_EQ(blob_index.file_number(), kGeneration); + EXPECT_EQ(blob_index.generation_file_number(), kGeneration); + EXPECT_EQ(blob_index.origin_file_number(), kOrigin); + EXPECT_EQ(blob_index.offset(), kOffset); + EXPECT_EQ(blob_index.original_value_offset(), kOffset); + EXPECT_EQ(blob_index.size(), kSize); + EXPECT_EQ(blob_index.compression(), kZSTD); + EXPECT_EQ(blob_index.stable_checksum(), kChecksum); + EXPECT_EQ(blob_index.stored_checksum(), kStoredChecksum); + + std::string reencoded; + blob_index.EncodeTo(&reencoded); + EXPECT_EQ(reencoded, encoded); +} + +// An uncompressed indirect index stores no stored CRC, so stored_checksum() +// derives it from the stable CRC. A compressed index stores it. In both cases +// the accessor, DebugString() and EncodeTo() must give the encoded values. +TEST(BlobIndexTest, IndirectStoredChecksumMatchesEncoding) { + constexpr uint32_t kStableChecksum = 0x89abcdefU; + constexpr uint32_t kEncodedStoredChecksum = 0x01234567U; + for (const CompressionType compression : {kNoCompression, kZSTD}) { + // Generation 301 is above origin 101, so it encodes a flattened index; + // generation 101 encodes an unflattened one. + for (const uint64_t generation : {uint64_t{101}, uint64_t{301}}) { + SCOPED_TRACE("compression=" + CompressionTypeToString(compression) + + " generation=" + std::to_string(generation)); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, generation, + {.origin_file_number = 101, + .original_value_offset = 123456, + .stored_value_size = 789, + .compression = compression, + .stable_checksum = kStableChecksum, + .stored_checksum = kEncodedStoredChecksum}); + const uint32_t expected_stored_checksum = + compression == kNoCompression + ? BlobIndex::ExtendChecksumWithCompression(kStableChecksum, + kNoCompression) + : kEncodedStoredChecksum; + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + EXPECT_EQ(blob_index.stable_checksum(), kStableChecksum); + EXPECT_EQ(blob_index.stored_checksum(), expected_stored_checksum); + EXPECT_TRUE(blob_index.fingerprint() == + (StableBlobFingerprint{789, compression, kStableChecksum, + expected_stored_checksum})); + EXPECT_NE(blob_index.DebugString(/*output_hex=*/false) + .find(" stored_crc:" + + std::to_string(expected_stored_checksum) + " "), + std::string::npos); + + std::string reencoded; + blob_index.EncodeTo(&reencoded); + EXPECT_EQ(reencoded, encoded); + } + } +} + +TEST(BlobIndexTest, IndirectBlobFieldOrder) { + // Hand-made bytes in the documented order: generation, origin, original + // offset. All three differ, so a swapped order fails. + std::string encoded; + encoded.push_back(static_cast(BlobIndex::Type::kIndirectFlattenedBlob)); + PutVarint64(&encoded, /*generation=*/7); + PutVarint64(&encoded, /*origin=*/3); + PutVarint64(&encoded, /*original_offset=*/100); + PutVarint64(&encoded, /*size=*/50); + encoded.push_back(static_cast(kZSTD)); + PutFixed32(&encoded, /*logical_crc=*/0x11223344U); + PutFixed32(&encoded, /*stored_crc=*/0x55667788U); + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + EXPECT_EQ(blob_index.generation_file_number(), 7); + EXPECT_EQ(blob_index.origin_file_number(), 3); + EXPECT_EQ(blob_index.original_value_offset(), 100); + EXPECT_EQ(blob_index.size(), 50); + EXPECT_EQ(blob_index.compression(), kZSTD); + EXPECT_EQ(blob_index.stable_checksum(), 0x11223344U); + EXPECT_EQ(blob_index.stored_checksum(), 0x55667788U); + + std::string reencoded; + BlobIndex::EncodeIndirectBlob(&reencoded, 7, + {.origin_file_number = 3, + .original_value_offset = 100, + .stored_value_size = 50, + .compression = kZSTD, + .stable_checksum = 0x11223344U, + .stored_checksum = 0x55667788U}); + EXPECT_EQ(reencoded, encoded); +} + +TEST(BlobIndexTest, UnflattenedBlobNormalizesGenerationToOrigin) { + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 77, + .original_value_offset = 88, + .stored_value_size = 99, + .compression = kLZ4Compression, + .stable_checksum = 1234, + .stored_checksum = 4321}); + // Type 3 stores no generation: type, origin, offset, size, compression and + // two CRCs. + ASSERT_EQ(encoded[0], static_cast(BlobIndex::Type::kIndirectBlob)); + ASSERT_EQ(encoded.size(), 1U + 1 + 1 + 1 + 1 + 4 + 4); + + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + EXPECT_EQ(blob_index.generation_file_number(), 77); + EXPECT_EQ(blob_index.origin_file_number(), 77); + std::string reencoded; + blob_index.EncodeTo(&reencoded); + EXPECT_EQ(reencoded, encoded); +} + +TEST(BlobIndexTest, FlattenedBlobRoundTripAndCanonicalGeneration) { + // Type 4 stores the generation, which must be above the origin. + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, 200, + {.origin_file_number = 77, + .original_value_offset = 88, + .stored_value_size = 99, + .compression = kNoCompression, + .stable_checksum = 1234, + .stored_checksum = 0}); + ASSERT_EQ(encoded[0], + static_cast(BlobIndex::Type::kIndirectFlattenedBlob)); + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + EXPECT_EQ(blob_index.generation_file_number(), 200); + EXPECT_EQ(blob_index.origin_file_number(), 77); + std::string reencoded; + blob_index.EncodeTo(&reencoded); + EXPECT_EQ(reencoded, encoded); + + for (const uint64_t generation : {uint64_t{77}, uint64_t{76}}) { + SCOPED_TRACE(generation); + std::string bad; + bad.push_back(static_cast(BlobIndex::Type::kIndirectFlattenedBlob)); + PutVarint64(&bad, generation); + bad.append(encoded.substr(2)); + BlobIndex decoded; + EXPECT_TRUE(decoded.DecodeFrom(bad).IsCorruption()); + } + { + std::string unknown(encoded); + unknown[0] = static_cast(BlobIndex::Type::kUnknown); + BlobIndex decoded; + EXPECT_TRUE(decoded.DecodeFrom(unknown).IsCorruption()); + } + for (size_t size = 0; size < encoded.size(); ++size) { + BlobIndex decoded; + EXPECT_TRUE(decoded.DecodeFrom(Slice(encoded.data(), size)).IsCorruption()) + << "accepted truncated size " << size; + } +} + +TEST(BlobIndexTest, IndirectBlobIndexRejectsInvalidFileNumbers) { + // A file number must be in [1, BlobIndex::kMaxFileNumber]. The test writes + // the type byte and the file numbers by hand and copies the other fields + // from a valid index of origin 77 (one varint byte). + std::string valid; + BlobIndex::EncodeUnflattenedIndirectBlob(&valid, + {.origin_file_number = 77, + .original_value_offset = 88, + .stored_value_size = 99, + .compression = kNoCompression, + .stable_checksum = 1234, + .stored_checksum = 0}); + const std::string other_fields = valid.substr(2); + const auto encode = [&](BlobIndex::Type type, + std::initializer_list file_numbers) { + std::string encoded(1, static_cast(type)); + for (uint64_t file_number : file_numbers) { + PutVarint64(&encoded, file_number); + } + return encoded + other_fields; + }; + constexpr uint64_t kMax = BlobIndex::kMaxFileNumber; + BlobIndex decoded; + ASSERT_OK(decoded.DecodeFrom(encode(BlobIndex::Type::kIndirectBlob, {kMax}))); + ASSERT_OK(decoded.DecodeFrom( + encode(BlobIndex::Type::kIndirectFlattenedBlob, {kMax, 77}))); + for (const std::string& bad : + {encode(BlobIndex::Type::kIndirectBlob, {0}), + encode(BlobIndex::Type::kIndirectBlob, {kMax + 1}), + encode(BlobIndex::Type::kIndirectBlob, + {std::numeric_limits::max()}), + encode(BlobIndex::Type::kIndirectFlattenedBlob, {kMax + 1, 77}), + encode(BlobIndex::Type::kIndirectFlattenedBlob, + {kMax + 2, kMax + 1})}) { + SCOPED_TRACE(Slice(bad).ToString(/*hex=*/true)); + EXPECT_TRUE(decoded.DecodeFrom(bad).IsCorruption()); + } +} + +TEST(BlobIndexTest, IndirectBlobIndexRejectsMalformedEncoding) { + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 77, + .original_value_offset = 88, + .stored_value_size = 99, + .compression = kNoCompression, + .stable_checksum = 1234, + .stored_checksum = 4321}); + + for (size_t size = 0; size < encoded.size(); ++size) { + BlobIndex blob_index; + EXPECT_TRUE( + blob_index.DecodeFrom(Slice(encoded.data(), size)).IsCorruption()) + << "accepted truncated size " << size; + } + + { + // An extra byte after the compression byte must make decoding fail. The + // decoder must not accept the shifted bytes as a valid index with other + // checksums. + std::string old_layout(encoded); + old_layout.insert(old_layout.begin() + 5, '\x02'); + BlobIndex blob_index; + EXPECT_TRUE(blob_index.DecodeFrom(old_layout).IsCorruption()); + } + { + std::string noncanonical_origin(encoded); + noncanonical_origin[1] = static_cast(0x80 | encoded[1]); + noncanonical_origin.insert(noncanonical_origin.begin() + 2, '\0'); + BlobIndex blob_index; + EXPECT_TRUE(blob_index.DecodeFrom(noncanonical_origin).IsCorruption()); + } + { + std::string trailing(encoded); + trailing.push_back('x'); + BlobIndex blob_index; + EXPECT_TRUE(blob_index.DecodeFrom(trailing).IsCorruption()); + } + { + std::string invalid_compression(encoded); + // Here the compression byte is at position 4, after the type byte and three + // one-byte varints (origin, offset and size). + invalid_compression[4] = static_cast(kDisableCompressionOption); + BlobIndex blob_index; + EXPECT_TRUE(blob_index.DecodeFrom(invalid_compression).IsCorruption()); + } +} + +namespace { + +// How BuildTableForTest() hands each entry to the table builder. +enum class TableAddModeForTest { + // TableBuilder::Add(). + kAdd, + // TableBuilder::TryAdd() without a decoded BlobIndex. + kTryAdd, + // TableBuilder::TryAdd() with the caller's decode of each kTypeBlobIndex + // value in TableBuilderAddContext::decoded_blob_index, as compaction does. + kTryAddWithDecode, +}; + +// Builds one block-based SST in memory from `entries` (internal keys in +// order) with exact blob reference metadata, and returns its bytes. A fixed +// clock keeps the time properties equal across builds, so two builds of the +// same entries give the same bytes. `*status` gets the builder status. +std::string BuildTableForTest( + const std::vector>& entries, + TableAddModeForTest mode, Status* status) { + Options options; + options.enable_blob_indirection = true; + // Format version 6 and above mix a random value into each block checksum + // (FormatVersionUsesContextChecksum()), so two builds would differ. Format + // version 5 has no such value. + BlockBasedTableOptions table_options; + table_options.format_version = 5; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + ImmutableOptions ioptions(options); + MockSystemClock clock(SystemClock::Default()); + ioptions.clock = &clock; + const MutableCFOptions moptions(options); + const InternalKeyComparator icmp(options.comparator); + const InternalTblPropCollFactories factories; + // TableBuilderOptions keeps references to these, so they must outlive the + // builder. + const ReadOptions read_options; + const WriteOptions write_options; + const CompressionOptions compression_options; + // Column family 0 and file number 12 are example values; the exact + // metadata needs a known column family and a file number. + TableBuilderOptions tbo( + ioptions, moptions, read_options, write_options, icmp, &factories, + kNoCompression, compression_options, /*_column_family_id=*/0, + kDefaultColumnFamilyName, /*_level=*/1, kUnknownNewestKeyTime, + /*_is_bottommost=*/false, TableFileCreationReason::kCompaction, + /*_oldest_key_time=*/0, /*_db_id=*/"db", /*_db_session_id=*/"session", + /*_target_file_size=*/0, /*_cur_file_num=*/12); + tbo.collect_blob_reference_metadata = true; + + auto sink = std::make_unique(); + test::StringSink* const sink_ptr = sink.get(); + WritableFileWriter file_writer(std::move(sink), "table", FileOptions()); + std::unique_ptr builder( + options.table_factory->NewTableBuilder(tbo, &file_writer)); + for (const auto& [key, value] : entries) { + BlobIndex decoded; + TableBuilderAddContext context; + if (mode == TableAddModeForTest::kTryAddWithDecode && + ExtractValueType(key) == kTypeBlobIndex && + decoded.DecodeFrom(value).ok()) { + context.decoded_blob_index = &decoded; + } + if (mode == TableAddModeForTest::kAdd) { + builder->Add(key, value); + } else { + EXPECT_EQ(builder->TryAdd(key, value, &context), + TableBuilderAddResult::kAdded); + } + } + if (builder->status().ok()) { + *status = builder->Finish(); + EXPECT_OK(file_writer.Flush(IOOptions())); + } else { + *status = builder->status(); + builder->Abandon(); + } + return sink_ptr->contents(); +} + +std::string InternalKeyForTest(const std::string& user_key, + SequenceNumber seqno, ValueType type) { + std::string encoded; + AppendInternalKey(&encoded, ParsedInternalKey(user_key, seqno, type)); + return encoded; +} + +} // namespace + +// Compaction passes its own decode of each BlobIndex to the table builder. +// The SST must have the same bytes (and so the same properties and blob +// reference metadata) as when the builder decodes the value itself. +TEST(BlobReferenceMetadataTest, SharedBlobIndexDecodeGivesSameTable) { + std::string legacy; + BlobIndex::EncodeBlob(&legacy, /*file_number=*/5, /*offset=*/100, + /*size=*/50, kNoCompression); + std::string unflattened; + BlobIndex::EncodeUnflattenedIndirectBlob(&unflattened, + {.origin_file_number = 20, + .original_value_offset = 200, + .stored_value_size = 60, + .compression = kNoCompression, + .stable_checksum = 1234, + .stored_checksum = 0}); + std::string flattened; + BlobIndex::EncodeIndirectBlob(&flattened, /*generation=*/40, + {.origin_file_number = 30, + .original_value_offset = 300, + .stored_value_size = 70, + .compression = kZSTD, + .stable_checksum = 5678, + .stored_checksum = 8765}); + const std::vector> entries = { + {InternalKeyForTest("a", 10, kTypeBlobIndex), legacy}, + {InternalKeyForTest("b", 11, kTypeValue), "value"}, + {InternalKeyForTest("c", 12, kTypeBlobIndex), unflattened}, + {InternalKeyForTest("d", 13, kTypeBlobIndex), flattened}, + }; + + Status add_status; + const std::string expected = + BuildTableForTest(entries, TableAddModeForTest::kAdd, &add_status); + ASSERT_OK(add_status); + // The builder wrote the exact reference metadata. + ASSERT_NE(expected.find(kBlobReferenceIndexBlockName), std::string::npos); + for (const TableAddModeForTest mode : + {TableAddModeForTest::kTryAdd, TableAddModeForTest::kTryAddWithDecode}) { + SCOPED_TRACE("mode=" + std::to_string(static_cast(mode))); + Status status; + EXPECT_EQ(BuildTableForTest(entries, mode, &status), expected); + EXPECT_OK(status); + } +} + +// A malformed BlobIndex value has no shared decode, so the builder decodes it +// itself and returns the same error as Add(). +TEST(BlobReferenceMetadataTest, MalformedBlobIndexFailsTheSameWay) { + const std::vector> entries = { + {InternalKeyForTest("a", 10, kTypeBlobIndex), "\xff"}, + }; + Status add_status; + BuildTableForTest(entries, TableAddModeForTest::kAdd, &add_status); + ASSERT_EQ(add_status.code(), Status::Code::kCorruption); + for (const TableAddModeForTest mode : + {TableAddModeForTest::kTryAdd, TableAddModeForTest::kTryAddWithDecode}) { + SCOPED_TRACE("mode=" + std::to_string(static_cast(mode))); + Status status; + BuildTableForTest(entries, mode, &status); + EXPECT_EQ(status.ToString(), add_status.ToString()); + } +} + +TEST(BlobReferenceMetadataTest, EmptyRoundTrip) { + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + EXPECT_TRUE(metadata.blocks.empty()); + EXPECT_TRUE(metadata.descriptor.IsEmpty()); + + std::string encoded_descriptor; + metadata.descriptor.EncodeTo(&encoded_descriptor); + EXPECT_EQ(encoded_descriptor.size(), + BlobReferenceMetadataDescriptor::kEncodedSize); + BlobReferenceMetadataDescriptor decoded_descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(encoded_descriptor, + &decoded_descriptor)); + EXPECT_TRUE(decoded_descriptor.IsEmpty()); + EXPECT_TRUE(decoded_descriptor.HasRepairMetadata()); + EXPECT_EQ(decoded_descriptor.column_family_id, 0U); + EXPECT_EQ(decoded_descriptor.sst_file_number, 1U); + + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + EXPECT_TRUE(index.blocks.empty()); + EXPECT_TRUE(index.groups.empty()); + EXPECT_TRUE(references.empty()); +} + +TEST(BlobReferenceMetadataTest, RejectsOtherDescriptorVersions) { + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + std::string encoded; + metadata.descriptor.EncodeTo(&encoded); + BlobReferenceMetadataDescriptor decoded; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(encoded, &decoded)); + + for (uint32_t version : {0U, 2U, 4U}) { + SCOPED_TRACE(version); + std::string other_version = encoded; + EncodeFixed32(other_version.data() + 4, version); + EncodeFixed32( + other_version.data() + other_version.size() - sizeof(uint32_t), + crc32c::Value(other_version.data(), + other_version.size() - sizeof(uint32_t))); + EXPECT_TRUE( + BlobReferenceMetadataDescriptor::DecodeFrom(other_version, &decoded) + .IsCorruption()); + } +} + +TEST(BlobReferenceMetadataTest, FinishReturnsSummaryOnlyOnRequest) { + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 42, + .original_value_offset = 700, + .stored_value_size = 800, + .compression = kZSTD, + .stable_checksum = 123, + .stored_checksum = 321}); + const BlobIndex blob_index = DecodeBlobIndexForTest(encoded); + for (const bool want_summary : {false, true}) { + SCOPED_TRACE(want_summary); + BlobReferenceMetadataCollector collector( + nullptr, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(blob_index)); + MetadataForTest metadata; + std::shared_ptr summary; + ASSERT_OK(FinishForTest(&collector, &metadata, + want_summary ? &summary : nullptr)); + // The descriptor's index handle points to the index block, which Finish() + // writes after the data blocks. + EXPECT_EQ(metadata.descriptor.index_handle.size(), metadata.index.size()); + EXPECT_EQ(metadata.descriptor.reference_count, 1U); + ASSERT_EQ(summary != nullptr, want_summary); + if (want_summary) { + EXPECT_EQ(summary->file_number, kTestSstFileNumber); + EXPECT_EQ(summary->groups.size(), 1U); + } + // Finish() runs only once. A second Finish() and a later Add() return + // InvalidArgument and write nothing. + MetadataForTest again; + EXPECT_TRUE(FinishForTest(&collector, &again).IsInvalidArgument()); + EXPECT_TRUE(again.blocks.empty()); + EXPECT_TRUE(collector.Add(blob_index).IsInvalidArgument()); + } +} + +TEST(BlobReferenceMetadataTest, CollectSortAndDecode) { + std::string flattened_encoded; + BlobIndex::EncodeIndirectBlob(&flattened_encoded, 99, + {.origin_file_number = 42, + .original_value_offset = 900, + .stored_value_size = 600, + .compression = kSnappyCompression, + .stable_checksum = 456, + .stored_checksum = 654}); + std::string same_file_encoded; + BlobIndex::EncodeBlob(&same_file_encoded, kCurrentFileBlobIndexFileNumber, + /*offset=*/100, /*size=*/200, kNoCompression); + std::string inline_encoded; + BlobIndex::EncodeInlinedTTL(&inline_encoded, /*expiration=*/123, "inline"); + + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(flattened_encoded))); + ASSERT_OK(collector.Add(UnflattenedForTest(42, 700, 800))); + ASSERT_OK(collector.Add(LegacyForTest(41, 500))); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(same_file_encoded))); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(inline_encoded))); + + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + EXPECT_EQ(metadata.descriptor.reference_count, 3); + EXPECT_EQ(metadata.descriptor.stored_value_bytes, 1500); + // The same-file and inlined indexes are skipped. The three small groups that + // remain share one block. + ASSERT_EQ(metadata.blocks.size(), 1U); + + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(index.groups.size(), 3U); + ASSERT_EQ(references.size(), 3); + EXPECT_EQ(references[0].encoding_class, BlobReferenceEncodingClass::kDirect); + EXPECT_EQ(references[0].generation_file_number, 41); + EXPECT_EQ(references[0].origin_file_number, 41); + EXPECT_EQ(references[1].encoding_class, + BlobReferenceEncodingClass::kIndirectUnflattened); + EXPECT_EQ(references[2].encoding_class, + BlobReferenceEncodingClass::kIndirectFlattened); + EXPECT_EQ(references[2].generation_file_number, 99); + EXPECT_EQ(references[2].origin_file_number, 42); + EXPECT_EQ(references[2].stable_checksum, 456); + EXPECT_EQ(references[2].stored_checksum, 654); +} + +// Builds metadata with known bytes for three groups: kDirect, unflattened and +// flattened. The runs contain a zero gap, a large gap, a compression that +// differs from the run default, and a flattened gap from the end of the +// previous value. The terms are defined in blob_reference_metadata.h. +class BlobReferenceLayoutTest : public testing::Test { + protected: + static constexpr uint64_t kKeySize = 8; + // Record header and user key of every record in these tests. + static constexpr uint64_t kExtra = BlobLogRecord::kHeaderSize + kKeySize; + static constexpr uint64_t kLargeOffset = uint64_t{1} << 40; + + static void AddReferences(BlobReferenceMetadataCollector* collector) { + std::string encoded; + for (uint64_t offset : {uint64_t{140}, uint64_t{190}, kLargeOffset}) { + BlobIndex::EncodeBlob(&encoded, 41, offset, 10, kNoCompression); + ASSERT_OK(collector->Add(DecodeBlobIndexForTest(encoded), kKeySize)); + } + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 42, + .original_value_offset = 140, + .stored_value_size = 20, + .compression = kZSTD, + .stable_checksum = 1, + .stored_checksum = 2}); + ASSERT_OK(collector->Add(DecodeBlobIndexForTest(encoded), kKeySize)); + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 42, + .original_value_offset = 200, + .stored_value_size = 30, + .compression = kLZ4Compression, + .stable_checksum = 3, + .stored_checksum = 4}); + ASSERT_OK(collector->Add(DecodeBlobIndexForTest(encoded), kKeySize)); + for (const auto& [offset, size, crc] : + {std::make_tuple(uint64_t{500}, uint64_t{7}, 5U), + std::make_tuple(uint64_t{600}, uint64_t{8}, 6U)}) { + BlobIndex::EncodeIndirectBlob( + &encoded, 99, + {.origin_file_number = 42, + .original_value_offset = offset, + .stored_value_size = size, + .compression = kNoCompression, + .stable_checksum = crc, + .stored_checksum = + BlobIndex::ExtendChecksumWithCompression(crc, kNoCompression)}); + ASSERT_OK(collector->Add(DecodeBlobIndexForTest(encoded), kKeySize)); + } + } + + // A run header has no group key. It holds the count, the first offset and the + // default compression. Each entry stores (stored size << 1) | differs. The + // gap of an unflattened or kDirect entry excludes the record header and the + // user key of that entry. + static std::string DirectRun(uint64_t count = 3) { + BytesForTest run; + run.Varint(count).Varint(140).Byte(kNoCompression); + run.Varint(10 << 1).Varint(kKeySize); + run.Varint(0).Varint(10 << 1).Varint(kKeySize); + if (count == 3) { + run.Varint(kLargeOffset - (190 + 10 + kExtra)) + .Varint(10 << 1) + .Varint(kKeySize); + } + return run.bytes; + } + static std::string UnflattenedRun() { + return BytesForTest() + .Varint(2) + .Varint(140) + .Byte(kZSTD) + .Varint(20 << 1) + .Varint(kKeySize) + .Fixed32(1) + .Fixed32(2) + .Varint(0) + .Varint((30 << 1) | 1) + .Byte(kLZ4Compression) + .Varint(kKeySize) + .Fixed32(3) + .Fixed32(4) + .bytes; + } + static std::string FlattenedRun() { + return BytesForTest() + .Varint(2) + .Varint(500) + .Byte(kNoCompression) + .Varint(7 << 1) + .Fixed32(5) + .Varint(600 - (500 + 7)) + .Varint(8 << 1) + .Fixed32(6) + .bytes; + } + + // The group table entries of the three groups. Each position is a (start + // block delta, run offset) pair. + static std::string GroupTable(std::pair direct, + std::pair unflattened, + std::pair flattened, + uint64_t direct_count = 3, + uint64_t flattened_generation = 99) { + return BytesForTest() + .Varint(3) + .Byte(0) + .Varint(41) + .Varint(direct.first) + .Varint(direct.second) + .Varint(direct_count) + .Varint(30) + .Varint(3 * kKeySize) + .Byte(1) + .Varint(42) + .Varint(unflattened.first) + .Varint(unflattened.second) + .Varint(2) + .Varint(50) + .Varint(2 * kKeySize) + .Byte(2) + .Varint(flattened_generation) + .Varint(42) + .Varint(flattened.first) + .Varint(flattened.second) + .Varint(2) + .Varint(15) + .bytes; + } + + // A block table for blocks of these sizes. The first block starts at offset + // 1000, and each block follows the trailer of the previous one. + static std::string BlockTable(const std::vector& blocks) { + BytesForTest table; + table.Varint(blocks.size()).Varint(1000); + for (const std::string& block : blocks) { + table.Varint(block.size()); + } + return table.bytes; + } + + // The index for one block that holds the three runs. + static std::string IndexFor(const std::string& block, size_t direct_size, + size_t unflattened_size, + uint64_t direct_count = 3) { + return BlockTable({block}) + GroupTable({0, 0}, {0, direct_size}, + {0, direct_size + unflattened_size}, + direct_count); + } + + // Metadata with these blocks and this index. The index directly follows the + // trailer of the last block. + static MetadataForTest MetadataFor(std::vector blocks, + std::string index, + uint64_t reference_count = 7, + uint64_t stored_value_bytes = 95) { + MetadataForTest metadata; + uint64_t next_offset = 1000; + for (const std::string& block : blocks) { + next_offset += block.size() + 5; + } + metadata.blocks = std::move(blocks); + metadata.index = std::move(index); + metadata.descriptor.reference_count = reference_count; + metadata.descriptor.stored_value_bytes = stored_value_bytes; + metadata.descriptor.index_handle = + BlockHandle(next_offset, metadata.index.size()); + return metadata; + } + + static MetadataForTest Good() { + const std::string block = DirectRun() + UnflattenedRun() + FlattenedRun(); + return MetadataFor( + {block}, IndexFor(block, DirectRun().size(), UnflattenedRun().size())); + } + + static bool DecodeFails(const MetadataForTest& metadata) { + BlobReferenceIndex index; + std::vector references; + const Status status = DecodeAllForTest(metadata, &index, &references); + EXPECT_TRUE(status.ok() || status.IsCorruption()) << status.ToString(); + return status.IsCorruption(); + } +}; + +TEST_F(BlobReferenceLayoutTest, RejectsCountThatDoesNotFitItsBlocks) { + // The decoder has no fixed limit on the reference count, but the descriptor's + // count must fit the data blocks. A block holds at most 64 KB of entries, and + // an entry takes at least 2 bytes, so one block cannot hold 64 KB / 2 + 1 + // references. + MetadataForTest metadata = Good(); + ASSERT_EQ(metadata.blocks.size(), 1U); + BlobReferenceIndex index; + metadata.descriptor.reference_count = 64 * 1024 / 2 + 1; + const Status status = + DecodeBlobReferenceIndex(metadata.index, metadata.descriptor, &index); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_NE(status.ToString().find("does not fit"), std::string::npos) + << status.ToString(); +} + +TEST_F(BlobReferenceLayoutTest, RejectsFlattenedGenerationBelowOrigin) { + // A flattened group needs generation > origin, as kIndirectFlattenedBlob + // does, so generation 40 with origin 42 is malformed. + const std::string block = DirectRun() + UnflattenedRun() + FlattenedRun(); + const std::string index = + BlockTable({block}) + + GroupTable({0, 0}, {0, DirectRun().size()}, + {0, DirectRun().size() + UnflattenedRun().size()}, + /*direct_count=*/3, /*flattened_generation=*/40); + EXPECT_TRUE(DecodeFails(MetadataFor({block}, index))); + EXPECT_FALSE(DecodeFails(Good())); +} + +TEST_F(BlobReferenceLayoutTest, EncodesKnownBytes) { + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + AddReferences(&collector); + MetadataForTest metadata; + std::shared_ptr summary; + ASSERT_OK(FinishForTest(&collector, &metadata, &summary)); + const MetadataForTest expected = Good(); + ASSERT_EQ(metadata.blocks, expected.blocks); + EXPECT_EQ(metadata.index, expected.index); + EXPECT_EQ(metadata.descriptor.reference_count, 7); + EXPECT_EQ(metadata.descriptor.stored_value_bytes, 95); + + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(references.size(), 7U); + EXPECT_EQ(ReferenceFieldsForTest(references[2]), + std::make_tuple(BlobReferenceEncodingClass::kDirect, uint64_t{41}, + uint64_t{41}, kLargeOffset, uint64_t{10}, + 10 + kExtra, kNoCompression, 0U, 0U)); + EXPECT_EQ( + ReferenceFieldsForTest(references[4]), + std::make_tuple(BlobReferenceEncodingClass::kIndirectUnflattened, + uint64_t{42}, uint64_t{42}, uint64_t{200}, uint64_t{30}, + 30 + kExtra, kLZ4Compression, 3U, 4U)); + EXPECT_EQ(ReferenceFieldsForTest(references[6]), + std::make_tuple( + BlobReferenceEncodingClass::kIndirectFlattened, uint64_t{99}, + uint64_t{42}, uint64_t{600}, uint64_t{8}, + 8 + kSimpleGen2BlobTrailerSize, kNoCompression, 6U, + BlobIndex::ExtendChecksumWithCompression(6, kNoCompression))); + + // The groups of the build-time summary equal the index groups without the + // kDirect group. + ASSERT_NE(summary, nullptr); + const auto group_fields = [](const std::vector& groups) { + std::vector> + fields; + for (const BlobReferenceGroup& group : groups) { + fields.emplace_back(group.encoding_class, group.generation_file_number, + group.origin_file_number, group.reference_count, + group.stored_value_bytes, group.record_bytes); + } + return fields; + }; + EXPECT_EQ(group_fields(IndirectGroupsForTest(index)), + group_fields(summary->groups)); + EXPECT_EQ(summary->groups.size(), 2U); +} + +TEST_F(BlobReferenceLayoutTest, RejectsCorruptMetadata) { + ASSERT_FALSE(DecodeFails(Good())); + const std::string direct = DirectRun(); + const std::string unflattened = UnflattenedRun(); + const std::string flattened = FlattenedRun(); + const std::string block = direct + unflattened + flattened; + const auto with_runs = [&](const std::string& d, const std::string& u, + const std::string& f) { + return MetadataFor({d + u + f}, IndexFor(d + u + f, d.size(), u.size())); + }; + // Replaces `old_bytes` bytes at `pos` of `run`. + const auto patch = [](std::string run, size_t pos, size_t old_bytes, + const std::string& bytes) { + return run.replace(pos, old_bytes, bytes); + }; + const size_t run_header_size = 4; // count, first offset (2), compression + { + // Non-canonical varints: the run count, and the first entry's size. + EXPECT_TRUE( + DecodeFails(with_runs(patch(direct, 0, 1, std::string("\x83\x00", 2)), + unflattened, flattened))); + EXPECT_TRUE(DecodeFails( + with_runs(patch(direct, run_header_size, 1, std::string("\x94\x00", 2)), + unflattened, flattened))); + } + { + // A compression flagged as different that equals the run default. + const std::string same_compression = + BytesForTest().Varint((20 << 1) | 1).Byte(kZSTD).bytes; + EXPECT_TRUE(DecodeFails(with_runs( + direct, patch(unflattened, run_header_size, 1, same_compression), + flattened))); + } + { + // A wrong run offset is caught, although runs hold no group key. Either the + // bytes do not decode as the group's runs, or the runs do not end where the + // next group starts. + for (const auto& [unflattened_offset, flattened_offset] : + {std::make_pair(direct.size() + 1, direct.size() + unflattened.size()), + std::make_pair(direct.size(), direct.size() + unflattened.size() - 1), + std::make_pair(direct.size(), + direct.size() + unflattened.size() + 1)}) { + SCOPED_TRACE(unflattened_offset); + SCOPED_TRACE(flattened_offset); + EXPECT_TRUE(DecodeFails(MetadataFor( + {block}, + BlockTable({block}) + GroupTable({0, 0}, {0, unflattened_offset}, + {0, flattened_offset})))); + } + } + { + // A repeated offset in a flattened group: stored size 0 and gap 0. + const std::string repeat = BytesForTest() + .Varint(2) + .Varint(500) + .Byte(kNoCompression) + .Varint(0) + .Fixed32(5) + .Varint(0) + .Varint(0) + .Fixed32(6) + .bytes; + const std::string index = BlockTable({repeat}) + BytesForTest() + .Varint(1) + .Byte(2) + .Varint(99) + .Varint(42) + .Varint(0) + .Varint(0) + .Varint(2) + .Varint(0) + .bytes; + EXPECT_TRUE(DecodeFails(MetadataFor({repeat}, index, 2, 0))); + } + { + // Index class rules: an unknown class, and a flattened group whose origin + // equals its generation. + std::string unknown_class = Good().index; + const size_t first_class = BlockTable({block}).size() + 1; + ASSERT_EQ(unknown_class[first_class], 0); + unknown_class[first_class] = 3; + EXPECT_TRUE(DecodeFails(MetadataFor({block}, unknown_class))); + std::string same_origin = Good().index; + const size_t origin_pos = same_origin.size() - 5; + ASSERT_EQ(same_origin[origin_pos], 42); + same_origin[origin_pos] = 99; + EXPECT_TRUE(DecodeFails(MetadataFor({block}, same_origin))); + } + { + // Run counts that do not match the index totals. + EXPECT_TRUE(DecodeFails(MetadataFor( + {block}, IndexFor(block, direct.size(), unflattened.size(), 4), 8, + 95))); + EXPECT_TRUE(DecodeFails(MetadataFor( + {block}, IndexFor(block, direct.size(), unflattened.size(), 2), 6, + 95))); + } + { + // A run offset outside the block, read through the group reader. + const MetadataForTest metadata = MetadataFor( + {block}, + BlockTable({block}) + + GroupTable({0, 0}, {0, block.size() + 10}, {0, block.size() + 20})); + BlobReferenceIndex index; + ASSERT_OK( + DecodeBlobReferenceIndex(metadata.index, metadata.descriptor, &index)); + std::vector references; + EXPECT_TRUE( + ReadGroupForTest(metadata, index, 1, &references).IsCorruption()); + EXPECT_TRUE(DecodeFails(metadata)); + } + { + // A group that continues in the next block, which starts with another + // group's run instead. + const std::string two_of_three = DirectRun(2); + const std::string block1 = unflattened + flattened; + const MetadataForTest metadata = + MetadataFor({two_of_three, block1}, + BlockTable({two_of_three, block1}) + + GroupTable({0, 0}, {1, 0}, {0, unflattened.size()})); + EXPECT_TRUE(DecodeFails(metadata)); + BlobReferenceIndex decoded_index; + ASSERT_OK(DecodeBlobReferenceIndex(metadata.index, metadata.descriptor, + &decoded_index)); + std::vector references; + EXPECT_TRUE(ReadGroupForTest(metadata, decoded_index, 0, &references) + .IsCorruption()); + } + { + // Index groups out of order, trailing index bytes, totals that differ from + // the descriptor, a block that overlaps the index, and an index that does + // not directly follow the last block. + const MetadataForTest good = Good(); + const std::string swapped = BlockTable({block}) + BytesForTest() + .Varint(2) + .Byte(1) + .Varint(42) + .Varint(0) + .Varint(0) + .Varint(2) + .Varint(50) + .Varint(2 * kKeySize) + .Byte(0) + .Varint(41) + .Varint(0) + .Varint(10) + .Varint(3) + .Varint(30) + .Varint(3 * kKeySize) + .bytes; + EXPECT_TRUE(DecodeFails(MetadataFor(good.blocks, swapped, 5, 80))); + EXPECT_TRUE( + DecodeFails(MetadataFor(good.blocks, good.index + std::string(1, 0)))); + EXPECT_TRUE(DecodeFails(MetadataFor(good.blocks, good.index, 8, 95))); + EXPECT_TRUE(DecodeFails(MetadataFor(good.blocks, good.index, 7, 96))); + for (int64_t shift : {-1, 1}) { + SCOPED_TRACE(shift); + MetadataForTest metadata = good; + metadata.descriptor.index_handle = BlockHandle( + good.descriptor.index_handle.offset() + shift, good.index.size()); + EXPECT_TRUE(DecodeFails(metadata)); + } + } + { + // Truncated and bit-flipped descriptors. + const MetadataForTest good = Good(); + BlobReferenceMetadataDescriptor descriptor = good.descriptor; + descriptor.column_family_id = kTestColumnFamilyId; + descriptor.sst_file_number = kTestSstFileNumber; + std::string encoded; + descriptor.EncodeTo(&encoded); + BlobReferenceMetadataDescriptor decoded; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(encoded, &decoded)); + // The decoder rejects a descriptor without repair metadata: an unknown + // column family or SST file number 0. + for (int field = 0; field < 2; ++field) { + SCOPED_TRACE(field); + BlobReferenceMetadataDescriptor incomplete = descriptor; + if (field == 0) { + incomplete.column_family_id = std::numeric_limits::max(); + } else { + incomplete.sst_file_number = 0; + } + std::string incomplete_encoded; + incomplete.EncodeTo(&incomplete_encoded); + EXPECT_TRUE(BlobReferenceMetadataDescriptor::DecodeFrom( + incomplete_encoded, &decoded) + .IsCorruption()); + } + EXPECT_TRUE(BlobReferenceMetadataDescriptor::DecodeFrom( + Slice(encoded.data(), encoded.size() - 1), &decoded) + .IsCorruption()); + encoded[10] ^= 1; + EXPECT_TRUE(BlobReferenceMetadataDescriptor::DecodeFrom(encoded, &decoded) + .IsCorruption()); + } +} + +TEST(BlobReferenceMetadataTest, OverlappingReferencesFailTheBuild) { + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + // The second value starts inside the first record. Add() finds an in-order + // overlap. The build fails, and Finish() returns the same error. + ASSERT_OK(collector.Add(UnflattenedForTest(42, 1000, 100))); + EXPECT_TRUE(collector.Add(UnflattenedForTest(42, 1050, 100)).IsCorruption()); + MetadataForTest metadata; + const Status status = FinishForTest(&collector, &metadata); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + + // An out-of-order overlap is found when Finish() sorts the group. + BlobReferenceMetadataCollector unsorted( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(unsorted.Add(UnflattenedForTest(42, 1050, 100))); + ASSERT_OK(unsorted.Add(UnflattenedForTest(42, 1000, 100))); + const Status unsorted_status = FinishForTest(&unsorted, &metadata); + EXPECT_TRUE(unsorted_status.IsCorruption()) << unsorted_status.ToString(); +} + +namespace { +// An unflattened reference with the given compression and consistent CRCs. +BlobIndex UnflattenedWithCompressionForTest(uint64_t origin, uint64_t offset, + uint64_t stored_size, + CompressionType compression) { + const uint32_t stable = static_cast(offset * 7 + origin); + const uint32_t stored = + compression == kNoCompression + ? BlobIndex::ExtendChecksumWithCompression(stable, kNoCompression) + : stable + 1; + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = origin, + .original_value_offset = offset, + .stored_value_size = stored_size, + .compression = compression, + .stable_checksum = stable, + .stored_checksum = stored}); + return DecodeBlobIndexForTest(encoded); +} + +// Returns the references that a collector should produce from these +// BlobIndexes, in sort order (class, generation, origin, original offset). +std::vector ExpectedReferencesForTest( + const std::vector& added, uint64_t user_key_size) { + std::vector expected; + for (const BlobIndex& blob_index : added) { + BlobReference reference; + bool included = false; + EXPECT_OK(BlobReference::FromBlobIndex(blob_index, &reference, &included)); + if (reference.encoding_class != + BlobReferenceEncodingClass::kIndirectFlattened) { + reference.record_bytes += + BlobLogRecord::CalculateAdjustmentForRecordHeader(user_key_size); + } + expected.push_back(reference); + } + std::sort( + expected.begin(), expected.end(), + [](const BlobReference& lhs, const BlobReference& rhs) { + return std::tie(lhs.encoding_class, lhs.generation_file_number, + lhs.origin_file_number, lhs.original_value_offset) < + std::tie(rhs.encoding_class, rhs.generation_file_number, + rhs.origin_file_number, rhs.original_value_offset); + }); + return expected; +} + +void ExpectSameReferencesForTest(const std::vector& actual, + const std::vector& expected) { + ASSERT_EQ(actual.size(), expected.size()); + for (size_t i = 0; i < actual.size(); ++i) { + ASSERT_EQ(ReferenceFieldsForTest(actual[i]), + ReferenceFieldsForTest(expected[i])) + << i; + } +} +} // namespace + +TEST(BlobReferenceMetadataTest, InterleavedGroupsDecodeInKeyOrder) { + // References of many groups arrive interleaved, each group in rising offset + // order, with mixed compression inside a group. + const uint32_t seed = 20261107; + SCOPED_TRACE("seed " + std::to_string(seed)); + Random rnd(seed); + constexpr uint64_t kGroups = 60; + std::vector next_offset(kGroups, 10); + std::vector added; + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (int i = 0; i < 20000; ++i) { + const uint64_t group = rnd.Uniform(kGroups); + const uint64_t size = 10 + rnd.Uniform(200); + const CompressionType compression = rnd.OneIn(4) ? kNoCompression : kZSTD; + added.push_back(UnflattenedWithCompressionForTest( + 100 + group, next_offset[group], size, compression)); + ASSERT_OK(collector.Add(added.back(), /*user_key_size=*/16)); + next_offset[group] += + size + BlobLogRecord::kHeaderSize + 16 + rnd.Uniform(3) * 50; + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + const std::vector expected = + ExpectedReferencesForTest(added, /*user_key_size=*/16); + ExpectSameReferencesForTest(references, expected); + ASSERT_EQ(index.groups.size(), kGroups); + ASSERT_GT(index.blocks.size(), 1U); + // Index totals equal the sums over each group's references. + size_t position = 0; + for (const BlobReferenceIndexEntry& entry : index.groups) { + uint64_t count = 0, stored = 0, record = 0; + while (position < expected.size() && + expected[position].origin_file_number == + entry.group.origin_file_number) { + ++count; + stored += expected[position].stored_value_size; + record += expected[position].record_bytes; + ++position; + } + EXPECT_EQ(entry.group.reference_count, count); + EXPECT_EQ(entry.group.stored_value_bytes, stored); + EXPECT_EQ(entry.group.record_bytes, record); + } +} + +// The collector does not keep the stored CRC of a kNoCompression indirect +// reference, because the metadata does not encode it. The reader derives it +// from the stable CRC. This test checks that every decoded reference still has +// the correct stored CRC: +// - kNoCompression: ExtendChecksumWithCompression(stable CRC, kNoCompression) +// - compressed (kLZ4Compression here): the stored CRC of the BlobIndex +// It uses unflattened and flattened references, and one group whose offsets +// arrive out of order, so that the unsorted fallback path is also checked. +TEST(BlobReferenceMetadataTest, DecodedStoredChecksumMatchesBlobIndex) { + struct Added { + BlobIndex blob_index; + uint32_t expected_stored_checksum; + }; + std::vector added; + // Adds one indirect reference. The file numbers, offsets and CRC values are + // example values. The stored CRC of a compressed reference differs from its + // stable CRC, so a mix-up of the two fields fails the test. + auto add = [&](uint64_t generation, uint64_t origin, uint64_t offset, + CompressionType compression) { + const uint32_t stable = static_cast(origin * 1000 + offset); + const uint32_t stored = stable + 7; + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, generation, + {.origin_file_number = origin, + .original_value_offset = offset, + .stored_value_size = 40, + .compression = compression, + .stable_checksum = stable, + .stored_checksum = stored}); + const BlobIndex blob_index = DecodeBlobIndexForTest(encoded); + const uint32_t expected = + compression == kNoCompression + ? BlobIndex::ExtendChecksumWithCompression(stable, kNoCompression) + : stored; + // The BlobIndex itself gives the same stored CRC. + EXPECT_EQ(blob_index.stored_checksum(), expected); + added.push_back({blob_index, expected}); + }; + // Unflattened group of origin 50, offsets in order. + add(50, 50, 100, kNoCompression); + add(50, 50, 300, kLZ4Compression); + add(50, 50, 500, kNoCompression); + // Unflattened group of origin 51, offsets out of order (fallback path). + add(51, 51, 900, kNoCompression); + add(51, 51, 100, kLZ4Compression); + add(51, 51, 500, kNoCompression); + // Flattened group: generation 70, origin 60. + add(70, 60, 100, kNoCompression); + add(70, 60, 300, kLZ4Compression); + + const uint64_t fallbacks_before = + BlobReferenceMetadataCollector::UnsortedGroupFallbackCount(); + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (const Added& entry : added) { + ASSERT_OK(collector.Add(entry.blob_index, /*user_key_size=*/8)); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + // Only the group of origin 51 took the fallback. + EXPECT_EQ(BlobReferenceMetadataCollector::UnsortedGroupFallbackCount(), + fallbacks_before + 1); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(references.size(), added.size()); + for (const BlobReference& reference : references) { + SCOPED_TRACE("origin=" + std::to_string(reference.origin_file_number) + + " offset=" + std::to_string(reference.original_value_offset)); + const Added* match = nullptr; + for (const Added& entry : added) { + if (entry.blob_index.origin_file_number() == + reference.origin_file_number && + entry.blob_index.generation_file_number() == + reference.generation_file_number && + entry.blob_index.original_value_offset() == + reference.original_value_offset) { + match = &entry; + } + } + ASSERT_NE(match, nullptr); + EXPECT_EQ(reference.compression, match->blob_index.compression()); + EXPECT_EQ(reference.stable_checksum, match->blob_index.stable_checksum()); + EXPECT_EQ(reference.stored_checksum, match->expected_stored_checksum); + } + // The full reference also equals the one BlobReference::FromBlobIndex() + // gives, which derives the stored CRC itself. + std::vector blob_indexes; + for (const Added& entry : added) { + blob_indexes.push_back(entry.blob_index); + } + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(blob_indexes, /*user_key_size=*/8)); +} + +namespace { +// Encodes one run of in-order references of one group, from the references +// themselves, with the layout that BlobReferenceLayoutTest describes: a run +// header (count, first offset, default compression), then one entry per +// reference. The stored CRC is in an entry only for a compressed reference. +std::string EncodeRunFromReferencesForTest( + const std::vector& references, uint64_t user_key_size) { + const CompressionType default_compression = references[0].compression; + const bool stores_key_size = references[0].encoding_class != + BlobReferenceEncodingClass::kIndirectFlattened; + const uint64_t record_overhead = + stores_key_size ? BlobLogRecord::kHeaderSize + user_key_size : 0; + BytesForTest bytes; + bytes.Varint(references.size()) + .Varint(references[0].original_value_offset) + .Byte(static_cast(default_compression)); + for (size_t i = 0; i < references.size(); ++i) { + const BlobReference& reference = references[i]; + if (i > 0) { + const BlobReference& previous = references[i - 1]; + bytes.Varint(reference.original_value_offset - + (previous.original_value_offset + + previous.stored_value_size + record_overhead)); + } + const bool differs = reference.compression != default_compression; + bytes.Varint((reference.stored_value_size << 1) | (differs ? 1 : 0)); + if (differs) { + bytes.Byte(static_cast(reference.compression)); + } + if (stores_key_size) { + bytes.Varint(user_key_size); + } + bytes.Fixed32(reference.stable_checksum); + if (reference.compression != kNoCompression) { + bytes.Fixed32(reference.stored_checksum); + } + } + return bytes.bytes; +} +} // namespace + +// The collector skips the stored CRC of a kNoCompression indirect reference. +// This test checks that the skip does not change the encoded metadata: the +// collector's block equals the bytes encoded from the canonical references +// that BlobReference::FromBlobIndex() gives, which carry the derived stored +// CRC. It uses one unflattened group and one flattened group, each with +// kNoCompression and kLZ4Compression references. +TEST(BlobReferenceMetadataTest, StoredChecksumSkipKeepsMetadataBytes) { + constexpr uint64_t kKeySize = 8; + constexpr uint64_t kStoredSize = 40; + std::vector unflattened; + std::vector flattened; + // The file numbers, offsets and CRC values are example values. Offsets + // leave a gap after each record, so no records overlap. + const CompressionType compressions[] = {kNoCompression, kLZ4Compression, + kNoCompression}; + for (size_t i = 0; i < 3; ++i) { + const uint64_t offset = 100 + i * 200; + const uint32_t stable = static_cast(1000 + i); + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&encoded, + {.origin_file_number = 50, + .original_value_offset = offset, + .stored_value_size = kStoredSize, + .compression = compressions[i], + .stable_checksum = stable, + .stored_checksum = stable + 7}); + unflattened.push_back(DecodeBlobIndexForTest(encoded)); + BlobIndex::EncodeIndirectBlob(&encoded, /*generation=*/70, + {.origin_file_number = 60, + .original_value_offset = offset, + .stored_value_size = kStoredSize, + .compression = compressions[i], + .stable_checksum = stable + 100, + .stored_checksum = stable + 107}); + flattened.push_back(DecodeBlobIndexForTest(encoded)); + } + + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (size_t i = 0; i < 3; ++i) { + ASSERT_OK(collector.Add(unflattened[i], kKeySize)); + ASSERT_OK(collector.Add(flattened[i], kKeySize)); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + ASSERT_EQ(metadata.blocks.size(), 1U); + + // The canonical references carry the derived stored CRC for kNoCompression. + const std::vector unflattened_references = + ExpectedReferencesForTest(unflattened, kKeySize); + const std::vector flattened_references = + ExpectedReferencesForTest(flattened, kKeySize); + for (const auto* references : + {&unflattened_references, &flattened_references}) { + for (const BlobReference& reference : *references) { + if (reference.compression == kNoCompression) { + EXPECT_EQ(reference.stored_checksum, + BlobIndex::ExtendChecksumWithCompression( + reference.stable_checksum, kNoCompression)); + } + } + } + // The unflattened group sorts before the flattened group, so its run comes + // first in the block. + EXPECT_EQ(metadata.blocks[0], + EncodeRunFromReferencesForTest(unflattened_references, kKeySize) + + EncodeRunFromReferencesForTest(flattened_references, kKeySize)); +} + +TEST(BlobReferenceMetadataTest, EstimateBoundsRealSizeClosely) { + // 200 groups arrive interleaved, with 16-byte user keys and mixed value sizes + // and compression. The estimate made before Finish() is at least the real + // metadata size and within 10% of it. + const uint32_t seed = 20261108; + SCOPED_TRACE("seed " + std::to_string(seed)); + Random rnd(seed); + constexpr uint64_t kGroups = 200; + std::vector next_offset(kGroups, 10); + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (int i = 0; i < 200000; ++i) { + const uint64_t group = rnd.Uniform(kGroups); + const uint64_t size = 512 + rnd.Uniform(4096); + ASSERT_OK(collector.Add(UnflattenedWithCompressionForTest( + 100 + group, next_offset[group], size, + rnd.OneIn(4) ? kNoCompression : kZSTD), + /*user_key_size=*/16)); + next_offset[group] += size + BlobLogRecord::kHeaderSize + 16; + } + const uint64_t estimate = collector.EstimatedEncodedSize(); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + uint64_t real = metadata.index.size(); + for (const std::string& block : metadata.blocks) { + real += block.size() + BlockBasedTable::kBlockTrailerSize; + } + EXPECT_GE(estimate, real); + EXPECT_LE(estimate, real + real / 10); +} + +TEST(BlobReferenceMetadataTest, RunsOfOneGroupMustNotOverlapAcrossBlocks) { + // One group spans two blocks. The second run stores its first offset as a + // full value, not as a gap, so the decoder must check it against the first + // run. A first offset whose record would overlap the last record of the first + // run is Corruption. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + uint64_t offset = 10; + for (int i = 0; i < 2000; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest(42, offset, 40))); + offset += 40 + BlobLogRecord::kHeaderSize; + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + ASSERT_EQ(metadata.blocks.size(), 2U); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + // The first reference of the second block and the one before it. + Slice header = metadata.blocks[1]; + uint64_t count = 0; + uint64_t first_offset = 0; + ASSERT_TRUE(GetVarint64(&header, &count)); + ASSERT_TRUE(GetVarint64(&header, &first_offset)); + const size_t second = references.size() - count; + ASSERT_EQ(references[second].original_value_offset, first_offset); + const BlobReference& last = references[second - 1]; + const auto with_first_offset = [&](uint64_t new_first) { + MetadataForTest changed = metadata; + std::string block; + PutVarint64(&block, count); + PutVarint64(&block, new_first); + block.append(header.data(), header.size()); + changed.blocks[1] = block; + // Keep the block table consistent with the new block size. + BlobReferenceIndex changed_index = index; + changed_index.blocks[1] = + BlockHandle(index.blocks[1].offset(), changed.blocks[1].size()); + return ForEachBlobReference( + changed_index, BlockReaderForTest(changed), + [](const BlobReference&) { return Status::OK(); }); + }; + ASSERT_OK(with_first_offset(first_offset)); + for (const uint64_t bad : + {last.original_value_offset, + last.original_value_offset + last.stored_value_size, + first_offset - 1}) { + SCOPED_TRACE(bad); + const Status status = with_first_offset(bad); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + } +} + +TEST(BlobReferenceMetadataTest, HugeIndexCountFailsBeforeAllocation) { + // A tiny index that claims many blocks or many groups is rejected by the + // input size check alone, before any reserve(). + const auto decode = [](const std::string& input) { + BlobReferenceMetadataDescriptor descriptor; + descriptor.index_handle = BlockHandle(1000, input.size()); + descriptor.reference_count = 1; + BlobReferenceIndex index; + return DecodeBlobReferenceIndex(input, descriptor, &index); + }; + std::string blocks; + PutVarint64(&blocks, 1 << 20); + PutVarint64(&blocks, 0); + EXPECT_TRUE(decode(blocks).IsCorruption()); + std::string groups; + PutVarint64(&groups, 1); + PutVarint64(&groups, 0); + PutVarint64(&groups, 995); + PutVarint64(&groups, 1 << 20); + EXPECT_TRUE(decode(groups).IsCorruption()); +} + +TEST(BlobReferenceMetadataTest, GroupSpanningChunksSplitsRunsAtBlockEnds) { + // One group spans many chunks and several blocks. Runs end only at block + // ends, and the gaps at chunk joins decode to the right offsets. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::vector added; + uint64_t offset = 10; + for (int i = 0; i < 10000; ++i) { + added.push_back(UnflattenedForTest(42, offset, 40)); + ASSERT_OK(collector.Add(added.back())); + // Every 7th record leaves a gap of unreferenced bytes. + offset += 40 + BlobLogRecord::kHeaderSize + (i % 7 == 0 ? 123 : 0); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(added, /*user_key_size=*/0)); + ASSERT_EQ(index.groups.size(), 1U); + ASSERT_GT(index.blocks.size(), 3U); + // Each block holds one run of the group. The first reference of a run is the + // one right after the last reference of the previous block. + BlobReferenceGroupDecoder decoder(index, /*group=*/0); + size_t next = 0; + for (size_t block = 0; block < metadata.blocks.size(); ++block) { + ASSERT_FALSE(decoder.done()); + ASSERT_EQ(decoder.next_block(), block); + const Slice contents = metadata.blocks[block]; + const size_t first = next; + ASSERT_OK(decoder.DecodeRun(contents, [&](const BlobReference& reference) { + EXPECT_EQ(reference.original_value_offset, + references[next++].original_value_offset); + return Status::OK(); + })); + ASSERT_GT(next, first); + // Each block except the last stays within the 16 KB target. A run takes + // only whole 512-byte chunks, so a block is about one chunk short at most. + // The test allows a block to be up to 1 KB short. + if (block + 1 < metadata.blocks.size()) { + EXPECT_LE(contents.size(), 16 * 1024U); + EXPECT_GT(contents.size(), 15 * 1024U); + } + } + ASSERT_EQ(next, references.size()); +} + +TEST(BlobReferenceMetadataTest, DecoderRejectsBlockOver64KBAfterDecompression) { + // The index decoder checks only the size on disk. A compressed block that + // decompresses to more than 64 KB passes that check, so the run decoder + // rejects it. Here the contents of a valid block are padded past 64 KB, as + // a corrupt compressed block could decompress. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(UnflattenedForTest(42, /*offset=*/10, 40))); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(metadata.blocks.size(), 1U); + std::string padded = metadata.blocks[0]; + padded.resize(64 * 1024 + 1, '\0'); + BlobReferenceGroupDecoder decoder(index, /*group=*/0); + const Status status = decoder.DecodeRun( + padded, [](const BlobReference& /*reference*/) { return Status::OK(); }); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("64 KB"), std::string::npos); +} + +TEST(BlobReferenceMetadataTest, SmallGroupsShareOneBlock) { + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (uint64_t origin = 100; origin < 200; ++origin) { + ASSERT_OK(collector.Add(UnflattenedForTest(origin, 10, 40))); + ASSERT_OK(collector.Add(UnflattenedForTest(origin, 200, 40))); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(index.blocks.size(), 1U); + ASSERT_EQ(index.groups.size(), 100U); + for (size_t i = 0; i < index.groups.size(); ++i) { + EXPECT_EQ(index.groups[i].start_block, 0U); + if (i > 0) { + EXPECT_GT(index.groups[i].run_offset, index.groups[i - 1].run_offset); + } + } +} + +TEST(BlobReferenceMetadataTest, UnsortedGroupTakesFallback) { + // An offset lower than the group's last one sends the group to the unsorted + // fallback. The group still decodes in offset order. + const uint64_t fallbacks_before = + BlobReferenceMetadataCollector::UnsortedGroupFallbackCount(); + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::vector added; + for (uint64_t i = 0; i < 2000; ++i) { + // Two interleaved offset sequences in one group. + const uint64_t offset = (i % 2 == 0 ? 0 : 1000000) + (i / 2) * 200; + added.push_back(UnflattenedForTest(42, offset, 40)); + ASSERT_OK(collector.Add(added.back())); + } + added.push_back(UnflattenedForTest(43, 10, 40)); + ASSERT_OK(collector.Add(added.back())); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + EXPECT_EQ(BlobReferenceMetadataCollector::UnsortedGroupFallbackCount(), + fallbacks_before + 1); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(added, /*user_key_size=*/0)); +} + +TEST(BlobReferenceMetadataTest, FallbackKeepsItsScratchCharged) { + // While Finish() encodes a fallback group again, the cache charge still + // includes the group's sort keys. No charge update during that time drops + // below them. + auto reservation_manager = std::make_shared( + std::numeric_limits::max()); + BlobReferenceMetadataCollector collector( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + constexpr uint64_t kCount = 20000; + for (uint64_t i = 0; i < kCount; ++i) { + // Two interleaved offset sequences in one group. + const uint64_t offset = (i % 2 == 0 ? 0 : 100000000) + (i / 2) * 200; + ASSERT_OK(collector.Add(UnflattenedForTest(42, offset, 40))); + } + bool encoding = false; + size_t lowest = std::numeric_limits::max(); + size_t updates = 0; + reservation_manager->on_update = [&](size_t charge) { + if (encoding) { + lowest = std::min(lowest, charge); + ++updates; + } + }; + SyncPoint::GetInstance()->SetCallBack( + "BlobReferenceMetadataCollector::SortUnsortedGroup", + [&](void*) { encoding = true; }); + SyncPoint::GetInstance()->SetCallBack( + "BlobReferenceMetadataCollector::SortUnsortedGroup:Encoded", + [&](void*) { encoding = false; }); + SyncPoint::GetInstance()->EnableProcessing(); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + reservation_manager->on_update = nullptr; + ASSERT_GT(updates, 1U); + // The charge covers one 16-byte sort key per reference plus the chunks. It + // stays below sizeof(BlobReference) (64 bytes) per reference, the cost of + // decoding every reference into a full BlobReference. + EXPECT_GE(lowest, kCount * 16); + EXPECT_LT(lowest, kCount * sizeof(BlobReference)); +} + +TEST(BlobReferenceMetadataTest, LargeFallbackGroupStaysCompact) { + // A large group has every other reference out of order. While the collector + // collects and sorts it, the charge is 16 bytes of sort key per reference + // plus the chunks. The chunks hold the entries (about 11 to 15 bytes each + // here, held twice while the group is encoded again) and the chunk headers. + // The peak is about 44 bytes per reference (estimate), and the test allows + // 56. Decoding each reference into a 64-byte BlobReference, in both an + // unsorted buffer and a sort buffer, would need more than 100. + auto reservation_manager = std::make_shared( + std::numeric_limits::max()); + size_t peak = 0; + reservation_manager->on_update = [&](size_t charge) { + peak = std::max(peak, charge); + }; + BlobReferenceMetadataCollector collector( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + constexpr uint64_t kCount = 200000; + std::vector added; + added.reserve(kCount); + for (uint64_t i = 0; i < kCount; ++i) { + // Two interleaved offset sequences in one group. + const uint64_t offset = (i % 2 == 0 ? 0 : 100000000) + (i / 2) * 200; + added.push_back(UnflattenedForTest(42, offset, 40)); + ASSERT_OK(collector.Add(added.back())); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + reservation_manager->on_update = nullptr; + EXPECT_LE(peak, kCount * 56); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(added, /*user_key_size=*/0)); +} + +TEST(BlobReferenceMetadataTest, FallbackKeepsDefaultAndStaysUnderEstimate) { + // In-order ZSTD references arrive, then one kNoCompression reference at a + // lower offset. The fallback keeps the group's default compression (ZSTD), so + // the ZSTD entries do not grow and the estimate stays an upper bound. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::vector added; + for (uint64_t i = 0; i < 2000; ++i) { + added.push_back( + UnflattenedWithCompressionForTest(42, 1000 + i * 100, 40, kZSTD)); + ASSERT_OK(collector.Add(added.back())); + } + added.push_back( + UnflattenedWithCompressionForTest(42, 10, 40, kNoCompression)); + ASSERT_OK(collector.Add(added.back())); + const uint64_t estimate = collector.EstimatedEncodedSize(); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + uint64_t real = metadata.index.size(); + for (const std::string& block : metadata.blocks) { + real += block.size() + BlockBasedTable::kBlockTrailerSize; + } + EXPECT_GE(estimate, real); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(added, /*user_key_size=*/0)); + // The first run's default byte is still ZSTD. + Slice header = metadata.blocks[0]; + uint64_t value = 0; + ASSERT_TRUE(GetVarint64(&header, &value)); + ASSERT_TRUE(GetVarint64(&header, &value)); + ASSERT_FALSE(header.empty()); + EXPECT_EQ(static_cast(header[0]), kZSTD); +} + +TEST(BlobReferenceMetadataTest, FallbackFindsRepeatInAnotherChunk) { + // A repeat of an offset from an early chunk arrives after many chunks. The + // fallback finds it. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (uint64_t i = 0; i < 2000; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest(42, 10 + i * 100, 40))); + } + ASSERT_OK(collector.Add(UnflattenedForTest(42, 10 + 5 * 100, 40))); + MetadataForTest metadata; + const Status status = FinishForTest(&collector, &metadata); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_NE(status.ToString().find("Repeated blob reference"), + std::string::npos); +} + +TEST(BlobReferenceMetadataTest, ChunkJoinsWithChangingKeySizes) { + // User key sizes change from entry to entry, also at the chunk joins. At a + // join, Add() stores the gap at the start of the new chunk. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::vector expected; + uint64_t offset = 10; + for (uint64_t i = 0; i < 3000; ++i) { + const uint64_t key_size = (i * 7) % 40; + const BlobIndex blob_index = UnflattenedForTest(42, offset, 40); + ASSERT_OK(collector.Add(blob_index, key_size)); + expected.push_back(ExpectedReferencesForTest({blob_index}, key_size)[0]); + // The next value starts after this value, the next record's header and the + // next record's key. Every 11th record leaves 5 unreferenced bytes. + offset += 40 + BlobLogRecord::kHeaderSize + ((i + 1) * 7) % 40 + + (i % 11 == 0 ? 5 : 0); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + ASSERT_GT(metadata.blocks.size(), 1U); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ExpectSameReferencesForTest(references, expected); +} + +TEST(BlobReferenceMetadataTest, DirectAndFlattenedGroupsSpanChunks) { + // A kDirect group and a flattened group each fill many chunks and blocks. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::vector added; + for (uint64_t i = 0; i < 3000; ++i) { + added.push_back(LegacyForTest(10, 1000 + i * 200)); + ASSERT_OK(collector.Add(added.back(), /*user_key_size=*/8)); + } + for (uint64_t i = 0; i < 3000; ++i) { + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, 99, + {.origin_file_number = 42, + .original_value_offset = 10 + i * 100, + .stored_value_size = 40, + .compression = kZSTD, + .stable_checksum = static_cast(i), + .stored_checksum = static_cast(i)}); + added.push_back(DecodeBlobIndexForTest(encoded)); + ASSERT_OK(collector.Add(added.back(), /*user_key_size=*/8)); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + ASSERT_GT(metadata.blocks.size(), 2U); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(index.groups.size(), 2U); + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(added, /*user_key_size=*/8)); +} + +TEST(BlobReferenceMetadataTest, ContinuationRunKeepsGroupDefault) { + // The group's first reference is ZSTD, and later references mix ZSTD and + // kNoCompression. The runs that continue the group in later blocks still have + // the group's default compression (ZSTD). + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::vector added; + for (uint64_t i = 0; i < 3000; ++i) { + added.push_back(UnflattenedWithCompressionForTest( + 42, 10 + i * 100, 40, i == 0 || i % 3 != 0 ? kZSTD : kNoCompression)); + ASSERT_OK(collector.Add(added.back())); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + ASSERT_GT(metadata.blocks.size(), 1U); + for (size_t block = 0; block < metadata.blocks.size(); ++block) { + Slice header = metadata.blocks[block]; + uint64_t value = 0; + ASSERT_TRUE(GetVarint64(&header, &value)); + ASSERT_TRUE(GetVarint64(&header, &value)); + ASSERT_FALSE(header.empty()); + EXPECT_EQ(static_cast(header[0]), kZSTD) << block; + } + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ExpectSameReferencesForTest( + references, ExpectedReferencesForTest(added, /*user_key_size=*/0)); +} + +TEST(BlobReferenceMetadataTest, SlabChargeFailureAllocatesNoSlab) { + // The charge for the first 64 KB slab fails. Add() returns the error, and the + // slab is not allocated. + auto reservation_manager = + std::make_shared(/*limit=*/32 * 1024); + BlobReferenceMetadataCollector collector( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + const Status status = collector.Add(UnflattenedForTest(42, 10, 40)); + EXPECT_TRUE(status.IsMemoryLimit()) << status.ToString(); + EXPECT_LT(collector.TEST_ReservedMemory(), 64 * 1024U); +} + +TEST(BlobReferenceMetadataTest, RunDefaultIsGroupFirstCompression) { + // Entries whose compression differs from the group's first entry carry the + // compression byte. Within one block, the bytes equal a direct encoding of + // the sorted references. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + const CompressionType compressions[] = {kNoCompression, kZSTD, kZSTD, + kNoCompression, kLZ4Compression}; + std::vector added; + uint64_t offset = 10; + for (size_t i = 0; i < 5; ++i) { + added.push_back( + UnflattenedWithCompressionForTest(42, offset, 30, compressions[i])); + ASSERT_OK(collector.Add(added.back(), /*user_key_size=*/8)); + offset += 30 + BlobLogRecord::kHeaderSize + 8 + (i == 2 ? 77 : 0); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + const std::vector expected = + ExpectedReferencesForTest(added, /*user_key_size=*/8); + ExpectSameReferencesForTest(references, expected); + + // The direct encoding of one run: header, then the entries. + BytesForTest bytes; + bytes.Varint(expected.size()) + .Varint(expected[0].original_value_offset) + .Byte(static_cast(compressions[0])); + for (size_t i = 0; i < expected.size(); ++i) { + const BlobReference& reference = expected[i]; + if (i > 0) { + const BlobReference& previous = expected[i - 1]; + bytes.Varint(reference.original_value_offset - + (previous.original_value_offset + + previous.stored_value_size + BlobLogRecord::kHeaderSize + + 8)); + } + const bool differs = reference.compression != compressions[0]; + bytes.Varint((reference.stored_value_size << 1) | (differs ? 1 : 0)); + if (differs) { + bytes.Byte(static_cast(reference.compression)); + } + bytes.Varint(8).Fixed32(reference.stable_checksum); + if (reference.compression != kNoCompression) { + bytes.Fixed32(reference.stored_checksum); + } + } + ASSERT_EQ(metadata.blocks.size(), 1U); + EXPECT_EQ(metadata.blocks[0], bytes.bytes); +} + +TEST(BlobReferenceMetadataTest, ChargeGrowsPerSlabAndIsReleased) { + auto reservation_manager = std::make_shared( + std::numeric_limits::max()); + { + BlobReferenceMetadataCollector collector( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(UnflattenedForTest(42, 10, 40))); + const size_t first = reservation_manager->GetTotalMemoryUsed(); + EXPECT_GE(first, 64 * 1024U); + // About 11 bytes per entry, so 20000 entries need several 64 KB slabs. + for (uint64_t i = 1; i < 20000; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest( + 42, 10 + i * (40 + BlobLogRecord::kHeaderSize), 40))); + } + const size_t grown = reservation_manager->GetTotalMemoryUsed(); + EXPECT_GE(grown, first + 2 * 64 * 1024U); + EXPECT_EQ(grown, collector.TEST_ReservedMemory()); + } + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), 0U); + { + BlobReferenceMetadataCollector collector( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(UnflattenedForTest(42, 10, 40))); + EXPECT_GT(reservation_manager->GetTotalMemoryUsed(), 0U); + } + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), 0U); +} + +TEST(BlobReferenceMetadataTest, GroupsSpanAndShareBlocks) { + // Build a kDirect group, many small groups that share blocks, and one large + // group that spans several blocks. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + uint64_t expected_count = 0; + for (uint64_t offset = 0; offset < 50; ++offset) { + ASSERT_OK(collector.Add(LegacyForTest(10, 1000 + offset * 200))); + ++expected_count; + } + for (uint64_t origin = 100; origin < 400; ++origin) { + const uint64_t refs = 1 + origin % 7; + for (uint64_t i = 0; i < refs; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest(origin, 10 + i * 80, 40))); + ++expected_count; + } + } + constexpr uint64_t kLargeOrigin = 1000; + for (uint64_t i = 0; i < 20000; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest(kLargeOrigin, 10 + i * 80, 40))); + ++expected_count; + } + MetadataForTest metadata; + std::shared_ptr summary; + ASSERT_OK(FinishForTest(&collector, &metadata, &summary)); + ASSERT_EQ(metadata.descriptor.reference_count, expected_count); + + BlobReferenceIndex index; + std::vector all; + ASSERT_OK(DecodeAllForTest(metadata, &index, &all)); + ASSERT_EQ(all.size(), expected_count); + ASSERT_EQ(index.groups.size(), 302U); + ASSERT_GE(metadata.blocks.size(), 3U); + + // Reading each group through the index gives the same references, in the same + // order, as walking every block. + std::vector by_group; + for (size_t group = 0; group < index.groups.size(); ++group) { + std::vector references; + ASSERT_OK(ReadGroupForTest(metadata, index, group, &references)); + ASSERT_EQ(references.size(), index.groups[group].group.reference_count); + by_group.insert(by_group.end(), references.begin(), references.end()); + } + ASSERT_EQ(by_group.size(), all.size()); + for (size_t i = 0; i < all.size(); ++i) { + ASSERT_EQ(ReferenceFieldsForTest(by_group[i]), + ReferenceFieldsForTest(all[i])) + << i; + } + + // Layout: some groups start inside a block, the large group spans at least + // three blocks, and some block holds only that group. + EXPECT_TRUE(std::any_of(index.groups.begin(), index.groups.end(), + [](const BlobReferenceIndexEntry& entry) { + return entry.run_offset > 0; + })); + const size_t large = FindBlobReferenceGroup( + index, {BlobReferenceEncodingClass::kIndirectUnflattened, kLargeOrigin, + kLargeOrigin}); + ASSERT_EQ(large, index.groups.size() - 1); + EXPECT_GE(index.blocks.size() - index.groups[large].start_block, 3U); + // A block after the large group's start block where no group starts holds + // only a run of the large group (the last group). + size_t single_run_blocks = 0; + for (size_t block = index.groups[large].start_block + 1; + block < index.blocks.size(); ++block) { + single_run_blocks += + std::none_of(index.groups.begin(), index.groups.end(), + [&](const BlobReferenceIndexEntry& entry) { + return entry.start_block == block; + }); + } + EXPECT_GE(single_run_blocks, 1U); + + // Group lookup: the first group, a group in the middle of the index, and a + // missing group. The lookup of the spanning group is checked above. + EXPECT_EQ(FindBlobReferenceGroup( + index, {BlobReferenceEncodingClass::kDirect, 10, 10}), + 0U); + EXPECT_EQ( + FindBlobReferenceGroup( + index, {BlobReferenceEncodingClass::kIndirectUnflattened, 200, 200}), + 101U); + EXPECT_EQ( + FindBlobReferenceGroup( + index, {BlobReferenceEncodingClass::kIndirectUnflattened, 999, 999}), + index.groups.size()); + + // The indirect groups rebuilt from the index equal the groups of the + // build-time summary. + const std::vector rebuilt = IndirectGroupsForTest(index); + ASSERT_EQ(rebuilt.size(), summary->groups.size()); + ASSERT_EQ(rebuilt.size(), 301U); + for (size_t i = 0; i < rebuilt.size(); ++i) { + EXPECT_EQ(rebuilt[i].origin_file_number, + summary->groups[i].origin_file_number); + EXPECT_EQ(rebuilt[i].reference_count, summary->groups[i].reference_count); + EXPECT_EQ(rebuilt[i].stored_value_bytes, + summary->groups[i].stored_value_bytes); + EXPECT_EQ(rebuilt[i].record_bytes, summary->groups[i].record_bytes); + } +} + +TEST(BlobReferenceMetadataTest, GroupSpansBlocksAtTargetSize) { + // A reference takes about 11 encoded bytes (gap, size, user key size and two + // CRCs), so 3000 references of one group need more than one 16 KB block. + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (uint64_t i = 0; i < 3000; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest(42, 10 + i * 80, 40))); + } + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(index.groups.size(), 1U); + EXPECT_GE(index.blocks.size(), 2U); + for (const std::string& block : metadata.blocks) { + EXPECT_LE(block.size(), 16U * 1024); + } + std::vector group; + ASSERT_OK(ReadGroupForTest(metadata, index, 0, &group)); + EXPECT_EQ(group.size(), 3000U); +} + +TEST(BlobReferenceMetadataTest, SingleBlockSst) { + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(UnflattenedForTest(42, 700, 800))); + ASSERT_OK(collector.Add(UnflattenedForTest(42, 1700, 800))); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + BlobReferenceIndex index; + std::vector references; + ASSERT_OK(DecodeAllForTest(metadata, &index, &references)); + ASSERT_EQ(index.blocks.size(), 1U); + ASSERT_EQ(index.groups.size(), 1U); + EXPECT_EQ(index.groups[0].group.reference_count, 2U); + EXPECT_EQ(index.groups[0].group.stored_value_bytes, 1600U); + EXPECT_EQ(references.size(), 2U); +} + +TEST(BlobReferenceMetadataTest, EstimateCoversOneLargeGroup) { + // EstimatedEncodedSize() is an upper bound of the metadata size. The table + // builder adds it to the SST size estimate that decides when to cut an output + // file. For one large group, the bound must not count one data block per + // reference: that would cut SST outputs far below their target size. The + // bound must still cover every block that Finish() writes. + constexpr uint64_t kReferenceCount = 20000; + constexpr size_t kBlockTrailerBytes = 5; + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + for (uint64_t i = 0; i < kReferenceCount; ++i) { + ASSERT_OK(collector.Add(UnflattenedForTest(42, 1000 + i * 200, 80))); + } + const uint64_t estimated = collector.EstimatedEncodedSize(); + EXPECT_LT(estimated, 224U * kReferenceCount / 2); + MetadataForTest metadata; + ASSERT_OK(FinishForTest(&collector, &metadata)); + ASSERT_GT(metadata.blocks.size(), 1U); + uint64_t written = metadata.index.size() + kBlockTrailerBytes; + for (const std::string& block : metadata.blocks) { + written += block.size() + kBlockTrailerBytes; + } + EXPECT_GE(estimated, written); +} + +TEST(BlobReferenceMetadataTest, EstimateCoversDistinctWideEncodedGroups) { + constexpr size_t kBlockTrailerBytes = 5; + const std::shared_ptr cache = NewLRUCache(16 * 1024 * 1024); + const std::shared_ptr reservation = + std::make_shared>(cache); + auto collector = std::make_unique( + reservation, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + constexpr uint64_t kReferences = 200; + for (uint64_t i = 0; i < kReferences; ++i) { + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, (1ULL << 60) + 1, + {.origin_file_number = (1ULL << 59) + i, + .original_value_offset = (1ULL << 63) + i, + .stored_value_size = 1ULL << 40, + .compression = kZSTD, + .stable_checksum = 123, + .stored_checksum = 456}); + ASSERT_OK(collector->Add(DecodeBlobIndexForTest(encoded))); + } + const uint64_t estimated = collector->EstimatedEncodedSize(); + EXPECT_GT(cache->GetPinnedUsage(), 0U); + uint64_t written = 0; + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(collector->Finish( + [&written](const Slice& block, bool /*is_index_block*/, + BlockHandle* handle) -> Status { + *handle = BlockHandle((1ULL << 60) + written, block.size()); + written += block.size() + kBlockTrailerBytes; + return Status::OK(); + }, + &descriptor, /*summary=*/nullptr)); + EXPECT_GE(estimated, written); + collector.reset(); + EXPECT_EQ(cache->GetPinnedUsage(), 0U); +} + +TEST(BlobReferenceMetadataTest, RepeatedReferenceFailsClosed) { + // Each physical record is referenced at most once per SST. A repeat, with + // equal or with different fields, fails the build. + std::string first_encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&first_encoded, + {.origin_file_number = 42, + .original_value_offset = 700, + .stored_value_size = 800, + .compression = kZSTD, + .stable_checksum = 123, + .stored_checksum = 4321}); + std::string conflicting_encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&conflicting_encoded, + {.origin_file_number = 42, + .original_value_offset = 700, + .stored_value_size = 800, + .compression = kZSTD, + .stable_checksum = 456, + .stored_checksum = 4321}); + std::string other_encoded; + BlobIndex::EncodeUnflattenedIndirectBlob(&other_encoded, + {.origin_file_number = 42, + .original_value_offset = 2000, + .stored_value_size = 800, + .compression = kZSTD, + .stable_checksum = 789, + .stored_checksum = 4321}); + std::string legacy_encoded; + BlobIndex::EncodeBlob(&legacy_encoded, /*file_number=*/41, /*offset=*/500, + /*size=*/1000, kLZ4Compression); + + for (const std::string* repeat : + {&first_encoded, &conflicting_encoded, &legacy_encoded}) { + BlobReferenceMetadataCollector collector( + nullptr, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(first_encoded))); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(other_encoded))); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(legacy_encoded))); + // Add() finds a repeat of the group's last offset. Finish() finds a repeat + // of an earlier offset when it sorts the group. + const Status add_status = collector.Add(DecodeBlobIndexForTest(*repeat)); + if (!add_status.ok()) { + EXPECT_TRUE(add_status.IsCorruption()) << add_status.ToString(); + EXPECT_NE(add_status.ToString().find("Repeated blob reference"), + std::string::npos) + << add_status.ToString(); + } + MetadataForTest metadata; + const Status status = FinishForTest(&collector, &metadata); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_NE(status.ToString().find("Repeated blob reference"), + std::string::npos) + << status.ToString(); + } +} + +TEST(BlobReferenceMetadataTest, ValidatesExcludedSameFileReference) { std::string encoded; BlobIndex::EncodeBlob(&encoded, kCurrentFileBlobIndexFileNumber, - /*offset=*/123, /*size=*/456, kNoCompression); - + /*offset=*/500, + /*size=*/1000, static_cast(0xff)); BlobIndex blob_index; ASSERT_OK(blob_index.DecodeFrom(encoded)); - EXPECT_TRUE(blob_index.IsSameFile()); - EXPECT_EQ(blob_index.file_number(), kCurrentFileBlobIndexFileNumber); - EXPECT_NE(blob_index.DebugString(false).find("file:same"), std::string::npos); + EXPECT_TRUE(blob_index.Validate().IsCorruption()); + + BlobReferenceMetadataCollector collector( + nullptr, BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + EXPECT_TRUE(collector.Add(blob_index).IsCorruption()); } -TEST(BlobIndexTest, EmptyBlobIndexIsCorruption) { - BlobIndex blob_index; - ASSERT_EQ(blob_index.DecodeFrom(Slice()).code(), Status::Code::kCorruption); +TEST(BlobReferenceMetadataTest, ReservationFailureFailsCollection) { + auto reservation_manager = + std::make_shared(/*limit=*/1); + auto collector = std::make_unique( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::string encoded; + BlobIndex::EncodeBlob(&encoded, /*file_number=*/41, /*offset=*/500, + /*size=*/1000, kNoCompression); + Status status = collector->Add(DecodeBlobIndexForTest(encoded)); + EXPECT_TRUE(status.IsMemoryLimit()) << status.ToString(); + EXPECT_GT(reservation_manager->GetTotalMemoryUsed(), 1); + collector.reset(); + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), 0); +} + +TEST(BlobReferenceMetadataTest, SummaryKeepsOnlyItsOwnCharge) { + auto reservation_manager = + std::make_shared(/*limit=*/1024 * 1024); + auto collector = std::make_unique( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + ASSERT_OK(collector->Add(UnflattenedForTest(42, 10, 40))); + const size_t working = reservation_manager->GetTotalMemoryUsed(); + EXPECT_GE(working, 64 * 1024U); + MetadataForTest metadata; + std::shared_ptr summary; + ASSERT_OK(FinishForTest(collector.get(), &metadata, &summary)); + // The working buffers are freed; the summary's charge covers only itself. + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), summary->MemoryUsage()); + collector.reset(); + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), summary->MemoryUsage()); + summary.reset(); + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), 0U); +} + +TEST(BlobReferenceMetadataTest, ReleasesReservationOnDestruction) { + auto reservation_manager = + std::make_shared(/*limit=*/1024 * 1024); + { + BlobReferenceMetadataCollector collector( + reservation_manager, + BlobReferenceSstIdentity{.column_family_id = kTestColumnFamilyId, + .sst_file_number = kTestSstFileNumber}); + std::string encoded; + BlobIndex::EncodeBlob(&encoded, /*file_number=*/41, /*offset=*/500, + /*size=*/1000, kNoCompression); + ASSERT_OK(collector.Add(DecodeBlobIndexForTest(encoded))); + EXPECT_GT(reservation_manager->GetTotalMemoryUsed(), 0); + } + EXPECT_EQ(reservation_manager->GetTotalMemoryUsed(), 0); +} + +namespace { + +BlobReferenceGroup MakeCensusGroup(uint64_t generation, uint64_t origin) { + BlobReferenceGroup group; + group.encoding_class = generation == origin + ? BlobReferenceEncodingClass::kIndirectUnflattened + : BlobReferenceEncodingClass::kIndirectFlattened; + group.generation_file_number = generation; + group.origin_file_number = origin; + group.reference_count = 1; + return group; +} + +std::shared_ptr MakeCensusFile( + uint64_t file_number, std::vector groups) { + auto file = std::make_shared(); + file->file_number = file_number; + file->groups = std::move(groups); + return file; +} + +} // namespace + +TEST(BlobReferenceCensusTest, GroupsOfOneGenerationMakeOneHolder) { + // A holder of a generation number is an SST that has at least one group with + // that generation number. The census must count each SST once per generation + // number, even if the SST has several groups with that generation number. + // + // Relocation file 30 holds values whose origins are blob files 10 and 20. + // References flattened to file 30 have generation 30 and origin 10 or 20, so + // both groups use the generation number 30. + // - SST 100 has both groups of generation number 30 and one group of blob + // file 40. + // - SST 200 has one group of generation number 30. + const auto group = [](uint64_t generation, uint64_t origin, + uint64_t references) { + BlobReferenceGroup result = MakeCensusGroup(generation, origin); + result.reference_count = references; + result.stored_value_bytes = 10 * references; + result.record_bytes = 20 * references; + return result; + }; + const auto first = MakeCensusFile( + 100, {group(30, 10, 2), group(30, 20, 3), group(40, 40, 1)}); + const auto second = MakeCensusFile(200, {group(30, 10, 4)}); + // Added in the same delta that removes SST 200 (see below). + const auto third = MakeCensusFile(300, {group(30, 10, 1), group(30, 20, 1)}); + + std::shared_ptr state; + ASSERT_OK(BlobReferenceCensus::BuildState(7, {first, second}, &state)); + ASSERT_EQ(state->generation_count(), 2U); + ASSERT_EQ(*state->FindSstFiles(30), (std::vector{100, 200})); + ASSERT_EQ(state->FindTotals(30)->reference_count, 9U); + ASSERT_EQ(state->FindTotals(30)->stored_value_bytes, 90U); + ASSERT_EQ(state->FindTotals(30)->record_bytes, 180U); + + // The census worker must find the same holders as BuildState(). This must + // also hold after FinalizeWorkerBuild(), when each delta updates the encoded + // holder sets. + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, nullptr)); + ASSERT_OK(census.AddStartingFile(token, first)); + ASSERT_OK(census.AddStartingFile(token, second)); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 7), + BlobReferenceCensus::PublishResult::kPublished); + std::string difference; + ASSERT_TRUE( + census.TEST_GetReadyState(7)->SameMemberships(*state, &difference)) + << difference; + + // Removing SST 100 removes it once from the holders of generation number 30, + // and removes generation number 40, which no SST uses any more. + BlobReferenceCensusDelta removal; + removal.base_version_number = 7; + removal.version_number = 8; + removal.removed_files = {100}; + removal.removed_summaries = {first}; + std::shared_ptr removed; + ASSERT_OK(BlobReferenceCensus::BuildState(8, {second}, &removed)); + ASSERT_EQ(*removed->FindSstFiles(30), (std::vector{200})); + ASSERT_EQ(removed->FindSstFiles(40), nullptr); + ASSERT_EQ(removed->FindTotals(30)->reference_count, 4U); + ASSERT_OK(census.ApplyWorkerDelta(token, removal)); + ASSERT_EQ(census.TryPublishWorker(token, 8, 8), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE( + census.TEST_GetReadyState(8)->SameMemberships(*removed, &difference)) + << difference; + + // One delta removes SST 200 and adds SST 300. Both groups of generation + // number 30 in SST 300 use one slot. Removals free their slots before + // additions take the lowest free slot, so SST 300 gets slot 0, which SST 100 + // used. + BlobReferenceCensusDelta replace; + replace.base_version_number = 8; + replace.version_number = 9; + replace.removed_files = {200}; + replace.removed_summaries = {second}; + replace.added_files = {third}; + std::shared_ptr replaced; + ASSERT_OK(BlobReferenceCensus::BuildState(9, {third}, &replaced)); + ASSERT_EQ(*replaced->FindSstFiles(30), (std::vector{300})); + ASSERT_EQ(replaced->FindTotals(30)->reference_count, 2U); + ASSERT_OK(census.ApplyWorkerDelta(token, replace)); + ASSERT_EQ(census.TryPublishWorker(token, 9, 9), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE( + census.TEST_GetReadyState(9)->SameMemberships(*replaced, &difference)) + << difference; + auto garbage = census.Cancel(); +} + +TEST(BlobReferenceCensusTest, SlotTableAndSlotSetsMatchModel) { + // Random SST additions and removals over kGenerations (6) generation + // numbers. Each SST holds a random subset of the generations. After each + // change, the slot table plus one slot set per generation must give the same + // SSTs per generation as a simple model. The sets must also switch from list + // to bitmap and back. + for (uint32_t seed : {1u, 7u, 42u, 2026u}) { + SCOPED_TRACE("seed " + std::to_string(seed)); + Random rnd(seed); + constexpr uint64_t kGenerations = 6; + BlobCensusSlotTable slots; + std::array sets; + std::map> model; // generation -> SSTs + std::map> + sst_generations; // SST -> generations + std::vector scratch; + uint64_t next_file = 1; + bool saw_bitmap = false; + bool saw_list_after_bitmap = false; + for (int step = 0; step < 3000; ++step) { + SCOPED_TRACE("step " + std::to_string(step)); + // One change: remove some SSTs and add some, like a compaction. + std::vector removed_files; + for (const auto& entry : sst_generations) { + // In some phases, remove more SSTs than are added, so sets shrink + // again. + const uint32_t remove_one_in = (step / 500) % 2 == 0 ? 6 : 2; + if (rnd.OneIn(remove_one_in)) { + removed_files.push_back(entry.first); + } + } + std::array, kGenerations> removed; + std::array, kGenerations> added; + for (uint64_t file : removed_files) { + uint32_t slot = 0; + ASSERT_OK(slots.Release(file, &slot)); + for (uint64_t generation : sst_generations[file]) { + removed[generation].push_back(slot); + model[generation].erase(file); + } + sst_generations.erase(file); + } + const uint32_t additions = rnd.Uniform(8); + for (uint32_t i = 0; i < additions; ++i) { + const uint64_t file = next_file++; + uint32_t slot = 0; + ASSERT_OK(slots.Assign(file, &slot)); + // Assign() must give the lowest free slot. + for (uint32_t lower = 0; lower < slot; ++lower) { + ASSERT_NE(slots.FileNumber(lower), 0U); + } + std::vector& generations = sst_generations[file]; + for (uint64_t generation = 0; generation < kGenerations; ++generation) { + if (rnd.OneIn(static_cast(generation) + 1)) { + generations.push_back(generation); + added[generation].push_back(slot); + model[generation].insert(file); + } + } + } + for (uint64_t generation = 0; generation < kGenerations; ++generation) { + std::sort(removed[generation].begin(), removed[generation].end()); + std::sort(added[generation].begin(), added[generation].end()); + const bool was_bitmap = sets[generation].is_bitmap(); + ASSERT_OK(sets[generation].Update(removed[generation], + added[generation], slots.slot_count(), + &scratch)); + saw_bitmap |= sets[generation].is_bitmap(); + saw_list_after_bitmap |= was_bitmap && !sets[generation].is_bitmap(); + std::vector held; + sets[generation].AppendTo(&held); + std::set files; + for (uint32_t slot : held) { + ASSERT_TRUE(sets[generation].Contains(slot)); + files.insert(slots.FileNumber(slot)); + } + ASSERT_EQ(files, model[generation]); + ASSERT_EQ(sets[generation].count(), model[generation].size()); + } + ASSERT_EQ(slots.live_count(), sst_generations.size()); + } + EXPECT_TRUE(saw_bitmap); + EXPECT_TRUE(saw_list_after_bitmap); + } +} + +TEST(BlobReferenceCensusTest, SlotSetInPlaceUpdatesMatchModel) { + // A dense set is a bitmap and is updated in place; a sparse set is a list. + // After every change, the set must hold the same slots as a model, and a new + // set built by Assign() from the model must hold the same slots. The dense + // case must switch between list and bitmap in both directions. + const uint32_t seed = 20261108; + SCOPED_TRACE("seed " + std::to_string(seed)); + Random rnd(seed); + constexpr uint32_t kSlots = 4096; + for (const uint32_t density_one_in : {2u, 300u}) { + SCOPED_TRACE(density_one_in); + BlobCensusSlotSet set; + std::set model; + std::vector scratch; + bool saw_bitmap = false; + bool saw_switch_to_list = false; + bool saw_switch_to_bitmap = false; + for (int step = 0; step < 400; ++step) { + // Grow for 200 steps, then shrink, so dense sets switch back to lists. + const bool shrinking = step >= 200; + std::vector removed; + std::vector added; + for (uint32_t slot : model) { + if (rnd.OneIn(shrinking ? 3 : 50)) { + removed.push_back(slot); + } + } + const int additions = + density_one_in == 2 ? (shrinking ? 4 : 64) : (shrinking ? 0 : 1); + for (int i = 0; i < additions; ++i) { + const uint32_t slot = rnd.Uniform(kSlots); + const bool held = + model.count(slot) != 0 && + !std::binary_search(removed.begin(), removed.end(), slot); + if (!held) { + added.push_back(slot); + } + } + std::sort(added.begin(), added.end()); + added.erase(std::unique(added.begin(), added.end()), added.end()); + const bool was_bitmap = set.is_bitmap(); + ASSERT_OK(set.Update(removed, added, kSlots, &scratch)); + for (uint32_t slot : removed) { + model.erase(slot); + } + model.insert(added.begin(), added.end()); + saw_bitmap |= set.is_bitmap(); + saw_switch_to_list |= was_bitmap && !set.is_bitmap(); + saw_switch_to_bitmap |= !was_bitmap && set.is_bitmap(); + std::vector held; + set.AppendTo(&held); + ASSERT_EQ(held, std::vector(model.begin(), model.end())); + ASSERT_EQ(set.count(), model.size()); + BlobCensusSlotSet reference; + reference.Assign(held, kSlots); + std::vector reference_held; + reference.AppendTo(&reference_held); + ASSERT_EQ(reference_held, held); + for (int i = 0; i < 16; ++i) { + const uint32_t slot = rnd.Uniform(kSlots); + ASSERT_EQ(set.Contains(slot), model.count(slot) != 0); + } + } + if (density_one_in == 2) { + EXPECT_TRUE(saw_bitmap); + EXPECT_TRUE(saw_switch_to_bitmap); + EXPECT_TRUE(saw_switch_to_list); + } + } +} + +TEST(BlobReferenceCensusTest, SlotSetListHasExactSize) { + // A list is encoded at its exact size, so its memory charge has no unused + // capacity. + BlobCensusSlotSet set; + std::vector scratch; + std::vector slots; + for (uint32_t slot = 0; slot < 4000; slot += 97) { + slots.push_back(slot); + } + ASSERT_OK(set.Update({}, slots, 1 << 20, &scratch)); + ASSERT_FALSE(set.is_bitmap()); + size_t list_bytes = 0; + for (size_t i = 0; i < slots.size(); ++i) { + list_bytes += VarintLength(i == 0 ? slots[i] : slots[i] - slots[i - 1]); + } + EXPECT_EQ(set.MemoryUsage(), list_bytes); + // After half of the slots are removed, the list still has its exact size. The + // charge is never below 15 bytes, the smallest std::string capacity in + // libstdc++. + std::vector removed; + for (size_t i = 0; i < slots.size(); i += 2) { + removed.push_back(slots[i]); + } + ASSERT_OK(set.Update(removed, {}, 1 << 20, &scratch)); + std::vector held; + set.AppendTo(&held); + list_bytes = 0; + for (size_t i = 0; i < held.size(); ++i) { + list_bytes += VarintLength(i == 0 ? held[i] : held[i] - held[i - 1]); + } + EXPECT_EQ(set.MemoryUsage(), std::max(list_bytes, 15)); +} + +TEST(BlobReferenceCensusTest, SlotSetRejectsMismatchedChanges) { + BlobCensusSlotSet set; + std::vector scratch; + ASSERT_OK(set.Update({}, {1, 5, 9}, 16, &scratch)); + // Removing a slot that the set does not hold, or adding a slot that it holds, + // is Corruption. + ASSERT_TRUE(set.Update({2}, {}, 16, &scratch).IsCorruption()); + ASSERT_TRUE(set.Update({}, {5}, 16, &scratch).IsCorruption()); + // A slot freed and reused by another SST in the same change stays held. + ASSERT_OK(set.Update({5}, {5}, 16, &scratch)); + std::vector held; + set.AppendTo(&held); + ASSERT_EQ(held, (std::vector{1, 5, 9})); + BlobCensusSlotTable slots; + uint32_t slot = 0; + ASSERT_OK(slots.Assign(10, &slot)); + ASSERT_TRUE(slots.Assign(10, &slot).IsCorruption()); + ASSERT_TRUE(slots.Release(11, &slot).IsCorruption()); +} + +TEST(BlobReferenceCensusTest, MutableWorkerPublishesOnlyCaughtUpEpoch) { + BlobReferenceCensus census; + size_t wakeups = 0; + census.SetBuildWakeup( + [](void* owner, void* count) { + // NeedsWorkerRun() locks the census mutex, so this call also shows that + // the wakeup runs without that mutex held. + EXPECT_TRUE( + static_cast(owner)->NeedsWorkerRun(9)); + ++*static_cast(count); + }, + &census, &wakeups); + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, nullptr)); + ASSERT_OK(census.AddStartingFile( + token, MakeCensusFile(100, {MakeCensusGroup(10, 10)}))); + auto delta = std::make_shared(); + delta->base_version_number = 7; + delta->version_number = 9; + delta->removed_files = {100}; + delta->removed_summaries = {MakeCensusFile(100, {MakeCensusGroup(10, 10)})}; + delta->added_files = {MakeCensusFile(200, {MakeCensusGroup(20, 10)})}; + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(delta, reservation, &prepared)); + ASSERT_NE(prepared.log_entry, nullptr); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 7), + BlobReferenceCensus::PublishResult::kInstallPending); + ASSERT_FALSE(census.NeedsWorkerRun(7)); + auto garbage = census.Install(prepared); + ASSERT_EQ(wakeups, 1U); + ASSERT_TRUE(census.NeedsWorkerRun(9)); + ASSERT_FALSE(census.IsReady(9)); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 9), + BlobReferenceCensus::PublishResult::kQueueNotEmpty); + + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_EQ(deltas.size(), 1U); + ASSERT_OK(census.ApplyWorkerDelta(token, *deltas.front())); + ASSERT_OK(census.AcknowledgeBuildDeltas(token, delta->QueueWeight())); + ASSERT_EQ(census.WorkerVersionNumber(), 9U); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 9, 9), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE(census.IsReady(9)); + + uint64_t gc_token = 0; + { + ASSERT_OK(census.BeginGcJob(9, &gc_token)); + std::shared_ptr selected; + ASSERT_OK(census.SnapshotSelected(gc_token, 9, {10, 20}, + /*gc_block_cache=*/nullptr, &selected)); + ASSERT_EQ(selected->GetSstCount(10), 0U); + ASSERT_EQ(selected->GetSstCount(20), 1U); + census.EndGcJob(gc_token); + } + + // After the publish, the same build cursor stays and receives the deltas of + // later installs. + auto empty_delta = std::make_shared(); + empty_delta->base_version_number = 9; + empty_delta->version_number = 12; + const uint64_t next_reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(empty_delta, next_reservation, &prepared)); + garbage = census.Install(prepared); + ASSERT_FALSE(census.IsReady(12)); + deltas.clear(); + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_OK(census.ApplyWorkerDelta(token, *deltas.front())); + ASSERT_OK(census.AcknowledgeBuildDeltas(token, empty_delta->QueueWeight())); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 12, 12), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE(census.IsReady(12)); + ASSERT_FALSE(census.IsReady(9)); +} + +TEST(BlobReferenceCensusTest, + CancelPendingInstallWakesOnlyMatchingReservation) { + BlobReferenceCensus census; + size_t wakeups = 0; + census.SetBuildWakeup( + [](void* owner, void* count) { + EXPECT_TRUE( + static_cast(owner)->NeedsWorkerRun(7)); + ++*static_cast(count); + }, + &census, &wakeups); + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, nullptr)); + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 7), + BlobReferenceCensus::PublishResult::kInstallPending); + ASSERT_FALSE(census.NeedsWorkerRun(7)); + census.CancelInstallPreparation(reservation + 1); + ASSERT_EQ(wakeups, 0U); + ASSERT_FALSE(census.NeedsWorkerRun(7)); + census.CancelInstallPreparation(reservation); + ASSERT_EQ(wakeups, 1U); + ASSERT_TRUE(census.NeedsWorkerRun(7)); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 7), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_FALSE(census.NeedsWorkerRun(7)); +} + +TEST(BlobReferenceCensusTest, InvalidatedInstallDetachesCanceledBuildContext) { + BlobReferenceCensus census; + const uint64_t old_token = census.BeginBuild(); + auto context = std::make_shared(); + context->token = old_token; + ASSERT_TRUE(census.TrySetBuildContext(context)); + ASSERT_OK(census.ResetWorkerState(old_token, 7, nullptr, nullptr)); + size_t wakeups = 0; + census.SetBuildWakeup( + [](void* owner, void* count) { + EXPECT_EQ(static_cast(owner)->GetBuildContext(), + nullptr); + ++*static_cast(count); + }, + &census, &wakeups); + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.FinalizeWorkerBuild(old_token)); + ASSERT_EQ(census.TryPublishWorker(old_token, 7, 7), + BlobReferenceCensus::PublishResult::kInstallPending); + BlobReferenceCensusPreparedInstall prepared; + ASSERT_OK(census.PrepareInvalidation(reservation, &prepared)); + auto garbage = census.Install(prepared); + ASSERT_EQ(garbage.build_context, context); + ASSERT_EQ(census.GetBuildContext(), nullptr); + ASSERT_EQ(wakeups, 1U); + ASSERT_EQ(census.TryPublishWorker(old_token, 7, 7), + BlobReferenceCensus::PublishResult::kStale); + const uint64_t new_token = census.BeginBuild(); + ASSERT_NE(new_token, old_token); + ASSERT_OK(census.ResetWorkerState(new_token, 8, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(new_token)); + ASSERT_EQ(census.TryPublishWorker(new_token, 8, 8), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE(census.IsReady(8)); +} + +TEST(BlobReferenceCensusTest, + CanceledWorkerSliceCannotInvalidateReplacementReadyState) { + BlobReferenceCensus census; + const uint64_t old_token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(old_token, 7, nullptr, nullptr)); + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->LoadDependency( + {{"BlobReferenceCensus::AddStartingFile:CapturedWorker", + "BlobReferenceCensusTest::OldWorkerCaptured"}, + {"BlobReferenceCensusTest::ReplacementReady", + "BlobReferenceCensus::AddStartingFile:BeforeWorkerLock"}}); + sync_point->EnableProcessing(); + auto cleanup = Defer([&] { + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Status old_status; + port::Thread old_slice([&] { + old_status = census.AddStartingFile( + old_token, MakeCensusFile(100, {MakeCensusGroup(10, 10)})); + }); + TEST_SYNC_POINT("BlobReferenceCensusTest::OldWorkerCaptured"); + auto garbage = census.FailBuild(old_token); + const uint64_t new_token = census.BeginBuild(); + const Status reset_status = + census.ResetWorkerState(new_token, 7, nullptr, nullptr); + const Status finalize_status = census.FinalizeWorkerBuild(new_token); + const BlobReferenceCensus::PublishResult result = + census.TryPublishWorker(new_token, 7, 7); + TEST_SYNC_POINT("BlobReferenceCensusTest::ReplacementReady"); + old_slice.join(); + ASSERT_OK(reset_status); + ASSERT_OK(finalize_status); + ASSERT_EQ(result, BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE(old_status.IsAborted()); + ASSERT_TRUE(census.IsReady(7)); +} + +TEST(BlobReferenceCensusTest, + InvalidationBeforeContextAttachmentRejectsOldToken) { + BlobReferenceCensus census; + const uint64_t old_token = census.BeginBuild(); + bool released = false; + auto context = std::shared_ptr( + new BlobReferenceCensusBuildContext, + [&](BlobReferenceCensusBuildContext* stale) { + // GetBuildContext() locks the census mutex, so this call shows that the + // census releases the rejected context after it unlocks. + EXPECT_EQ(census.GetBuildContext(), nullptr); + released = true; + delete stale; + }); + auto cleanup = Defer([&] { + context.reset(); + auto canceled = census.Cancel(); + }); + context->token = old_token; + ASSERT_OK(census.ResetWorkerState(old_token, 7, nullptr, nullptr)); + const uint64_t reservation = census.BeginInstallPreparation(); + BlobReferenceCensusPreparedInstall prepared; + ASSERT_OK(census.PrepareInvalidation(reservation, &prepared)); + auto garbage = census.Install(prepared); + ASSERT_EQ(garbage.build_context, nullptr); + ASSERT_FALSE(census.TrySetBuildContext(std::move(context))); + ASSERT_TRUE(released); + ASSERT_EQ(census.GetBuildContext(), nullptr); + garbage = census.FailBuild(old_token); + const uint64_t new_token = census.BeginBuild(); + auto fresh = std::make_shared(); + fresh->token = new_token; + ASSERT_TRUE(census.TrySetBuildContext(fresh)); + ASSERT_OK(census.ResetWorkerState(new_token, 8, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(new_token)); + ASSERT_EQ(census.TryPublishWorker(new_token, 8, 8), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE(census.IsReady(8)); +} + +TEST(BlobGcCandidateHeapTest, RanksUpdatesAndRetiresPhysicalFiles) { + BlobGcCandidateHeap heap; + heap.SetPhysicalFile({10, 1000, true}); + heap.SetPhysicalFile({20, 2000, false}); + heap.SetPhysicalFile({30, 1000, false}); + ASSERT_OK(heap.AdjustLiveBytes(10, 300, true)); + ASSERT_OK(heap.AdjustLiveBytes(20, 200, true)); + ASSERT_OK(heap.AdjustLiveBytes(30, 800, true)); + ASSERT_EQ(heap.Select()[0], 20U); + ASSERT_EQ(heap.Select()[1], 10U); + ASSERT_EQ(heap.Select()[2], 0U); + ASSERT_OK(heap.AdjustLiveBytes(30, 800, false)); + ASSERT_EQ(heap.Select()[0], 30U); + heap.DeletePhysicalFile(30); + ASSERT_EQ(heap.Select()[0], 20U); + ASSERT_EQ(heap.size(), 2U); + ASSERT_TRUE(heap.IsIndirectOrigin(10)); + ASSERT_FALSE(heap.IsIndirectOrigin(20)); +} + +TEST(BlobReferenceCensusTest, MetadataCommitIsExclusive) { + // Route retirement and the indirect blob GC install both make metadata-only + // commits. Only one may run at a time, so a second BeginMetadataCommit() + // returns TryAgain until the first one ends. + BlobReferenceCensus census; + ASSERT_OK(census.BeginMetadataCommit()); + ASSERT_TRUE(census.BeginMetadataCommit().IsTryAgain()); + census.EndMetadataCommit(true); + ASSERT_OK(census.BeginMetadataCommit()); + census.EndMetadataCommit(true); +} + +TEST(BlobReferenceCensusTest, MetadataCommitFencesGcJobStart) { + // While a metadata-only commit (BeginMetadataCommit()) is in progress, + // BeginGcJob() returns TryAgain, because the commit can change routes that + // the indirect blob GC job depends on. After a commit with an unknown result + // (EndMetadataCommit(false)), BeginGcJob() returns Busy until + // ResolveMetadataCommitAmbiguity() runs. + BlobReferenceCensus census; + const uint64_t build_token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(build_token, 7, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(build_token)); + ASSERT_EQ(census.TryPublishWorker(build_token, 7, 7), + BlobReferenceCensus::PublishResult::kPublished); + + ASSERT_OK(census.BeginMetadataCommit()); + uint64_t gc_token = 0; + ASSERT_TRUE(census.BeginGcJob(7, &gc_token).IsTryAgain()); + census.EndMetadataCommit(true); + ASSERT_OK(census.BeginGcJob(7, &gc_token)); + // Only one GC job is registered at a time. + uint64_t second_token = 0; + ASSERT_TRUE(census.BeginGcJob(7, &second_token).IsBusy()); + census.EndGcJob(gc_token); + + ASSERT_OK(census.BeginMetadataCommit()); + census.EndMetadataCommit(false); + ASSERT_TRUE(census.BeginGcJob(7, &gc_token).IsBusy()); + census.ResolveMetadataCommitAmbiguity(); + ASSERT_OK(census.BeginGcJob(7, &gc_token)); + census.EndGcJob(gc_token); +} + +TEST(BlobGcCandidateHeapTest, ExactPortableRatioOrdering) { + const uint64_t max = std::numeric_limits::max(); + ASSERT_GT(BlobGcCandidateHeap::CompareRatios(max - 1, max, max - 2, max), 0); + ASSERT_LT(BlobGcCandidateHeap::CompareRatios(max - 2, max, max - 1, max), 0); + ASSERT_EQ(BlobGcCandidateHeap::CompareRatios(1, 3, 2, 6), 0); + ASSERT_GT(BlobGcCandidateHeap::CompareRatios(2, 3, 1, 2), 0); + ASSERT_LT(BlobGcCandidateHeap::CompareRatios(1, 3, 1, 2), 0); + ASSERT_EQ(BlobGcCandidateHeap::CompareRatios(0, max, 0, 1), 0); + ASSERT_LT(BlobGcCandidateHeap::CompareRatios(0, max, 1, max), 0); +} + +TEST(BlobReferenceCensusTest, AmbiguousMetadataCommitBlocksNextCommit) { + // After a metadata-only commit with an unknown result + // (EndMetadataCommit(false)), the durable state is not known. No new + // metadata-only commit may start until ResolveMetadataCommitAmbiguity(), + // which runs after a new MANIFEST was written. After Cancel() (column family + // drop), BeginMetadataCommit() returns ColumnFamilyDropped. + BlobReferenceCensus census; + ASSERT_OK(census.BeginMetadataCommit()); + census.EndMetadataCommit(false); + ASSERT_TRUE(census.BeginMetadataCommit().IsTryAgain()); + census.ResolveMetadataCommitAmbiguity(); + ASSERT_OK(census.BeginMetadataCommit()); + census.EndMetadataCommit(true); + auto garbage = census.Cancel(); + ASSERT_TRUE(census.BeginMetadataCommit().IsColumnFamilyDropped()); +} + +TEST(BlobReferenceCensusTest, RepeatedMismatchIsTrackedPerFile) { + BlobReferenceCensus census; + // A first report for a file requests a rebuild. + ASSERT_FALSE(census.ReportMismatch(10)); + // A report for another file is not a repeat: it requests a rebuild too. + ASSERT_FALSE(census.ReportMismatch(20)); + // The same file again, with no match in between, is a repeat. + ASSERT_TRUE(census.ReportMismatch(20)); + ASSERT_EQ(census.gc_mismatch_suppress_count(), 1U); + // A matching indirect blob GC check clears it. + census.ClearMismatch(20); + ASSERT_FALSE(census.ReportMismatch(20)); +} + +TEST(BlobReferenceCensusTest, ClearMismatchForgetsOnlyItsOwnFile) { + // Indirect blob GC calls ClearMismatch() for each checked input of a job when + // no input of that job mismatched. A match of another file must keep the + // remembered mismatch, so that a match of one file does not hide a repeat + // mismatch of a different file. A match of the same file forgets it, so its + // next mismatch requests a rebuild again. + BlobReferenceCensus census; + ASSERT_FALSE(census.ReportMismatch(20)); + // A match of file 30 keeps the mismatch of file 20: the next mismatch of + // file 20 is a repeat. + census.ClearMismatch(30); + ASSERT_TRUE(census.ReportMismatch(20)); + ASSERT_EQ(census.gc_mismatch_suppress_count(), 1U); + // A match of file 20 forgets it: the next mismatch of file 20 requests a + // rebuild and is not counted as suppressed. + census.ClearMismatch(20); + ASSERT_FALSE(census.ReportMismatch(20)); + ASSERT_EQ(census.gc_mismatch_suppress_count(), 1U); +} + +TEST(BlobReferenceCensusTest, MismatchReportBlocksPublish) { + // After indirect blob GC reports a mismatch, the census is not ready, and the + // worker of the same build cannot publish it again (for example from the + // apply after an install) before the census job rebuilds it. + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 3, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 3, 3), + BlobReferenceCensus::PublishResult::kPublished); + ASSERT_TRUE(census.IsReady(3)); + ASSERT_FALSE(census.ReportMismatch(10)); + ASSERT_FALSE(census.IsReady(3)); + ASSERT_EQ(census.TryPublishWorker(token, 3, 3), + BlobReferenceCensus::PublishResult::kStale); + ASSERT_FALSE(census.IsReady(3)); + // The census job takes the report and fails the build. The next build can + // publish again. + ASSERT_TRUE(census.TakeMismatchReport()); + auto garbage = census.FailBuild(token); + const uint64_t next_token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(next_token, 3, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(next_token)); + ASSERT_EQ(census.TryPublishWorker(next_token, 3, 3), + BlobReferenceCensus::PublishResult::kPublished); +} + +TEST(BlobGcCandidateHeapTest, FrontierRiseRetriesSuppressedRelocationFiles) { + // Indirect blob GC suppressed relocation files 15, 30 and 50, for example + // because a coarse SST pinned a generation number routed to them. When the + // lowest coarse frontier of the Version rises from 20 to 40 (the coarse + // frontier of an SST is the oldest indirect-origin file number a coarse SST + // may reference), the heap retries every suppressed relocation file above the + // old frontier (30 and 50). File 15 stays suppressed. + BlobGcCandidateHeap heap; + heap.SetCoarseIndirectOriginFrontier(20); + heap.SetPhysicalFile({15, 1000, false}); + heap.SetPhysicalFile({30, 1000, false}); + heap.SetPhysicalFile({50, 1000, false}); + heap.Suppress(15, /*retry_micros=*/0); + heap.Suppress(30, /*retry_micros=*/0); + heap.Suppress(50, /*retry_micros=*/0); + ASSERT_EQ(heap.Select()[0], 0U); + heap.SetCoarseIndirectOriginFrontier(40); + const BlobGcCandidateHeap::Selection selected = heap.Select(); + ASSERT_NE(selected[0], 0U); + ASSERT_NE(selected[1], 0U); + ASSERT_EQ(selected[2], 0U); + ASSERT_NE(selected[0], 15U); + ASSERT_NE(selected[1], 15U); +} + +TEST(BlobGcCandidateHeapTest, SuppressionIsRelevantAndRetryable) { + BlobGcCandidateHeap heap; + heap.SetPhysicalFile({10, 1000, false}); + heap.SetPhysicalFile({20, 1000, false}); + heap.Suppress(10, /*retry_micros=*/100); + ASSERT_EQ(heap.Select()[0], 20U); + ASSERT_EQ(heap.RetryDue(99), 100U); + ASSERT_EQ(heap.Select()[0], 20U); + ASSERT_EQ(heap.RetryDue(100), 0U); + ASSERT_EQ(heap.Select()[0], 10U); + heap.Suppress(10, /*retry_micros=*/0); + heap.SetPhysicalFile({20, 2000, false}); + ASSERT_EQ(heap.Select()[0], 20U); + // File 10 is suppressed with no retry time, as after a corruption. A change + // to another file (file 20) does not clear the suppression. A live-byte + // change of file 10 itself does. + ASSERT_OK(heap.AdjustLiveBytes(10, 100, true)); + ASSERT_EQ(heap.Select()[1], 10U); +} + +TEST(BlobReferenceCensusTest, HeapMovesLiveBytesWithChangedForwardRoutes) { + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, nullptr)); + ASSERT_OK(census.SetWorkerPhysicalFiles(token, {{10, 1000, true}})); + BlobReferenceGroup group = MakeCensusGroup(10, 10); + group.reference_count = 1; + group.stored_value_bytes = 100; + group.record_bytes = 150; + ASSERT_OK(census.AddStartingFile(token, MakeCensusFile(100, {group}))); + ASSERT_EQ(census.SelectWorkerCandidates(token)[0], 10U); + + BlobReferenceCensusDelta delta; + delta.base_version_number = 7; + delta.version_number = 8; + delta.physical_files = {{20, 1000, false}, {10, 0, true}}; + auto forward = std::make_shared(); + forward->SetRoute(BlobRelocationRoute(10, 20)); + forward->Seal(); + delta.forward_state = forward; + delta.changed_routes = {10}; + ASSERT_OK(census.ApplyWorkerDelta(token, delta)); + ASSERT_EQ(census.SelectWorkerCandidates(token)[0], 20U); + ASSERT_EQ(census.SelectWorkerCandidates(token)[1], 0U); +} + +TEST(BlobReferenceCensusTest, CandidateFailureUsesRelevantRevision) { + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, nullptr)); + ASSERT_OK(census.SetWorkerPhysicalFiles( + token, {{10, 1000, true}, {20, 1000, true}})); + BlobGcCandidateStamp stamp; + ASSERT_OK(census.CaptureCandidateStamp(7, {10}, &stamp)); + BlobReferenceCensusDelta delta; + delta.base_version_number = 7; + delta.version_number = 8; + delta.physical_files = {{20, 2000, true}}; + ASSERT_OK(census.ApplyWorkerDelta(token, delta)); + // The delta changed only file 20, so the failure report for file 10, made + // with the stamp of Version 7, still applies and suppresses file 10. + census.SuppressWorkerCandidates(stamp, {10}, {}, 0); + ASSERT_EQ(census.SelectWorkerCandidates(token)[0], 20U); + ASSERT_EQ(census.SelectWorkerCandidates(token)[1], 0U); + + ASSERT_OK(census.CaptureCandidateStamp(8, {20}, &stamp)); + delta.base_version_number = 8; + delta.version_number = 9; + delta.physical_files = {{20, 3000, true}}; + ASSERT_OK(census.ApplyWorkerDelta(token, delta)); + // File 20 changed after the stamp was taken, so this old failure report must + // not suppress it, even though the file number is the same. + census.SuppressWorkerCandidates(stamp, {20}, {}, 0); + ASSERT_EQ(census.SelectWorkerCandidates(token)[0], 20U); + ASSERT_OK(census.CaptureCandidateStamp(9, {20}, &stamp)); + census.SuppressWorkerCandidates(stamp, {20}, {20}, 100); + ASSERT_EQ(census.SelectWorkerCandidates(token)[0], 0U); + ASSERT_EQ(census.RetryWorkerCandidates(token, 1000099), 1000100U); + ASSERT_EQ(census.RetryWorkerCandidates(token, 1000100), 0U); + ASSERT_EQ(census.SelectWorkerCandidates(token)[0], 20U); + // A second transient failure in a row doubles the retry delay, from 1 second + // to 2 seconds. A change to the file resets the failure count. The test + // passes the current time as an argument, so it never sleeps. + census.SuppressWorkerCandidates(stamp, {20}, {20}, 2000000); + ASSERT_EQ(census.RetryWorkerCandidates(token, 2000001), 4000000U); + auto canceled = census.FailBuild(token); + const uint64_t replacement = census.BeginBuild(); + ASSERT_NE(replacement, token); + ASSERT_OK(census.ResetWorkerState(replacement, 10, nullptr, nullptr)); + ASSERT_OK(census.SetWorkerPhysicalFiles(replacement, {{20, 1000, true}})); + census.SuppressWorkerCandidates(stamp, {20}, {}, 0); + ASSERT_EQ(census.SelectWorkerCandidates(replacement)[0], 20U); +} + +TEST(BlobReferenceCensusTest, SelectedSnapshotOwnsItsCacheReservation) { + const std::shared_ptr cache = NewLRUCache(16 * 1024 * 1024); + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, cache)); + ASSERT_OK(census.AddStartingFile( + token, MakeCensusFile(100, {MakeCensusGroup(10, 10)}))); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 7), + BlobReferenceCensus::PublishResult::kPublished); + const size_t worker_charge = cache->GetPinnedUsage(); + ASSERT_GT(worker_charge, 0U); + std::shared_ptr selected; + ASSERT_OK(census.ReadSelectedSnapshot(7, {10}, cache, &selected)); + ASSERT_GT(cache->GetPinnedUsage(), worker_charge); + selected.reset(); + ASSERT_EQ(cache->GetPinnedUsage(), worker_charge); +} + +TEST(BlobReferenceCensusTest, SelectedSnapshotRejectsUnchargedAllocation) { + const std::shared_ptr cache = + NewLRUCache(2 * 1024 * 1024, /*num_shard_bits=*/0, + /*strict_capacity_limit=*/true); + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 7, nullptr, cache)); + ASSERT_OK(census.AddStartingFile( + token, MakeCensusFile(100, {MakeCensusGroup(10, 10)}))); + cache->SetCapacity(cache->GetUsage()); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 7, 7), + BlobReferenceCensus::PublishResult::kPublished); + std::shared_ptr selected; + ASSERT_TRUE( + census.ReadSelectedSnapshot(7, {10}, cache, &selected).IsMemoryLimit()); + ASSERT_EQ(selected, nullptr); + ASSERT_TRUE(census.IsReady(7)); +} + +TEST(BlobReferenceCensusTest, LogSharesCacheChargeAndReleasesOnAck) { + const std::shared_ptr cache = NewLRUCache(16 * 1024 * 1024); + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + size_t first_charge = 0; + BlobReferenceCensusPreparedInstall prepared; + for (uint64_t version = 1; version <= 4; ++version) { + auto delta = std::make_shared(); + delta->base_version_number = version; + delta->version_number = version + 1; + delta->retained_forward_state_bytes = 32 * 1024; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK( + census.PrepareInstallWithCache(delta, reservation, cache, &prepared)); + auto garbage = census.Install(prepared); + if (version == 1) { + first_charge = cache->GetPinnedUsage(); + ASSERT_GT(first_charge, 0U); + } else { + ASSERT_EQ(cache->GetPinnedUsage(), first_charge); + } + } + prepared = {}; + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + for (const std::shared_ptr& delta : deltas) { + ASSERT_OK(census.AcknowledgeBuildDeltas(token, delta->QueueWeight())); + } + ASSERT_EQ(cache->GetPinnedUsage(), 0U); +} + +TEST(BlobReferenceCensusTest, GcJobStaysRegisteredAcrossInvalidation) { + // An indirect blob GC job reads no deltas after its snapshot, so an install + // that invalidates the census does not end it. The job still holds the + // registration (one GC job per census) until EndGcJob(). A new job cannot + // start until the census is ready again. + BlobReferenceCensus census; + const uint64_t build_token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(build_token, 3, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(build_token)); + ASSERT_EQ(census.TryPublishWorker(build_token, 3, 3), + BlobReferenceCensus::PublishResult::kPublished); + + uint64_t gc_token = 0; + ASSERT_OK(census.BeginGcJob(3, &gc_token)); + ASSERT_TRUE(census.IsSnapshotPending()); + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInvalidation(reservation, &prepared)); + auto garbage = census.Install(prepared); + garbage = {}; + ASSERT_TRUE(census.IsGcJobActive(gc_token)); + ASSERT_TRUE(census.IsSnapshotPending()); + census.EndGcJob(gc_token); + ASSERT_FALSE(census.IsGcJobActive(gc_token)); + ASSERT_FALSE(census.IsSnapshotPending()); + ASSERT_TRUE(census.BeginGcJob(3, &gc_token).IsTryAgain()); +} + +TEST(BlobReferenceCensusTest, GcJobPausesWorkerUntilSnapshot) { + // While a registered indirect blob GC job has not copied its snapshot, the + // worker applies no delta: ApplyWorkerDelta() returns TryAgain and changes + // nothing, so the copy shows the census at the job's input Version. After + // SnapshotSelected() the worker applies deltas again. + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 3, nullptr, nullptr)); + ASSERT_OK(census.AddStartingFile( + token, MakeCensusFile(100, {MakeCensusGroup(10, 10)}))); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 3, 3), + BlobReferenceCensus::PublishResult::kPublished); + + uint64_t gc_token = 0; + ASSERT_OK(census.BeginGcJob(3, &gc_token)); + ASSERT_TRUE(census.IsSnapshotPending()); + BlobReferenceCensusDelta delta; + delta.base_version_number = 3; + delta.version_number = 4; + delta.added_files = {MakeCensusFile(200, {MakeCensusGroup(10, 10)})}; + ASSERT_TRUE(census.ApplyWorkerDelta(token, delta).IsTryAgain()); + ASSERT_EQ(census.WorkerVersionNumber(), 3U); + + std::shared_ptr selected; + ASSERT_OK(census.SnapshotSelected(gc_token, 3, {10}, + /*gc_block_cache=*/nullptr, &selected)); + ASSERT_EQ(selected->GetSstCount(10), 1U); + ASSERT_FALSE(census.IsSnapshotPending()); + ASSERT_OK(census.ApplyWorkerDelta(token, delta)); + ASSERT_EQ(census.WorkerVersionNumber(), 4U); + census.EndGcJob(gc_token); + auto garbage = census.FailBuild(token); +} + +TEST(BlobReferenceCensusTest, SharedLogHasBoundedReadsAndReclaimsOnDrain) { + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + std::weak_ptr first_entry; + BlobReferenceCensusPreparedInstall prepared; + for (uint64_t version = 1; version <= 130; ++version) { + auto delta = std::make_shared(); + delta->base_version_number = version; + delta->version_number = version + 1; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(delta, reservation, &prepared)); + if (version == 1) { + first_entry = prepared.log_entry; + } + auto garbage = census.Install(prepared); + } + prepared = {}; + ASSERT_FALSE(first_entry.expired()); + size_t total = 0; + for (;;) { + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_LE(deltas.size(), BlobReferenceCensus::kMaxDeltasPerRead); + if (deltas.empty()) { + break; + } + for (const std::shared_ptr& delta : + deltas) { + ASSERT_OK(census.AcknowledgeBuildDeltas(token, delta->QueueWeight())); + ++total; + } + } + ASSERT_EQ(total, 130U); + ASSERT_TRUE(first_entry.expired()); + auto garbage = census.FailBuild(token); +} + +namespace { +// The tests below that need a full change log lower its limit +// (BlobReferenceCensus::kMaxLogMemory, 256 MB) to kTestLogLimit (1 MB, an +// example size) through the sync point +// "BlobReferenceCensus::AppendChange:Limit", so an oversized delta needs only +// about 1 MB of memory. +constexpr size_t kTestLogLimit = size_t{1} << 20; + +class ScopedCensusLogLimit { + public: + ScopedCensusLogLimit() { + SyncPoint::GetInstance()->SetCallBack( + "BlobReferenceCensus::AppendChange:Limit", + [](void* arg) { *static_cast(arg) = kTestLogLimit; }); + SyncPoint::GetInstance()->EnableProcessing(); + } + ~ScopedCensusLogLimit() { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + } +}; +} // namespace + +TEST(BlobReferenceCensusTest, SingleOversizedDeltaDoesNotBlockProgress) { + ScopedCensusLogLimit log_limit; + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + std::vector groups( + kTestLogLimit / sizeof(BlobReferenceGroup) + 1, MakeCensusGroup(5, 5)); + auto delta = std::make_shared(); + delta->base_version_number = 3; + delta->version_number = 4; + delta->added_files = {MakeCensusFile(30, std::move(groups))}; + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(delta, reservation, &prepared)); + auto install_garbage = census.Install(prepared); + + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_EQ(deltas.size(), 1U); + // A delta larger than the limit is kept because the cursor's queued weight + // was below the limit (0 here), so one large install cannot block progress. + // The queued weight is now above the limit, so the next delta makes this + // lagging cursor overflow: the census drops the cursor instead of keeping + // more data. + auto next = std::make_shared(); + next->base_version_number = 4; + next->version_number = 5; + const uint64_t next_reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(next, next_reservation, &prepared)); + install_garbage = census.Install(prepared); + deltas.clear(); + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_TRUE(overflowed); + ASSERT_TRUE(deltas.empty()); + auto context = std::make_shared(); + context->token = token; + ASSERT_TRUE(census.TrySetBuildContext(context)); + auto garbage = census.FailBuild(token); + ASSERT_EQ(garbage.build_context, context); + ASSERT_EQ(census.IsReady(4), false); +} + +namespace { +// Installs a delta from `version` to `version + 1`. With `oversized`, its +// QueueWeight() is larger than kTestLogLimit; otherwise the delta is empty and +// its weight is small. Callers lower the limit with ScopedCensusLogLimit. +void InstallCensusDelta(BlobReferenceCensus* census, uint64_t version, + bool oversized) { + auto delta = std::make_shared(); + delta->base_version_number = version; + delta->version_number = version + 1; + if (oversized) { + delta->added_files = {MakeCensusFile( + 100 + version, std::vector( + kTestLogLimit / sizeof(BlobReferenceGroup) + 1, + MakeCensusGroup(5, 5)))}; + } + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census->BeginInstallPreparation(); + ASSERT_OK(census->PrepareInstall(delta, reservation, &prepared)); + auto garbage = census->Install(prepared); +} + +// Returns a summary of SST `file_number` with `group_count` groups. With a +// non-null `charge_cache`, the summary has its own cache charge on that cache +// (a reservation manager and a reservation handle for its MemoryUsage()), as +// the summaries that table builders and compaction input reads return do. +std::shared_ptr MakeCensusFileWithGroups( + uint64_t file_number, size_t group_count, + const std::shared_ptr& charge_cache) { + auto file = std::make_shared(); + file->file_number = file_number; + file->groups.assign(group_count, MakeCensusGroup(5, 5)); + if (charge_cache != nullptr) { + file->reservation_manager = + std::make_shared( + std::make_shared>(charge_cache)); + EXPECT_OK(file->reservation_manager->MakeCacheReservation( + file->MemoryUsage(), &file->reservation_handle)); + } + return file; +} +} // namespace + +TEST(BlobReferenceCensusTest, OverflowRestartUsesRestartLimit) { + // After an overflow restart, each build until the next publish has the + // restart limit: 4 x the normal limit (kMaxRestartLogMemory / kMaxLogMemory; + // the test lowers the normal limit to kTestLogLimit). The limit is checked + // before each append, so the cursor keeps four deltas that are each a bit + // larger than kTestLogLimit, and the fifth overflows it. A build that + // overflows the restart limit fails; the next build still has the restart + // limit. + ScopedCensusLogLimit log_limit; + BlobReferenceCensus census; + uint64_t token = census.BeginBuild(); + // The first overflow restarts the build; a second one before any publish + // does not restart it again. + ASSERT_TRUE(census.UseOverflowRestart()); + ASSERT_FALSE(census.UseOverflowRestart()); + ASSERT_EQ(census.overflow_restart_count(), 1U); + auto garbage = census.FailBuild(token); + token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 3, nullptr, nullptr)); + for (uint64_t version = 3; version < 7; ++version) { + std::vector groups( + kTestLogLimit / sizeof(BlobReferenceGroup) + 1, MakeCensusGroup(5, 5)); + auto delta = std::make_shared(); + delta->base_version_number = version; + delta->version_number = version + 1; + delta->added_files = {MakeCensusFile(30 + version, std::move(groups))}; + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(delta, reservation, &prepared)); + auto install_garbage = census.Install(prepared); + } + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_EQ(deltas.size(), 4U); + deltas.clear(); + // The cursor now holds more than 4 x kTestLogLimit, so the next delta + // overflows it. + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 7, /*oversized=*/false)); + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_TRUE(overflowed); + garbage = census.FailBuild(token); + // The next build (after the backoff of the census job) still has the restart + // limit: two oversized deltas are kept. + token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 8, nullptr, nullptr)); + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 8, /*oversized=*/true)); + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 9, /*oversized=*/true)); + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 10, /*oversized=*/false)); + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_EQ(deltas.size(), 3U); + deltas.clear(); + garbage = census.FailBuild(token); + // A later build publishes, which resets the one-restart limit. + token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 5, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 5, 5), + BlobReferenceCensus::PublishResult::kPublished); + // After a publish, a later overflow may restart again. + ASSERT_TRUE(census.UseOverflowRestart()); + ASSERT_EQ(census.overflow_restart_count(), 2U); +} + +TEST(BlobReferenceCensusTest, OversizedDeltaBehindSmallDeltaDoesNotOverflow) { + // The limit is checked before each append. A small queued delta (a flush) + // does not make a following large delta (a compaction that rewrites a + // level) overflow the cursor. Once the queued weight has reached the limit, + // the next delta overflows it. + ScopedCensusLogLimit log_limit; + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 1, /*oversized=*/false)); + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 2, /*oversized=*/true)); + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_EQ(deltas.size(), 2U); + + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 3, /*oversized=*/false)); + deltas.clear(); + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_TRUE(overflowed); + auto garbage = census.FailBuild(token); +} + +// Summaries made with kCensusTestGroupCount have 128 groups each (about 6 KB, +// an example size), so 200 of them hold about 1.2 MB. +constexpr size_t kCensusTestGroupCount = 128; + +TEST(BlobReferenceCensusTest, ChargedSummariesCountInQueueWeightNotInCharge) { + // A compaction delta that adds and removes 100 SSTs. Each summary has its own + // cache charge. QueueWeight() counts all summaries, so the log limit covers + // them. ChargeBytes() counts only the delta's own memory, so the change log + // does not charge the summaries a second time. The same delta with + // uncharged summaries counts them in both. + std::shared_ptr charge_cache = NewLRUCache(64 << 20); + BlobReferenceCensusDelta charged; + BlobReferenceCensusDelta uncharged; + size_t summary_bytes = 0; + for (uint64_t i = 0; i < 100; ++i) { + charged.added_files.push_back(MakeCensusFileWithGroups( + 1000 + i, kCensusTestGroupCount, charge_cache)); + charged.removed_summaries.push_back(MakeCensusFileWithGroups( + 2000 + i, kCensusTestGroupCount, charge_cache)); + uncharged.added_files.push_back( + MakeCensusFileWithGroups(1000 + i, kCensusTestGroupCount, nullptr)); + uncharged.removed_summaries.push_back( + MakeCensusFileWithGroups(2000 + i, kCensusTestGroupCount, nullptr)); + summary_bytes += 2 * charged.added_files.back()->MemoryUsage(); + } + ASSERT_GT(charged.QueueWeight(), summary_bytes); + ASSERT_LT(charged.ChargeBytes(), size_t{64} << 10); + ASSERT_GT(uncharged.QueueWeight(), summary_bytes); + ASSERT_EQ(uncharged.ChargeBytes(), uncharged.QueueWeight()); +} + +TEST(BlobReferenceCensusTest, LogChargeLeavesOutChargedSummaries) { + // PrepareInstallWithCache() charges the delta's ChargeBytes() to the block + // cache of the change log. A delta with one charged and one uncharged summary + // (1 MB each, example sizes) charges only the uncharged one, because the + // charged one already has its own charge (on another cache here). The + // reservation manager charges in steps of its dummy entry size (256 KB), so + // the test checks a range. + std::shared_ptr log_cache = NewLRUCache(64 << 20); + std::shared_ptr summary_cache = NewLRUCache(64 << 20); + constexpr size_t kGroups = (size_t{1} << 20) / sizeof(BlobReferenceGroup); + auto charged = MakeCensusFileWithGroups(10, kGroups, summary_cache); + auto uncharged = MakeCensusFileWithGroups(11, kGroups, nullptr); + auto delta = std::make_shared(); + delta->base_version_number = 1; + delta->version_number = 2; + delta->added_files = {charged, uncharged}; + BlobReferenceCensus census; + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK( + census.PrepareInstallWithCache(delta, reservation, log_cache, &prepared)); + const size_t log_charge = log_cache->GetUsage(); + EXPECT_GE(log_charge, uncharged->MemoryUsage()); + EXPECT_LT(log_charge, uncharged->MemoryUsage() + charged->MemoryUsage()); + auto garbage = census.Install(prepared); +} + +TEST(BlobReferenceCensusTest, ChargedSummariesCountTowardLogLimit) { + // A compaction delta whose summaries all have their own cache charge (about + // 1.2 MB) is followed by a small flush delta before the census reads either. + // The test lowers the cursor limit to 256 KB (sync point + // "BlobReferenceCensus::AppendChange:Limit"). QueueWeight() counts the + // charged summaries, so the compaction delta reaches the limit and the flush + // delta overflows the build cursor. The test also counts the sync point + // calls, so it fails if the sync point is renamed. + int limit_calls = 0; + SyncPoint::GetInstance()->SetCallBack( + "BlobReferenceCensus::AppendChange:Limit", [&limit_calls](void* arg) { + ++limit_calls; + *static_cast(arg) = size_t{256} << 10; + }); + SyncPoint::GetInstance()->EnableProcessing(); + auto cleanup = Defer([] { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + }); + std::shared_ptr charge_cache = NewLRUCache(64 << 20); + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + auto compaction = std::make_shared(); + compaction->base_version_number = 1; + compaction->version_number = 2; + size_t summary_bytes = 0; + for (uint64_t i = 0; i < 100; ++i) { + compaction->added_files.push_back(MakeCensusFileWithGroups( + 1000 + i, kCensusTestGroupCount, charge_cache)); + compaction->removed_summaries.push_back(MakeCensusFileWithGroups( + 2000 + i, kCensusTestGroupCount, charge_cache)); + summary_bytes += 2 * compaction->added_files.back()->MemoryUsage(); + } + ASSERT_GT(summary_bytes, size_t{256} << 10); + ASSERT_GT(compaction->QueueWeight(), summary_bytes); + auto flush = std::make_shared(); + flush->base_version_number = 2; + flush->version_number = 3; + flush->added_files = {MakeCensusFile(3000, {})}; + for (const auto& delta : {compaction, flush}) { + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(delta, reservation, &prepared)); + auto garbage = census.Install(prepared); + } + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_TRUE(overflowed); + ASSERT_TRUE(deltas.empty()); + ASSERT_GT(limit_calls, 0); + auto garbage = census.FailBuild(token); +} + +TEST(BlobReferenceCensusTest, PublishEndsRestartLimit) { + // After an overflow restart, the build has the restart limit only until it + // publishes. A later delta that arrives when the queued weight has reached + // the normal limit overflows the cursor again. + ScopedCensusLogLimit log_limit; + BlobReferenceCensus census; + uint64_t token = census.BeginBuild(); + ASSERT_TRUE(census.UseOverflowRestart()); + auto garbage = census.FailBuild(token); + token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(token, 3, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(token)); + ASSERT_EQ(census.TryPublishWorker(token, 3, 3), + BlobReferenceCensus::PublishResult::kPublished); + + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 3, /*oversized=*/true)); + ASSERT_NO_FATAL_FAILURE(InstallCensusDelta(&census, 4, /*oversized=*/true)); + BlobReferenceCensusDeltaQueue deltas; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &deltas, &overflowed)); + ASSERT_TRUE(overflowed); + garbage = census.FailBuild(token); +} + +TEST(BlobReferenceCensusTest, DetachedDeltasRemainChargedUntilApplied) { + ScopedCensusLogLimit log_limit; + BlobReferenceCensus census; + const uint64_t token = census.BeginBuild(); + + auto first = std::make_shared(); + first->base_version_number = 1; + first->version_number = 2; + // The first delta alone reaches the limit, so the cursor overflows at the + // next delta unless the first one is acknowledged. + first->added_files = { + MakeCensusFile(10, std::vector( + kTestLogLimit / sizeof(BlobReferenceGroup) + 1, + MakeCensusGroup(5, 5)))}; + BlobReferenceCensusPreparedInstall prepared; + uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(first, reservation, &prepared)); + auto garbage = census.Install(prepared); + + BlobReferenceCensusDeltaQueue detached; + bool overflowed = false; + ASSERT_OK(census.TakeBuildDeltas(token, &detached, &overflowed)); + ASSERT_FALSE(overflowed); + ASSERT_EQ(detached.size(), 1); + + auto second = std::make_shared(); + second->base_version_number = 2; + second->version_number = 3; + second->added_files = { + MakeCensusFile(11, std::vector( + kTestLogLimit / (2 * sizeof(BlobReferenceGroup)), + MakeCensusGroup(6, 6)))}; + reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(second, reservation, &prepared)); + garbage = census.Install(prepared); + + BlobReferenceCensusDeltaQueue later; + ASSERT_OK(census.TakeBuildDeltas(token, &later, &overflowed)); + ASSERT_TRUE(overflowed); + ASSERT_TRUE(later.empty()); + garbage = census.FailBuild(token); +} + +TEST(BlobReferenceCensusTest, CancelDetachesReadyStateAndQueuedDeltas) { + BlobReferenceCensus census; + std::shared_ptr state; + ASSERT_OK(BlobReferenceCensus::BuildState(1, {}, &state)); + const uint64_t catch_up_token = census.BeginBuild(); + ASSERT_OK(census.ResetWorkerState(catch_up_token, 1, nullptr, nullptr)); + ASSERT_OK(census.FinalizeWorkerBuild(catch_up_token)); + ASSERT_EQ(census.TryPublishWorker(catch_up_token, 1, 1), + BlobReferenceCensus::PublishResult::kPublished); + auto delta = std::make_shared(); + delta->base_version_number = 1; + delta->version_number = 2; + BlobReferenceCensusPreparedInstall prepared; + const uint64_t reservation = census.BeginInstallPreparation(); + ASSERT_OK(census.PrepareInstall(delta, reservation, &prepared)); + auto install_garbage = census.Install(prepared); + + auto canceled = census.Cancel(); + ASSERT_NE(canceled.log_entries[0], nullptr); + ASSERT_EQ(canceled.log_entries[0]->delta, delta); + ASSERT_EQ(census.IsReady(2), false); + ASSERT_EQ(census.TryPublishWorker(catch_up_token, 1, 2), + BlobReferenceCensus::PublishResult::kStale); } // kTypeBlobIndex is a value type used by BlobDB only. The base rocksdb @@ -229,6 +4125,254 @@ TEST_F(DBBlobIndexTest, EmptyBlobIndexFailsFlushWithCorruption) { // Note: the following test case pertains to the StackableDB-based BlobDB // implementation. We should be able to write kTypeBlobIndex to memtables and // SST files. +TEST_F(DBBlobIndexTest, PrefixGateRejectsBlobIndirection) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + DestroyAndReopen(options); + ColumnFamilyData::TEST_SetAllowBlobIndirection(false); + struct RestoreGate { + ~RestoreGate() { ColumnFamilyData::TEST_SetAllowBlobIndirection(true); } + } restore_gate; + + ASSERT_TRUE(dbfull() + ->SetOptions({{"enable_blob_indirection", "true"}}) + .IsNotSupported()); + Options gated = options; + gated.enable_blob_indirection = true; + ColumnFamilyHandle* handle = nullptr; + ASSERT_TRUE( + db_->CreateColumnFamily(gated, "gated", &handle).IsNotSupported()); + ASSERT_EQ(handle, nullptr); + Close(); + ASSERT_TRUE(TryReopen(gated).IsNotSupported()); +} + +TEST_F(DBBlobIndexTest, SstFileWriterOutputStaysLegacy) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + + // SstFileWriter output has no DB file number, and without a CF handle it has + // no CF ID. So the table builder writes no exact reference metadata for it, + // with or without a CF handle. + const auto write_external = [&](ColumnFamilyHandle* handle, + const std::string& key) { + const std::string path = dbname_ + "_" + key + ".sst"; + SstFileWriter writer(EnvOptions(), options, handle); + EXPECT_OK(writer.Open(path)); + EXPECT_OK(writer.Put(key, "value")); + EXPECT_OK(writer.Finish()); + return path; + }; + ASSERT_OK( + db_->IngestExternalFile({write_external(/*handle=*/nullptr, "no_handle")}, + IngestExternalFileOptions())); + ASSERT_OK(db_->IngestExternalFile( + {write_external(db_->DefaultColumnFamily(), "with_handle")}, + IngestExternalFileOptions())); + + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 2U); + for (const auto& table : properties) { + EXPECT_EQ(table.second->user_collected_properties.count( + kBlobReferenceDescriptorPropertyName), + 0U); + } + + // RepairDB sets aside an SST that has no CF name. The SST written with a CF + // handle must survive repair. + Close(); + ASSERT_OK(RepairDB(dbname_, options)); + Reopen(options); + ASSERT_EQ(Get("with_handle"), "value"); +} + +TEST_F(DBBlobIndexTest, PrepopulateSkipsReferenceBlocks) { + // With prepopulate_block_cache, flush and compaction warm the blocks they + // write into the block cache. No reader looks up the blob reference blocks + // (BlockType::kUserDefinedIndex) in the block cache, so they are not warmed. + // The test counts the blocks that the table builder warms, by block type. + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + BlockBasedTableOptions table_options; + table_options.block_cache = NewLRUCache(8 << 20); + table_options.prepopulate_block_cache = + BlockBasedTableOptions::PrepopulateBlockCache::kFlushAndCompaction; + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + DestroyAndReopen(options); + std::atomic warmed_data_blocks{0}; + std::atomic warmed_reference_blocks{0}; + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTableBuilder::InsertBlockInCacheHelper", [&](void* arg) { + const BlockType type = *static_cast(arg); + if (type == BlockType::kData) { + ++warmed_data_blocks; + } else if (type == BlockType::kUserDefinedIndex) { + ++warmed_reference_blocks; + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + for (int i = 0; i < 100; ++i) { + ASSERT_OK(Put("key" + std::to_string(i), std::string(100, 'v'))); + } + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + EXPECT_GT(warmed_data_blocks.load(), 0); + EXPECT_EQ(warmed_reference_blocks.load(), 0); + // The compaction output has reference blocks. + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1U); + EXPECT_NE(properties.begin()->second->user_collected_properties.find( + kBlobReferenceDescriptorPropertyName), + properties.begin()->second->user_collected_properties.end()); +} + +TEST_F(DBBlobIndexTest, FlushRecordsExactReferenceMetadata) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + CreateAndReopenWithCF({"exact"}, options); + const std::string key = "exact-key"; + const std::string value(100, 'v'); + ASSERT_OK(Put(1, key, value)); + ASSERT_OK(Flush(1)); + // Flush writes only the classification. The compaction output (level 1 or + // higher) has the exact descriptor and reference blocks. + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &properties)); + ASSERT_EQ(properties.size(), 1U); + const std::string& sst_path = properties.begin()->first; + const TableProperties& table_properties = *properties.begin()->second; + const auto encoded = table_properties.user_collected_properties.find( + kBlobReferenceDescriptorPropertyName); + ASSERT_NE(encoded, table_properties.user_collected_properties.end()); + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(encoded->second, + &descriptor)); + // The descriptor names the real CF ID and SST file number, not placeholder + // values. + ASSERT_TRUE(descriptor.HasRepairMetadata()); + EXPECT_EQ(descriptor.column_family_id, handles_[1]->GetID()); + EXPECT_EQ(descriptor.sst_file_number, table_properties.orig_file_number); + ASSERT_EQ(descriptor.reference_count, 1U); + + // The index block is never compressed, so the test decodes it straight from + // the file bytes. It names one group. The group's record bytes are those of a + // normal blob record: its header, the user key and the value. + std::string sst; + ASSERT_OK(ReadFileToString(env_, sst_path, &sst)); + const auto block = [&](const BlockHandle& handle) { + return Slice(sst.data() + handle.offset(), + static_cast(handle.size())); + }; + BlobReferenceIndex index; + ASSERT_OK(DecodeBlobReferenceIndex(block(descriptor.index_handle), descriptor, + &index)); + ASSERT_EQ(index.blocks.size(), 1U); + ASSERT_EQ(index.groups.size(), 1U); + EXPECT_EQ(index.groups[0].group.record_bytes, + BlobLogRecord::kHeaderSize + key.size() + value.size()); + // The reference blocks are written before the SST tail, so a table open does + // not prefetch them. + EXPECT_GE( + table_properties.tail_start_offset, + descriptor.index_handle.offset() + descriptor.index_handle.size() + 5); + EXPECT_GT(index.blocks[0].offset(), 0U); +} + +TEST_F(DBBlobIndexTest, ReferenceDataBlocksUseTableCompression) { + // Equal values give equal entries, so a reference data block compresses well + // when the table has a compressor, and stays raw when it does not. + std::vector compressions = {kNoCompression}; + if (Snappy_Supported()) { + compressions.push_back(kSnappyCompression); + } + if (LZ4_Supported()) { + compressions.push_back(kLZ4Compression); + } + for (CompressionType compression : compressions) { + SCOPED_TRACE(static_cast(compression)); + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.compression = compression; + DestroyAndReopen(options); + for (int i = 0; i < 2000; ++i) { + ASSERT_OK(Put(Key(i), std::string(100, 'v'))); + } + ASSERT_OK(Flush()); + // Flush writes only the classification. The compaction output (level 1 or + // higher) is exact. + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 1U); + const auto encoded = + properties.begin()->second->user_collected_properties.find( + kBlobReferenceDescriptorPropertyName); + ASSERT_NE(encoded, + properties.begin()->second->user_collected_properties.end()); + BlobReferenceMetadataDescriptor descriptor; + ASSERT_OK(BlobReferenceMetadataDescriptor::DecodeFrom(encoded->second, + &descriptor)); + std::string sst; + ASSERT_OK(ReadFileToString(env_, properties.begin()->first, &sst)); + BlobReferenceIndex index; + ASSERT_OK(DecodeBlobReferenceIndex( + Slice(sst.data() + descriptor.index_handle.offset(), + static_cast(descriptor.index_handle.size())), + descriptor, &index)); + ASSERT_FALSE(index.blocks.empty()); + // Each block trailer starts with the compression type. + MetadataForTest metadata; + for (const BlockHandle& handle : index.blocks) { + EXPECT_EQ(static_cast( + sst[static_cast(handle.offset() + handle.size())]), + compression); + metadata.blocks.emplace_back(sst.data() + handle.offset(), + static_cast(handle.size())); + } + if (compression == kNoCompression) { + uint64_t count = 0; + ASSERT_OK(ForEachBlobReference(index, BlockReaderForTest(metadata), + [&](const BlobReference&) { + ++count; + return Status::OK(); + })); + EXPECT_EQ(count, 2000U); + } + } +} + TEST_F(DBBlobIndexTest, Write) { for (auto tier : kAllTiers) { DestroyAndReopen(GetTestOptions()); @@ -505,6 +4649,65 @@ TEST_F(DBBlobIndexTest, EmbeddedBlobIteratorWithFirstKeyIndex) { EXPECT_EQ(iter->value(), big2); } +TEST_F(DBBlobIndexTest, CompressedSameFileRangeIsRejectedWithBlobSource) { + // As in base RocksDB, a range of a compressed value is not supported. The + // table reader of a DB-owned SST (it has a BlobSource) returns Corruption for + // one, from the scalar and the batched read, before any other check, so a + // kBlockCacheTier read gets Corruption too. Base SSTs never hold a compressed + // embedded blob, so the test uses a made-up same-file index. + Options options = GetTestOptions(); + options.max_open_files = -1; + DestroyAndReopen(options); + const std::string sst_path = dbname_ + "/embedded_compressed_range.sst"; + SstFileWriterEmbeddedBlobOptions embedded_blob_options; + embedded_blob_options.min_blob_size = 8; + SstFileWriter writer(EnvOptions(), options); + ASSERT_OK(writer.OpenWithEmbeddedBlobs(sst_path, embedded_blob_options)); + ASSERT_OK(writer.Put("k1", std::string(1024, 'x'))); + ASSERT_OK(writer.Finish()); + ASSERT_OK(db_->IngestExternalFile({sst_path}, IngestExternalFileOptions())); + + ColumnFamilyData* const cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + const BlockBasedTable* table = nullptr; + for (int level = 0; level < cfd->NumberLevels() && table == nullptr; + ++level) { + for (const FileMetaData* file : + cfd->current()->storage_info()->LevelFiles(level)) { + table = static_cast(file->fd.pinned_reader.Get()); + } + } + ASSERT_NE(table, nullptr); + + std::string encoded; + BlobIndex::EncodeBlob(&encoded, kCurrentFileBlobIndexFileNumber, + /*offset=*/0, /*size=*/16, kSnappyCompression); + BlobIndex index; + ASSERT_OK(index.DecodeFrom(encoded)); + for (const ReadTier read_tier : {kReadAllTier, kBlockCacheTier}) { + ReadOptions read_options; + read_options.read_tier = read_tier; + PinnableSlice value; + EXPECT_TRUE(table + ->GetSameFileBlob( + read_options, index, /*range_offset=*/1, + /*range_length=*/4, + BlobVerifyPolicy::kVerifyIfNoAmplification, &value) + .IsCorruption()); + PinnableSlice batch_value; + Status batch_status; + SameFileBlobReadRequest request; + request.blob_index = &index; + request.range_offset = 1; + request.range_length = 4; + request.result = &batch_value; + request.status = &batch_status; + table->MultiGetSameFileBlob(read_options, /*num_reads=*/1, &request); + EXPECT_TRUE(batch_status.IsCorruption()) << batch_status.ToString(); + } + ASSERT_EQ(Get("k1"), std::string(1024, 'x')); +} + // Backward iteration (Prev / SeekForPrev) over an embedded-blob SST must work, // including for wide-column entities whose same-file blob columns are resolved // into an iterator-owned buffer. DBIter requires the underlying iterator's diff --git a/db/blob/same_file_blob_reader.h b/db/blob/same_file_blob_reader.h index 74ab87f41f26..e5db0df6f24b 100644 --- a/db/blob/same_file_blob_reader.h +++ b/db/blob/same_file_blob_reader.h @@ -7,17 +7,45 @@ #include #include +#include +#include +#include #include "db/blob/blob_constants.h" +#include "db/blob/blob_gen2_format.h" #include "rocksdb/rocksdb_namespace.h" #include "rocksdb/status.h" namespace ROCKSDB_NAMESPACE { +class BlobContents; class BlobIndex; +class MemoryAllocator; class PinnableSlice; struct ReadOptions; +// The inputs and outputs of one SameFileBlobReadRequest that only a +// BlockBasedTable without a BlobSource uses, that is, a standalone table reader +// such as the relocation file reader. Other readers ignore it. The caller owns +// one per request and points the request's `standalone` at it. A request with +// a non-null `standalone` gets its value in `contents` instead of in `result`. +struct SameFileBlobStandaloneRead { + // Input: allocates the read and decompress buffers. May be null. + MemoryAllocator* allocator = nullptr; + // Output: the value or range as an owned buffer, so a caller can hand the + // buffer to a cache without a copy. + std::unique_ptr contents; + // Outputs of the record read, passed down to the SimpleGen2 reader as is: + // - gen2.payload_crc32c: the CRC32C of a whole uncompressed value whose + // record checksum was checked with kCRC32c and passed. The SimpleGen2 + // reader sets it for every checked record. MultiGetSameFileBlob() then + // clears it unless the request reads a whole uncompressed value, because + // only then is the payload CRC the logical CRC. + // - gen2.copied: true when this request was an exact duplicate in the batch + // and got a copy instead of its own read. + SimpleGen2ReadExtras gen2; +}; + // One request for SameFileBlobReader::MultiGetSameFileBlob (the batched // counterpart of GetSameFileBlob). See GetSameFileBlob for the per-field // semantics (range_length == kWholeBlobLength selects the whole value). @@ -28,6 +56,23 @@ struct SameFileBlobReadRequest { BlobVerifyPolicy verify_policy = BlobVerifyPolicy::kVerifyIfNoAmplification; PinnableSlice* result = nullptr; Status* status = nullptr; + // Standalone readers only (see SameFileBlobStandaloneRead); not owned. May be + // null. + SameFileBlobStandaloneRead* standalone = nullptr; +}; + +// One stored-byte read, used when indirect blob GC copies a value out of a +// relocation file. The read checks the stored CRC of the value and, unless the +// file uses kNoChecksum, the record checksum in the SimpleGen2 trailer. It +// never decompresses the value. When stored_payload is not null, it receives +// the exact stored bytes (which may be compressed or not). It points into the +// call's CoalescedReadBuffers. When stored_payload is null, the read only +// checks the value. +struct StoredSameFileBlobReadRequest { + const BlobIndex& blob_index; + Slice* stored_payload; + Status& status; + uint32_t stored_checksum = 0; }; // Narrow interface for resolving a same-file ("embedded") blob reference diff --git a/db/builder.cc b/db/builder.cc index f55a1b463301..de25d87b6e11 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -16,6 +16,7 @@ #include "db/blob/blob_counting_iterator.h" #include "db/blob/blob_file_builder.h" #include "db/blob/blob_garbage_meter.h" +#include "db/blob/blob_reference_metadata.h" #include "db/column_family.h" #include "db/compaction/compaction_iterator.h" #include "db/dbformat.h" @@ -71,6 +72,47 @@ void ExtractTimestampFromTableProperties(const TableProperties& tp, } } +void ExtractBlobReferenceDescriptorFromTableProperties( + const TableProperties& tp, FileMetaData* meta) { + assert(meta != nullptr); + meta->blob_reference_legacy_metadata = false; + const auto descriptor = + tp.user_collected_properties.find(kBlobReferenceDescriptorPropertyName); + if (descriptor == tp.user_collected_properties.end()) { + meta->blob_reference_descriptor.clear(); + } else { + meta->blob_reference_descriptor = descriptor->second; + } +} + +Status ExtractBlobReferenceClassificationFromTableProperties( + const TableProperties& tp, FileMetaData* meta) { + meta->oldest_coarse_indirect_origin_file_number = kInvalidBlobFileNumber; + const auto property = tp.user_collected_properties.find( + kBlobReferenceClassificationPropertyName); + if (property == tp.user_collected_properties.end()) { + return Status::OK(); + } + if (!meta->blob_reference_descriptor.empty()) { + return Status::Corruption( + "SST has both exact blob reference metadata and a coarse " + "classification"); + } + Status status = DecodeBlobReferenceClassification( + property->second, &meta->oldest_coarse_indirect_origin_file_number); + if (status.ok()) { + // A classification with frontier 0 means that the SST has no indirect + // references (see "Coarse SST" in db/blob/blob_reference_metadata.h). It + // needs no pin, so we store it like a legacy SST + // (blob_reference_legacy_metadata = true). Recovery then accepts it without + // exact metadata. + meta->blob_reference_legacy_metadata = + meta->oldest_coarse_indirect_origin_file_number == + kInvalidBlobFileNumber; + } + return status; +} + Status BuildTable( const std::string& dbname, VersionSet* versions, const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions, @@ -201,7 +243,8 @@ Status BuildTable( event_logger, ioptions.listeners, dbname, tboptions.column_family_name, fname, job_id, meta->fd, kInvalidBlobFileNumber, tp, tboptions.reason, s, file_checksum, - file_checksum_func_name); + file_checksum_func_name, + /*blob_references=*/nullptr); return s; } @@ -240,7 +283,8 @@ Status BuildTable( tboptions.column_family_name, write_hint, io_tracer, blob_callback, blob_creation_reason, ioptions.cf_paths.front().path, &blob_file_paths, - blob_file_additions) + blob_file_additions, + version != nullptr ? version->cfd()->blob_source() : nullptr) : nullptr); const std::atomic kManualCompactionCanceledFalse{false}; @@ -432,6 +476,15 @@ Status BuildTable( tp = builder ->GetTableProperties(); // refresh now that builder is finished ExtractTimestampFromTableProperties(tp, meta); + ExtractBlobReferenceDescriptorFromTableProperties(tp, meta); + s = ExtractBlobReferenceClassificationFromTableProperties(tp, meta); + meta->blob_reference_summary = builder->GetBlobReferenceSstSummary(); + TEST_SYNC_POINT_CALLBACK("BuildTable:BlobReferenceSummary", meta); + if (meta->blob_reference_summary != nullptr && + meta->blob_reference_summary->file_number != meta->fd.GetNumber()) { + s = Status::Corruption( + "Blob reference summary is for another SST file number"); + } if (memtable_payload_bytes != nullptr && memtable_garbage_bytes != nullptr) { const CompactionIterationStats& ci_stats = c_iter.iter_stats(); @@ -610,7 +663,8 @@ Status BuildTable( event_logger, ioptions.listeners, dbname, tboptions.column_family_name, fname, job_id, meta->fd, meta->oldest_blob_file_number, tp, tboptions.reason, status_for_listener, file_checksum, - file_checksum_func_name); + file_checksum_func_name, + /*blob_references=*/nullptr); return s; } diff --git a/db/builder.h b/db/builder.h index ab5864273016..a72cc36421f9 100644 --- a/db/builder.h +++ b/db/builder.h @@ -49,6 +49,22 @@ TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions, void ExtractTimestampFromTableProperties(const TableProperties& tp, FileMetaData* meta); +// Copies the exact blob reference descriptor of a newly built SST from its +// table properties (kBlobReferenceDescriptorPropertyName) to `meta`, which the +// MANIFEST stores. If the SST has no descriptor, it clears +// meta->blob_reference_descriptor. It also sets +// meta->blob_reference_legacy_metadata to false. +// ExtractBlobReferenceClassificationFromTableProperties() sets it again for an +// SST whose classification has frontier 0 (see +// FileMetaData::blob_reference_legacy_metadata). The frontier is the oldest +// indirect-origin file number a coarse SST may reference, and frontier 0 means +// that the SST has no indirect references (see "Coarse SST" in +// db/blob/blob_reference_metadata.h). +void ExtractBlobReferenceDescriptorFromTableProperties( + const TableProperties& tp, FileMetaData* meta); +Status ExtractBlobReferenceClassificationFromTableProperties( + const TableProperties& tp, FileMetaData* meta); + // Build a Table file from the contents of *iter. The generated file // will be named according to number specified in meta. On success, the rest of // *meta will be filled with metadata about the generated table. diff --git a/db/c.cc b/db/c.cc index a7003600f170..f39ac1d57657 100644 --- a/db/c.cc +++ b/db/c.cc @@ -10815,6 +10815,36 @@ unsigned char rocksdb_options_get_enable_blob_files(rocksdb_options_t* opt) { return opt->rep.enable_blob_files; } +void rocksdb_options_set_enable_blob_indirection(rocksdb_options_t* opt, + unsigned char v) { + opt->rep.enable_blob_indirection = v; +} + +unsigned char rocksdb_options_get_enable_blob_indirection( + rocksdb_options_t* opt) { + return opt->rep.enable_blob_indirection; +} + +void rocksdb_options_set_pin_blob_relocation_location_blocks( + rocksdb_options_t* opt, unsigned char v) { + opt->rep.pin_blob_relocation_location_blocks = v; +} + +unsigned char rocksdb_options_get_pin_blob_relocation_location_blocks( + rocksdb_options_t* opt) { + return opt->rep.pin_blob_relocation_location_blocks; +} + +void rocksdb_options_set_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt, uint64_t v) { + opt->rep.blob_indirection_max_references_per_sst = v; +} + +uint64_t rocksdb_options_get_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt) { + return opt->rep.blob_indirection_max_references_per_sst; +} + void rocksdb_options_set_min_blob_size(rocksdb_options_t* opt, uint64_t v) { opt->rep.min_blob_size = v; } @@ -10876,6 +10906,16 @@ double rocksdb_options_get_blob_gc_force_threshold(rocksdb_options_t* opt) { return opt->rep.blob_garbage_collection_force_threshold; } +void rocksdb_options_set_indirect_blob_gc_read_coalesce_bytes( + rocksdb_options_t* opt, uint64_t v) { + opt->rep.indirect_blob_gc_read_coalesce_bytes = v; +} + +uint64_t rocksdb_options_get_indirect_blob_gc_read_coalesce_bytes( + rocksdb_options_t* opt) { + return opt->rep.indirect_blob_gc_read_coalesce_bytes; +} + void rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt, uint64_t v) { opt->rep.blob_compaction_readahead_size = v; diff --git a/db/column_family.cc b/db/column_family.cc index 669388c61964..bf545857e9a0 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -20,6 +20,8 @@ #include "db/blob/blob_file_cache.h" #include "db/blob/blob_file_partition_manager.h" +#include "db/blob/blob_reference_census.h" +#include "db/blob/blob_reference_metadata.h" #include "db/blob/blob_source.h" #include "db/compaction/compaction_picker.h" #include "db/compaction/compaction_picker_fifo.h" @@ -40,6 +42,7 @@ #include "port/port.h" #include "rocksdb/convenience.h" #include "rocksdb/table.h" +#include "table/adaptive/adaptive_table_factory.h" #include "table/merging_iterator.h" #include "util/autovector.h" #include "util/cast_util.h" @@ -686,7 +689,10 @@ ColumnFamilyData::ColumnFamilyData( db_session_id, fast_sst_open, name_)); blob_file_cache_.reset( new BlobFileCache(_table_cache, &ioptions(), soptions(), id_, - internal_stats_->GetBlobFileReadHist(), io_tracer)); + internal_stats_->GetBlobFileReadHist(), io_tracer, + GetBlockBasedTableWriterOptions( + mutable_cf_options_.table_factory.get()), + db_session_id, block_cache_tracer)); blob_source_.reset(new BlobSource(ioptions_, mutable_cf_options_, db_id, db_session_id, blob_file_cache_.get())); // Let the table cache route same-file ("embedded") blob reads through the @@ -752,6 +758,26 @@ ColumnFamilyData::ColumnFamilyData( } } +BlobReferenceCensus* ColumnFamilyData::GetOrCreateBlobReferenceCensus() { + if (blob_reference_census_storage_ == nullptr) { + blob_reference_census_storage_ = std::make_unique(); + // A new census has no published view yet. Mark it unready now, so that a + // census whose first build never starts (the job is not scheduled, or + // background work is paused) reports its unready time instead of 0. + blob_reference_census_storage_->NoteUnready(ioptions_.clock->NowMicros()); + blob_reference_census_.store(blob_reference_census_storage_.get(), + std::memory_order_release); + } + return blob_reference_census_storage_.get(); +} + +BlobReferenceCensusGarbage ColumnFamilyData::CancelBlobReferenceCensus() { + if (blob_reference_census_storage_ == nullptr) { + return {}; + } + return blob_reference_census_storage_->Cancel(); +} + // DB mutex held ColumnFamilyData::~ColumnFamilyData() { assert(refs_.load(std::memory_order_relaxed) == 0); @@ -1283,9 +1309,24 @@ void ColumnFamilyData::CreateNewMemtable(SequenceNumber earliest_seq) { } bool ColumnFamilyData::NeedsCompaction() const { - return is_table_builder_supported_ && - !mutable_cf_options_.disable_auto_compactions && - compaction_picker_->NeedsCompaction(current_->storage_info()); + if (!is_table_builder_supported_ || + mutable_cf_options_.disable_auto_compactions) { + return false; + } + Version* const current = current_; + if (current == nullptr) { + return false; + } + if (compaction_picker_->NeedsCompaction(current->storage_info())) { + return true; + } + return mutable_cf_options_.enable_blob_files && + mutable_cf_options_.enable_blob_garbage_collection && + mutable_cf_options_.enable_blob_indirection && + current->storage_info()->BlobPhysicalFileForIndirectBlobGC() != + kInvalidBlobFileNumber && + blob_reference_census() != nullptr && + blob_reference_census()->IsReady(current->GetVersionNumber()); } Compaction* ColumnFamilyData::PickCompaction( @@ -1530,9 +1571,49 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() { } } +Status CheckBlobIndirectionCompactionStyle(CompactionStyle style, + bool has_blob_indirection) { + if (has_blob_indirection && style == kCompactionStyleFIFO) { + return Status::NotSupported("Blob indirection is incompatible with FIFO"); + } + return Status::OK(); +} + +Status CheckBlobIndirectionCompactionStyle( + const std::vector& column_families) { + for (const ColumnFamilyDescriptor& cf : column_families) { + Status s = CheckBlobIndirectionCompactionStyle( + cf.options.compaction_style, cf.options.enable_blob_indirection); + if (!s.ok()) { + return s; + } + } + return Status::OK(); +} + +namespace { +std::atomic allow_blob_indirection_for_test{false}; +} // namespace + +void ColumnFamilyData::TEST_SetAllowBlobIndirection(bool allow) { + allow_blob_indirection_for_test.store(allow, std::memory_order_relaxed); +} + Status ColumnFamilyData::ValidateOptions( const DBOptions& db_options, const ColumnFamilyOptions& cf_options) { - Status s; + // Blob indirection is not complete in this part of the indirect blob GC + // stack, and data written with it may not stay readable. So the option is + // rejected unless a test allows it through TEST_SetAllowBlobIndirection(). + if (cf_options.enable_blob_indirection && + !allow_blob_indirection_for_test.load(std::memory_order_relaxed)) { + return Status::NotSupported( + "enable_blob_indirection is not supported in this revision"); + } + Status s = CheckBlobIndirectionCompactionStyle( + cf_options.compaction_style, cf_options.enable_blob_indirection); + if (!s.ok()) { + return s; + } s = CheckCompressionSupported(cf_options); if (s.ok() && db_options.allow_concurrent_memtable_write) { s = CheckConcurrentWritesSupported(cf_options); @@ -1605,6 +1686,30 @@ Status ColumnFamilyData::ValidateOptions( "Blob direct write does not support user-defined timestamps."); } } + if (cf_options.enable_blob_indirection && !cf_options.enable_blob_files) { + return Status::InvalidArgument( + "enable_blob_indirection requires enable_blob_files=true"); + } + if (cf_options.blob_indirection_max_references_per_sst < + kMinBlobIndirectionMaxReferencesPerSst || + cf_options.blob_indirection_max_references_per_sst > + kMaxBlobIndirectionMaxReferencesPerSst) { + return Status::InvalidArgument( + "blob_indirection_max_references_per_sst must be in [" + + std::to_string(kMinBlobIndirectionMaxReferencesPerSst) + ", " + + std::to_string(kMaxBlobIndirectionMaxReferencesPerSst) + "]"); + } + if (cf_options.enable_blob_indirection && + cf_options.enable_blob_direct_write) { + return Status::NotSupported( + "Blob indirection and blob direct write cannot be enabled together"); + } + if (cf_options.enable_blob_indirection && + !cf_options.table_factory->IsInstanceOf( + TableFactory::kBlockBasedTableName())) { + return Status::NotSupported( + "Blob indirection requires Block-Based Table format"); + } if (ucmp->timestamp_size() > 0 && !cf_options.persist_user_defined_timestamps) { if (db_options.atomic_flush) { @@ -1737,11 +1842,13 @@ Status ColumnFamilyData::ValidateOptions( return s; } -Status ColumnFamilyData::SetOptions( +Status ColumnFamilyData::PrepareMutableCFOptions( const DBOptions& db_opts, - const std::unordered_map& options_map) { + const std::unordered_map& options_map, + MutableCFOptions* mutable_cf_options) const { + assert(mutable_cf_options != nullptr); ColumnFamilyOptions cf_opts = - BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_); + BuildColumnFamilyOptions(initial_cf_options_, *mutable_cf_options); ConfigOptions config_opts; config_opts.mutable_options_only = true; #ifndef NDEBUG @@ -1757,9 +1864,25 @@ Status ColumnFamilyData::SetOptions( s = ValidateOptions(db_opts, cf_opts); } if (s.ok()) { - mutable_cf_options_ = MutableCFOptions(cf_opts); - mutable_cf_options_.RefreshDerivedOptions(ioptions_); - options_install_pending_ = true; + *mutable_cf_options = MutableCFOptions(cf_opts); + mutable_cf_options->RefreshDerivedOptions(ioptions_); + } + return s; +} + +void ColumnFamilyData::PublishMutableCFOptions( + const MutableCFOptions& mutable_cf_options) { + mutable_cf_options_ = mutable_cf_options; + options_install_pending_ = true; +} + +Status ColumnFamilyData::SetOptions( + const DBOptions& db_opts, + const std::unordered_map& options_map) { + MutableCFOptions mutable_cf_options(mutable_cf_options_); + Status s = PrepareMutableCFOptions(db_opts, options_map, &mutable_cf_options); + if (s.ok()) { + PublishMutableCFOptions(mutable_cf_options); } return s; } diff --git a/db/column_family.h b/db/column_family.h index af691d90c9f2..7b4f87be33fc 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -37,6 +37,8 @@ namespace ROCKSDB_NAMESPACE { class Version; class VersionSet; class VersionStorageInfo; +class BlobReferenceCensus; +struct BlobReferenceCensusGarbage; class MemTable; class MemTableListVersion; class CompactionPicker; @@ -284,6 +286,16 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options); Status CheckCFPathsSupported(const DBOptions& db_options, const ColumnFamilyOptions& cf_options); +// Returns NotSupported when `has_blob_indirection` is true and `style` is +// kCompactionStyleFIFO, because blob indirection does not support FIFO +// compaction. ColumnFamilyData::ValidateOptions() calls it for one CF. +Status CheckBlobIndirectionCompactionStyle(CompactionStyle style, + bool has_blob_indirection); +// Runs the check above for each CF of an open request. Read-only and secondary +// opens call it, because they do not call ValidateOptions(). +Status CheckBlobIndirectionCompactionStyle( + const std::vector& column_families); + ColumnFamilyOptions SanitizeCfOptions(const ImmutableDBOptions& db_options, bool read_only, const ColumnFamilyOptions& src); @@ -377,6 +389,32 @@ class ColumnFamilyData { // Validate CF options against DB options static Status ValidateOptions(const DBOptions& db_options, const ColumnFamilyOptions& cf_options); + + // Test-only, not public API. Until the indirect blob GC stack is complete, + // ValidateOptions rejects enable_blob_indirection unless a test allows it. + static void TEST_SetAllowBlobIndirection(bool allow); + + // DBImpl::SetOptions() must change the options of all its column families or + // none, so it changes them in two steps: prepare every column family, then + // publish them all together. This is the prepare step. It applies + // `options_map` on top of `*mutable_cf_options`, validates the result, and + // stores the result back in `*mutable_cf_options`. It does not change the + // options of this column family. REQUIRES: DB mutex held. The caller starts + // `*mutable_cf_options` from a copy of the latest options of this column + // family. It must make sure that no one changes those options before the + // publish step. DBImpl::SetOptions() holds options_mutex_ for this. + Status PrepareMutableCFOptions( + const DBOptions& db_options, + const std::unordered_map& options_map, + MutableCFOptions* mutable_cf_options) const; + + // The publish step: makes options from PrepareMutableCFOptions() the latest + // options of this column family. LogAndApply() calls it for + // DBImpl::SetOptions(). Readers see the new options only after + // DBImpl::SetOptions() installs a new SuperVersion. Then + // DBImpl::SetOptions() calls MarkOptionsInstalled(). + // REQUIRES: DB mutex held. + void PublishMutableCFOptions(const MutableCFOptions& mutable_cf_options); // REQUIRES: DB mutex held Status SetOptions( const DBOptions& db_options, @@ -397,6 +435,55 @@ class ColumnFamilyData { InternalStats* internal_stats() { return internal_stats_.get(); } + // The blob reference census of this CF (see db/blob/blob_reference_census.h). + // It is null until the CF first needs it. Two paths create it, both through + // GetOrCreateBlobReferenceCensus(): + // - a MANIFEST install for a CF whose current Version has the per-CF format + // fence, or for a CF with enable_blob_indirection when the DB has the + // DB-global format fence after the install (for example, the install that + // activates the format, or a later install after the CF turns the option + // on). Both fences are described under "MANIFEST records" in + // db/blob/blob_relocation_manifest.h; + // - census scheduling for a CF recovered from the MANIFEST with the format + // fence. + // Once created, the census lives as long as the CFD. + BlobReferenceCensus* blob_reference_census() { + return blob_reference_census_.load(std::memory_order_acquire); + } + const BlobReferenceCensus* blob_reference_census() const { + return blob_reference_census_.load(std::memory_order_acquire); + } + // The number of failed census builds in a row (see + // blob_reference_census_failures_). REQUIRES: DB mutex held. + uint32_t blob_reference_census_failures() const { + return blob_reference_census_failures_; + } + // Indirect blob GC idle time: the DB clock time since which this column + // family has had GC candidates and no GC job began, or 0. A census publish + // with candidates while no GC job runs, and the end of a GC job that leaves + // candidates, start it; a GC job start and a publish without candidates + // clear it. + void NoteIndirectBlobGcCandidates(uint64_t now_micros) { + uint64_t expected = 0; + indirect_blob_gc_candidates_since_micros_.compare_exchange_strong( + expected, now_micros, std::memory_order_relaxed); + } + void ClearIndirectBlobGcCandidates() { + indirect_blob_gc_candidates_since_micros_.store(0, + std::memory_order_relaxed); + } + uint64_t IndirectBlobGcCandidatesSinceMicros() const { + return indirect_blob_gc_candidates_since_micros_.load( + std::memory_order_relaxed); + } + + // REQUIRES: DB mutex held. + BlobReferenceCensus* GetOrCreateBlobReferenceCensus(); + + // REQUIRES: DB mutex held. The caller must destroy the returned state after + // it releases the DB mutex. + BlobReferenceCensusGarbage CancelBlobReferenceCensus(); + MemTableList* imm() { return &imm_; } MemTable* mem() { return mem_; } @@ -676,6 +763,7 @@ class ColumnFamilyData { private: friend class ColumnFamilySet; + friend class DBImpl; ColumnFamilyData( uint32_t id, const std::string& name, Version* dummy_versions, Cache* table_cache, WriteBufferManager* write_buffer_manager, @@ -716,9 +804,10 @@ class ColumnFamilyData { const bool is_delete_range_supported_; const bool is_table_builder_supported_; - // True from a successful SetOptions() until MarkOptionsInstalled(). A - // DB::SetOptions() call that fails on a later column family leaves it set - // for the column families it already changed. + // True from PublishMutableCFOptions() until MarkOptionsInstalled(). + // DBImpl::SetOptions() installs the options of all the column families it + // changes together, inside LogAndApply, after every one of them is prepared. + // So it sets this flag for every column family it changes, or for none. bool options_install_pending_ = false; std::unique_ptr table_cache_; @@ -726,6 +815,10 @@ class ColumnFamilyData { std::unique_ptr blob_source_; // Per-CF manager for write-path blob direct-write files. std::shared_ptr blob_partition_manager_; + // Owns the census. blob_reference_census_ points to the same object, so + // readers that do not hold the DB mutex can load it. + std::unique_ptr blob_reference_census_storage_; + std::atomic blob_reference_census_{nullptr}; std::unique_ptr internal_stats_; @@ -770,6 +863,24 @@ class ColumnFamilyData { // compaction set. bool queued_for_compaction_; + // State for the background census worker, protected by the DB mutex. DBImpl + // keeps two intrusive lists of CFs, linked by next_blob_reference_census_: + // the run queue and the retry list. While queued_for_blob_reference_census_ + // is true, the CF is in one of the lists or its census is running, and the + // census scheduler holds one reference to the CF. A running census worker + // releases that reference when it is done, or keeps it when it moves the CF + // to the retry list. blob_reference_census_retry_micros_ is the time when the + // retry list moves the CF back to the run queue. A CF with no failed build + // can move back earlier when its census has new work. + // blob_reference_census_failures_ counts failed builds in a row (capped at + // 6). After a failure, the retry delay is 1 second, doubled for each failure + // in a row, so at most 32 seconds. + bool queued_for_blob_reference_census_ = false; + ColumnFamilyData* next_blob_reference_census_ = nullptr; + uint64_t blob_reference_census_retry_micros_ = 0; + uint32_t blob_reference_census_failures_ = 0; + std::atomic indirect_blob_gc_candidates_since_micros_{0}; + // Read outside the DB mutex by compaction workers (acquire-loaded roughly // every 1024 records in ProcessKeyValueCompaction). Given its own cache line // via CacheAlignedWrapper to avoid false sharing with the DB-mutex-written diff --git a/db/compaction/compaction.cc b/db/compaction/compaction.cc index 1062a6cb837b..4a4629f7567e 100644 --- a/db/compaction/compaction.cc +++ b/db/compaction/compaction.cc @@ -475,6 +475,17 @@ void Compaction::PopulateProximalLevelOutputRange() { } } +bool Compaction::MustRewriteCoarseL0File(const FileMetaData& file) { + // A coarse file that leaves L0 must be rewritten with exact reference + // metadata. Moving it would keep its coarse pin until a later compaction + // rewrites it, which can take a long time. An L0 file with frontier 0 (see + // "Coarse SST" in db/blob/blob_reference_metadata.h) is not coarse: it has + // exact metadata, or it is a legacy SST (only legacy references, or none). It + // pins nothing and can move. + return file.oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber; +} + Compaction::~Compaction() { if (input_version_ != nullptr) { input_version_->Unref(); @@ -593,6 +604,14 @@ bool Compaction::IsTrivialMove() const { return false; } + if (start_level_ == 0 && output_level_ != 0) { + for (const FileMetaData* file : inputs_.front().files) { + if (MustRewriteCoarseL0File(*file)) { + return false; + } + } + } + // Used in universal compaction, where trivial move can be done if the // input files are non overlapping if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) && @@ -639,6 +658,23 @@ bool Compaction::IsTrivialMove() const { return true; } +bool Compaction::CollectExactBlobReferences() const { + if (!UsesBlobIndirection(mutable_cf_options_, input_vstorage_)) { + return false; + } + if (output_level_ != 0 || bottommost_level_) { + return true; + } + for (const CompactionInputFiles& level : inputs_) { + for (const FileMetaData* file : level.files) { + if (!file->blob_reference_descriptor.empty()) { + return true; + } + } + } + return false; +} + void Compaction::AddInputDeletions(VersionEdit* out_edit) { for (size_t which = 0; which < num_input_levels(); which++) { for (size_t i = 0; i < inputs_[which].size(); i++) { diff --git a/db/compaction/compaction.h b/db/compaction/compaction.h index 915187fd2608..c8739f3f1256 100644 --- a/db/compaction/compaction.h +++ b/db/compaction/compaction.h @@ -229,6 +229,10 @@ class Compaction { // moving a single input file to the next level (no merging or splitting) bool IsTrivialMove() const; + // True if L0 file `file` holds coarse (pinning) blob metadata that must be + // rewritten as exact metadata when it leaves L0, so it cannot be moved. + static bool MustRewriteCoarseL0File(const FileMetaData& file); + // The split user key in the output level if this compaction is required to // split the output files according to the existing cursor in the output // level under round-robin compaction policy. Empty indicates no required @@ -454,6 +458,24 @@ class Compaction { return enable_blob_garbage_collection_; } + // Returns true when the output must have exact reference metadata: the DB + // uses blob indirection (the option is on, or the Version has the format), + // and one of these holds: the output level is L1 or deeper, the compaction + // is bottommost (at any level, including L0), or an L0 input already has + // exact metadata. Otherwise the output gets at most the coarse + // classification. Called during unlocked job preparation. + // + // Why: a flush and an L0 compaction that is not bottommost stay coarse to + // save collector memory. Their data gets exact metadata when a later + // compaction rewrites it into L1 or deeper, or in a bottommost compaction. + // An output never loses exact metadata that an input has, because a coarse + // output would pin files again that the census already tracks exactly. A + // bottommost compaction always makes exact output, so coarse data gets exact + // metadata at the latest when a bottommost compaction rewrites it. This also + // holds with universal compaction and num_levels = 1, where no output goes + // to L1. + bool CollectExactBlobReferences() const; + double blob_garbage_collection_age_cutoff() const { return blob_garbage_collection_age_cutoff_; } diff --git a/db/compaction/compaction_iterator.cc b/db/compaction/compaction_iterator.cc index 793b7d4743be..41e1d0b9a176 100644 --- a/db/compaction/compaction_iterator.cc +++ b/db/compaction/compaction_iterator.cc @@ -2178,7 +2178,7 @@ Status CompactionIterator::OutputHasExternalBlobReferences( return Status::OK(); } if (ikey_.type == kTypeWideColumnEntity) { - return WideColumnSerialization::ForEachBlobFileNumber( + return WideColumnSerialization::ForEachBlobIndex( value_, [has_external_refs](const BlobIndex& index) { if (!index.IsInlined() && !index.IsSameFile()) { *has_external_refs = true; diff --git a/db/compaction/compaction_iterator_test.cc b/db/compaction/compaction_iterator_test.cc index af5b45b14fc9..4df5a1574549 100644 --- a/db/compaction/compaction_iterator_test.cc +++ b/db/compaction/compaction_iterator_test.cc @@ -3042,7 +3042,8 @@ class WideColumnEntityBlobExtractionTest : public testing::Test { write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kFlush, immutable_options.cf_paths.front().path, &blob_file_paths_, - &blob_file_additions_); + &blob_file_additions_, + /*blob_source=*/nullptr); // Set up the input iterator const Comparator* cmp = BytewiseComparator(); @@ -3485,7 +3486,8 @@ class WideColumnEntityBlobGCTest : public testing::Test { nullptr /*BlobFileCompletionCallback*/, BlobFileCreationReason::kCompaction, immutable_options.cf_paths.front().path, &blob_file_paths_, - &blob_file_additions_); + &blob_file_additions_, + /*blob_source=*/nullptr); } // Set up the input iterator diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 12b26cf504d4..3d4f41dcf58d 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -20,9 +21,12 @@ #include #include +#include "cache/cache_reservation_manager.h" #include "db/blob/blob_counting_iterator.h" #include "db/blob/blob_file_addition.h" #include "db/blob/blob_file_builder.h" +#include "db/blob/blob_reference_census.h" +#include "db/blob/blob_reference_metadata.h" #include "db/blob/missing_blob_repair_iterator.h" #include "db/builder.h" #include "db/compaction/clipping_iterator.h" @@ -56,6 +60,7 @@ #include "rocksdb/status.h" #include "rocksdb/table.h" #include "rocksdb/utilities/options_type.h" +#include "table/adaptive/adaptive_table_factory.h" #include "table/format.h" #include "table/merging_iterator.h" #include "table/meta_blocks.h" @@ -1283,8 +1288,58 @@ void CompactionJob::FinalizeCompactionRun( const_cast(&input_status)); } +void CompactionJob::ReadInputBlobReferenceSummaries() { + Compaction* const compaction = compact_->compaction; + Version* const input = compaction->input_version(); + ColumnFamilyData* const cfd = compaction->column_family_data(); + if (input == nullptr || cfd->blob_reference_census() == nullptr || + !input->storage_info()->HasBlobIndirectionFormat()) { + return; + } + std::shared_ptr reservation_manager; + const BlockBasedTableOptions* const table_options = + GetBlockBasedTableWriterOptions( + compaction->mutable_cf_options().table_factory.get()); + if (table_options != nullptr && table_options->block_cache != nullptr && + table_options->cache_usage_options.options_overrides + .at(CacheEntryRole::kBlobReferenceCensus) + .charged != CacheEntryRoleOptions::Decision::kDisabled) { + reservation_manager = std::make_shared( + std::make_shared< + CacheReservationManagerImpl>( + table_options->block_cache)); + } + ReadOptions read_options(Env::IOActivity::kCompaction); + read_options.fill_cache = false; + read_options.rate_limiter_priority = GetRateLimiterPriority(); + for (const CompactionInputFiles& level : *compaction->inputs()) { + for (const FileMetaData* file : level.files) { + if (file->blob_reference_descriptor.empty()) { + continue; + } + std::shared_ptr summary; + Status status = cfd->table_cache()->ReadBlobReferenceSstSummary( + read_options, file_options_, cfd->internal_comparator(), *file, + compaction->mutable_cf_options(), &summary, reservation_manager); + TEST_SYNC_POINT_CALLBACK( + "CompactionJob::ReadInputBlobReferenceSummaries:Status", &status); + if (!status.ok()) { + // Not a compaction failure: the census rebuilds after the install. + ROCKS_LOG_WARN(db_options_.info_log, + "[%s] [JOB %d] Cannot read blob reference groups of " + "input #%" PRIu64 ": %s", + cfd->GetName().c_str(), job_id_, file->fd.GetNumber(), + status.ToString().c_str()); + return; + } + input_blob_reference_summaries_.push_back(std::move(summary)); + } + } +} + Status CompactionJob::Run() { InitializeCompactionRun(); + ReadInputBlobReferenceSummaries(); const uint64_t start_micros = db_options_.clock->NowMicros(); @@ -1835,7 +1890,9 @@ void CompactionJob::CreateBlobFileBuilder( cfd->GetID(), cfd->GetName(), write_hint_, io_tracer_, blob_callback_, BlobFileCreationReason::kCompaction, GetBlobOutputPath(), sub_compact->Current().GetOutputFilePathsPtr(), - sub_compact->Current().GetBlobFileAdditionsPtr()); + sub_compact->Current().GetBlobFileAdditionsPtr(), + // Compaction output never prepopulates the blob cache (flush only). + /*blob_source=*/nullptr); } else { blob_file_builder = nullptr; } @@ -2492,7 +2549,7 @@ Status CompactionJob::FinishCompactionOutputFile( if (s.ok() && (current_entries > 0 || tp.num_range_deletions > 0)) { // Output to event logger and fire events. - outputs.UpdateTableProperties(); + s = outputs.UpdateTableProperties(); ROCKS_LOG_INFO(db_options_.info_log, "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64 " keys, %" PRIu64 " bytes%s, temperature: %s", @@ -2505,10 +2562,24 @@ Status CompactionJob::FinishCompactionOutputFile( FileDescriptor output_fd; uint64_t oldest_blob_file_number = kInvalidBlobFileNumber; Status status_for_listener = s; + TableFileBlobReferenceInfo blob_references; + bool has_blob_references = false; if (meta != nullptr) { fname = GetTableFileName(meta->fd.GetNumber()); output_fd = meta->fd; oldest_blob_file_number = meta->oldest_blob_file_number; + BlobReferenceMetadataDescriptor descriptor; + if (s.ok() && !meta->blob_reference_descriptor.empty() && + BlobReferenceMetadataDescriptor::DecodeFrom( + meta->blob_reference_descriptor, &descriptor) + .ok()) { + has_blob_references = true; + blob_references.count = descriptor.reference_count; + blob_references.closed_by_limit = + outputs.CurrentOutputClosedByBlobReferenceLimit(); + RecordInHistogram(stats_, COMPACTION_OUTPUT_BLOB_REFERENCES, + descriptor.reference_count); + } } else { fname = "(nil)"; if (s.ok()) { @@ -2519,7 +2590,8 @@ Status CompactionJob::FinishCompactionOutputFile( event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(), fname, job_id_, output_fd, oldest_blob_file_number, tp, TableFileCreationReason::kCompaction, status_for_listener, file_checksum, - file_checksum_func_name); + file_checksum_func_name, + has_blob_references ? &blob_references : nullptr); // Report new file to SstFileManagerImpl auto sfm = @@ -2765,6 +2837,20 @@ Status CompactionJob::InstallCompactionResults(bool* compaction_released) { const std::unordered_map::const_iterator blob_file = blob_files.find(blob_file_number); if (blob_file == blob_files.end()) { + // Skip a generation number that has no BlobFileMetaData garbage + // counts: a routed generation number (a blob file removed by blob + // file retirement, or a deleted relocation file), or a live + // relocation file that a flattened reference uses as its generation + // number. Indirect blob GC finds the live values of these files + // from the exact reference metadata of SSTs, not from garbage + // counts. + if (blob_file_number != kInvalidBlobFileNumber && + (storage_info->GetBlobRelocationRouteTarget(blob_file_number) != + kInvalidBlobFileNumber || + storage_info->PeekBlobRelocationFileMetaData( + blob_file_number) != nullptr)) { + continue; + } if (blob_file_number != kInvalidBlobFileNumber && (compaction->skip_missing_blob_files() || storage_info->IsDroppedBlobFile(blob_file_number))) { @@ -2813,6 +2899,8 @@ Status CompactionJob::InstallCompactionResults(bool* compaction_released) { // Add compaction inputs and outputs only after all blob metadata has been // validated across subcompactions. compaction->AddInputDeletions(edit); + edit->SetBlobReferenceSummariesOfDeletedFiles( + std::move(input_blob_reference_summaries_)); for (const auto& sub_compact : compact_->sub_compact_states) { sub_compact.AddOutputsEdit(edit); @@ -2821,6 +2909,9 @@ Status CompactionJob::InstallCompactionResults(bool* compaction_released) { } } + edit->MaybeSetBlobIndirectionFormatFence( + compaction->mutable_cf_options().enable_blob_indirection); + for (const auto& pair : blob_total_garbage) { const uint64_t blob_file_number = pair.first; const BlobGarbageMeter::BlobStats& stats = pair.second; @@ -2940,7 +3031,8 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact, event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(), fname, job_id_, FileDescriptor(), kInvalidBlobFileNumber, TableProperties(), TableFileCreationReason::kCompaction, s, - kUnknownFileChecksum, kUnknownFileChecksumFuncName); + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + /*blob_references=*/nullptr); return s; } @@ -3045,6 +3137,19 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact, sub_compact->compaction->max_output_file_size(), file_number, proximal_after_seqno_ /*last_level_inclusive_max_seqno_threshold*/, dbname_ /*db_name*/, IsRemoteCompaction() /*is_remote_compaction*/); + const Compaction* compaction = sub_compact->compaction; + const Version* input_version = compaction->input_version(); + tboptions.track_blob_reference_classification = UsesBlobIndirection( + compaction->mutable_cf_options(), + input_version != nullptr ? input_version->storage_info() : nullptr); + tboptions.collect_blob_reference_metadata = + compaction->CollectExactBlobReferences(); + if (input_version != nullptr && input_version->storage_info() != nullptr) { + tboptions.blob_route_snapshot_fingerprint_high = + input_version->storage_info()->GetBlobRelocationStateFingerprintHigh(); + tboptions.blob_route_snapshot_fingerprint_low = + input_version->storage_info()->GetBlobRelocationStateFingerprintLow(); + } outputs.NewBuilder(tboptions); @@ -3467,7 +3572,7 @@ Status CompactionJob::ReadOutputFilesTableProperties( return s; } -void CompactionJob::RestoreCompactionOutputs( +Status CompactionJob::RestoreCompactionOutputs( const ColumnFamilyData* cfd, const std::vector>& output_files_table_properties, @@ -3491,13 +3596,17 @@ void CompactionJob::RestoreCompactionOutputs( cfd->internal_comparator(), enable_output_hash, true /* finished */); - outputs_to_restore->UpdateTableProperties( + const Status status = outputs_to_restore->UpdateTableProperties( *output_files_table_properties[i]); + if (!status.ok()) { + return status; + } } outputs_to_restore->SetNumOutputRecords( subcompaction_progress_per_level.GetNumProcessedOutputRecords()); outputs_to_restore->AddRestoredBytesWritten(restored_bytes); + return Status::OK(); } Status CompactionJob::RestoreBlobCompactionProgress( @@ -3624,9 +3733,12 @@ Status CompactionJob::MaybeResumeSubcompactionProgressOnInputIterator( s.ToString()); } - RestoreCompactionOutputs(cfd, output_files_table_properties, - subcompaction_progress_per_level, - sub_compact->Outputs(is_proximal_level)); + s = RestoreCompactionOutputs(cfd, output_files_table_properties, + subcompaction_progress_per_level, + sub_compact->Outputs(is_proximal_level)); + if (!s.ok()) { + return s; + } // Skip past all the used file numbers to avoid creating new output files // after resumption that conflict with the existing output files diff --git a/db/compaction/compaction_job.h b/db/compaction/compaction_job.h index 0bc0a8c0b061..892edabd44f7 100644 --- a/db/compaction/compaction_job.h +++ b/db/compaction/compaction_job.h @@ -50,6 +50,7 @@ namespace ROCKSDB_NAMESPACE { class Arena; +struct BlobReferenceSstSummary; class CompactionState; class ErrorHandler; class MemTable; @@ -258,6 +259,12 @@ class CompactionJob { CompactionJobStats* job_stats_; + // Reference summaries of the exact input SSTs. Their memory is charged to + // CacheEntryRole::kBlobReferenceCensus when that role is charged. The job + // keeps them until the install edit hands them to the census. + std::vector> + input_blob_reference_summaries_; + // Lower bound for the minimal sequence number whose write time is // preserved. See Prepare(). SequenceNumber preserve_time_min_seqno_floor_ = 0; @@ -328,6 +335,14 @@ class CompactionJob { void ReleaseSubcompactionResources(); void InitializeCompactionRun(); + + // To remove an SST, the census delta needs the SST's reference summary. This + // reads the summary of every exact input SST while the job holds its inputs, + // so the install edit can hand them to the census. Every compaction reads + // them, also a full compaction, so every install updates the census with a + // delta. If a read fails, it stops, and the missing summary makes the census + // rebuild instead. + void ReadInputBlobReferenceSummaries(); void RunSubcompactions(); void RecordRemoteCompactionInputBlobFileCount(); void UpdateTimingStats(uint64_t start_micros); @@ -627,7 +642,7 @@ class CompactionJob { const FileMetaData* file_meta, const ReadOptions& read_options, std::shared_ptr* tp); - void RestoreCompactionOutputs( + Status RestoreCompactionOutputs( const ColumnFamilyData* cfd, const std::vector>& output_files_table_properties, diff --git a/db/compaction/compaction_outputs.cc b/db/compaction/compaction_outputs.cc index 5ed242930574..2a5dd8d0a5eb 100644 --- a/db/compaction/compaction_outputs.cc +++ b/db/compaction/compaction_outputs.cc @@ -12,8 +12,11 @@ #include +#include "db/blob/blob_index.h" +#include "db/blob/blob_reference_metadata.h" #include "db/builder.h" #include "logging/logging.h" +#include "monitoring/statistics_impl.h" namespace ROCKSDB_NAMESPACE { @@ -56,6 +59,14 @@ Status CompactionOutputs::Finish( meta->user_defined_timestamps_persisted = static_cast(tp.user_defined_timestamps_persisted); ExtractTimestampFromTableProperties(tp, meta); + ExtractBlobReferenceDescriptorFromTableProperties(tp, meta); + s = ExtractBlobReferenceClassificationFromTableProperties(tp, meta); + meta->blob_reference_summary = builder_->GetBlobReferenceSstSummary(); + if (meta->blob_reference_summary != nullptr && + meta->blob_reference_summary->file_number != meta->fd.GetNumber()) { + s = Status::Corruption( + "Blob reference summary is for another SST file number"); + } } current_output().finished = true; stats_.bytes_written += current_bytes; @@ -66,6 +77,21 @@ Status CompactionOutputs::Finish( return s; } +Status CompactionOutputs::UpdateTableProperties( + const TableProperties& table_properties) { + current_output().table_properties = + std::make_shared(table_properties); + ExtractBlobReferenceDescriptorFromTableProperties(table_properties, + ¤t_output().meta); + return ExtractBlobReferenceClassificationFromTableProperties( + table_properties, ¤t_output().meta); +} + +Status CompactionOutputs::UpdateTableProperties() { + TableProperties table_properties = GetTableProperties(); + return UpdateTableProperties(table_properties); +} + IOStatus CompactionOutputs::WriterSyncClose(const Status& input_status, SystemClock* clock, Statistics* statistics, @@ -406,6 +432,22 @@ void CompactionOutputs::PrepareNewOutputForFirstKey( } } +void CompactionOutputs::NoteBlobReferenceLimitCut(const Slice& next_key) { + current_output().closed_by_blob_reference_limit = true; + const ImmutableOptions& ioptions = compaction_->immutable_options(); + RecordTick(ioptions.stats, COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS); + ROCKS_LOG_INFO( + ioptions.info_log, + "[%s] Compaction output table builder requested a new table before " + "key %s for file #%" PRIu64 ". reason: %s", + compaction_->column_family_data()->GetName().c_str(), + ioptions.allow_data_in_errors ? next_key.ToString(/*hex=*/true).c_str() + : "hidden", + current_output().meta.fd.GetNumber(), + TableBuilderNewTableReasonToString( + TableBuilderNewTableReason::kBlobReferenceLimit)); +} + Status CompactionOutputs::AddToOutput( CompactionIterator& c_iter, const CompactionFileOpenFunc& open_file_func, const CompactionFileCloseFunc& close_file_func, @@ -428,6 +470,18 @@ Status CompactionOutputs::AddToOutput( cut_before = true; } else { cut_before = ShouldStopBefore(c_iter) && HasBuilder(); + // An output is full when it holds blob_indirection_max_references_per_sst + // references. A full output takes no new user key, so the history of one + // user key (all its kept versions) is never split. An output may pass the + // limit by one key's history. + if (!cut_before && HasBuilder() && builder_->BlobReferenceLimitReached() && + !LargestUserKeyEquals(c_iter.user_key())) { + // The LargestUserKeyEquals() check above keeps the versions of one user + // key in one file. Version::Get() in L0 depends on it: it stops at the + // first file that holds the key. + NoteBlobReferenceLimitCut(c_iter.key()); + cut_before = true; + } } if (cut_before) { const Slice next_table_min_key = c_iter.key(); @@ -479,6 +533,15 @@ Status CompactionOutputs::AddToOutput( assert(builder_ != nullptr); const Slice& value = c_iter.value(); + // Decode a kTypeBlobIndex value once, and share the decode with the table + // builder, the blob garbage meter and the file metadata update below. If the + // decode fails, each of them decodes again and reports the error as usual. + BlobIndex decoded_blob_index; + const BlobIndex* shared_blob_index = nullptr; + if (c_iter.ikey().type == kTypeBlobIndex && + decoded_blob_index.DecodeFrom(value).ok()) { + shared_blob_index = &decoded_blob_index; + } bool retried_after_new_table = false; while (true) { s = current_output().validator.CanAdd(key); @@ -487,6 +550,7 @@ Status CompactionOutputs::AddToOutput( } TableBuilderAddContext add_context; + add_context.decoded_blob_index = shared_blob_index; const TableBuilderAddResult add_result = builder_->TryAdd(key, value, &add_context); @@ -580,7 +644,7 @@ Status CompactionOutputs::AddToOutput( current_output_file_size_ = builder_->EstimatedFileSize(); if (blob_garbage_meter_) { - s = blob_garbage_meter_->ProcessOutFlow(key, value); + s = blob_garbage_meter_->ProcessOutFlow(key, value, shared_blob_index); } if (!s.ok()) { @@ -594,7 +658,7 @@ Status CompactionOutputs::AddToOutput( std::min(smallest_preferred_seqno_, preferred_seqno); } s = current_output().meta.UpdateBoundaries(key, value, ikey.sequence, - ikey.type); + ikey.type, shared_blob_index); return s; } diff --git a/db/compaction/compaction_outputs.h b/db/compaction/compaction_outputs.h index 88dd0128bee1..041c766d72d7 100644 --- a/db/compaction/compaction_outputs.h +++ b/db/compaction/compaction_outputs.h @@ -41,6 +41,11 @@ class CompactionOutputs { OutputValidator validator; bool finished; bool is_proximal_level; + // True when the output was closed because it already held + // blob_indirection_max_references_per_sst blob references when the next + // user key arrived (see CompactionOutputs::AddToOutput()). Used by the + // event log. + bool closed_by_blob_reference_limit = false; std::shared_ptr table_properties; }; @@ -128,15 +133,10 @@ class CompactionOutputs { // Update output table properties from already populated TableProperties. // Used for remote compaction - void UpdateTableProperties(const TableProperties& table_properties) { - current_output().table_properties = - std::make_shared(table_properties); - } + Status UpdateTableProperties(const TableProperties& table_properties); + // Update output table properties from table builder - void UpdateTableProperties() { - current_output().table_properties = - std::make_shared(GetTableProperties()); - } + Status UpdateTableProperties(); IOStatus WriterSyncClose(const Status& intput_status, SystemClock* clock, Statistics* statistics, bool use_fsync); @@ -186,6 +186,10 @@ class CompactionOutputs { FileMetaData* GetMetaData() { return ¤t_output().meta; } + bool CurrentOutputClosedByBlobReferenceLimit() const { + return outputs_.back().closed_by_blob_reference_limit; + } + bool HasOutput() const { return !outputs_.empty(); } uint64_t NumEntries() const { return builder_->NumEntries(); } @@ -296,6 +300,10 @@ class CompactionOutputs { uint64_t GetCurrentKeyGrandparentOverlappedBytes( const Slice& internal_key) const; + // Counts and logs a cut of the current output by the blob reference limit + // before `next_key`, and marks the output for the event log. + void NoteBlobReferenceLimitCut(const Slice& next_key); + // Add current key from compaction_iterator to the output file. If needed // close and open new compaction output with the functions provided. Status AddToOutput(CompactionIterator& c_iter, diff --git a/db/compaction/compaction_picker.cc b/db/compaction/compaction_picker.cc index 87f97358475d..7b58cad8cc5a 100644 --- a/db/compaction/compaction_picker.cc +++ b/db/compaction/compaction_picker.cc @@ -283,10 +283,21 @@ bool CompactionPicker::ExpandInputsToCleanCut(const std::string& /*cf_name*/, assert(!inputs->empty()); const int level = inputs->level; - // GetOverlappingInputs will always do the right thing for level-0. - // So we don't need to do any expansion if level == 0. + // For L0, the normal overlap expansion (GetOverlappingInputs) is enough for + // legacy files. Later picks must take all files of an epoch run (see + // VersionStorageInfo::ExpandExactL0Inputs()). A file of the run can widen the + // key range, so L0 is searched again for all files that overlap the wider + // range, until the set stops changing. Without exact files nothing is added. if (level == 0) { - return true; + bool expanded = false; + if (!ExpandL0ToWholeRuns(vstorage, inputs, /*search_first=*/false, + /*starting_l0_file=*/nullptr, &expanded)) { + return false; + } + // The caller already checked the original inputs. So, as in base RocksDB, + // an L0 set that this function does not change is returned without the + // being_compacted check; only a set that grew is checked. + return !expanded || !AreFilesInCompaction(inputs->files); } InternalKey smallest; @@ -499,6 +510,10 @@ Status CompactionPicker::GetCompactionInputsFromFileNumbers( return Status::InvalidArgument(message); } + if (!vstorage->ExpandExactL0Inputs(&matched_input_files[0].files)) { + return Status::Aborted("Exact L0 input run is unavailable for compaction"); + } + for (int level = first_non_empty_level; level <= last_non_empty_level; ++level) { matched_input_files[level].level = level; @@ -1311,25 +1326,51 @@ void CompactionPicker::PickFilesMarkedForCompaction( start_level_inputs->files.clear(); } +bool CompactionPicker::ExpandL0ToWholeRuns(VersionStorageInfo* vstorage, + CompactionInputFiles* inputs, + bool search_first, + const FileMetaData* starting_l0_file, + bool* grew) { + *grew = false; + bool search = search_first; + while (true) { + const size_t before = inputs->size(); + if (search) { + // GetOverlappingInputs() replaces the set with the L0 files that overlap + // its key range. The set keeps the starting file. + InternalKey smallest; + InternalKey largest; + GetRange(*inputs, &smallest, &largest); + inputs->files.clear(); + vstorage->GetOverlappingInputs(0, &smallest, &largest, &inputs->files, + /*hint_index=*/-1, /*file_index=*/nullptr, + /*expand_range=*/true, starting_l0_file); + } + // ExpandExactL0Inputs() also puts the files back in epoch order. + if (!vstorage->ExpandExactL0Inputs(&inputs->files)) { + return false; + } + if (inputs->size() <= before) { + return true; + } + *grew = true; + search = true; + } +} + bool CompactionPicker::GetOverlappingL0Files( VersionStorageInfo* vstorage, CompactionInputFiles* start_level_inputs, int output_level, int* parent_index, const FileMetaData* starting_l0_file) { // Two level 0 compaction won't run at the same time, so don't need to worry // about files on level 0 being compacted. assert(level0_compactions_in_progress()->empty()); + bool grew = false; + if (!ExpandL0ToWholeRuns(vstorage, start_level_inputs, /*search_first=*/true, + starting_l0_file, &grew)) { + return false; + } InternalKey smallest; InternalKey largest; - GetRange(*start_level_inputs, &smallest, &largest); - // Note that the next call will discard the file we placed in - // c->inputs_[0] earlier and replace it with an overlapping set - // which will include the picked file. - start_level_inputs->files.clear(); - vstorage->GetOverlappingInputs(0, &smallest, &largest, - &(start_level_inputs->files), - /*hint_index=*/-1, - /*file_index=*/nullptr, - /*expand_range=*/true, - /*starting_l0_file=*/starting_l0_file); // If we include more L0 files in the same compaction run it can // cause the 'smallest' and 'largest' key to get extended to a diff --git a/db/compaction/compaction_picker.h b/db/compaction/compaction_picker.h index 1d93d1a3fca8..25442b646dac 100644 --- a/db/compaction/compaction_picker.h +++ b/db/compaction/compaction_picker.h @@ -227,6 +227,17 @@ class CompactionPicker { CompactionInputFiles* start_level_inputs, std::function skip_marked_file); + // Grows the L0 set `inputs` until two rules add no file: (1) every L0 file + // that overlaps the key range of the set (only `starting_l0_file` and older + // files, when it is not null), and (2) every file of an epoch run that has a + // file in the set (VersionStorageInfo::ExpandExactL0Inputs()). A file of a + // run can widen the key range, so both steps repeat. With `search_first` + // false, the first round skips the overlap search. Sets `*grew` if the set + // got a new file. Returns false if ExpandExactL0Inputs() fails. + bool ExpandL0ToWholeRuns(VersionStorageInfo* vstorage, + CompactionInputFiles* inputs, bool search_first, + const FileMetaData* starting_l0_file, bool* grew); + // @param starting_l0_file If not null, restricts L0 file selection to only // include files at or older than starting_l0_file. bool GetOverlappingL0Files(VersionStorageInfo* vstorage, diff --git a/db/compaction/compaction_picker_level.cc b/db/compaction/compaction_picker_level.cc index 98b36219fce4..9e16a8e8f384 100644 --- a/db/compaction/compaction_picker_level.cc +++ b/db/compaction/compaction_picker_level.cc @@ -339,6 +339,13 @@ void LevelCompactionBuilder::SetupInitialFiles() { } bool LevelCompactionBuilder::SetupOtherL0FilesIfNeeded() { + // An L0 trivial move takes L0 files with disjoint key ranges and already + // keeps whole epoch runs (see TryPickL0TrivialMove()), so it skips + // ExpandExactL0Inputs(). + if (start_level_ == 0 && !is_l0_trivial_move_ && + !vstorage_->ExpandExactL0Inputs(&start_level_inputs_.files)) { + return false; + } if (start_level_ == 0 && output_level_ != 0 && !is_l0_trivial_move_) { return compaction_picker_->GetOverlappingL0Files( vstorage_, &start_level_inputs_, output_level_, &parent_index_); @@ -682,6 +689,13 @@ bool LevelCompactionBuilder::TryPickL0TrivialMove() { CompactionInputFiles output_level_inputs; output_level_inputs.level = output_level_; FileMetaData* file = *it; + // A coarse file must be rewritten when it leaves L0, so the move stops + // before it. Files picked so far still move. The coarse file waits for a + // normal compaction. + if (Compaction::MustRewriteCoarseL0File(*file)) { + RecordTick(ioptions_.stats, COMPACTION_COARSE_L0_TRIVIAL_MOVE_BLOCKED); + break; + } if (it == level_files.rbegin()) { my_smallest = file->smallest; my_largest = file->largest; @@ -705,6 +719,23 @@ bool LevelCompactionBuilder::TryPickL0TrivialMove() { break; } } + // The files of an epoch run (see VersionStorageInfo::ExpandExactL0Inputs()) + // must move together. The loop visits files oldest first, from the back + // of level_files (which is newest first). cut_epoch is the epoch of the + // next newer file, where the loop stopped. If the newest picked files have + // that epoch, the loop cut their run: drop them. This also drops a group + // whose run has no exact file. That is safe, because it only makes the + // trivial move smaller. + if (vstorage_->HasBlobIndirectionFormat() && + start_level_inputs_.size() < level_files.size()) { + const uint64_t cut_epoch = + level_files[level_files.size() - start_level_inputs_.size() - 1] + ->epoch_number; + while (!start_level_inputs_.empty() && + start_level_inputs_.files.back()->epoch_number == cut_epoch) { + start_level_inputs_.files.pop_back(); + } + } } if (!start_level_inputs_.empty()) { @@ -928,18 +959,35 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() { start_level_inputs_.clear(); const std::vector& level_files = vstorage_->LevelFiles(0 /* level */); - if (level_files.size() < - static_cast( - mutable_cf_options_.level0_file_num_compaction_trigger + 2) || + // The trigger counts sorted runs: an exact epoch run counts once (see + // VersionStorageInfo::NumL0SortedRuns()). + if (level_files.empty() || + vstorage_->NumL0SortedRuns() < + mutable_cf_options_.level0_file_num_compaction_trigger + 2 || level_files[0]->being_compacted) { // If L0 isn't accumulating much files beyond the regular trigger, don't // resort to L0->L0 compaction yet. return false; } - return PickCostBasedIntraL0Compaction( - level_files, kMinFilesForIntraL0Compaction, - std::numeric_limits::max(), - mutable_cf_options_.max_compaction_bytes, &start_level_inputs_); + if (!PickCostBasedIntraL0Compaction( + level_files, kMinFilesForIntraL0Compaction, + std::numeric_limits::max(), + mutable_cf_options_.max_compaction_bytes, &start_level_inputs_)) { + return false; + } + // The minimum also counts sorted runs, so that one exact epoch run is never + // picked alone and rewritten into itself. It counts the whole runs that + // SetupOtherL0FilesIfNeeded() expands the pick to. + if (vstorage_->HasBlobIndirectionFormat()) { + std::vector expanded = start_level_inputs_.files; + if (!vstorage_->ExpandExactL0Inputs(&expanded) || + vstorage_->CountL0SortedRuns(expanded) < + kMinFilesForIntraL0Compaction) { + start_level_inputs_.clear(); + return false; + } + } + return true; } bool LevelCompactionBuilder::PickSizeBasedIntraL0Compaction() { @@ -950,9 +998,12 @@ bool LevelCompactionBuilder::PickSizeBasedIntraL0Compaction() { } const std::vector& l0_files = vstorage_->LevelFiles(/*level=*/0); + // The minimum counts sorted runs: an exact epoch run counts once (see + // VersionStorageInfo::NumL0SortedRuns()). Counting its files would pick the + // same run again after each compaction that the reference cap cuts. size_t min_num_file = std::max(2, mutable_cf_options_.level0_file_num_compaction_trigger); - if (l0_files.size() < min_num_file) { + if (static_cast(vstorage_->NumL0SortedRuns()) < min_num_file) { return false; } uint64_t l0_size = 0; @@ -988,7 +1039,7 @@ bool LevelCompactionBuilder::PickSizeBasedIntraL0Compaction() { } start_level_inputs_.files.push_back(file); } - if (start_level_inputs_.files.size() < min_num_file) { + if (vstorage_->CountL0SortedRuns(start_level_inputs_.files) < min_num_file) { start_level_inputs_.clear(); return false; } diff --git a/db/compaction/compaction_picker_test.cc b/db/compaction/compaction_picker_test.cc index 065541e8a69a..86a7a564a4fa 100644 --- a/db/compaction/compaction_picker_test.cc +++ b/db/compaction/compaction_picker_test.cc @@ -4,6 +4,7 @@ // (found in the LICENSE.Apache file in the root directory). #include +#include #include #include @@ -297,6 +298,374 @@ class CompactionPickerTest : public CompactionPickerTestBase { } }; +TEST_F(CompactionPickerTest, ExactL0ClosureUsesEpochNotFileNumber) { + NewVersionStorage(3, kCompactionStyleLevel); + Add(0, 1, "a", "b"); + Add(0, 100, "c", "d"); + Add(0, 2, "e", "f"); + files_[0]->epoch_number = 20; + files_[1]->epoch_number = 10; + files_[2]->epoch_number = 10; + // The test checks which files ExpandExactL0Inputs adds, not descriptor + // decoding, so any non-empty descriptor marks a file as exact. + files_[2]->blob_reference_descriptor = "exact"; + vstorage_->SetBlobIndirectionFormatVersion(1); + std::vector selected{files_[2].get()}; + ASSERT_TRUE(vstorage_->ExpandExactL0Inputs(&selected)); + ASSERT_EQ(selected.size(), 2U); + ASSERT_EQ(selected[0]->fd.GetNumber(), 100U); + ASSERT_EQ(selected[1]->fd.GetNumber(), 2U); + files_[1]->being_compacted = true; + selected = {files_[2].get()}; + ASSERT_FALSE(vstorage_->ExpandExactL0Inputs(&selected)); + ASSERT_EQ(selected.size(), 1U); + files_[1]->being_compacted = false; + vstorage_->SetBlobIndirectionFormatVersion(0); + ASSERT_TRUE(vstorage_->ExpandExactL0Inputs(&selected)); + ASSERT_EQ(selected.size(), 1U); +} + +TEST_F(CompactionPickerTest, ExactL0ClosureSortsOutOfOrderInputs) { + // Files 100 and 2 share epoch 10, and file 2 is exact, so the two files form + // an exact epoch run that must stay together. The inputs are given oldest + // first, out of the Version's epoch order. ExpandExactL0Inputs keeps every + // input, adds file 100, and returns the files newest epoch first. + NewVersionStorage(3, kCompactionStyleLevel); + Add(0, 1, "a", "b"); + Add(0, 100, "c", "d"); + Add(0, 2, "e", "f"); + files_[0]->epoch_number = 20; + files_[1]->epoch_number = 10; + files_[2]->epoch_number = 10; + files_[2]->blob_reference_descriptor = "exact"; + vstorage_->SetBlobIndirectionFormatVersion(1); + std::vector selected{files_[2].get(), files_[0].get()}; + ASSERT_TRUE(vstorage_->ExpandExactL0Inputs(&selected)); + ASSERT_EQ(selected.size(), 3U); + EXPECT_EQ(selected[0]->fd.GetNumber(), 1U); + EXPECT_EQ(selected[1]->fd.GetNumber(), 100U); + EXPECT_EQ(selected[2]->fd.GetNumber(), 2U); +} + +TEST_F(CompactionPickerTest, ExactL0CompactRangeExpandedOutOfEpochOrder) { + NewVersionStorage(6, kCompactionStyleLevel); + // Both L0 files are exact, and the range is [100, 200]. The older file 1 + // (100-550) overlaps the range and widens it to 550. The newer file 2 + // (500-600) starts after 200, the end of the requested range, so the first + // overlap search misses it. The search after the widening adds it second, out + // of epoch order. The pick still lists the inputs newest first. + Add(0, 2U, "500", "600"); + Add(0, 1U, "100", "550"); + files_[0]->epoch_number = 20; + files_[1]->epoch_number = 10; + for (size_t i = 0; i < 2; ++i) { + files_[i]->blob_reference_descriptor = "exact"; + } + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + const InternalKey begin("100", kMaxSequenceNumber, kValueTypeForSeek); + const InternalKey end("200", 0, kValueTypeForSeek); + bool manual_conflict = false; + InternalKey manual_end; + InternalKey* manual_end_ptr = &manual_end; + std::unique_ptr compaction( + level_compaction_picker.PickCompactionForCompactRange( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + /*input_level=*/0, /*output_level=*/1, + /*compact_range_options*/ {}, &begin, &end, &manual_end_ptr, + &manual_conflict, + /*max_file_num_to_ignore=*/std::numeric_limits::max(), + /*trim_ts=*/"", /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + ASSERT_FALSE(manual_conflict); + ASSERT_EQ(compaction->num_input_files(0), 2U); + EXPECT_EQ(compaction->input(0, 0)->fd.GetNumber(), 2U); + EXPECT_EQ(compaction->input(0, 1)->fd.GetNumber(), 1U); +} + +TEST_F(CompactionPickerTest, ExactL0SiblingWidensRangeToOlderFile) { + // S1[100-130] and S2[400-460] form an exact run (epoch 10). O[450-470] is + // older (epoch 5). CompactRange(100, 110) picks S1, the run adds S2, and the + // wider range must also take O; otherwise O would stay in L0 under the newer + // values that move to L1. + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "100", "130"); + Add(0, 2U, "400", "460"); + Add(0, 3U, "450", "470"); + files_[0]->epoch_number = 10; + files_[1]->epoch_number = 10; + files_[2]->epoch_number = 5; + files_[0]->blob_reference_descriptor = "exact"; + files_[1]->blob_reference_descriptor = "exact"; + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + const InternalKey begin("100", kMaxSequenceNumber, kValueTypeForSeek); + const InternalKey end("110", 0, kValueTypeForSeek); + bool manual_conflict = false; + InternalKey manual_end; + InternalKey* manual_end_ptr = &manual_end; + std::unique_ptr compaction( + level_compaction_picker.PickCompactionForCompactRange( + cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(), + /*input_level=*/0, /*output_level=*/1, + /*compact_range_options*/ {}, &begin, &end, &manual_end_ptr, + &manual_conflict, + /*max_file_num_to_ignore=*/std::numeric_limits::max(), + /*trim_ts=*/"", /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + ASSERT_FALSE(manual_conflict); + std::set picked; + for (size_t i = 0; i < compaction->num_input_files(0); ++i) { + picked.insert(compaction->input(0, i)->fd.GetNumber()); + } + EXPECT_EQ(picked, (std::set{1, 2, 3})); + // The inputs are in epoch order, newest first, so O is last. + EXPECT_EQ( + compaction->input(0, compaction->num_input_files(0) - 1)->fd.GetNumber(), + 3U); +} + +TEST_F(CompactionPickerTest, ExactL0SiblingWidensRangeInAutomaticPick) { + // Automatic L0 -> L1: the picker starts from the oldest L0 file X[300-310] + // and widens L0 inputs to the range of its L1 parent Y[100-320]. That adds + // S1[100-130]; its run sibling S2[400-460] widens the range again, so the + // older O[450-470] must be added too. + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "100", "130"); + Add(0, 2U, "400", "460"); + Add(0, 3U, "450", "470"); + Add(0, 4U, "300", "310"); + Add(1, 10U, "100", "320"); + files_[0]->epoch_number = 10; + files_[1]->epoch_number = 10; + files_[2]->epoch_number = 5; + files_[3]->epoch_number = 1; + files_[0]->blob_reference_descriptor = "exact"; + files_[1]->blob_reference_descriptor = "exact"; + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, + /*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, vstorage_.get(), + &log_buffer_, /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + ASSERT_EQ(compaction->start_level(), 0); + std::set picked; + for (size_t i = 0; i < compaction->num_input_files(0); ++i) { + picked.insert(compaction->input(0, i)->fd.GetNumber()); + } + // The pick must widen (S1 overlaps the parent), and then it has the whole run + // and the older overlapping file. + EXPECT_EQ(picked, (std::set{1, 2, 3, 4})); +} + +TEST_F(CompactionPickerTest, ExactL0TrivialMoveOfSequentialRuns) { + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.level0_file_num_compaction_trigger = 4; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + // Sequential inserts: newer L0 files hold larger keys, so key order is the + // reverse of the Version's newest-first L0 order. + Add(0, 4U, "451", "500"); + Add(0, 3U, "351", "400"); + Add(0, 2U, "251", "300"); + Add(0, 1U, "151", "200"); + Add(1, 10U, "100", "150"); + for (size_t i = 0; i < 4; ++i) { + files_[i]->epoch_number = 40 - 10 * i; + files_[i]->blob_reference_descriptor = "exact"; + } + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, + /*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, vstorage_.get(), + &log_buffer_, /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(compaction->num_input_files(0), 4U); + for (size_t i = 0; i < 4; ++i) { + ASSERT_EQ(compaction->input(0, i)->fd.GetNumber(), i + 1); + } +} + +TEST_F(CompactionPickerTest, ExactL0TrivialMoveKeepsEpochRunWhole) { + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + // L0 has 3 sorted runs: the epoch run of files 3 and 2 counts once (see + // VersionStorageInfo::NumL0SortedRuns()). + mutable_cf_options_.level0_file_num_compaction_trigger = 3; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 4U, "700", "800"); + // Files 3 and 2 share epoch 20: one exact epoch run. File 3 overlaps L1 file + // 10, so it cannot be trivially moved, and file 2 must stay in L0 with it. + // Only file 1 is moved. + Add(0, 3U, "550", "560"); + Add(0, 2U, "300", "400"); + Add(0, 1U, "100", "200"); + Add(1, 10U, "500", "600"); + const uint64_t epochs[] = {30, 20, 20, 10}; + for (size_t i = 0; i < 4; ++i) { + files_[i]->epoch_number = epochs[i]; + files_[i]->blob_reference_descriptor = "exact"; + } + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, + /*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, vstorage_.get(), + &log_buffer_, /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + ASSERT_TRUE(compaction->IsTrivialMove()); + ASSERT_EQ(compaction->num_input_files(0), 1U); + ASSERT_EQ(compaction->input(0, 0)->fd.GetNumber(), 1U); +} + +TEST_F(CompactionPickerTest, CostBasedIntraL0NeverPicksOneExactRunAlone) { + // The cost-based intra-L0 pick (PickIntraL0Compaction) runs when L0 -> L1 is + // blocked. It takes the newest L0 files while the work per removed file + // does not grow and the total stays within max_compaction_bytes, and it needs + // at least 4 inputs (kMinFilesForIntraL0Compaction). It must count sorted + // runs, not files: if its inputs were one exact epoch run, the compaction + // would rewrite the run into itself and remove no sorted run. + // + // Setup: L0 holds 8 files of 200000 bytes each. The 5 newest (files 1..5, + // epoch 50) form one exact epoch run. Files 6, 7 and 8 are older single + // files. With max_compaction_bytes = 1000000, the pick stops after the 5 + // run files. L0 has 4 sorted runs, which passes the trigger + 2 check with a + // trigger of 2. The L1 file covers all L0 keys and is being compacted, so + // L0 -> L1 is blocked. + // + // With blob indirection, the 5 files are one sorted run, so the cost-based + // pick must not take them alone. Without blob indirection, the same 5 files + // are 5 sorted runs, and the pick takes them. That case shows that the + // layout reaches the cost-based pick. + for (const bool indirection : {false, true}) { + SCOPED_TRACE(indirection); + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_compaction_bytes = 1000000u; + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 1U, "100", "150", 200000U, 0, 180, 181); + Add(0, 2U, "151", "200", 200000U, 0, 170, 171); + Add(0, 3U, "201", "250", 200000U, 0, 160, 161); + Add(0, 4U, "251", "300", 200000U, 0, 150, 151); + Add(0, 5U, "301", "350", 200000U, 0, 140, 141); + Add(0, 6U, "100", "350", 200000U, 0, 130, 131); + Add(0, 7U, "100", "350", 200000U, 0, 120, 121); + Add(0, 8U, "100", "350", 200000U, 0, 110, 111); + Add(1, 9U, "100", "350", 200000U, 0, 100, 101); + const uint64_t epochs[] = {50, 50, 50, 50, 50, 40, 30, 20}; + for (size_t i = 0; i < 8; ++i) { + files_[i]->epoch_number = epochs[i]; + if (indirection) { + files_[i]->blob_reference_descriptor = "exact"; + } + } + if (indirection) { + vstorage_->SetBlobIndirectionFormatVersion(1); + } + vstorage_->LevelFiles(1)[0]->being_compacted = true; + UpdateVersionStorageInfo(); + ASSERT_EQ(vstorage_->NumL0SortedRuns(), indirection ? 4 : 8); + + std::unique_ptr compaction( + level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, + /*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, + vstorage_.get(), &log_buffer_, /*full_history_ts_low=*/"")); + if (!indirection) { + ASSERT_NE(compaction, nullptr); + ASSERT_EQ(compaction->output_level(), 0); + ASSERT_EQ(compaction->num_input_files(0), 5U); + continue; + } + // No compaction, or an intra-L0 compaction with at least 4 sorted runs. + if (compaction != nullptr) { + ASSERT_EQ(compaction->start_level(), 0); + std::vector inputs = *compaction->inputs(0); + EXPECT_GE(vstorage_->CountL0SortedRuns(inputs), 4U); + } + } +} + +TEST_F(CompactionPickerTest, CoarseL0FilesAreNotTriviallyMoved) { + // Coarse L0 files must be rewritten with exact metadata when they leave L0. + // The L0 trivial move stops at them, so the normal pick runs and sets + // grandparents for output cutting. + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 2U, "300", "400"); + Add(0, 1U, "100", "200"); + Add(1, 10U, "500", "600"); + Add(2, 20U, "100", "450"); + files_[0]->epoch_number = 20; + files_[1]->epoch_number = 10; + files_[0]->oldest_coarse_indirect_origin_file_number = 50; + files_[1]->oldest_coarse_indirect_origin_file_number = 50; + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, + /*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, vstorage_.get(), + &log_buffer_, /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + EXPECT_EQ(compaction->start_level(), 0); + EXPECT_FALSE(compaction->IsTrivialMove()); + EXPECT_FALSE(compaction->grandparents().empty()); +} + +TEST_F(CompactionPickerTest, CoarseL0FileStopsTrivialMoveAfterExactFile) { + // File 1, the oldest L0 file, has no coarse pin. File 2, the next one, is + // coarse with frontier 50 (an example file number; see "Coarse SST" in + // db/blob/blob_reference_metadata.h). A trivial move may take only the files + // older than the coarse file. + mutable_cf_options_.max_bytes_for_level_base = 10000000u; + mutable_cf_options_.level0_file_num_compaction_trigger = 2; + mutable_cf_options_.max_compaction_bytes = 10000000u; + ioptions_.level_compaction_dynamic_level_bytes = false; + NewVersionStorage(6, kCompactionStyleLevel); + Add(0, 2U, "300", "400"); + Add(0, 1U, "100", "200"); + Add(2, 20U, "500", "600"); + files_[0]->epoch_number = 20; + files_[1]->epoch_number = 10; + files_[0]->oldest_coarse_indirect_origin_file_number = 50; + vstorage_->SetBlobIndirectionFormatVersion(1); + UpdateVersionStorageInfo(); + + std::unique_ptr compaction(level_compaction_picker.PickCompaction( + cf_name_, mutable_cf_options_, mutable_db_options_, + /*existing_snapshots=*/{}, /*snapshot_checker=*/nullptr, vstorage_.get(), + &log_buffer_, /*full_history_ts_low=*/"")); + ASSERT_NE(compaction, nullptr); + EXPECT_EQ(compaction->start_level(), 0); + if (compaction->IsTrivialMove()) { + for (const FileMetaData* file : *compaction->inputs(0)) { + EXPECT_EQ(file->oldest_coarse_indirect_origin_file_number, 0U); + } + } + bool has_coarse_input = false; + for (const FileMetaData* file : *compaction->inputs(0)) { + has_coarse_input |= file->oldest_coarse_indirect_origin_file_number != 0; + } + EXPECT_TRUE(!has_coarse_input || !compaction->IsTrivialMove()); +} + class CompactionPickerU64TsTest : public CompactionPickerTestBase { public: explicit CompactionPickerU64TsTest() diff --git a/db/compaction/compaction_picker_universal.cc b/db/compaction/compaction_picker_universal.cc index 8ea8bb324086..7b30618353df 100644 --- a/db/compaction/compaction_picker_universal.cc +++ b/db/compaction/compaction_picker_universal.cc @@ -9,6 +9,7 @@ #include "db/compaction/compaction_picker_universal.h" +#include #include #include #include @@ -93,8 +94,12 @@ class UniversalCompactionBuilder { int level; // `file` Will be null for level > 0. For level = 0, the sorted run is - // for this file. + // for this file, or it is the newest file of an exact epoch run (see + // VersionStorageInfo::NumL0SortedRuns()). The run is the + // `l0_file_count` L0 files from L0 position `l0_position`. FileMetaData* file; + size_t l0_position = 0; + size_t l0_file_count = 0; // For level > 0, `size` and `compensated_file_size` are sum of sizes all // files in the level. `being_compacted` should be the same for all files // in a non-zero level. Use the value here. @@ -330,6 +335,19 @@ class UniversalCompactionBuilder { Compaction* PickCompactionWithSortedRunRange( size_t start_index, size_t end_index, CompactionReason compaction_reason); + // Checks that the L0 sorted runs in [start_index, end_index] of sorted_runs_ + // can be compacted: VersionStorageInfo::ExpandExactL0Inputs() accepts their + // files and adds none, because each L0 sorted run holds a whole exact epoch + // run. Returns false, with a WARN in the LOG, when a needed file is being + // compacted, has kUnknownEpochNumber, or the expansion would add a file. + // Returns true when the range starts at a non-L0 sorted run. + bool CheckExactL0RunRange(size_t start_index, size_t end_index) const; + + // Appends the L0 files of sorted run `sr` (one file, or every file of an + // exact epoch run) to `files`, in L0 order. + void AppendL0RunFiles(const SortedRun& sr, + std::vector* files) const; + // Try to pick periodic compaction. The caller should only call it // if there is at least one file marked for periodic compaction. // null will be returned if no such a compaction can be formed @@ -664,12 +682,17 @@ void UniversalCompactionBuilder::SortedRun::DumpSizeInfo( char* out_buf, size_t out_buf_size, size_t sorted_run_count) const { if (level == 0) { assert(file != nullptr); - snprintf(out_buf, out_buf_size, - "file %" PRIu64 "[%" ROCKSDB_PRIszt - "] " - "with size %" PRIu64 " (compensated size %" PRIu64 ")", - file->fd.GetNumber(), sorted_run_count, file->fd.GetFileSize(), - file->compensated_file_size); + const int written = snprintf( + out_buf, out_buf_size, + "file %" PRIu64 "[%" ROCKSDB_PRIszt + "] " + "with size %" PRIu64 " (compensated size %" PRIu64 ")", + file->fd.GetNumber(), sorted_run_count, size, compensated_file_size); + if (l0_file_count > 1 && written > 0 && + static_cast(written) < out_buf_size) { + snprintf(out_buf + written, out_buf_size - written, + " (epoch run of %" ROCKSDB_PRIszt " files)", l0_file_count); + } } else { snprintf(out_buf, out_buf_size, "level %d[%" ROCKSDB_PRIszt @@ -686,15 +709,35 @@ UniversalCompactionBuilder::CalculateSortedRuns( assert(max_run_size); *max_run_size = 0; std::vector ret; - for (FileMetaData* f : vstorage.LevelFiles(0)) { - if (earliest_snapshot_.has_value() && f->marked_for_compaction) { - file_marked_for_compaction_to_sorted_run_index_.emplace(f->fd.GetNumber(), - ret.size()); + // Each L0 file is a sorted run, except that an exact epoch run is one sorted + // run (see VersionStorageInfo::NumL0SortedRuns()). Its size is the sum of + // its files, and picking it picks all its files. + const std::vector& l0_files = vstorage.LevelFiles(0); + for (size_t first = 0; first < l0_files.size();) { + const size_t end = vstorage.L0SortedRunEnd(first); + uint64_t size = 0; + uint64_t compensated_size = 0; + bool being_compacted = false; + bool has_marked_standalone_rangedel = false; + for (size_t i = first; i < end; ++i) { + FileMetaData* const f = l0_files[i]; + if (earliest_snapshot_.has_value() && f->marked_for_compaction) { + file_marked_for_compaction_to_sorted_run_index_.emplace( + f->fd.GetNumber(), ret.size()); + } + size += f->fd.GetFileSize(); + compensated_size += f->compensated_file_size; + being_compacted = being_compacted || f->being_compacted; + has_marked_standalone_rangedel = + has_marked_standalone_rangedel || + (f->marked_for_compaction && f->FileIsStandAloneRangeTombstone()); } - ret.emplace_back( - 0, f, f->fd.GetFileSize(), f->compensated_file_size, f->being_compacted, - f->marked_for_compaction && f->FileIsStandAloneRangeTombstone()); - *max_run_size = std::max(*max_run_size, f->fd.GetFileSize()); + ret.emplace_back(0, l0_files[first], size, compensated_size, + being_compacted, has_marked_standalone_rangedel); + ret.back().l0_position = first; + ret.back().l0_file_count = end - first; + *max_run_size = std::max(*max_run_size, size); + first = end; } for (int level = 1; level <= last_level; level++) { uint64_t total_compensated_size = 0U; @@ -1031,7 +1074,11 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns( if (!done || candidate_count <= 1) { return nullptr; } - size_t first_index_after = start_index + candidate_count; + size_t end_index = start_index + candidate_count - 1; + if (!CheckExactL0RunRange(start_index, end_index)) { + return nullptr; + } + size_t first_index_after = end_index + 1; // Compression is enabled if files compacted earlier already reached // size ratio of compression. bool enable_compression = true; @@ -1082,8 +1129,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns( for (size_t i = start_index; i < first_index_after; i++) { auto& picking_sr = sorted_runs_[i]; if (picking_sr.level == 0) { - FileMetaData* picking_file = picking_sr.file; - inputs[0].files.push_back(picking_file); + AppendL0RunFiles(picking_sr, &inputs[0].files); } else { auto& files = inputs[picking_sr.level - start_level].files; for (auto* f : vstorage_->LevelFiles(picking_sr.level)) { @@ -1509,9 +1555,16 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() { if (sr->being_compacted) { continue; } - FileMetaData* f = vstorage_->LevelFiles(0)[loop]; - if (f->marked_for_compaction && !ShouldSkipMarkedFile(f)) { - start_level_inputs.files.push_back(f); + // A sorted run is one L0 file or one exact epoch run. Pick the whole + // run when one of its files is marked. + std::vector run_files; + AppendL0RunFiles(*sr, &run_files); + if (std::any_of(run_files.begin(), run_files.end(), + [this](const FileMetaData* f) { + return f->marked_for_compaction && + !ShouldSkipMarkedFile(f); + })) { + start_level_inputs.files = std::move(run_files); start_index = static_cast(loop); // Consider this as the first candidate. break; @@ -1528,8 +1581,14 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() { break; } - FileMetaData* f = vstorage_->LevelFiles(0)[loop]; - start_level_inputs.files.push_back(f); + AppendL0RunFiles(*sr, &start_level_inputs.files); + } + // Each sorted run holds a whole exact epoch run (see + // VersionStorageInfo::NumL0SortedRuns()), so ExpandExactL0Inputs() adds + // no file here. It still fails the pick when a run file has + // kUnknownEpochNumber. + if (!vstorage_->ExpandExactL0Inputs(&start_level_inputs.files)) { + return nullptr; } } else { int start_level; @@ -1558,9 +1617,51 @@ Compaction* UniversalCompactionBuilder::PickCompactionToOldest( compaction_reason); } +bool UniversalCompactionBuilder::CheckExactL0RunRange(size_t start_index, + size_t end_index) const { + std::vector inputs; + for (size_t index = start_index; + index <= end_index && sorted_runs_[index].level == 0; ++index) { + AppendL0RunFiles(sorted_runs_[index], &inputs); + } + const size_t picked_count = inputs.size(); + if (!vstorage_->ExpandExactL0Inputs(&inputs)) { + ROCKS_LOG_WARN(ioptions_.logger, + "[%s] Universal: skipped a pick of %" ROCKSDB_PRIszt + " L0 files: an exact epoch run has a file that is being " + "compacted or has no epoch number", + cf_name_.c_str(), picked_count); + return false; + } + if (inputs.size() != picked_count) { + // A sorted run that holds only part of an exact epoch run. The pick would + // write an output that overlaps the files it leaves out, so it stops. + ROCKS_LOG_WARN(ioptions_.logger, + "[%s] Universal: skipped a pick of %" ROCKSDB_PRIszt + " L0 files that holds part of an exact epoch run (the run " + "has %" ROCKSDB_PRIszt " files in the pick range)", + cf_name_.c_str(), picked_count, inputs.size()); + return false; + } + return true; +} + +void UniversalCompactionBuilder::AppendL0RunFiles( + const SortedRun& sr, std::vector* files) const { + assert(sr.level == 0); + const std::vector& l0_files = vstorage_->LevelFiles(0); + assert(sr.l0_position + sr.l0_file_count <= l0_files.size()); + assert(l0_files[sr.l0_position] == sr.file); + files->insert(files->end(), l0_files.begin() + sr.l0_position, + l0_files.begin() + sr.l0_position + sr.l0_file_count); +} + Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange( size_t start_index, size_t end_index, CompactionReason compaction_reason) { assert(start_index < sorted_runs_.size()); + if (!CheckExactL0RunRange(start_index, end_index)) { + return nullptr; + } // Estimate total file size uint64_t estimated_total_size = 0; @@ -1578,8 +1679,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange( for (size_t loop = start_index; loop <= end_index; loop++) { auto& picking_sr = sorted_runs_[loop]; if (picking_sr.level == 0) { - FileMetaData* f = picking_sr.file; - inputs[0].files.push_back(f); + AppendL0RunFiles(picking_sr, &inputs[0].files); } else { auto& files = inputs[picking_sr.level - start_level].files; for (auto* f : vstorage_->LevelFiles(picking_sr.level)) { @@ -1681,7 +1781,10 @@ Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() { if (start_index == sorted_runs_.size() - 1) { bool included_file_marked = false; int start_level = sorted_runs_[start_index].level; - FileMetaData* start_file = sorted_runs_[start_index].file; + std::vector start_run_files; + if (start_level == 0) { + AppendL0RunFiles(sorted_runs_[start_index], &start_run_files); + } for (const std::pair& level_file_pair : vstorage_->FilesMarkedForPeriodicCompaction()) { if (start_level != 0) { @@ -1691,8 +1794,9 @@ Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() { break; } } else { - // Last sorted run is a L0 file. - if (start_file == level_file_pair.second) { + // Last sorted run is a L0 file or an exact epoch run. + if (std::find(start_run_files.begin(), start_run_files.end(), + level_file_pair.second) != start_run_files.end()) { included_file_marked = true; break; } diff --git a/db/compaction/compaction_service_job.cc b/db/compaction/compaction_service_job.cc index 5197b1350cf1..f29bd89959b3 100644 --- a/db/compaction/compaction_service_job.cc +++ b/db/compaction/compaction_service_job.cc @@ -1010,7 +1010,20 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService( assert(compaction_outputs); compaction_outputs->AddOutput(std::move(meta), cfd->internal_comparator(), false, true, file.paranoid_hash); - compaction_outputs->UpdateTableProperties(file.table_properties); + s = compaction_outputs->UpdateTableProperties(file.table_properties); + if (!s.ok()) { + fs_->DeleteFile(tgt_file, IOOptions(), /*dbg=*/nullptr) + .PermitUncheckedError(); + cleanup_installed_sst_files(); + const Status cleanup_status = cleanup_installed_blob_files(); + if (!cleanup_status.ok()) { + s = cleanup_status; + } + sub_compact->status = s; + db_options_.compaction_service->OnInstallation( + response.scheduled_job_id, CompactionServiceJobStatus::kFailure); + return CompactionServiceJobStatus::kFailure; + } installed_sst_files.push_back(tgt_file); } diff --git a/db/db_filesnapshot.cc b/db/db_filesnapshot.cc index daf0fb9b258f..c2544cd4312a 100644 --- a/db/db_filesnapshot.cc +++ b/db/db_filesnapshot.cc @@ -406,28 +406,38 @@ Status DBImpl::GetLiveFilesStorageInfoImpl( info.temperature = meta->temperature; } } - const auto& blob_files = vsi.GetBlobFiles(); - for (const auto& meta : blob_files) { - assert(meta); - + const auto AddBlobFile = [&](uint64_t file_number, uint64_t file_size, + const std::string& checksum_method, + const std::string& checksum) { results.emplace_back(); LiveFileStorageInfo& info = results.back(); - - info.relative_filename = BlobFileName(meta->GetBlobFileNumber()); - info.directory = GetDir(/* path_id */ 0); - info.file_number = meta->GetBlobFileNumber(); + info.relative_filename = BlobFileName(file_number); + info.directory = GetDir(/*path_id=*/0); + info.file_number = file_number; info.file_type = kBlobFile; - info.size = meta->GetBlobFileSize(); + info.size = file_size; if (opts.include_checksum_info) { - info.file_checksum_func_name = meta->GetChecksumMethod(); - info.file_checksum = meta->GetChecksumValue(); + info.file_checksum_func_name = checksum_method; + info.file_checksum = checksum; if (info.file_checksum_func_name.empty()) { info.file_checksum_func_name = kUnknownFileChecksumFuncName; info.file_checksum = kUnknownFileChecksum; } } + }; + const auto& blob_files = vsi.GetBlobFiles(); + for (const auto& meta : blob_files) { + assert(meta); + AddBlobFile(meta->GetBlobFileNumber(), meta->GetBlobFileSize(), + meta->GetChecksumMethod(), meta->GetChecksumValue()); // TODO?: info.temperature } + // Relocation files live in the first cf_path, like normal blob files. + for (const auto& meta : vsi.GetBlobRelocationFiles()) { + assert(meta != nullptr); + AddBlobFile(meta->GetFileNumber(), meta->GetFileSize(), + meta->GetChecksumMethod(), meta->GetChecksumValue()); + } } // Release-build validation: every id the subset-checkpoint caller asked for diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index 57368c8b92c5..41a4a87b8204 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -3219,6 +3219,142 @@ TEST_P(DBAtomicFlushTest, FlushMultipleCFsDropSomeBeforeRequestFlush) { Destroy(options); } +TEST_P(DBAtomicFlushTest, AllColumnFamiliesDroppedWhileInstallWaits) { + // An atomic flush of column family 1 writes its SST and queues its MANIFEST + // writer. A drop of that column family is queued ahead of it, so when the + // writer reaches the head of the queue every column family of the group is + // dropped. LogAndApply() then returns ColumnFamilyDropped without calling + // any writer callback, so InstallMemtableAtomicFlushResults() updates the + // memtable list itself, once, with that error (the test counts the sync + // point + // MemTableList::InstallMemtableAtomicFlushResults:PublishWithoutCallback). + // The DB stays usable and reopens. (A second drop cannot be queued at the + // same time: DropColumnFamily() holds the DB options mutex while it waits.) + // + // Order of events: + // 1. An empty edit W0 blocks inside its MANIFEST write. + // 2. The flush switches the memtable (DropColumnFamily() would hold the + // write thread it needs), and its background job waits at its start. + // 3. The drop queues behind W0. + // 4. The flush job writes its SST and queues its install behind the drop. + // 5. W0 finishes, the drop is applied, then the install finds the column + // family dropped. + if (!GetParam()) { + return; + } + Options options = CurrentOptions(); + options.create_if_missing = true; + options.atomic_flush = true; + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + WriteOptions write_options; + write_options.disableWAL = true; + for (int cf = 0; cf < 3; ++cf) { + ASSERT_OK(Put(cf, "key", "value", write_options)); + } + + std::mutex state_mutex; + std::condition_variable state_changed; + int queued_callers = 0; + bool first_write_blocked = false; + bool release_first_write = false; + bool flush_job_waiting = false; + bool release_flush_job = false; + std::atomic first_write_seen{false}; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) { + std::lock_guard lock(state_mutex); + ++queued_callers; + state_changed.notify_all(); + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifestStart", [&](void*) { + if (first_write_seen.exchange(true)) { + return; + } + std::unique_lock lock(state_mutex); + first_write_blocked = true; + state_changed.notify_all(); + state_changed.wait(lock, [&] { return release_first_write; }); + }); + int publishes_without_callback = 0; + SyncPoint::GetInstance()->SetCallBack( + "MemTableList::InstallMemtableAtomicFlushResults:PublishWithoutCallback", + [&](void* arg) { + EXPECT_TRUE(static_cast(arg)->IsColumnFamilyDropped()); + ++publishes_without_callback; + }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::BackgroundCallFlush:start", [&](void*) { + std::unique_lock lock(state_mutex); + flush_job_waiting = true; + state_changed.notify_all(); + state_changed.wait(lock, [&] { return release_flush_job; }); + }); + SyncPoint::GetInstance()->EnableProcessing(); + const auto wait_for = [&](const std::function& done) { + std::unique_lock lock(state_mutex); + state_changed.wait(lock, done); + }; + + ColumnFamilyData* const default_cfd = + static_cast(handles_[0])->cfd(); + Status first_status; + port::Thread first_writer([&] { + VersionEdit edit; + edit.SetColumnFamily(default_cfd->GetID()); + dbfull()->TEST_LockMutex(); + first_status = dbfull()->GetVersionSet()->LogAndApply( + default_cfd, ReadOptions(), WriteOptions(), &edit, + dbfull()->TEST_Mutex(), /*dir_contains_current_file=*/nullptr); + dbfull()->TEST_UnlockMutex(); + }); + wait_for([&] { return first_write_blocked; }); + Status flush_status; + port::Thread flusher([&] { + FlushOptions flush_options; + flush_options.wait = true; + flush_status = dbfull()->Flush(flush_options, handles_[1]); + }); + wait_for([&] { return flush_job_waiting; }); + Status drop_status; + port::Thread dropper( + [&] { drop_status = dbfull()->DropColumnFamily(handles_[1]); }); + wait_for([&] { return queued_callers >= 2; }); + { + std::lock_guard lock(state_mutex); + release_flush_job = true; + state_changed.notify_all(); + } + wait_for([&] { return queued_callers >= 3; }); + { + std::lock_guard lock(state_mutex); + release_first_write = true; + state_changed.notify_all(); + } + first_writer.join(); + dropper.join(); + flusher.join(); + // Flush() returns ColumnFamilyDropped as soon as the drop is applied. The + // flush job can still be inside its install at that time. Wait for it, so + // that its sync point callbacks run before they are cleared. + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_OK(first_status); + ASSERT_OK(drop_status); + ASSERT_TRUE(flush_status.IsColumnFamilyDropped()) << flush_status.ToString(); + EXPECT_EQ(publishes_without_callback, 1); + ASSERT_EQ(Get(0, "key"), "value"); + ASSERT_EQ(Get(2, "key"), "value"); + ASSERT_OK(dbfull()->DestroyColumnFamilyHandle(handles_[1])); + handles_.erase(handles_.begin() + 1); + ReopenWithColumnFamilies({kDefaultColumnFamilyName, "eevee"}, options); + ASSERT_EQ(Get(0, "key"), "value"); +} + TEST_P(DBAtomicFlushTest, FlushMultipleCFsDropSomeAfterScheduleFlushBeforeFlushJobRun) { bool atomic_flush = GetParam(); diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index c3fe1bb77626..d33fb25c6669 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "db/blob/blob_fetcher.h" #include "db/blob/blob_file_partition_manager.h" #include "db/blob/blob_index.h" +#include "db/blob/blob_reference_census.h" #include "db/builder.h" #include "db/coalescing_iterator.h" #include "db/compaction/compaction_job.h" @@ -830,7 +832,9 @@ void DBImpl::WaitForBackgroundWork() { while (remote_compaction_pool_jobs_ || CompactionsScheduled() || bg_flush_scheduled_ || bg_wbm_flush_scheduled_ || bg_pressure_callback_in_progress_ || - bg_async_file_open_state_ == AsyncFileOpenState::kScheduled) { + bg_async_file_open_state_ == AsyncFileOpenState::kScheduled || + bg_blob_reference_census_state_ == + BlobReferenceCensusJobState::kScheduled) { bg_cv_.Wait(); } } @@ -847,6 +851,49 @@ void DBImpl::WaitForAsyncFileOpen() { } } +#ifndef NDEBUG +void DBImpl::TEST_WaitForBlobReferenceCensus() { + // Move CFs on the retry list that can run now back to the LOW queue. + // Otherwise waiting only for the scheduled task can return before a CF's + // catch-up slice runs. + RetryBlobReferenceCensus(); + InstrumentedMutexLock lock(&mutex_); + // A test can request a candidate refresh without a Version installation (for + // example, by ending a metadata-only commit). Drive that queued work before + // waiting; production uses its retry timer. + for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) { + ScheduleBlobReferenceCensus(cfd); + } + for (;;) { + // Also wait for the applies that flush and compaction jobs owe: a manual + // compaction can return before its job ran + // ApplyBlobReferenceCensusAfterInstall(). + while ((bg_blob_reference_census_state_ == + BlobReferenceCensusJobState::kScheduled || + blob_reference_census_pending_applies_ > 0) && + !shutting_down_.load(std::memory_order_acquire)) { + bg_cv_.Wait(); + } + bool deferred_catch_up = false; + for (ColumnFamilyData* cfd = blob_reference_census_retry_head_; + cfd != nullptr; cfd = cfd->next_blob_reference_census_) { + // A CF whose build failed may stay unavailable on purpose. Wait only for + // CFs without failures that still have work to do. + deferred_catch_up |= !cfd->IsDropped() && + cfd->blob_reference_census_failures_ == 0 && + cfd->blob_reference_census()->NeedsWorkerRun( + cfd->current()->GetVersionNumber()); + } + if (!deferred_catch_up || shutting_down_.load(std::memory_order_acquire)) { + break; + } + mutex_.Unlock(); + RetryBlobReferenceCensus(); + mutex_.Lock(); + } +} +#endif // NDEBUG + void DBImpl::NotifyOnDBShutdownBegin() { mutex_.AssertHeld(); if (shutdown_notification_sent_ || immutable_db_options_.listeners.empty()) { @@ -914,10 +961,23 @@ void DBImpl::CancelAllBackgroundWork(bool wait) { shutting_down_.store(true, std::memory_order_release); bg_cv_.SignalAll(); env_->UnSchedule(&bg_async_file_open_state_, Env::Priority::HIGH); + env_->UnSchedule(&bg_blob_reference_census_state_, Env::Priority::LOW); if (!wait) { return; } WaitForBackgroundWork(); + // A census worker registers its retry task on first use, which can be after + // the periodic tasks were canceled, so unregister it again here. + // WaitForBackgroundWork() above also waited for the worker's cleanup without + // the mutex (CFD pins and change log). + mutex_.Unlock(); + periodic_task_scheduler_.Unregister(PeriodicTaskType::kBlobReferenceCensus) + .PermitUncheckedError(); + mutex_.Lock(); + if (blob_reference_census_queue_head_ != nullptr || + blob_reference_census_retry_head_ != nullptr) { + UnscheduleBlobReferenceCensus(this); + } } Status DBImpl::MaybeReleaseTimestampedSnapshotsAndCheck() { @@ -1199,6 +1259,8 @@ Status DBImpl::CloseHelper() { bg_flush_scheduled_ || bg_wbm_flush_scheduled_ || bg_purge_scheduled_ || bg_pressure_callback_in_progress_ || bg_async_file_open_state_ == AsyncFileOpenState::kScheduled || + bg_blob_reference_census_state_ == + BlobReferenceCensusJobState::kScheduled || async_wal_precreate_state_ == AsyncWALPrecreateState::kScheduled || pending_purge_obsolete_files_ || error_handler_.IsRecoveryInProgress()) { @@ -1206,7 +1268,29 @@ Status DBImpl::CloseHelper() { bg_cv_.Wait(); } + // A census worker registers its retry task on first use, which can be after + // the first CancelPeriodicTaskScheduler() call, so unregister it again. No + // worker remains after the wait above. The census state can be as large as + // the live SST set, so it is freed below without the DB mutex. mutex_.Unlock(); + periodic_task_scheduler_.Unregister(PeriodicTaskType::kBlobReferenceCensus) + .PermitUncheckedError(); + mutex_.Lock(); + if (blob_reference_census_queue_head_ != nullptr || + blob_reference_census_retry_head_ != nullptr) { + UnscheduleBlobReferenceCensus(this); + } + std::vector census_garbage; + const size_t column_family_count = + versions_->GetColumnFamilySet()->NumberOfColumnFamilies(); + mutex_.Unlock(); + census_garbage.reserve(column_family_count); + mutex_.Lock(); + for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) { + census_garbage.emplace_back(cfd->CancelBlobReferenceCensus()); + } + mutex_.Unlock(); + std::vector().swap(census_garbage); ReleaseRemoteCompactionPool(); mutex_.Lock(); @@ -1864,6 +1948,7 @@ void DBImpl::DumpStats() { // Also probe block cache(s) for problems, dump to info log UnorderedSet probed_caches; + std::vector blob_gc_warnings; TEST_SYNC_POINT("DBImpl::DumpStats:StartRunning"); { InstrumentedMutexLock l(&mutex_); @@ -1904,13 +1989,52 @@ void DBImpl::DumpStats() { property_info = GetPropertyInfo(*property); assert(property_info != nullptr); assert(!property_info->need_out_of_mutex); + // Indirect blob GC that stays idle while it has candidates for longer than + // one stats period is starved, for example by LSM compactions. + const uint64_t period_micros = + uint64_t{mutable_db_options_.stats_dump_period_sec} * 1000 * 1000; for (auto cfd : *versions_->GetColumnFamilySet()) { if (cfd->initialized()) { cfd->internal_stats()->GetStringProperty(*property_info, *property, &stats); + if (cfd->blob_reference_census() != nullptr) { + // kCensusUnreadyWarnMicros (10 minutes, a fixed choice): a census + // that stays unready this long blocks the work that needs it. This + // check runs here, not in the census job, so it also fires when that + // job is stuck, paused or never scheduled. + constexpr uint64_t kCensusUnreadyWarnMicros = + 10ULL * 60 * 1000 * 1000; + const uint64_t unready_micros = + cfd->internal_stats()->BlobReferenceCensusUnreadyMicros(); + if (unready_micros > kCensusUnreadyWarnMicros) { + blob_gc_warnings.push_back( + "[" + cfd->GetName() + + "] Blob reference census has been unready for " + + std::to_string(unready_micros / 1000000) + " s (state: " + + (cfd->blob_reference_census_failures() > 0 ? "failed" + : "building") + + ", failed builds in a row: " + + std::to_string(cfd->blob_reference_census_failures()) + + "); the work that needs the census waits"); + } + } + if (period_micros != 0 && cfd->blob_reference_census() != nullptr) { + const uint64_t idle_micros = + cfd->internal_stats()->IndirectBlobGcIdleMicrosWithCandidates(); + if (idle_micros > period_micros) { + blob_gc_warnings.push_back("[" + cfd->GetName() + + "] Indirect blob GC has had candidates " + "for " + + std::to_string(idle_micros / 1000000) + + " s without starting a job"); + } + } } } } + for (const std::string& warning : blob_gc_warnings) { + ROCKS_LOG_WARN(immutable_db_options_.info_log, "%s", warning.c_str()); + } TEST_SYNC_POINT("DBImpl::DumpStats:2"); ROCKS_LOG_INFO(immutable_db_options_.info_log, "------- DUMPING STATS -------"); @@ -1987,22 +2111,68 @@ Status DBImpl::SetOptions( return Status::OK(); } - for (const auto& cf_opts : column_families_opts_map) { - if (cf_opts.second.empty()) { + // SetOptions() changes the options of all the given column families, or of + // none. Readers never see a mix of old and new options and Versions. The + // steps: + // 1. Group the input maps by column family (ColumnFamilyOptionsUpdate). + // 2. Return early if no map changes any option, unless an earlier call left + // the OPTIONS file or the seqno-to-time worker out of date. + // 3. Under the DB mutex, read a new phasing anchor if any map sets + // periodic_compaction_seconds, even to its current value (see + // VersionSet::PrepareCompactionPhaseReanchor()). Build one dummy edit + // per column family, and one MutableCFOptionsPublication that holds an + // options entry for each column family. + // 4. Make one LogAndApply() call. pre_cb prepares the options of every + // column family. LogAndApply() builds the new Versions and publishes them + // with the options and the anchor. The comment above + // MutableCFOptionsPublication (version_set.h) lists how. + // 5. Install the new SuperVersions before the DB mutex is released. + // 6. Write one OPTIONS file. WriteOptionsFile() releases the DB mutex + // during the file write. + // 7. Update the seqno-to-time worker if any map sets + // preserve_internal_time_seconds or preclude_last_level_data_seconds, or + // if an earlier update of the worker failed. + + // One column family and all the options maps given for it, in the order they + // are applied. + using OptionsMap = std::unordered_map; + struct ColumnFamilyOptionsUpdate { + std::reference_wrapper cfd; + std::vector> options_maps; + }; + // Group the input by column family. Two different handles can point to the + // same column family, and the single LogAndApply call below takes each column + // family only once. So one writer applies all maps of a column family, in the + // iteration order of the input map. The same pass rejects an empty map and + // notes whether any map sets periodic_compaction_seconds (new phasing + // anchor), or sets preserve_internal_time_seconds or + // preclude_last_level_data_seconds (seqno-to-time worker update). It reads + // only the caller's handles and maps, so it runs before any lock is taken. + std::vector column_family_updates; + UnorderedMap update_index_by_cfd; + bool changing_periodic_compaction_seconds = false; + bool needs_seqno_worker = false; + for (const auto& [handle, options_map] : column_families_opts_map) { + if (options_map.empty()) { ROCKS_LOG_WARN(immutable_db_options_.info_log, "SetOptions() on column family [%s], empty input", - cf_opts.first->GetName().c_str()); + handle->GetName().c_str()); return Status::InvalidArgument("empty input"); } - } - - autovector*>> - column_family_datas; - for (const auto& cf_opts : column_families_opts_map) { - column_family_datas.push_back( - {static_cast_with_check(cf_opts.first)->cfd(), - &cf_opts.second}); + auto* cfd = static_cast_with_check(handle)->cfd(); + const auto [it, inserted] = + update_index_by_cfd.emplace(cfd, column_family_updates.size()); + if (inserted) { + column_family_updates.push_back({*cfd, {std::cref(options_map)}}); + } else { + column_family_updates[it->second].options_maps.push_back( + std::cref(options_map)); + } + changing_periodic_compaction_seconds |= + options_map.count("periodic_compaction_seconds") > 0; + needs_seqno_worker |= + options_map.count("preserve_internal_time_seconds") > 0 || + options_map.count("preclude_last_level_data_seconds") > 0; } DeferredInstrumentedMutexLock ol( @@ -2018,9 +2188,15 @@ Status DBImpl::SetOptions( bool unchanged = true; { InstrumentedMutexLock l(&mutex_); - for (const auto& cfd_opts : column_family_datas) { - if (!cfd_opts.first->SetOptionsIsNoOp(db_options, *cfd_opts.second)) { - unchanged = false; + for (const auto& update : column_family_updates) { + for (const auto& options_map : update.options_maps) { + if (!update.cfd.get().SetOptionsIsNoOp(db_options, + options_map.get())) { + unchanged = false; + break; + } + } + if (!unchanged) { break; } } @@ -2029,90 +2205,107 @@ Status DBImpl::SetOptions( ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetOptions() on [%zu] column families, input option " "values are not changed, skipping updating.", - column_family_datas.size()); + column_family_updates.size()); return Status::OK(); } } - autovector - new_options_copy; // For logging outside of DB mutex + // The new options of each column family, in the order of + // column_family_updates. Each entry starts as a copy of the latest options, + // and pre_cb applies the input maps to it under the DB mutex. The entries are + // also used for logging after the DB mutex is released. + MutableCFOptionsPublication options_publication; + autovector& prepared_options = + options_publication.new_mutable_cf_options; Status s; Status persist_options_status; SuperVersionContext sv_context(/* create_superversion */ true); { InstrumentedMutexLock l(&mutex_); - // Manifest writers + Version appenders like flush and compaction use - // LogAndApply, which releases DB mutex to wait for other manifest writers - // and for the manifest write. We need to append a Version for the options - // to take full effect (e.g. compaction scores), but we don't want to - // interleave with other callers of LogAndApply, which could at least - // temporarily roll back option changes. Thus, we use a special call to - // LogAndApply that allows us to - // - // (a) Apply the options update when we know we are the exclusive version - // appender + (fake) manifest writer, and + // Steps 3 to 6 of the list at the top of this function. If the options were + // published before LogAndApply(), another caller that already built its + // Version (for example, a flush that is writing the MANIFEST with the DB + // mutex released) could install that Version afterwards. That Version has + // the old options, so readers would see new options with an old Version. So + // LogAndApply() publishes the options together with the new Versions. + // pre_cb runs when this call reaches the head of the MANIFEST writer queue, + // so a column family drop that was queued before this call is already + // applied, and the IsDropped() check in pre_cb sees it. pre_cb also + // prepares the options. If either step fails, the call fails before + // anything is published. // - // (b) Append a new Version without manifest write nor DB mutex release - // - // Thus aren't releasing the DB mutex from LogAndApply calling pre_cb, - // through installing the new Version until the end of this block, after - // installing the new SuperVersion. - VersionEdit dummy_edit; - dummy_edit.MarkNoManifestWriteDummy(); - TEST_SYNC_POINT_CALLBACK("DBImpl::SetOptions:dummy_edit", &dummy_edit); - // If any CF is changing periodic_compaction_seconds, (re)anchor phasing to - // now BEFORE the new Version is built below, so the new Version's scoring - // spreads a turn-down's newly past-due cohort over the phase grid (within - // ~N/4 of now) instead of firing it all at once (a herd). Anchoring is - // DB-level and benign to CFs that are not changing their interval. - bool changing_periodic_compaction_seconds = false; - for (const auto& cfd_opts : column_family_datas) { - if (cfd_opts.second->count("periodic_compaction_seconds") > 0) { - changing_periodic_compaction_seconds = true; - break; - } - } - if (changing_periodic_compaction_seconds) { - versions_->ReanchorCompactionPhase(); - } - for (const auto& cfd_opts : column_family_datas) { - auto* cfd = cfd_opts.first; - const auto* options_map_ptr = cfd_opts.second; - auto pre_cb = [&]() -> Status { - Status cb_s = cfd->SetOptions(db_options, *options_map_ptr); - if (cb_s.ok()) { - new_options_copy.emplace_back(cfd->GetLatestMutableCFOptions()); + // If any map sets periodic_compaction_seconds, read a new phasing anchor + // (the current time). LogAndApply() publishes it later. See + // VersionSet::PrepareCompactionPhaseReanchor(). + options_publication.compaction_phase_anchor_time = + changing_periodic_compaction_seconds + ? versions_->PrepareCompactionPhaseReanchor() + : std::nullopt; + + autovector cfds; + autovector> edit_lists; + std::vector> dummy_edits; + const size_t num_cfds = column_family_updates.size(); + cfds.reserve(num_cfds); + edit_lists.reserve(num_cfds); + dummy_edits.reserve(num_cfds); + prepared_options.reserve(num_cfds); + // Build one edit per column family for the single LogAndApply call. Each + // edit is a dummy that writes nothing to the MANIFEST. Each prepared entry + // starts as a copy of the latest options. options_mutex_ keeps other + // SetOptions calls from changing the latest options before pre_cb applies + // the input maps on top of this copy. + for (const auto& update : column_family_updates) { + cfds.push_back(&update.cfd.get()); + dummy_edits.emplace_back(new VersionEdit()); + dummy_edits.back()->MarkNoManifestWriteDummy(); + TEST_SYNC_POINT_CALLBACK("DBImpl::SetOptions:dummy_edit", + dummy_edits.back().get()); + edit_lists.emplace_back(); + edit_lists.back().push_back(dummy_edits.back().get()); + prepared_options.emplace_back( + update.cfd.get().GetLatestMutableCFOptions()); + } + + // LogAndApply runs this under the DB mutex, once this call is at the head + // of the manifest writer queue. It applies every input map to the prepared + // options. A dropped column family or an invalid map fails the whole call + // before any options or Version is published, so a failure leaves every + // column family unchanged. + auto pre_cb = [&]() -> Status { + for (size_t i = 0; i < num_cfds; ++i) { + auto& cfd = column_family_updates[i].cfd.get(); + if (cfd.IsDropped()) { + return Status::ColumnFamilyDropped(); } - return cb_s; - }; - - s = versions_->LogAndApply( - cfd, read_options, write_options, &dummy_edit, &mutex_, - directories_.GetDbDir(), false /*new_descriptor_log=*/, - nullptr /*new_opts*/, {} /*manifest_wcb*/, pre_cb); - if (!versions_->io_status().ok()) { - assert(!s.ok()); - error_handler_.SetBGError(versions_->io_status(), - BackgroundErrorReason::kManifestWrite); - } - if (!s.ok()) { - if (!new_options_copy.empty()) { - // The options that this call already changed are not in the OPTIONS - // file or the seqno-to-time worker. A failure that changed nothing - // leaves both up to date. - options_file_stale_ = true; - seqno_time_worker_stale_ = true; + for (const auto& options_map : column_family_updates[i].options_maps) { + Status cb_s = cfd.PrepareMutableCFOptions( + db_options, options_map.get(), &prepared_options[i]); + if (!cb_s.ok()) { + return cb_s; + } } - break; } + return Status::OK(); + }; + + s = versions_->LogAndApply(cfds, read_options, write_options, edit_lists, + &mutex_, directories_.GetDbDir(), + /*new_descriptor_log=*/false, + /*new_cf_options=*/nullptr, /*manifest_wcbs=*/{}, + pre_cb, /*max_file_opening_threads=*/1, + &options_publication); + if (!versions_->io_status().ok()) { + assert(!s.ok()); + error_handler_.SetBGError(versions_->io_status(), + BackgroundErrorReason::kManifestWrite); } - if (s.ok()) { // Trigger possible flush/compactions. This has to be before we persist // options to file, otherwise there will be a deadlock with writer // thread. - for (const auto& cfd_opts : column_family_datas) { - auto* cfd = cfd_opts.first; + for (const auto& update : column_family_updates) { + auto* cfd = &update.cfd.get(); assert(cfd != nullptr); InstallSuperVersionForConfigChange(cfd, &sv_context); if (auto* blob_partition_manager = cfd->blob_partition_manager()) { @@ -2122,15 +2315,20 @@ Status DBImpl::SetOptions( } cfd->MarkOptionsInstalled(); } + // Every SuperVersion is installed, so readers can no longer see a mix of + // new options and Versions with old SuperVersions. Now free the census + // payloads that LogAndApply deferred. This unlocks the DB mutex while it + // frees them and then locks it again. + versions_->ReleaseDeferredPublicationCleanup(&mutex_); persist_options_status = WriteOptionsFile(write_options, true /*db_mutex_already_held*/); bg_cv_.SignalAll(); #ifndef NDEBUG - for (size_t i = 0; i < column_family_datas.size(); ++i) { - auto* cfd = column_family_datas[i].first; - assert(new_options_copy[i] == cfd->GetLatestMutableCFOptions()); + for (size_t i = 0; i < column_family_updates.size(); ++i) { + auto* cfd = &column_family_updates[i].cfd.get(); + assert(prepared_options[i] == cfd->GetLatestMutableCFOptions()); assert(cfd->GetLatestMutableCFOptions() == cfd->GetCurrentMutableCFOptions()); assert(cfd->GetCurrentMutableCFOptions() == @@ -2141,44 +2339,35 @@ Status DBImpl::SetOptions( } sv_context.Clean(); - if (s.ok()) { - // A stale worker can lag the time options of any column family, so update - // it even if this request sets no time option. - bool needs_seqno_worker = update_seqno_worker; - for (const auto& cf_opts : column_families_opts_map) { - if (cf_opts.second.count("preserve_internal_time_seconds") > 0 || - cf_opts.second.count("preclude_last_level_data_seconds") > 0) { - needs_seqno_worker = true; - break; - } - } - if (needs_seqno_worker) { - s = RegisterRecordSeqnoTimeWorker(); - } + // If an earlier RegisterRecordSeqnoTimeWorker() call failed + // (seqno_time_worker_stale_), the worker can lag the time options of any + // column family. So update it even when this request sets no time option. + if (s.ok() && (needs_seqno_worker || update_seqno_worker)) { + s = RegisterRecordSeqnoTimeWorker(); } ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetOptions() on [%zu] column families, inputs:", - column_family_datas.size()); - for (size_t i = 0; i < column_family_datas.size(); ++i) { - const auto* cfd = column_family_datas[i].first; - const auto* options_map_ptr = column_family_datas[i].second; + column_families_opts_map.size()); + size_t input_index = 0; + for (const auto& [handle, options_map] : column_families_opts_map) { ROCKS_LOG_INFO(immutable_db_options_.info_log, "Set options on column family [%s] (%zu/%zu), inputs:", - cfd->GetName().c_str(), i, column_family_datas.size()); - for (const auto& o : *options_map_ptr) { + handle->GetName().c_str(), input_index++, + column_families_opts_map.size()); + for (const auto& o : options_map) { ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(), o.second.c_str()); } } if (s.ok()) { - for (size_t i = 0; i < column_family_datas.size(); ++i) { - const auto* cfd = column_family_datas[i].first; + for (size_t i = 0; i < column_family_updates.size(); ++i) { + const auto& cfd = column_family_updates[i].cfd.get(); ROCKS_LOG_INFO(immutable_db_options_.info_log, "Set options on column family [%s] (%zu/%zu) succeeded, " "updated CF options:", - cfd->GetName().c_str(), i, column_family_datas.size()); - new_options_copy[i].Dump(immutable_db_options_.info_log.get()); + cfd.GetName().c_str(), i, column_family_updates.size()); + prepared_options[i].Dump(immutable_db_options_.info_log.get()); } if (!persist_options_status.ok()) { // NOTE: WriteOptionsFile already logs on failure @@ -6724,6 +6913,11 @@ void DBImpl::CleanupSuperVersion(SuperVersion* sv) { RecordTick(stats_, NUMBER_SUPERVERSION_RELEASES); } +void DBImpl::ReleaseColumnFamilyDataRef(ColumnFamilyData* cfd) { + InstrumentedMutexLock lock(&mutex_); + cfd->UnrefAndTryDelete(); +} + void DBImpl::ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd, SuperVersion* sv) { if (!cfd->ReturnThreadLocalSuperVersion(sv)) { @@ -8750,8 +8944,8 @@ Status DBImpl::CommitFileIngestionHandles( // the lock-free path. status = versions_->LogAndApply( cfds_to_commit, read_options, write_options, edit_lists, &mutex_, - directories_.GetDbDir(), false /* new_descriptor_log */, - nullptr /* new_cf_options */, manifest_wcbs, {} /* pre_cb */, + directories_.GetDbDir(), /*new_descriptor_log=*/false, + /*new_cf_options=*/nullptr, manifest_wcbs, /*pre_cb=*/{}, max_file_opening_threads); // It is safe to update VersionSet last seqno here after LogAndApply since // LogAndApply persists last sequence number from VersionEdits, diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index a78882f53800..2fbc73dff5ae 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -102,6 +102,7 @@ class Version; class VersionEdit; class VersionSet; class WriteCallback; +struct BlobReferenceCensusBuildContext; struct JobContext; struct ExternalSstFileInfo; struct MemTableInfo; @@ -1209,6 +1210,19 @@ class DBImpl : public DB // Un-reference the super version and clean it up if it is the last reference. void CleanupSuperVersion(SuperVersion* sv); + // Releases a column family reference that the blob reference census took + // with cfd->Ref(). The census takes one in two places: + // - ScheduleBlobReferenceCensus() takes one for each column family that it + // queues. The column family keeps it while it waits in the run queue or + // the retry list. The census job releases it when the column family leaves + // both, and the cleanup of both lists at close releases the rest. + // - Each census build takes one when it starts, for its starting Version + // (see BuildBlobReferenceCensus()). The deleter of that Version releases + // it when the build context is freed. + // This function takes the DB mutex, so the caller must not hold it. The + // column family is deleted when its last reference is released. + void ReleaseColumnFamilyDataRef(ColumnFamilyData* cfd); + // Un-reference the super version and return it to thread local cache if // needed. If it is the last reference of the super version. Clean it up // after un-referencing it. @@ -1471,8 +1485,13 @@ class DBImpl : public DB Status TEST_WaitForCompact( const WaitForCompactOptions& wait_for_compact_options); + Status TEST_RunIndirectBlobGCBatch( + ColumnFamilyData* cfd, + const std::vector& candidate_physical_file_numbers); + // Wait for any background purge Status TEST_WaitForPurge(); + void TEST_WaitForBlobReferenceCensus(); // Get the background error status Status TEST_GetBGError(); @@ -2268,6 +2287,64 @@ class DBImpl : public DB // shutdown begins. void WaitForAsyncFileOpen(); + // Schedules the background job that builds and updates the blob reference + // census of each fenced CF (see db/blob/blob_reference_census.h). Only + // indirect blob GC uses the census; reads never wait for it. The overload + // without a CF queues every CF that needs a run. + void ScheduleBlobReferenceCensus(); + void ScheduleBlobReferenceCensus(ColumnFamilyData* requested_cfd); + static void BGWorkBlobReferenceCensus(void* arg); + static void UnscheduleBlobReferenceCensus(void* arg); + + // `*incomplete` is true when this run did not make the census ready: the scan + // has more slices, catch-up must wait while an indirect blob GC job copies + // its snapshot, or an install is being prepared. The job runs again later + // (see "Scheduling" in db/blob/blob_reference_census.h). `*overflowed` + // reports a change-log overflow. The first overflow restarts the build at + // once with the restart limit. A second overflow before a publish fails the + // build, which then backs off. + Status BuildBlobReferenceCensus(ColumnFamilyData* cfd, bool* incomplete, + bool* overflowed); + + // Selects the indirect blob GC candidates of the census worker of `cfd` and + // publishes the census for the current Version if the worker has applied + // every install. Sets `*incomplete` if the publish must wait (an install is + // pending or not yet applied). Returns Aborted if the build is stale, + // ShutdownInProgress during shutdown or after a drop. Takes the DB mutex. + // REQUIRES: the caller holds the census's worker_run_mutex() and not the DB + // mutex. + Status PublishBlobReferenceCensusWorker( + ColumnFamilyData* cfd, BlobReferenceCensusBuildContext* context, + bool* incomplete); + + // Called at the end of each background flush and compaction + // (BackgroundCallFlush() and BackgroundCallCompaction()), with the DB mutex + // held. `cfds` are the column families with a census that this job flushed or + // compacted (also when the job failed; a dropped column family is left out), + // each with a reference (`census_cfds`): BackgroundFlush() keeps the flush + // job's reference, and BackgroundCompaction() takes a new one. It releases + // the DB mutex and applies the queued census deltas of each of them (see + // "Apply after an install" in db/blob/blob_reference_census.h). It takes the + // DB mutex again, drops the references and returns. It never fails the + // caller's job. A column family that is dropped while this function has the + // DB mutex released gets its last Unref here, after the job's + // FindObsoleteFiles(). Its files are then deleted by the next obsolete-file + // pass or at close, so they do not leak. + void ApplyBlobReferenceCensusAfterInstall( + const autovector& cfds); + + // Applies the queued deltas of one column family. If it applied at least one, + // it then tries to publish the census. Does nothing if another thread runs + // the worker, if the build has not started or has not finished its scan of + // the SSTs, or if the build has a stored failure + // (BlobReferenceCensusBuildContext::failure). REQUIRES: DB mutex not held. + void ApplyBlobReferenceCensusDeltas(ColumnFamilyData* cfd); + + // Debug builds: rebuilds the expected per-generation-number records of the + // published Version from its SSTs and compares them with the census. Each + // call runs the check with a random 1 in 16 chance, unless a test forces it. + void CheckBlobReferenceCensus(ColumnFamilyData* cfd); + void RetryBlobReferenceCensus(); void InvokeWalFilterIfNeededOnColumnFamilyToWalNumberMap(); // Return true to proceed with current WAL record whose content is stored in @@ -3269,11 +3346,13 @@ class DBImpl : public DB Env::Priority thread_pri, bool separate_remote_pool, int* deferred_compactions, - uint64_t min_output_file_number); + uint64_t min_output_file_number, + autovector* census_cfds); Status BackgroundFlush(bool* madeProgress, JobContext* job_context, LogBuffer* log_buffer, FlushReason* reason, bool* flush_rescheduled_to_retain_udt, - Env::Priority thread_pri); + Env::Priority thread_pri, + autovector* census_cfds); Compaction* CreateIntendedCompactionForwardedToBottomPriorityPool( Compaction* c); @@ -3281,6 +3360,58 @@ class DBImpl : public DB const std::vector& inputs, bool* sfm_bookkeeping, LogBuffer* log_buffer); + // Runs one indirect blob GC job. The job collects 1 to 8 input files (normal + // blob files or relocation files) of `cfd`, whose file numbers are + // `candidate_physical_file_numbers`, into new relocation files, and installs + // the result with one MANIFEST edit. BackgroundCompaction() calls it. The + // overview at the top of db/blob/blob_relocation_gc.h lists the job steps and + // the suppression rules. IndirectBlobGCJob in db_impl_compaction_flush.cc + // holds the code. + // + // DB mutex: held on entry and held on return. The job releases it while it + // reads and writes files, and takes it again for short steps. + // + // Out parameters (each must be non-null; otherwise the call returns + // InvalidArgument): + // - `*made_progress` is set to true when the MANIFEST edit is installed. + // - `*sfm_reserved_bytes` must be 0 on entry. It is set to the bytes that + // the job reserved in the SstFileManager for its outputs. The caller + // releases that reservation. + // - `*retryable_stale_input` is set to false when the job starts, and to true + // when the job returns TryAgain because an input or the census changed + // while the job ran (a stale job). A later job can then retry the same + // inputs. It is not changed when the call returns InvalidArgument. + // + // Other parameters: + // - `log_buffer` receives the job's progress lines: the live bytes of each + // input that DecideInputs() checks, the skipped inputs when the job + // installs nothing, and the install summary. Errors go to the info log. It + // can be null. + // - `job_id` names the job in the logs and in the blob file listener calls. + // + // Return codes: + // - OK: the edit is installed, or no input was collected because every + // input was suppressed or deferred, or another GC job or an earlier + // metadata commit with an unknown outcome blocks the job (Busy from + // BlobReferenceCensus::BeginGcJob()). + // - TryAgain: the job installs nothing. `*retryable_stale_input` is set for a + // stale job, when the census is not ready, and when a metadata-only commit + // is in progress. + // - CompactionTooLarge: the SstFileManager has not enough disk space for + // the outputs, even with only one input left. + // - InvalidArgument: a null argument or a wrong number of inputs. + // - Others: ShutdownInProgress, ColumnFamilyDropped, Incomplete (the + // compaction was aborted), and I/O or Corruption errors. + Status RunIndirectBlobGCBatch( + ColumnFamilyData* cfd, + const std::vector& candidate_physical_file_numbers, + bool* made_progress, uint64_t* sfm_reserved_bytes, + bool* retryable_stale_input, LogBuffer* log_buffer, int job_id); + + // The job of RunIndirectBlobGCBatch(), with one method per phase. It is + // defined in db_impl_compaction_flush.cc. + class IndirectBlobGCJob; + // Request compaction tasks token from compaction thread limiter. // It always succeeds if force = true or limiter is disable. bool RequestCompactionToken(ColumnFamilyData* cfd, bool force, @@ -3743,8 +3874,7 @@ class DBImpl : public DB InstrumentedMutex options_mutex_; // True when the OPTIONS file can lag the options or column families in - // effect: the last OPTIONS file write failed, SetOptions() changed options - // and then failed before it wrote the file, or a column family batch with + // effect: the last OPTIONS file write failed, or a column family batch with // track_options_file_number_in_manifest stopped part way. Then SetOptions() // and SetDBOptions() write the file even for a request that changes no // option. A successful WriteOptionsFile() clears it, and so does a tracked @@ -3753,8 +3883,7 @@ class DBImpl : public DB bool options_file_stale_ = false; // True when the seqno-to-time worker can lag the time options of the column - // families: the last RegisterRecordSeqnoTimeWorker() failed, or SetOptions() - // changed options and then failed before it updated the worker. Then + // families, because the last RegisterRecordSeqnoTimeWorker() failed. Then // SetOptions() updates the worker even for a request that changes no option. // A successful RegisterRecordSeqnoTimeWorker() clears it. Protected by // options_mutex_. @@ -4136,6 +4265,11 @@ class DBImpl : public DB // CompactionService::Wait(), counted per executed subcompaction. std::atomic num_running_remote_compactions_ = 0; + // An indirect blob GC job removes whole relocation files and re-points the + // routes into them, so only one GC job runs per column family. Protected by + // mutex_. + std::set indirect_blob_gc_cfs_in_progress_; + // number of background memtable flush jobs, submitted to the HIGH pool int bg_flush_scheduled_ = 0; @@ -4164,6 +4298,36 @@ class DBImpl : public DB AsyncFileOpenState bg_async_file_open_state_ = AsyncFileOpenState::kNotScheduled; + enum class BlobReferenceCensusJobState : uint8_t { + kNotScheduled = 0, + kScheduled, + kComplete, + }; + BlobReferenceCensusJobState bg_blob_reference_census_state_ = + BlobReferenceCensusJobState::kNotScheduled; + // Two intrusive lists of CFs, linked through + // ColumnFamilyData::next_blob_reference_census_: the run queue + // (queue_head/tail) and the retry list (retry_head/tail). Adding a CF is O(1) + // and allocates nothing, so option changes and SuperVersion installs can + // enqueue a CF under the DB mutex without scanning all CFs. + ColumnFamilyData* blob_reference_census_queue_head_ = nullptr; + ColumnFamilyData* blob_reference_census_queue_tail_ = nullptr; + ColumnFamilyData* blob_reference_census_retry_head_ = nullptr; + ColumnFamilyData* blob_reference_census_retry_tail_ = nullptr; + size_t blob_reference_census_retry_count_ = 0; + std::atomic blob_reference_census_retry_registered_{false}; + // The census job registers its retry task in periodic_task_scheduler_, so + // it may start only after DB::Open() has started that scheduler. Until then, + // ScheduleBlobReferenceCensus() only queues column families. Protected by + // mutex_. + bool blob_reference_census_job_allowed_ = false; + // The number of census applies that flush and compaction jobs owe: each job + // adds one per column family when it installs, and + // ApplyBlobReferenceCensusAfterInstall() removes them. Such an apply never + // waits, so TEST_WaitForBlobReferenceCensus() can wait for it. Protected by + // mutex_. + size_t blob_reference_census_pending_applies_ = 0; + // State machine for the single async WAL precreation slot protected by // mutex_. Background precreation failure returns to kNotScheduled; foreground // rotation handles it the same as no prepared WAL and creates one diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 9f559031f8c9..27d555f4c541 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -7,19 +7,36 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include +#include #include #include +#include #include +#include #include +#include #include +#include "cache/cache_reservation_manager.h" +#include "db/blob/blob_file_cache.h" #include "db/blob/blob_file_partition_manager.h" +#include "db/blob/blob_file_reader.h" +#include "db/blob/blob_gen2_format.h" +#include "db/blob/blob_index.h" +#include "db/blob/blob_log_format.h" +#include "db/blob/blob_reference_census.h" +#include "db/blob/blob_relocation_file.h" +#include "db/blob/blob_relocation_gc.h" +#include "db/blob/blob_relocation_manifest.h" #include "db/builder.h" #include "db/db_impl/db_impl.h" #include "db/error_handler.h" #include "db/event_helpers.h" #include "file/file_util.h" +#include "file/filename.h" +#include "file/read_write_util.h" #include "file/sst_file_manager_impl.h" +#include "file/writable_file_writer.h" #include "logging/logging.h" #include "monitoring/iostats_context_imp.h" #include "monitoring/perf_context_imp.h" @@ -30,6 +47,10 @@ #include "rocksdb/io_status.h" #include "rocksdb/options.h" #include "rocksdb/table.h" +#include "table/adaptive/adaptive_table_factory.h" +#include "table/block_based/block_based_table_builder.h" +#include "table/embedded_blob_sst.h" +#include "table/table_builder.h" #include "test_util/sync_point.h" #include "util/cast_util.h" #include "util/coding.h" @@ -41,6 +62,64 @@ namespace ROCKSDB_NAMESPACE { namespace { +// Indirect blob GC reads every SST that holds a generation number of its +// inputs, once per SST. It checks for shutdown or a compaction abort before it +// reads the first SST, and then once every 1024 SSTs (when the SST index is a +// multiple of 1024). The interval of 1024 is an estimate, not measured. +constexpr uint64_t kIndirectBlobGCAbortCheckMask = (1 << 10) - 1; + +// Stored bytes of one read batch (1 MB, an estimate). A value larger than this +// is streamed in kBlobRelocationStreamChunkSize (256 KB) chunks instead. One +// coalesced read is also at most this size. +constexpr uint64_t kIndirectBlobGCReadBatchBytes = 1024 * 1024; +// kIndirectBlobGCBatchBytesPerValue (512 bytes, an estimate) is the fixed +// memory of one value in a read batch, beyond its stored bytes: the request, +// status and payload entries, and the record trailer. With the 1 MB batch, this +// bounds a batch to at most 2048 values. +constexpr uint64_t kIndirectBlobGCBatchBytesPerValue = 512; +// At most 8 inputs per indirect blob GC job (a fixed limit, not tuned). It +// matches kBlobGcMaxSelectedFiles (8), the number of candidates that the census +// publishes to the Version. +constexpr size_t kIndirectBlobGCMaxInputs = 8; +// At most 8 outputs per indirect blob GC job (a fixed limit, not tuned). When +// the inputs need more outputs, the largest input waits for a later job. The +// estimated size of each output is at most the target size (see +// kIndirectBlobGCMaxOutputBytes). An input with a larger estimate gets an +// output of its own. The SstFileManager space check reserves the estimated size +// of all outputs, capped at the total input size. +constexpr size_t kIndirectBlobGCMaxOutputs = 8; +// Cap for blob_file_size in indirect blob GC (1 GB, a fixed choice, not +// measured). The target output size is 90% of min(blob_file_size, +// kIndirectBlobGCMaxOutputBytes). The 90% is a fixed choice that leaves room +// for error in the output size estimate. An input larger than the target gets +// an output of its own, never larger than the input. +constexpr uint64_t kIndirectBlobGCMaxOutputBytes = uint64_t{1} << 30; + +Status EstimateIndirectBlobGCPartitionBytes(uint64_t physical_live_record_bytes, + uint64_t reference_count, + uint64_t* estimated_bytes) { + if (estimated_bytes == nullptr) { + return Status::InvalidArgument("Null blob GC output size estimate"); + } + // Output size estimate: 64 KB for the table tail and index, plus 48 bytes per + // value for its share of the location entries (estimates). One location + // entry packs up to 32 values of one origin, so a value needs much less than + // 48 bytes. The high value keeps the estimate on the safe side. + constexpr uint64_t kFixedOverhead = 64 * 1024; + constexpr uint64_t kPerEntryOverhead = 48; + if (reference_count > + (std::numeric_limits::max() - kFixedOverhead) / + kPerEntryOverhead || + physical_live_record_bytes > std::numeric_limits::max() - + kFixedOverhead - + reference_count * kPerEntryOverhead) { + return Status::MemoryLimit("Blob relocation output estimate overflow"); + } + *estimated_bytes = physical_live_record_bytes + kFixedOverhead + + reference_count * kPerEntryOverhead; + return Status::OK(); +} + void RecordAtomicFlushRequestReason(Statistics* stats, FlushReason flush_reason) { // Keep this aligned with the existing rocksdb.flush.reason.* counters: they @@ -77,8 +156,2711 @@ std::vector MemTableIds(const autovector& mems) { return ids; } +// The block cache to charge for indirect blob GC working memory, or null when +// there is no block-based writer cache or charging for the role is disabled. +std::shared_ptr GetIndirectBlobGcWorkingMemoryCache( + const MutableCFOptions& mutable_cf_options) { + const BlockBasedTableOptions* const options = + GetBlockBasedTableWriterOptions(mutable_cf_options.table_factory.get()); + if (options == nullptr || options->block_cache == nullptr) { + return nullptr; + } + const CacheEntryRoleOptions::Decision charged = + options->cache_usage_options.options_overrides + .at(CacheEntryRole::kIndirectBlobGcWorkingMemory) + .charged; + return charged == CacheEntryRoleOptions::Decision::kEnabled || + charged == CacheEntryRoleOptions::Decision::kFallback + ? options->block_cache + : nullptr; +} + } // namespace +// Background: indirect blob GC collects whole blob files (normal blob files +// or relocation files) into new relocation files, without an LSM compaction. +// The overview at the top of db/blob/blob_relocation_gc.h lists the job steps +// and the suppression rules. +// +// DBImpl::RunIndirectBlobGCBatch() checks its arguments, creates one +// IndirectBlobGCJob on its stack and calls Run(). Run() calls one method per +// phase, in order. The job reads DBImpl private members through db_. +// +// Terms: input, generation numbers of an input, input Version, census snapshot +// and stale job are defined in the overview. Two more terms: +// - GC job registration: the job registers with the census +// (BlobReferenceCensus::BeginGcJob()) and stays registered until Finish() +// (EndGcJob()). The census worker does not apply deltas while the job copies +// its census snapshot, and at most one GC job is registered per census. The +// job does not follow the Version changes after its snapshot (see "Install +// protocol" in Install()). +// - Candidate stamp (BlobGcCandidateStamp): the census worker and the file +// revision of each candidate when the job starts. The census ignores a +// suppression with a stamp from an older census worker or an older file +// revision (see "Suppression" in the overview). +// +// State: the phases share their state through the data members below. A data +// member has the name that the phase code uses for it, so the data members +// have no trailing underscore. +// +// Job status: the job has two status channels. The return value of Start() says +// only whether the job began: when it is not OK, or when gc_job_token is 0, the +// job did not begin, and Run() returns that value at once. After Start() began +// the job, the data member `status` is the job result until Run() returns. +// Start() can set it to an error and still return OK. Each phase after Start() +// also runs when `status` is not OK. Its loops check `status` first, so the +// phase does no new work, but it still runs its cleanup and its test sync +// points. Install() and Finish() decide what the job returns. +// +// Input outcomes: the outcome of one input is recorded in the places below. +// For each outcome, the list says where it is recorded, which methods set it, +// and which methods read it. +// - Accepted (relocate, or remove when it has no live value): eligible_inputs +// and PhysicalInput::eligible. DecideInputs() sets them. PackOutputs(), +// OpenInputReaders(), WriteOutputs() and Install() read them. +// - Suppressed: suppressed_files and suppression_reasons, two parallel vectors +// that each phase must keep in step. Any phase adds to them. Install() and +// Finish() read them. When `status` is CompactionTooLarge, Finish() adds +// capacity_deferred_files to suppressed_files (reason "deferred: outputs or +// disk space") and to transient_suppressions. Install() runs before that, so +// it does not log these reasons. +// - Suppressed with a backoff delay: also in transient_suppressions. Start(), +// SelectInputs(), DecideInputs() and Finish() add to it. Finish() reads it. +// - Failed in the read phase (only this input fails): InputPlan::failed. +// PlanLiveSetReads() and ReadLiveSets() set it (fail_input()). ReadLiveSets() +// skips the reads of a failed input, and DecideInputs() suppresses it. +// - Waits for a later job (memory budget): deferred_input and plan_end. +// PlanLiveSetReads() sets both. DecideInputs() and PackOutputs() also set +// deferred_input. ReadLiveSets() and DecideInputs() handle only the first +// plan_end inputs. Install() reads deferred_input and schedules the column +// family again. +// - Waits for a later job (outputs or disk space): capacity_deferred_files. +// PackOutputs() sets it, and also sets deferred_input. Install() and Finish() +// read it. +// - Candidate-local failure (ends the whole job): candidate_local_failure_input +// and candidate_local_failure_reason. OpenInputReaders() and WriteOutputs() +// set them. WriteOutputs() reads them and adds the input to suppressed_files. +// +// Lifecycle of the memory charges: the job charges most of its working memory +// to memory_tracker. The total is at most +// BlobRelocationMemoryTracker::kMaxBytes (64 MB). The charges are: +// - I/O workspace: the buffers that the job uses to read and stream the input +// values. Its charge is 2 x kIndirectBlobGCReadBatchBytes (1 MB each) plus +// 2 x kBlobRelocationStreamChunkSize (256 KB each), taken in SelectInputs(). +// Its share is held in a local variable (io_workspace_bytes), so it is freed +// only when the last copy of memory_tracker goes away, when the job is +// destroyed. The release Defers below keep copies of memory_tracker. +// - Planned read list (group_reads): taken in PlanLiveSetReads(). It grows by +// doubling, from 16 entries. release_group_read_charge frees it when the job +// is destroyed, after Run() returns, with the DB mutex held. +// - SST reader (selected_reader_bytes): 576 KB plus the decoded reference +// index, taken in ReadLiveSets() for one SST at a time. The 576 KB is +// kReaderBytes in ReadLiveSets(): a 64 KB reader estimate plus 2 x 256 KB +// read batches. It is freed after each SST and in Finish(). +// release_selected_reader_charge is only a safety net. +// - Live sets: 40 bytes per census reference, reserved in PlanLiveSetReads(). +// WriteOutputs() frees them. PlanLiveSetReads() frees the live set of a +// deferred or failed input at once. +// The census snapshot is charged to the same cache role, but not to +// memory_tracker. It is taken in Start(), and Finish() frees it. Neither +// release Defer is set when Start() ends the job. +class DBImpl::IndirectBlobGCJob { + public: + // The arguments are those of RunIndirectBlobGCBatch(), already checked. + IndirectBlobGCJob( + DBImpl* db, ColumnFamilyData* cfd_arg, + const std::vector& candidate_physical_file_numbers_arg, + bool* made_progress_arg, uint64_t* sfm_reserved_bytes_arg, + bool* retryable_stale_input_arg, LogBuffer* log_buffer_arg, + int job_id_arg) + : db_(db), + cfd(cfd_arg), + candidate_physical_file_numbers(candidate_physical_file_numbers_arg), + made_progress(made_progress_arg), + sfm_reserved_bytes(sfm_reserved_bytes_arg), + log_buffer(log_buffer_arg), + job_id(job_id_arg), + column_family(*cfd_arg), + stale_input_is_retryable(*retryable_stale_input_arg) {} + + // Runs the job and returns its status. The contract (DB mutex, out + // parameters, return codes) is on RunIndirectBlobGCBatch() in db_impl.h. + // Run() calls the methods below in this order. The word in brackets is the + // name of the job step in the overview of db/blob/blob_relocation_gc.h that + // the method belongs to. The first job step (the candidate pick) runs before + // Run(), in BackgroundCompaction(). + // 1. Start() [Plan]: check the census and the input Version, take a reference + // to the input Version, register the GC job and release the DB mutex. Then + // take the candidate stamp and the census snapshot. When the job cannot + // begin, Run() returns at once. + // 2. SelectInputs() [Plan]: build the inputs and their generation numbers, + // and suppress inputs with bad metadata or inputs that a coarse SST pins + // (see the coarse pin comment in SelectInputs()). + // 3. PlanLiveSetReads() [Plan]: plan the SST reads of each input and reserve + // its live set. + // 4. ReadLiveSets() [Read, Check]: read the planned generation numbers SST by + // SST and check the census totals. + // 5. DecideInputs() [Decide]: decide each input in order. + // 6. PackOutputs() [Pack]: pack the inputs into outputs, check the disk + // space, and get the output file numbers. + // 7. OpenInputReaders() [Copy]: open a reader for each input with live + // values. + // 8. WriteOutputs() [Copy]: write, sync and rename the outputs, then release + // the readers and the live sets. + // 9. Install() [Install]: take the DB mutex, build the MANIFEST edit and + // install it with one LogAndApply() call. + // 10. Finish() [no job step, cleanup]: end the GC job registration, release + // the input Version, suppress the skipped inputs, delete the outputs that + // are not installed, and schedule the census worker. + // + // REQUIRES: DB mutex held. Returns with the DB mutex held. + Status Run(); + + private: + // One indirect blob GC input: a normal blob file or a relocation file. + // `generations` holds its generation numbers: the input itself and, for a + // relocation file, every old file routed to it. + struct PhysicalInput { + uint64_t file_number = 0; + uint64_t physical_bytes = 0; + uint64_t total_record_bytes = 0; + std::shared_ptr normal_metadata; + std::shared_ptr relocation_metadata; + // Sorted: the input and the old files routed to it at job start. + std::vector generations; + // Sorted: the old files routed to the input at job start. + std::vector routed_generations; + std::unique_ptr live_set; + size_t live_count = 0; + uint64_t physical_live_record_bytes = 0; + uint64_t estimated_bytes = 0; + size_t output_index = std::numeric_limits::max(); + bool eligible = false; + }; + struct OutputState { + uint64_t file_number = 0; + std::string path; + std::string temporary_path; + std::string checksum_method; + std::string checksum_value; + BlobRelocationFileMetadata metadata; + BlobRelocationDirectory directory; + uint64_t file_size = 0; + uint64_t input_bytes = 0; + uint64_t value_bytes = 0; + size_t live_count = 0; + bool creation_started = false; + // A rename can fail after it took effect, so after a tried rename the final + // name may exist even when `renamed` is false. + bool rename_attempted = false; + bool renamed = false; + }; + // The live sets are planned, read and checked with the types below. See the + // comment at the start of PlanLiveSetReads(). + struct InputPlan { + bool planned = false; + bool failed = false; + bool transient = false; + const char* reason = nullptr; + }; + // One read: every group of one generation number in one SST, and the census + // record it is checked against. + struct GroupRead { + uint64_t sst_file_number = 0; + BlobReferenceGroup group; + size_t input_index = 0; + size_t check_index = 0; + }; + // The census record of one generation number, and what the SST reads found. + // After the reads, both must be equal for the same census Version. + struct GenerationCheck { + uint64_t generation = 0; + size_t input_index = 0; + BlobCensusGenerationTotals expected; + size_t expected_holders = 0; + BlobCensusGenerationTotals found; + size_t holders_found = 0; + }; + // The open readers of one input. Only one of the two is set. + struct InputReader { + std::unique_ptr normal; + CacheHandleGuard relocation; + }; + + // Takes a reference to the input Version (Ref()), checks that the census is + // ready for it, registers the GC job and releases the DB mutex. Then it + // takes the candidate stamp, finds the old files routed to each relocation + // input, and takes the census snapshot. When the job cannot begin, it + // releases the input Version if it took a reference, and returns the job's + // final status with the DB mutex held and no GC job registration + // (gc_job_token is 0): + // 1. TryAgain when there is no census or no input Version, or the census is + // not ready for the input Version. + // 2. Corruption when the table cache or the blob file cache is missing. + // 3. OK when BeginGcJob() returns Busy. For any other error, it returns that + // error after locked_census_status(). + // Otherwise it returns OK, sets `status` and returns without the DB mutex. + // REQUIRES: DB mutex held. + Status Start(); + + // Builds `inputs` and their generation numbers. It sets `status` to a stale + // TryAgain when a candidate changed. It suppresses an input with bad metadata + // or with a generation number pinned by a coarse SST. It also creates the + // memory tracker and charges the I/O workspace. When that charge fails, it + // gives every input a transient suppression. When the table format is not + // block-based, it gives every input a normal suppression. Called without the + // DB mutex. + void SelectInputs(); + + // Plans the SST reads of each input (group_reads, generation_checks) and + // reserves its live set. An input that does not fit the memory budget waits + // for a later job, with all later inputs (deferred_input), when an earlier + // input already holds a reservation. Otherwise its plan is marked failed and + // transient, so that DecideInputs() gives it a transient suppression. It sets + // census_mismatch when an SST that the census lists is not in the input + // Version. Called without the DB mutex. + void PlanLiveSetReads(); + + // Reads the planned generation numbers SST by SST into the live sets, then + // checks the totals found against the census. On a first census mismatch it + // takes the DB mutex for a short time to schedule a census rebuild. A + // mismatch that remains after a rebuild suppresses only that input. Called + // without the DB mutex. + void ReadLiveSets(); + + // Decides each planned input: relocate it, remove it (no live value), or + // suppress it. An accepted input joins eligible_inputs. When an input does + // not fit the memory budget after earlier inputs, it sets deferred_input. + // Called without the DB mutex. + void DecideInputs(); + + // Packs whole inputs into at most kIndirectBlobGCMaxOutputs (8) outputs and + // checks the disk space. When the inputs need more outputs or more space, the + // largest input waits (it moves from eligible_inputs to + // capacity_deferred_files). When only one input is left and the disk space + // check still fails, it sets `status` to CompactionTooLarge. It takes the DB + // mutex for a short time to get the output file numbers. Called without the + // DB mutex. + void PackOutputs(); + + // Opens a reader for each eligible input that has live values (`readers`). An + // input that cannot be opened (Corruption, MemoryLimit or NotSupported) + // becomes candidate_local_failure_input. A candidate-local failure in this + // phase or in WriteOutputs() ends the whole job: no output is installed, only + // the failed input is suppressed, and the other inputs stay candidates for a + // later job. Failures in the read phase (fail_input()) skip only their input. + // Called without the DB mutex. + void OpenInputReaders(); + + // Writes each output, then syncs and renames the outputs. Then it reports the + // outputs to the listeners, deletes the outputs that are not kept, and + // releases the readers and the live sets. A candidate-local failure here ends + // the whole job (see OpenInputReaders()). Called without the DB mutex. + // + // Error priority: a candidate-local failure blames one input. It is a source + // value that is corrupt or unsupported (Corruption or NotSupported; in the + // batched path also MemoryLimit, for a value that is too large), or an output + // that is larger than its inputs (the first input of that output is blamed). + // A source error of another kind (for example an I/O error) and an output + // write error have priority over it: the job returns that error, and + // candidate_local_failure_input stays unset. In the body, the test + // `candidate_local_failure_input == SIZE_MAX` means "no candidate-local + // failure yet". + void WriteOutputs(); + + // Takes the DB mutex. It ends the job without an edit when `status` is not + // OK, when the column family was dropped, or when the job is interrupted. On + // CompactionTooLarge it also defers the waiting inputs in the current + // Version, if the Version did not change. When no input is left to install, + // it suppresses the skipped inputs in the current Version and keeps `status` + // OK. Otherwise it releases the DB mutex to build the MANIFEST edit, and + // installs the edit with one LogAndApply() call that has a precondition (see + // "Install protocol" in the body). It marks in delete_renamed_outputs the + // outputs that Finish() must delete. Called without the DB mutex. Returns + // with the DB mutex held. + void Install(); + // Install() helpers: BuildEdit() builds the MANIFEST edit, and + // ValidatePhysicalInputs() is the precondition of its LogAndApply() call. + // See their definitions. + Status BuildEdit(); + Status ValidatePhysicalInputs(); + + // Ends the job: ends the GC job registration (EndGcJob()), releases the + // input Version, and suppresses the skipped inputs in the census heap. On + // CompactionTooLarge the inputs in capacity_deferred_files get a transient + // suppression. It deletes the renamed outputs marked in + // delete_renamed_outputs, releases most of the job's memory, and schedules + // the census worker (ScheduleBlobReferenceCensus()). It releases the DB mutex + // during this cleanup. The I/O workspace charge and the group read charge + // stay until the job is destroyed, with the DB mutex held (see "Lifecycle" + // above the class). Returns `status`. + // REQUIRES: DB mutex held. Returns with the DB mutex held. + Status Finish(); + + // Helpers that several phases use. Their comments are at their + // definitions. + Status stale_input(const char* message); + Status invariant_failed(const char* message); + Status unlocked_census_status(Status census_status); + Status locked_census_status(Status census_status); + Status interrupted(); + void suppress_candidates_transiently(const char* reason); + void fail_input(size_t input_index, const char* reason); + + DBImpl* const db_; + + // The arguments of RunIndirectBlobGCBatch(). + ColumnFamilyData* const cfd; + const std::vector& candidate_physical_file_numbers; + bool* const made_progress; + uint64_t* const sfm_reserved_bytes; + LogBuffer* const log_buffer; + const int job_id; + + // column_family is *cfd, and stale_input_is_retryable is the + // retryable_stale_input argument of RunIndirectBlobGCBatch(). + ColumnFamilyData& column_family; + bool& stale_input_is_retryable; + + // Set by Start(), except `inputs`, which SelectInputs() fills. input_version + // is the input Version: Start() takes a reference to it (Ref()), and Finish() + // releases it. census is the job's census snapshot (defined in the overview + // in db/blob/blob_relocation_gc.h), not the census itself, which is + // blob_reference_census. + BlobReferenceCensus* blob_reference_census = nullptr; + Version* input_version = nullptr; + uint64_t input_version_number = 0; + std::shared_ptr census; + VersionStorageInfo* input_storage = nullptr; + std::vector inputs; + MutableCFOptions mutable_cf_options; + bool allow_value_relocation = false; + TableCache* table_cache = nullptr; + BlobFileCache* blob_file_cache = nullptr; + FileOptions blob_gc_file_options; + std::string output_directory; + + // read_options is for the MANIFEST write (LogAndApply()). gc_read_options is + // for the SST and blob reads and for the output table builder: it does not + // fill the block cache and uses Env::IO_LOW. + ReadOptions read_options; + ReadOptions gc_read_options; + WriteOptions write_options; + WriteOptions manifest_write_options; + + // A copy of the background error, taken under the DB mutex in Start(). + // PackOutputs() runs without the DB mutex and passes it to the + // SstFileManager. + Status sfm_background_error; + + // The token of the job's census registration (BeginGcJob()). 0 means that + // the job is not registered. + uint64_t gc_job_token = 0; + + // The job status (see "Job status" in the class comment). + Status status; + std::shared_ptr gc_block_cache; + + // The inputs to suppress when the job ends, and the reason for each. The + // files in transient_suppressions get a transient suppression. Any phase can + // add to them. Install() and Finish() use them. + std::vector suppressed_files; + std::vector suppression_reasons; + std::set transient_suppressions; + BlobGcCandidateStamp candidate_stamp; + + // The old files routed to each relocation input, sorted. + std::map> routed_generations_by_input; + + // source_table_options, memory_tracker and release_selected_reader_charge are + // set by SelectInputs(). eligible_inputs holds the indexes in `inputs` of the + // inputs that DecideInputs() accepts. DecideInputs() (on a live-set failure), + // PackOutputs() and WriteOutputs() can remove inputs from it. + // selected_reader_bytes is the charge of the SST reader that ReadLiveSets() + // holds open. deferred_input is true when an input waits for a later job (set + // by PlanLiveSetReads(), DecideInputs() or PackOutputs()). + std::optional source_table_options; + std::shared_ptr memory_tracker; + std::vector eligible_inputs; + size_t selected_reader_bytes = 0; + std::optional release_selected_reader_charge; + bool deferred_input = false; + + // Set by PlanLiveSetReads() and ReadLiveSets(). The planned inputs are the + // first plan_end inputs. + std::vector generation_checks; + bool census_mismatch = false; + + // The input whose census check failed (SIZE_MAX: none). + size_t mismatched_input = std::numeric_limits::max(); + std::vector plans; + std::vector group_reads; + size_t group_read_bytes = 0; + std::optional release_group_read_charge; + size_t plan_end = 0; + + // Set by PackOutputs(). bins[i] lists the inputs of outputs[i]. + uint64_t aggregate_input_bytes = 0; + std::vector capacity_deferred_files; + std::vector bins; + std::vector outputs; + + // Set by OpenInputReaders() and WriteOutputs(). An input that fails during + // reading or writing is candidate_local_failure_input (SIZE_MAX: none). + std::vector readers; + size_t candidate_local_failure_input = std::numeric_limits::max(); + const char* candidate_local_failure_reason = nullptr; + + // Set by WriteOutputs() and Install(). + std::unique_ptr edit; + std::array output_needed{}; + size_t published_output_count = 0; + size_t published_relocated_count = 0; + uint64_t published_relocated_value_bytes = 0; + // read_input_count is the number of inputs that WriteOutputs() read and + // packed. published_input_count is the number of inputs that the installed + // edit removed; it stays 0 when the job installs nothing. + size_t read_input_count = 0; + size_t published_input_count = 0; + std::array delete_renamed_outputs{}; + + // Monitoring (tickers, the job histogram and the end-of-job line). Run() + // sets start_micros and reads the others after Finish(). WriteOutputs() adds + // to input_bytes_read and Install() sets bytes_written and installed. + uint64_t start_micros = 0; + uint64_t input_bytes_read = 0; + uint64_t bytes_written = 0; + bool installed = false; + // Inputs skipped because of corruption: file number -> what was found. + // `logged` is true when the code that found it already wrote an ERROR line. + // Finish() writes the other lines and counts INDIRECT_BLOB_GC_CORRUPT_INPUTS. + struct CorruptInput { + std::string detail; + bool logged = false; + }; + std::map corrupt_inputs; + void note_corruption(uint64_t file_number, const Status& s, + bool logged = false) { + CorruptInput& entry = corrupt_inputs[file_number]; + if (entry.detail.empty()) { + entry.detail = s.ToString(); + } + entry.logged = entry.logged || logged; + } + // Records the job outcome (tickers, histogram, end-of-job line). REQUIRES: + // DB mutex held. + void RecordOutcome(const Status& result, size_t skipped_count); +}; + +Status DBImpl::RunIndirectBlobGCBatch( + ColumnFamilyData* cfd, + const std::vector& candidate_physical_file_numbers, + bool* made_progress, uint64_t* sfm_reserved_bytes, + bool* retryable_stale_input, LogBuffer* log_buffer, int job_id) { + mutex_.AssertHeld(); + if (cfd == nullptr || made_progress == nullptr || + sfm_reserved_bytes == nullptr || retryable_stale_input == nullptr) { + return Status::InvalidArgument("Null indirect blob GC batch argument"); + } + if (candidate_physical_file_numbers.empty() || + candidate_physical_file_numbers.size() > kIndirectBlobGCMaxInputs) { + return Status::InvalidArgument("Invalid indirect blob GC batch size"); + } + IndirectBlobGCJob job(this, cfd, candidate_physical_file_numbers, + made_progress, sfm_reserved_bytes, + retryable_stale_input, log_buffer, job_id); + return job.Run(); +} + +Status DBImpl::IndirectBlobGCJob::Run() { + db_->mutex_.AssertHeld(); + Status start_status = Start(); + if (!start_status.ok() || gc_job_token == 0) { + // The job stops before SelectInputs() and Finish(), so nothing reads + // `status` or the copy of the background error. + status.PermitUncheckedError(); + sfm_background_error.PermitUncheckedError(); + return start_status; + } + RecordTick(db_->stats_, INDIRECT_BLOB_GC_JOBS_STARTED); + start_micros = db_->immutable_db_options_.clock->NowMicros(); + cfd->ClearIndirectBlobGcCandidates(); + SelectInputs(); + PlanLiveSetReads(); + ReadLiveSets(); + DecideInputs(); + PackOutputs(); + OpenInputReaders(); + WriteOutputs(); + Install(); + // Finish() clears suppressed_files, so count the skipped inputs first. The + // inputs that Finish() defers for outputs or disk space count too. + const size_t skipped_count = + suppressed_files.size() + + (status.IsCompactionTooLarge() ? capacity_deferred_files.size() : 0); + Status result = Finish(); + RecordOutcome(result, skipped_count); + return result; +} + +Status DBImpl::IndirectBlobGCJob::Start() { + assert(*sfm_reserved_bytes == 0); + stale_input_is_retryable = false; + + blob_reference_census = cfd->blob_reference_census(); + if (blob_reference_census == nullptr) { + return stale_input("Blob reference census is unavailable"); + } + + input_version = cfd->current(); + if (input_version == nullptr) { + return stale_input("Indirect blob GC input Version is unavailable"); + } + input_version->Ref(); + input_version_number = input_version->GetVersionNumber(); + if (!blob_reference_census->IsReady(input_version_number)) { + input_version->UnrefAndFreeBlobRelocationState(&db_->mutex_); + return stale_input("Blob reference census is not ready"); + } + + input_storage = input_version->storage_info(); + inputs.reserve(candidate_physical_file_numbers.size()); + mutable_cf_options = input_version->GetMutableCFOptions(); + const MutableCFOptions& latest_options = cfd->GetLatestMutableCFOptions(); + allow_value_relocation = latest_options.enable_blob_files && + latest_options.enable_blob_indirection && + latest_options.enable_blob_garbage_collection; + table_cache = cfd->table_cache(); + blob_file_cache = cfd->blob_file_cache(); + if (table_cache == nullptr || blob_file_cache == nullptr) { + input_version->UnrefAndFreeBlobRelocationState(&db_->mutex_); + return Status::Corruption("Missing indirect blob GC reader cache"); + } + blob_gc_file_options = db_->file_options_for_compaction_; + output_directory = cfd->ioptions().cf_paths.front().path; + read_options = ReadOptions(Env::IOActivity::kCompaction); + gc_read_options = read_options; + gc_read_options.fill_cache = false; + gc_read_options.rate_limiter_priority = Env::IO_LOW; + write_options = WriteOptions(Env::IO_LOW, Env::IOActivity::kCompaction); + // Indirect blob GC data writes use IO_LOW. The MANIFEST write uses the + // default priority, so a MANIFEST write at the head of the writer queue does + // not wait behind the background rate limiter. + manifest_write_options = WriteOptions(Env::IOActivity::kCompaction); + sfm_background_error = db_->error_handler_.GetBGError(); + status = + blob_reference_census->BeginGcJob(input_version_number, &gc_job_token); + if (!status.ok()) { + input_version->UnrefAndFreeBlobRelocationState(&db_->mutex_); + if (status.IsBusy()) { + // Busy: an earlier metadata commit has an unknown outcome until recovery, + // or another indirect blob GC job is registered. Retrying now would only + // spin; a later Version picks GC again. + return Status::OK(); + } + // TryAgain: a metadata commit is in progress, or the census is not ready + // for this Version; retry later. + stale_input_is_retryable = status.IsTryAgain(); + return locked_census_status(std::move(status)); + } + + db_->mutex_.Unlock(); + + // Indirect blob GC working memory, including the selected census snapshot, is + // charged to its own cache role, + // CacheEntryRole::kIndirectBlobGcWorkingMemory. + gc_block_cache = GetIndirectBlobGcWorkingMemoryCache(mutable_cf_options); + + status = blob_reference_census->CaptureCandidateStamp( + input_version_number, candidate_physical_file_numbers, &candidate_stamp); + if (status.IsTryAgain()) { + // The census changed or was canceled (for example, the CF was dropped). + stale_input_is_retryable = true; + } + // The old files routed to each relocation input, from one scan of the input + // Version's routes (without the DB mutex). selected_sources holds the + // generation numbers that the census snapshot copies: each candidate, and + // each old file routed to a relocation candidate. + std::set selected_sources; + for (uint64_t file_number : candidate_physical_file_numbers) { + selected_sources.insert(file_number); + if (input_storage->PeekBlobRelocationFileMetaData(file_number) != nullptr) { + routed_generations_by_input[file_number]; + } + } + if (!routed_generations_by_input.empty()) { + input_storage->ForEachBlobRelocationRoute( + [&](const BlobRelocationRoute& route) { + const auto input = + routed_generations_by_input.find(route.GetTargetFileNumber()); + if (input != routed_generations_by_input.end()) { + input->second.push_back(route.GetOldFileNumber()); + selected_sources.insert(route.GetOldFileNumber()); + } + }); + for (auto& input : routed_generations_by_input) { + std::sort(input.second.begin(), input.second.end()); + } + } + if (status.ok()) { + status = blob_reference_census->SnapshotSelected( + gc_job_token, input_version_number, selected_sources, gc_block_cache, + &census); + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:SnapshotSelected", + &status); + if (status.IsMemoryLimit()) { + suppress_candidates_transiently( + "selected census exceeds its memory budget"); + status = stale_input("Blob GC census snapshot exceeds the memory budget"); + } else { + status = unlocked_census_status(std::move(status)); + if (status.IsTryAgain()) { + stale_input_is_retryable = true; + } + } + } + selected_sources.clear(); + return Status::OK(); +} + +void DBImpl::IndirectBlobGCJob::SelectInputs() { + if (status.ok()) { + status = interrupted(); + } + for (uint64_t file_number : candidate_physical_file_numbers) { + if (!status.ok()) { + break; + } + PhysicalInput input; + input.file_number = file_number; + input.normal_metadata = input_storage->GetBlobFileMetaData(file_number); + input.relocation_metadata = + input_storage->GetBlobRelocationFileMetaData(file_number); + if ((input.normal_metadata == nullptr) == + (input.relocation_metadata == nullptr)) { + status = stale_input("Indirect blob GC candidate changed"); + break; + } + input.generations.push_back(file_number); + if (input.normal_metadata != nullptr) { + if (!input.normal_metadata->IsIndirectOrigin() || + input.normal_metadata->GetTotalBlobCount() == 0) { + status = stale_input("Indirect blob GC indirect-origin source changed"); + break; + } + input.physical_bytes = input.normal_metadata->GetBlobFileSize(); + input.total_record_bytes = input.normal_metadata->GetTotalBlobBytes(); + } else { + // The input Version is this column family's, so it owns the file. + input.physical_bytes = input.relocation_metadata->GetFileSize(); + input.total_record_bytes = + input.relocation_metadata->GetTotalRecordBytes(); + input.routed_generations = routed_generations_by_input[file_number]; + input.generations.insert(input.generations.end(), + input.routed_generations.begin(), + input.routed_generations.end()); + std::sort(input.generations.begin(), input.generations.end()); + } + // A coarse SST pins indirect-origin blob file numbers (see "Coarse SST" in + // blob_reference_metadata.h), so indirect blob GC skips a pinned input: + // a normal input whose file number is pinned, or a relocation input with + // a pinned routed generation number (an old indirect-origin file number + // that routes to it). In normal operation a coarse SST does not reference + // a routed number, but GC keeps this pin so that it never collects a file + // that a coarse SST may still read. When the pin goes away, the heap + // gives the input a new revision and GC can pick it again. Suppressing + // the input also removes it from the Version's GC candidate list, so a + // job with no output does not reschedule itself. + const bool coarse_pinned = + input.normal_metadata != nullptr + ? input_storage->IsCoarseIndirectOriginBlobPinned(file_number) + : std::any_of( + input.routed_generations.begin(), + input.routed_generations.end(), [&](uint64_t generation) { + return input_storage->IsCoarseIndirectOriginBlobPinned( + generation); + }); + if (coarse_pinned) { + suppressed_files.push_back(file_number); + suppression_reasons.emplace_back( + "coarse SST pins a generation number of the input"); + continue; + } + if (input.physical_bytes == 0 || input.total_record_bytes == 0) { + // Bad metadata affects only this input: suppress it, and keep other + // background work running. + ROCKS_LOG_ERROR(db_->immutable_db_options_.info_log, + "[%s] Indirect blob GC skipped blob file #%" PRIu64 + ": invalid input metadata", + cfd->GetName().c_str(), file_number); + note_corruption(file_number, Status::Corruption("invalid input metadata"), + /*logged=*/true); + suppressed_files.push_back(file_number); + suppression_reasons.emplace_back("invalid input metadata"); + continue; + } + inputs.push_back(std::move(input)); + } + + const TableFactory* const table_factory = + mutable_cf_options.table_factory.get(); + if (table_factory != nullptr) { + const BlockBasedTableOptions* options = + table_factory->GetOptions(); + const AdaptiveTableFactory* const adaptive_table_factory = + table_factory->CheckedCast(); + if (adaptive_table_factory != nullptr) { + options = adaptive_table_factory->GetBlockBasedTableWriterOptions(); + } + if (options != nullptr) { + source_table_options.emplace(*options); + } + } + + std::shared_ptr reservation_manager; + if (gc_block_cache != nullptr) { + reservation_manager = std::make_shared>(gc_block_cache); + } + memory_tracker = std::make_shared( + std::move(reservation_manager)); + bool workspace_charge_failed = false; + // Charge the I/O workspace, the buffers that the job uses to read and stream + // the input values. The charge has 2 x kIndirectBlobGCReadBatchBytes for the + // read buffer of one batch, which holds the batch's stored bytes plus the + // dead gaps that coalescing merges into its reads (at most one more batch). + // Payloads are slices of this buffer. The charge also has 2 x + // kBlobRelocationStreamChunkSize (256 KB each) for values that are streamed + // in chunks, one value at a time: the chunk buffer of the stream reader + // (StreamAndVerifyBlobStoredPayload), and, with direct I/O, the aligned + // buffer that RandomAccessFileReader::Read allocates for each chunk. A + // streamed value is copied between batches, never during one. Charge the + // workspace before allocating exact live sets or read buffers. + size_t io_workspace_bytes = 0; + if (status.ok() && allow_value_relocation) { + const Status reservation_status = memory_tracker->SetReservation( + &io_workspace_bytes, + 2 * kIndirectBlobGCReadBatchBytes + 2 * kBlobRelocationStreamChunkSize); + if (!reservation_status.ok()) { + for (const PhysicalInput& input : inputs) { + suppressed_files.push_back(input.file_number); + transient_suppressions.insert(input.file_number); + suppression_reasons.emplace_back( + "blob GC I/O workspace cache pressure"); + } + workspace_charge_failed = true; + source_table_options.reset(); + } + } + + release_selected_reader_charge.emplace( + [memory_tracker = memory_tracker, + &selected_reader_bytes = selected_reader_bytes] { + memory_tracker->SetReservation(&selected_reader_bytes, 0) + .PermitUncheckedError(); + }); + // The workspace charge failure above already suppressed every input + // (transient), so suppress them for the table format only when the format + // is the reason. + if (status.ok() && !source_table_options.has_value() && + !workspace_charge_failed) { + for (const PhysicalInput& input : inputs) { + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back("block-based table format is required"); + } + } +} + +void DBImpl::IndirectBlobGCJob::PlanLiveSetReads() { + // The live sets are read SST by SST. First each input's reads are planned and + // its live set reserved, in input order; then each selected SST is opened + // once, its reference index decoded once and each needed data block read at + // most once; then each input is decided in order. + plans.resize(inputs.size()); + release_group_read_charge.emplace([memory_tracker = memory_tracker, + &group_read_bytes = group_read_bytes] { + memory_tracker->SetReservation(&group_read_bytes, 0).PermitUncheckedError(); + }); + bool reserved_any = false; + + // Plan the reads of each input and reserve its live set from the census + // reference counts of its generation numbers. When an input does not fit the + // memory budget, it and all later inputs wait for a later job if an earlier + // input holds a reservation. Otherwise the input is suppressed with a retry + // time (transient). + for (size_t input_index = 0; + status.ok() && source_table_options.has_value() && + input_index < inputs.size(); + ++input_index) { + status = interrupted(); + if (!status.ok()) { + break; + } + PhysicalInput& input = inputs[input_index]; + plan_end = input_index + 1; + InputPlan& plan = plans[input_index]; + plan.planned = true; + input.live_set = std::make_unique(memory_tracker, + input.generations); + const size_t first_group_read = group_reads.size(); + const size_t first_generation_check = generation_checks.size(); + uint64_t live_count = 0; + for (uint64_t generation : input.generations) { + const std::vector* const sst_files = + census->FindSstFiles(generation); + const BlobCensusGenerationTotals* const totals = + census->FindTotals(generation); + if (sst_files == nullptr) { + continue; + } + // The live set is reserved from the census totals. + if (totals->reference_count > std::numeric_limits::max() || + totals->reference_count > + std::numeric_limits::max() - live_count) { + status = Status::Corruption("Exact blob reference count overflow"); + fail_input(input_index, "exact SST metadata is inconsistent"); + break; + } + live_count += totals->reference_count; + const size_t check_index = generation_checks.size(); + generation_checks.push_back( + {generation, input_index, *totals, sst_files->size(), {}, 0}); + BlobReferenceGroup key; + key.generation_file_number = generation; + for (size_t i = 0; status.ok() && i < sst_files->size(); ++i) { + if ((i & kIndirectBlobGCAbortCheckMask) == 0) { + status = interrupted(); + if (!status.ok()) { + break; + } + } + const uint64_t sst_file_number = (*sst_files)[i]; + if (input_storage->GetFileMetaDataByNumber(sst_file_number) == + nullptr) { + census_mismatch = true; + mismatched_input = input_index; + break; + } + if (group_reads.size() == group_reads.capacity()) { + const size_t capacity = + group_reads.capacity() == 0 ? 16 : 2 * group_reads.capacity(); + status = memory_tracker->SetReservation(&group_read_bytes, + capacity * sizeof(GroupRead)); + if (!status.ok()) { + break; + } + group_reads.reserve(capacity); + } + group_reads.push_back({sst_file_number, key, input_index, check_index}); + } + if (!status.ok() || census_mismatch) { + break; + } + } + if (status.ok() && !census_mismatch) { + status = input.live_set->Reserve(static_cast(live_count)); + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:ReserveLiveSet", + &status); + } + if (!status.ok()) { + // A deferred or failed input reads nothing, so it has no totals to check + // either. + group_reads.resize(first_group_read); + generation_checks.resize(first_generation_check); + input.live_set.reset(); + } + // Only the reference count overflow check above sets both Corruption and + // plan.failed. The cleanup above has run, so only this input fails. + if (status.IsCorruption() && plan.failed) { + status = Status::OK(); + continue; + } + if (status.IsMemoryLimit()) { + status = Status::OK(); + if (reserved_any) { + plan.planned = false; + plan_end = input_index; + deferred_input = true; + break; + } + plan.failed = true; + plan.transient = true; + plan.reason = "exact live set exceeds its memory budget"; + continue; + } + if (!status.ok() || census_mismatch) { + break; + } + reserved_any = true; + } +} + +void DBImpl::IndirectBlobGCJob::ReadLiveSets() { + // Read the planned generation numbers SST by SST. Corruption in a group or in + // an SST's metadata fails only the inputs that read it. + std::sort(group_reads.begin(), group_reads.end(), + [](const GroupRead& lhs, const GroupRead& rhs) { + return std::make_tuple(lhs.sst_file_number, + lhs.group.generation_file_number) < + std::make_tuple(rhs.sst_file_number, + rhs.group.generation_file_number); + }); + std::vector batch; + std::vector batch_inputs; + std::vector batch_reads; + for (size_t begin = 0; + status.ok() && !census_mismatch && begin < group_reads.size();) { + size_t end = begin; + while (end < group_reads.size() && group_reads[end].sst_file_number == + group_reads[begin].sst_file_number) { + ++end; + } + status = interrupted(); + if (!status.ok()) { + break; + } + batch.clear(); + batch_inputs.clear(); + batch_reads.clear(); + for (size_t i = begin; status.ok() && i < end; ++i) { + const GroupRead& read = group_reads[i]; + if (plans[read.input_index].failed) { + continue; + } + BlobReferenceGroupRead group_read; + group_read.group = &read.group; + group_read.whole_generation = true; + group_read.sink = inputs[read.input_index].live_set.get(); + batch.push_back(std::move(group_read)); + batch_inputs.push_back(read.input_index); + batch_reads.push_back(i); + } + const uint64_t sst_file_number = group_reads[begin].sst_file_number; + begin = end; + if (!status.ok() || batch.empty()) { + continue; + } + const FileMetaData* const file = + input_storage->GetFileMetaDataByNumber(sst_file_number); + // One reader is held at a time. Its charge is 64 KB for the table reader + // itself (an estimate, also charged when the reader comes from the table + // cache, to keep the charge simple), plus 2 x kBlobReferenceReadBatchBytes + // (256 KB each, 512 KB in total) for one batch of raw blocks and its + // decompressed copies. The decoded reference index is added at its real + // size once it is decoded: it grows with the groups and blocks of the SST + // (one BlobReferenceIndexEntry per group), so an SST with many groups needs + // more than any fixed estimate. + constexpr size_t kReaderBytes = + 64 * 1024 + 2 * kBlobReferenceReadBatchBytes; + status = + memory_tracker->SetReservation(&selected_reader_bytes, kReaderBytes); + TableCache::BlobReferenceReader reader; + if (status.ok()) { + status = table_cache->GetBlobReferenceReader( + gc_read_options, blob_gc_file_options, cfd->internal_comparator(), + *file, mutable_cf_options, &reader); + } + if (status.ok()) { + status = reader->ReadBlobReferences( + gc_read_options, file->blob_reference_descriptor, + [&](size_t index_bytes) { + if (index_bytes > + std::numeric_limits::max() - kReaderBytes) { + return Status::MemoryLimit("Blob reference index is too large"); + } + return memory_tracker->SetReservation(&selected_reader_bytes, + kReaderBytes + index_bytes); + }, + &batch); + } + reader.Reset(); + Status release_status = + memory_tracker->SetReservation(&selected_reader_bytes, 0); + release_status.PermitUncheckedError(); + // Reports the batch size and the reader charge left after closing. + std::pair batch_report{batch.size(), selected_reader_bytes}; + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:SstBatchRead", + &batch_report); + if (status.IsCorruption()) { + for (size_t input_index : batch_inputs) { + fail_input(input_index, "exact SST metadata is corrupt"); + note_corruption(inputs[input_index].file_number, status); + } + status = Status::OK(); + continue; + } + if (!status.ok()) { + break; + } + for (size_t i = 0; i < batch.size(); ++i) { + if (batch[i].status.IsNotFound() || batch[i].status.IsAborted()) { + // The SST lacks the generation number, or has more references than the + // census. + census_mismatch = true; + mismatched_input = batch_inputs[i]; + break; + } + if (batch[i].status.IsCorruption()) { + fail_input(batch_inputs[i], "exact SST metadata is corrupt"); + note_corruption(inputs[batch_inputs[i]].file_number, batch[i].status); + } else if (!batch[i].status.ok()) { + status = batch[i].status; + break; + } else { + GenerationCheck& check = + generation_checks[group_reads[batch_reads[i]].check_index]; + check.found.reference_count += batch[i].found_reference_count; + check.found.stored_value_bytes += batch[i].found_stored_value_bytes; + check.found.record_bytes += batch[i].found_record_bytes; + ++check.holders_found; + } + } + } + // The census totals check: for each generation number of an input that read + // all its SSTs, the number of SSTs found that hold the generation number, and + // their totals, must equal the census record. A first mismatch rebuilds the + // census and retries the job. A mismatch that remains after a rebuild + // suppresses only that input (see below). Only inputs that were planned, read + // and not failed are checked. + const GenerationCheck* mismatched_check = nullptr; + if (status.ok() && !census_mismatch) { + // A normal input that no SST holds in the census, while an SST still links + // to it as its oldest blob file: do not trust the census to remove the + // input (blob file retirement) with no output. + for (size_t input_index = 0; input_index < plan_end; ++input_index) { + const PhysicalInput& input = inputs[input_index]; + if (plans[input_index].planned && !plans[input_index].failed && + input.normal_metadata != nullptr && + !input.normal_metadata->GetLinkedSsts().empty() && + census->FindSstFiles(input.file_number) == nullptr) { + census_mismatch = true; + mismatched_input = input_index; + break; + } + } + } + if (status.ok() && !census_mismatch) { + for (const GenerationCheck& check : generation_checks) { + if (check.input_index >= plan_end || !plans[check.input_index].planned || + plans[check.input_index].failed) { + continue; + } + if (check.holders_found != check.expected_holders || + check.found.reference_count != check.expected.reference_count || + check.found.stored_value_bytes != check.expected.stored_value_bytes || + check.found.record_bytes != check.expected.record_bytes) { + census_mismatch = true; + mismatched_check = ✓ + mismatched_input = check.input_index; + break; + } + } + } + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:CensusTotalsCheck", + &census_mismatch); + if (status.ok() && !census_mismatch) { + // Every checked input matched the census. Forget a remembered mismatch of + // such an input, so that a later mismatch of it requests a rebuild again + // (see BlobReferenceCensus::ReportMismatch()). + for (size_t input_index = 0; input_index < plan_end; ++input_index) { + if (plans[input_index].planned && !plans[input_index].failed) { + blob_reference_census->ClearMismatch(inputs[input_index].file_number); + } + } + } + if (census_mismatch && mismatched_input >= inputs.size() && plan_end > 0) { + // Every real mismatch names its input; a test-injected one is charged to + // the first input. + mismatched_input = 0; + } + if (census_mismatch) { + const uint64_t mismatched_file = mismatched_input < inputs.size() + ? inputs[mismatched_input].file_number + : 0; + RecordTick(db_->stats_, INDIRECT_BLOB_GC_CENSUS_MISMATCHES); + if (blob_reference_census->ReportMismatch(mismatched_file)) { + // The rebuild did not clear the mismatch of this input. Skip only this + // input until the census changes its live bytes (that is, until its + // references change), rather than rebuilding at every Version. This job + // decides no input and publishes nothing; it is not retried at once, so + // the next job, after the census drops this input from its candidates, + // takes the other inputs. + if (mismatched_check != nullptr) { + ROCKS_LOG_ERROR( + db_->immutable_db_options_.info_log, + "[%s] Blob GC skips blob file #%" PRIu64 + " until its references change: the census rebuild did not clear " + "a mismatch for generation number %" PRIu64 + ": holders %" ROCKSDB_PRIszt "/%" ROCKSDB_PRIszt + ", references %" PRIu64 "/%" PRIu64 " (found/expected)", + cfd->GetName().c_str(), mismatched_file, + mismatched_check->generation, mismatched_check->holders_found, + mismatched_check->expected_holders, + mismatched_check->found.reference_count, + mismatched_check->expected.reference_count); + } else { + ROCKS_LOG_ERROR(db_->immutable_db_options_.info_log, + "[%s] Blob GC skips blob file #%" PRIu64 + " until its references change: the census rebuild " + "did not clear a mismatch", + cfd->GetName().c_str(), mismatched_file); + } + note_corruption( + mismatched_file, + Status::Corruption("census mismatch remains after a rebuild"), + /*logged=*/true); + suppressed_files.push_back(mismatched_file); + suppression_reasons.emplace_back( + "census mismatch remains after a rebuild"); + plan_end = 0; + } else { + ROCKS_LOG_ERROR(db_->immutable_db_options_.info_log, + "[%s] Blob GC found SSTs that do not match the blob " + "reference census for blob file #%" PRIu64 + "; the census is rebuilt", + cfd->GetName().c_str(), mismatched_file); + { + InstrumentedMutexLock lock(&db_->mutex_); + db_->ScheduleBlobReferenceCensus(cfd); + } + status = stale_input("Blob reference census does not match the SSTs"); + } + } + if (status.IsMemoryLimit()) { + // Reader and request charges come from the same budget as the live sets. + status = Status::OK(); + for (size_t input_index = 0; input_index < plan_end; ++input_index) { + if (plans[input_index].planned && !plans[input_index].failed) { + fail_input(input_index, "exact live set exceeds its memory budget"); + plans[input_index].transient = true; + } + } + } + + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:PlanEnd", &plan_end); +} + +void DBImpl::IndirectBlobGCJob::DecideInputs() { + // Decide each input in order. + for (size_t input_index = 0; status.ok() && input_index < plan_end; + ++input_index) { + PhysicalInput& input = inputs[input_index]; + InputPlan& plan = plans[input_index]; + if (!plan.planned) { + continue; + } + if (plan.transient) { + transient_suppressions.insert(input.file_number); + } + bool candidate_local_failure = plan.failed; + const char* candidate_reason = plan.reason; + if (!candidate_local_failure) { + status = input.live_set->Finalize(); + if (status.IsCorruption()) { + candidate_local_failure = true; + candidate_reason = "exact SST metadata is corrupt"; + note_corruption(input.file_number, status); + } + } + uint64_t input_live_record_bytes = 0; + if (status.ok() && !candidate_local_failure) { + input.live_count = input.live_set->references().size(); + input_live_record_bytes = input.live_set->live_record_bytes(); + if (input.relocation_metadata != nullptr) { + // A relocation file holds each value as stored bytes and a trailer. + if (input.live_count > (std::numeric_limits::max() - + input.live_set->live_stored_value_bytes()) / + kSimpleGen2BlobTrailerSize) { + status = Status::Corruption("Relocation live byte count overflow"); + candidate_local_failure = true; + candidate_reason = "exact SST metadata is corrupt"; + note_corruption(input.file_number, status); + } else { + input_live_record_bytes = + input.live_set->live_stored_value_bytes() + + input.live_count * kSimpleGen2BlobTrailerSize; + } + } + input.physical_live_record_bytes = input_live_record_bytes; + } + if (status.IsMemoryLimit()) { + if (!eligible_inputs.empty()) { + status = Status::OK(); + deferred_input = true; + break; + } + candidate_local_failure = true; + candidate_reason = "exact live set exceeds its memory budget"; + transient_suppressions.insert(input.file_number); + } + if (candidate_local_failure) { + status = Status::OK(); + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back(candidate_reason); + continue; + } + if (!status.ok()) { + break; + } + if (input_live_record_bytes > input.total_record_bytes) { + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back( + "exact live bytes exceed physical record bytes"); + continue; + } + if (input.relocation_metadata != nullptr && + input.relocation_metadata->IsMarkedLost() && + input_live_record_bytes != 0) { + // A lost relocation file (an accepted loss, recorded with + // VersionEdit::MarkBlobPhysicalFileLost()) is never read in this + // Version, so GC cannot copy the values that SSTs still reference. + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back( + "lost relocation file still has SST references"); + continue; + } + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Indirect blob GC exact candidate #%" PRIu64 + ": live=%" PRIu64 " total=%" PRIu64, + cfd->GetName().c_str(), input.file_number, + input_live_record_bytes, input.total_record_bytes); + if (!allow_value_relocation && input_live_record_bytes != 0) { + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back("live-value relocation is disabled"); + continue; + } + if (input.total_record_bytes - input_live_record_bytes < + input_live_record_bytes) { + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back( + "exact garbage ratio is below 50 percent"); + continue; + } + if (input.live_count != 0) { + Status estimate_status = EstimateIndirectBlobGCPartitionBytes( + input.physical_live_record_bytes, input.live_count, + &input.estimated_bytes); + if (!estimate_status.ok()) { + estimate_status.PermitUncheckedError(); + suppressed_files.push_back(input.file_number); + suppression_reasons.emplace_back( + "relocation output size estimate overflow"); + continue; + } + } + input.eligible = true; + eligible_inputs.push_back(input_index); + } + + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:AfterLiveSet", &status); + if (status.IsCorruption() || status.IsMemoryLimit()) { + // This phase read only input and SST metadata, and created no output. So + // its failures suppress the affected inputs and do not set a background + // error. + for (size_t input_index : eligible_inputs) { + suppressed_files.push_back(inputs[input_index].file_number); + suppression_reasons.emplace_back("source live-set validation failed"); + if (status.IsMemoryLimit()) { + transient_suppressions.insert(inputs[input_index].file_number); + } + } + eligible_inputs.clear(); + status = Status::OK(); + } + // Lets concurrency tests install a new SST after the job's live set is fixed + // but before output work begins. + TEST_SYNC_POINT("DBImpl::RunIndirectBlobGC:BeforeOutputWork"); +} + +void DBImpl::IndirectBlobGCJob::PackOutputs() { + capacity_deferred_files.reserve(eligible_inputs.size()); + auto* const sfm = static_cast( + db_->immutable_db_options_.sst_file_manager.get()); + while (status.ok() && !eligible_inputs.empty()) { + // Whole inputs are packed; empty inputs need no output. + std::vector items; + std::vector packed_inputs; + for (size_t input_index : eligible_inputs) { + PhysicalInput& input = inputs[input_index]; + input.output_index = std::numeric_limits::max(); + if (input.estimated_bytes == 0) { + continue; + } + BlobRelocationPackingItem item; + item.input_file_number = input.file_number; + item.estimated_bytes = input.estimated_bytes; + for (const BlobRelocationOriginRun& run : input.live_set->origin_runs()) { + item.origins.push_back(run.origin_file_number); + } + items.push_back(std::move(item)); + packed_inputs.push_back(input_index); + } + if (items.empty()) { + bins.clear(); + break; + } + const uint64_t configured_output_bytes = + mutable_cf_options.blob_file_size == 0 + ? 1 + : std::min(mutable_cf_options.blob_file_size, + kIndirectBlobGCMaxOutputBytes); + const uint64_t target_output_bytes = std::max( + 1, configured_output_bytes - configured_output_bytes / 10); + status = PlanBlobRelocationPacking(items, target_output_bytes, + kIndirectBlobGCMaxOutputs, &bins); + if (status.ok()) { + for (size_t output_index = 0; output_index < bins.size(); + ++output_index) { + for (size_t& item_index : bins[output_index].item_indexes) { + item_index = packed_inputs[item_index]; + inputs[item_index].output_index = output_index; + } + } + aggregate_input_bytes = 0; + uint64_t aggregate_estimated_bytes = 0; + for (size_t input_index : eligible_inputs) { + if (inputs[input_index].physical_bytes > + std::numeric_limits::max() - aggregate_input_bytes || + inputs[input_index].estimated_bytes > + std::numeric_limits::max() - + aggregate_estimated_bytes) { + status = Status::Corruption("Indirect blob GC input byte overflow"); + break; + } + aggregate_input_bytes += inputs[input_index].physical_bytes; + aggregate_estimated_bytes += inputs[input_index].estimated_bytes; + } + // The outputs need about their estimated size, never more than the + // inputs, so the space check reserves the smaller of the two. + const uint64_t reserved_bytes = + std::min(aggregate_estimated_bytes, aggregate_input_bytes); + if (!status.ok() || sfm == nullptr || + sfm->EnoughRoomForCompaction(reserved_bytes, output_directory, + sfm_background_error)) { + if (status.ok() && sfm != nullptr) { + *sfm_reserved_bytes = reserved_bytes; + } + break; + } + if (eligible_inputs.size() == 1) { + capacity_deferred_files.push_back( + inputs[eligible_inputs.front()].file_number); + status = Status::CompactionTooLarge(); + RecordTick(db_->stats_, COMPACTION_CANCELLED, 1); + break; + } + status = Status::OK(); + deferred_input = true; + bins.clear(); + } + // Two paths reach the code below. (1) PlanBlobRelocationPacking() returned + // MemoryLimit: the inputs need more than kIndirectBlobGCMaxOutputs + // (8) outputs. That is not a memory error. (2) The disk space check above + // failed with more than one input left, and `status` was set back to OK. + if (!status.ok() && !status.IsMemoryLimit()) { + break; + } + // Too many outputs or not enough space: the largest input waits for a later + // job. + status = Status::OK(); + const auto removed = std::max_element( + eligible_inputs.begin(), eligible_inputs.end(), + [&](size_t lhs, size_t rhs) { + return std::tie(inputs[lhs].physical_bytes, inputs[lhs].file_number) < + std::tie(inputs[rhs].physical_bytes, inputs[rhs].file_number); + }); + assert(removed != eligible_inputs.end()); + const size_t removed_input = *removed; + capacity_deferred_files.push_back(inputs[removed_input].file_number); + eligible_inputs.erase(removed); + inputs[removed_input].eligible = false; + inputs[removed_input].output_index = std::numeric_limits::max(); + deferred_input = true; + } + sfm_background_error.PermitUncheckedError(); + + outputs = std::vector(bins.size()); + if (status.ok() && !outputs.empty()) { + db_->mutex_.Lock(); + Version* const current = cfd->current(); + if (cfd->IsDropped() || current == nullptr) { + status = stale_input("Indirect blob GC column family changed"); + } else { + for (OutputState& output : outputs) { + output.file_number = db_->versions_->NewFileNumber(); + output.path = BlobFileName(output_directory, output.file_number); + output.temporary_path = + TempFileName(output_directory, output.file_number); + } + } + db_->mutex_.Unlock(); + } +} + +void DBImpl::IndirectBlobGCJob::OpenInputReaders() { + readers = std::vector(inputs.size()); + // Indirect blob GC opens a relocation input through the blob file cache, and + // GC's own open never pins its location blocks + // (pin_blob_relocation_location_blocks): GC is about to reclaim the file, and + // a pin would read every location block at IO_LOW while it holds a relocation + // stripe mutex, and keep them charged until the purge. A user read may + // already have cached a pinned reader, which GC then uses. If GC does not + // reclaim the file, its unpinned reader stays cached. + MutableCFOptions gc_open_options = mutable_cf_options; + gc_open_options.pin_blob_relocation_location_blocks = false; + for (size_t input_index : eligible_inputs) { + if (!status.ok()) { + break; + } + if (inputs[input_index].live_count == 0) { + // Exact empty-file cleanup does not read source data or create output. + continue; + } + const PhysicalInput& input = inputs[input_index]; + if (input.normal_metadata != nullptr) { + const BlobFileOpenInfo source_file{ + input.file_number, input.normal_metadata->GetChecksumValue(), + input.normal_metadata->GetChecksumMethod()}; + // Indirect blob GC input is cold data, and buffered io_uring reads of + // coalesced ranges can read more device bytes than needed. So GC prefers + // direct reads for normal blob file inputs (see + // OpenBlobFileReaderUncached()). User reads keep their cached buffered + // readers. + status = blob_file_cache->OpenBlobFileReaderUncached( + gc_read_options, source_file, &readers[input_index].normal, + /*allow_footer_skip_retry=*/false, /*prefer_direct_reads=*/true); + if (status.ok() && readers[input_index].normal == nullptr) { + status = Status::Corruption("Blob file cache returned a null reader"); + } + } else { + const BlobPhysicalFile source_file{ + BlobFileOpenInfo{input.file_number, + input.relocation_metadata->GetChecksumValue(), + input.relocation_metadata->GetChecksumMethod()}, + input.relocation_metadata->GetFileSize(), /*is_relocation=*/true, + input.relocation_metadata->GetTailSize(), + input.relocation_metadata->GetDirectoryDigest()}; + status = blob_file_cache->GetRelocationFileReader( + gc_read_options, source_file, gc_open_options, + &readers[input_index].relocation); + if (status.ok() && + readers[input_index].relocation.GetValue() == nullptr) { + status = Status::Corruption( + "Blob file cache returned a null relocation reader"); + } + } + if (status.IsCorruption() || status.IsMemoryLimit() || + status.IsNotSupported()) { + if (status.IsCorruption()) { + note_corruption(inputs[input_index].file_number, status); + } + candidate_local_failure_input = input_index; + candidate_local_failure_reason = + "the input file cannot be opened for relocation"; + status = Status::OK(); + break; + } + } + TEST_SYNC_POINT("DBImpl::RunIndirectBlobGC:InputsOpened"); +} + +void DBImpl::IndirectBlobGCJob::WriteOutputs() { + uint64_t aggregate_output_bytes = 0; + uint64_t total_relocated_value_bytes = 0; + size_t total_relocated_count = 0; + // This comparator must outlive each output's table builder. + const InternalKeyComparator relocation_comparator(BytewiseComparator()); + for (size_t output_index = 0; + status.ok() && + candidate_local_failure_input == std::numeric_limits::max() && + output_index < outputs.size(); + ++output_index) { + OutputState& output = outputs[output_index]; + status = interrupted(); + if (!status.ok()) { + break; + } + output.creation_started = true; + db_->blob_callback_.OnBlobFileCreationStarted( + output.path, cfd->GetName(), job_id, + BlobFileCreationReason::kIndirectBlobGC); + FileOptions file_options = blob_gc_file_options; + file_options.open_contract = FileOpenContract::kNoReopenForWrite | + FileOpenContract::kNoReadersWhileOpenForWrite; + std::unique_ptr file; + status = NewWritableFile(db_->fs_.get(), output.temporary_path, &file, + file_options); + std::unique_ptr file_writer; + std::unique_ptr table_builder; + if (status.ok()) { + if (file == nullptr) { + status = Status::Corruption("File system returned a null blob GC file"); + } else { + FSWritableFile& writable_file = *file; + writable_file.SetIOPriority(write_options.rate_limiter_priority); + FileTypeSet checksum_handoff_file_types = + db_->immutable_db_options_.checksum_handoff_file_types; + file_writer = std::make_unique( + std::move(file), output.temporary_path, file_options, + db_->immutable_db_options_.clock, db_->io_tracer_, db_->stats_, + Histograms::BLOB_DB_BLOB_FILE_WRITE_MICROS, + db_->immutable_db_options_.listeners, + db_->immutable_db_options_.file_checksum_gen_factory.get(), + checksum_handoff_file_types.Contains(FileType::kBlobFile), false); + BlobRelocationTableFileInfo file_info; + file_info.column_family_id = cfd->GetID(); + file_info.column_family_name = cfd->GetName(); + file_info.db_id = db_->db_id_; + file_info.db_session_id = db_->db_session_id_; + file_info.db_name = db_->dbname_; + file_info.file_number = output.file_number; + file_info.target_file_size = bins[output_index].estimated_bytes; + status = NewBlobRelocationTableBuilder( + cfd->ioptions(), mutable_cf_options, *source_table_options, + gc_read_options, write_options, relocation_comparator, file_info, + file_writer.get(), &table_builder); + } + } + + // The output gets whole inputs. Its values must be added in (origin, + // original offset) order: each input's live set is in that order, and the + // inputs of one output share no origin, so the inputs' origin runs are + // merged by origin. + struct OutputRun { + size_t input_index = 0; + BlobRelocationOriginRun run; + }; + std::vector output_runs; + // The old file numbers this output can be routed from: its inputs and the + // old files routed to them at job start. The builder stores them in the + // file's directory as its served list. After the build, the output check + // below compares that list with these file numbers. + std::vector served_generation_numbers; + for (size_t input_index : bins[output_index].item_indexes) { + const PhysicalInput& input = inputs[input_index]; + output.input_bytes += input.physical_bytes; + served_generation_numbers.insert(served_generation_numbers.end(), + input.generations.begin(), + input.generations.end()); + for (const BlobRelocationOriginRun& run : input.live_set->origin_runs()) { + output_runs.push_back({input_index, run}); + } + } + std::sort(output_runs.begin(), output_runs.end(), + [](const OutputRun& lhs, const OutputRun& rhs) { + return lhs.run.origin_file_number < rhs.run.origin_file_number; + }); + std::sort(served_generation_numbers.begin(), + served_generation_numbers.end()); + for (size_t i = 0; status.ok() && i < served_generation_numbers.size(); + ++i) { + if (i == 0 || + served_generation_numbers[i - 1] != served_generation_numbers[i]) { + status = table_builder->AddBlobRelocationServedFile( + served_generation_numbers[i]); + } + } + for (const OutputRun& output_run : output_runs) { + if (!status.ok() || + candidate_local_failure_input != std::numeric_limits::max()) { + break; + } + const size_t run_input = output_run.input_index; + const PhysicalInput& input = inputs[run_input]; + const std::vector& references = + input.live_set->references(); + const uint64_t origin = output_run.run.origin_file_number; + const size_t run_end = output_run.run.end; + size_t next_reference = output_run.run.begin; + while (status.ok() && next_reference < run_end) { + status = interrupted(); + if (!status.ok()) { + break; + } + const BlobRelocationLiveValue& next = references[next_reference]; + if (next.stored_value_size > kIndirectBlobGCReadBatchBytes) { + BlobFileReader* const reader = + input.normal_metadata != nullptr + ? readers[run_input].normal.get() + : readers[run_input].relocation.GetValue(); + if (reader == nullptr || table_builder == nullptr) { + status = Status::Corruption("Missing streaming blob GC dependency"); + break; + } + std::string encoded; + // Streaming looks the value up by its stable identity. + BlobIndex::EncodeIndirectBlob( + &encoded, origin, + {.origin_file_number = origin, + .original_value_offset = next.original_value_offset, + .stored_value_size = next.stored_value_size, + .compression = next.compression, + .stable_checksum = next.stable_checksum, + .stored_checksum = next.stored_checksum}); + BlobIndex index; + status = index.DecodeFrom(encoded); + if (!status.ok()) { + break; + } + Status source_status; + Status delivery_status; + status = table_builder->AddBlobRelocation( + origin, next.original_value_offset, next.stored_value_size, + next.compression, next.stored_checksum, + [&](const BlobChunkConsumer& consumer) { + const auto deliver = [&](const Slice& chunk) { + const Status interrupt_status = interrupted(); + delivery_status = interrupt_status.ok() ? consumer(chunk) + : interrupt_status; + return delivery_status; + }; + source_status = reader->relocation() != nullptr + ? reader->relocation()->StreamStoredBlob( + gc_read_options, index, deliver) + : reader->StreamStoredBlobForRelocation( + gc_read_options, index, deliver); + return source_status; + }); + const bool source_fault = + source_status.IsCorruption() || source_status.IsNotSupported(); + bool output_failed = + !delivery_status.ok() || (!status.ok() && source_status.ok()); +#ifndef NDEBUG + const std::string stream_status = status.ToString(); +#endif + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RunIndirectBlobGC:AfterAddBlobRelocation", &status); +#ifndef NDEBUG + // In debug builds, a test-injected destination failure also counts as + // an output failure. Status equality compares only the error code, so + // the status text is compared instead. + output_failed |= status.ToString() != stream_status; +#endif + if (!source_status.ok() && !source_fault) { + // A source error other than Corruption or NotSupported (for example + // I/O or abort) is returned before a later output error, as in the + // batch path below. + status = source_status; + } + if (!status.ok()) { + // A corrupt or unsupported source value fails its input, and the + // unfinished output is not published. Keep a destination failure + // even when the source also failed. + if (source_fault && !output_failed) { + if (source_status.IsCorruption()) { + note_corruption(inputs[run_input].file_number, source_status); + } + candidate_local_failure_input = run_input; + candidate_local_failure_reason = "blob stream source is corrupt"; + status = Status::OK(); + } + break; + } + output.value_bytes += next.stored_value_size; + ++output.live_count; + ++next_reference; + continue; + } + const size_t batch_begin = next_reference; + // A batch counts each value's stored bytes plus + // kIndirectBlobGCBatchBytesPerValue (see its comment), so it stays + // within the charged workspace also for tiny values. + // ReadCoalescedRanges() counts direct-read alignment in its buffer + // limit. + const uint64_t per_value_bytes = kIndirectBlobGCBatchBytesPerValue; + uint64_t batch_bytes = 0; + while (next_reference < run_end) { + const uint64_t value_size = + references[next_reference].stored_value_size; + if (value_size > kIndirectBlobGCReadBatchBytes) { + // A large value is streamed on its own by the branch above. + assert(next_reference != batch_begin); + break; + } + const uint64_t value_cost = value_size + per_value_bytes; + if (next_reference != batch_begin && + value_cost > + kIndirectBlobGCReadBatchBytes - + std::min(batch_bytes, kIndirectBlobGCReadBatchBytes)) { + break; + } + batch_bytes += value_cost; + ++next_reference; + if (batch_bytes >= kIndirectBlobGCReadBatchBytes) { + break; + } + } + const size_t batch_size = next_reference - batch_begin; + std::vector stored_payloads(batch_size); + std::vector read_statuses(batch_size); + CoalescedReadBuffers read_buffers; + const uint64_t coalesce_bytes = + mutable_cf_options.indirect_blob_gc_read_coalesce_bytes; + uint64_t bytes_read = 0; + if (input.normal_metadata != nullptr) { + std::vector requests(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + const BlobRelocationLiveValue& reference = + references[batch_begin + i]; + requests[i] = + StoredNormalBlobReadRequest{reference.original_value_offset, + reference.stored_value_size, + reference.compression, + &stored_payloads[i], + &read_statuses[i], + reference.stored_checksum}; + } + BlobFileReader* const reader = readers[run_input].normal.get(); + if (reader == nullptr) { + status = + Status::Corruption("Indirect blob GC reader is unavailable"); + break; + } + reader->MultiGetStoredBlobsForRelocation( + gc_read_options, coalesce_bytes, requests.size(), requests.data(), + kIndirectBlobGCReadBatchBytes, &read_buffers, &bytes_read); + } else { + std::vector requests(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + const BlobRelocationLiveValue& reference = + references[batch_begin + i]; + requests[i] = + StoredRelocationFileReadRequest{origin, + reference.original_value_offset, + reference.stored_value_size, + reference.compression, + reference.stable_checksum, + &stored_payloads[i], + &read_statuses[i], + reference.stored_checksum}; + } + const BlobFileReader* const reader = + readers[run_input].relocation.GetValue(); + if (reader == nullptr || reader->relocation() == nullptr) { + status = Status::Corruption( + "Indirect blob GC relocation reader is unavailable"); + break; + } + reader->relocation()->MultiGetStoredBlobs( + gc_read_options, coalesce_bytes, requests.size(), requests.data(), + kIndirectBlobGCReadBatchBytes, &read_buffers, &bytes_read); + } + input_bytes_read += bytes_read; + // A read past the buffer limit is deferred (Incomplete). Keep the + // values before the first deferred one; the next batch starts there. + size_t read_count = batch_size; + for (size_t i = 0; i < batch_size; ++i) { + if (read_statuses[i].IsIncomplete()) { + read_count = i; + break; + } + } + if (read_count == 0) { + status = Status::Corruption("Indirect blob GC read made no progress"); + break; + } + next_reference = batch_begin + read_count; + Status candidate_status; + Status external_status; + for (size_t i = 0; i < read_count; ++i) { + const Status& read_status = read_statuses[i]; + if (read_status.ok()) { + continue; + } + if (read_status.IsCorruption() || read_status.IsMemoryLimit() || + read_status.IsNotSupported()) { + if (candidate_status.ok()) { + candidate_status = read_status; + } + } else if (external_status.ok()) { + external_status = read_status; + } + } + // A corrupt, too large or unsupported source record fails only its + // input. Record that first. A source I/O error or an output error still + // has priority and is returned. + const bool candidate_failed = !candidate_status.ok(); + if (!external_status.ok()) { + status = external_status; + break; + } + if (table_builder == nullptr) { + status = + Status::Corruption("Indirect blob GC builder is unavailable"); + break; + } + for (size_t i = 0; status.ok() && i < read_count; ++i) { + if (!read_statuses[i].ok()) { + continue; + } + const BlobRelocationLiveValue& reference = + references[batch_begin + i]; + status = table_builder->AddBlobRelocation( + origin, reference.original_value_offset, stored_payloads[i], + reference.compression); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RunIndirectBlobGC:AfterAddBlobRelocation", &status); + if (status.ok()) { + output.value_bytes += reference.stored_value_size; + ++output.live_count; + } + } + // Keep an output failure even when another record in this batch is + // corrupt. All read statuses were checked first, so a source I/O error + // also keeps its priority. + if (status.ok() && candidate_failed) { + if (candidate_status.IsCorruption()) { + note_corruption(inputs[run_input].file_number, candidate_status); + } + candidate_local_failure_input = run_input; + candidate_local_failure_reason = + "blob source record is corrupt or too large"; + break; + } + } + } + + if (table_builder != nullptr) { + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max()) { + status = table_builder->Finish(); + output.file_size = table_builder->FileSize(); + if (status.ok()) { + status = + table_builder->GetBlobRelocationFileMetadata(&output.metadata); + } + if (status.ok()) { + status = BlobRelocationDirectory::DecodeFrom( + Slice(output.metadata.encoded_directory), &output.directory); + } + // Install check: the output holds exactly the live values of its + // inputs. Its directory has one origin entry per origin run, with the + // same count, first and last offset and record bytes, and it serves + // exactly the generation numbers of its inputs. + size_t expected_live_count = 0; + for (size_t input_index : bins[output_index].item_indexes) { + expected_live_count += inputs[input_index].live_count; + } + if (status.ok() && + (output.live_count != expected_live_count || + output.directory.total_entries() != expected_live_count || + output.directory.origins().size() != output_runs.size())) { + status = Status::Corruption( + "Blob relocation output does not match the live values"); + } + for (size_t k = 0; status.ok() && k < output_runs.size(); ++k) { + const BlobRelocationOriginRun& run = output_runs[k].run; + const std::vector& values = + inputs[output_runs[k].input_index].live_set->references(); + uint64_t run_record_bytes = 0; + for (size_t i = run.begin; i < run.end; ++i) { + run_record_bytes += uint64_t{values[i].stored_value_size} + + kSimpleGen2BlobTrailerSize; + } + const BlobRelocationOrigin& origin = output.directory.origins()[k]; + if (run.begin >= run.end || + origin.origin_file_number != run.origin_file_number || + origin.entry_count != run.end - run.begin || + origin.first_original_offset != + values[run.begin].original_value_offset || + origin.last_original_offset != + values[run.end - 1].original_value_offset || + origin.record_bytes != run_record_bytes) { + status = Status::Corruption( + "Blob relocation output does not match the live values"); + } + } + if (status.ok()) { + std::vector expected_served = served_generation_numbers; + expected_served.erase( + std::unique(expected_served.begin(), expected_served.end()), + expected_served.end()); + if (output.directory.served_file_numbers() != expected_served) { + status = Status::Corruption( + "Blob relocation output does not serve its input generation " + "numbers"); + } + } + } else { + table_builder->Abandon(); + } + IOStatus builder_io_status = table_builder->io_status(); + if (status.ok() && !builder_io_status.ok()) { + status = builder_io_status; + } else { + builder_io_status.PermitUncheckedError(); + } + table_builder.reset(); + } + if (file_writer != nullptr) { + IOOptions io_options; + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max()) { + status = + WritableFileWriter::PrepareIOOptions(write_options, io_options); + } + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max()) { + status = db_->immutable_db_options_.use_fsync + ? file_writer->Fsync(io_options) + : file_writer->Sync(io_options); + } + IOStatus close_status = file_writer->Close(io_options); + if (status.ok() && !close_status.ok()) { + status = close_status; + } else { + close_status.PermitUncheckedError(); + } + if (status.ok()) { + output.checksum_value = file_writer->GetFileChecksum(); + output.checksum_method = file_writer->GetFileChecksumFuncName(); + if (output.checksum_value == kUnknownFileChecksum) { + output.checksum_value.clear(); + } + if (output.checksum_method == kUnknownFileChecksumFuncName) { + output.checksum_method.clear(); + } + if (output.checksum_method.empty() != output.checksum_value.empty()) { + status = Status::Corruption( + "Incomplete blob relocation file checksum metadata"); + } + } + file_writer.reset(); + } + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max()) { + // Check that the output file is not larger than its inputs. + if (output.file_size > output.input_bytes || + output.file_size > + std::numeric_limits::max() - aggregate_output_bytes || + output.value_bytes > std::numeric_limits::max() - + total_relocated_value_bytes || + output.live_count > + std::numeric_limits::max() - total_relocated_count) { + candidate_local_failure_input = bins[output_index].item_indexes.front(); + candidate_local_failure_reason = + "relocation output exceeds its bounded packing plan"; + } else { + aggregate_output_bytes += output.file_size; + total_relocated_value_bytes += output.value_bytes; + total_relocated_count += output.live_count; + } + } + } + + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max() && + !outputs.empty() && aggregate_output_bytes >= aggregate_input_bytes) { + for (size_t input_index : eligible_inputs) { + suppressed_files.push_back(inputs[input_index].file_number); + suppression_reasons.emplace_back( + "packed relocation outputs would not reclaim bytes"); + inputs[input_index].eligible = false; + } + eligible_inputs.clear(); + } + + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max() && + !eligible_inputs.empty()) { + IOOptions io_options; + TEST_SYNC_POINT("DBImpl::RunIndirectBlobGC:BeforeRenameOutputs"); + for (OutputState& output : outputs) { + output.rename_attempted = true; + status = + db_->fs_->RenameFile(output.temporary_path, output.path, io_options, + /*dbg=*/nullptr); + if (!status.ok()) { + break; + } + output.renamed = true; + } + if (status.ok() && !outputs.empty()) { + FSDirectory* const data_dir = db_->GetDataDir(cfd, 0); + status = data_dir == nullptr + ? Status::Corruption("Missing blob GC data directory") + : data_dir->FsyncWithDirOptions( + IOOptions(), nullptr, + DirFsyncOptions( + DirFsyncOptions::FsyncReason::kNewFileSynced)); + } + } + + for (OutputState& output : outputs) { + if (!output.creation_started) { + continue; + } + Status callback_input_status = status; + if (callback_input_status.ok() && + (candidate_local_failure_input != std::numeric_limits::max() || + eligible_inputs.empty())) { + callback_input_status = + Status::TryAgain("Blob relocation batch was not published"); + } + if (!output.renamed) { + // The output was not renamed, or the rename reported failure. Report a + // failed creation and do not register the file with the SstFileManager. + // The cleanup deletes the final name of a tried rename (see + // rename_attempted). + db_->blob_callback_.OnBlobFileAbandoned( + output.path, cfd->GetName(), job_id, output.file_number, + BlobFileCreationReason::kIndirectBlobGC, + callback_input_status.ok() + ? Status::TryAgain("Blob relocation output was not renamed") + : callback_input_status); + callback_input_status.PermitUncheckedError(); + continue; + } + const Status completion_status = db_->blob_callback_.OnBlobFileCompleted( + output.path, cfd->GetName(), job_id, output.file_number, + BlobFileCreationReason::kIndirectBlobGC, callback_input_status, + output.checksum_value, output.checksum_method, output.live_count, + output.file_size, /*set_bg_error_on_space_limit=*/true); + callback_input_status.PermitUncheckedError(); + if (status.ok() && + candidate_local_failure_input == std::numeric_limits::max() && + !eligible_inputs.empty()) { + status = completion_status; + } else { + completion_status.PermitUncheckedError(); + } + } + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:AfterOutputWork", + &status); + + if (candidate_local_failure_input != std::numeric_limits::max()) { + suppressed_files.push_back( + inputs[candidate_local_failure_input].file_number); + suppression_reasons.emplace_back(candidate_local_failure_reason); + eligible_inputs.clear(); + } + + const bool keep_outputs = status.ok() && !eligible_inputs.empty(); + if (!keep_outputs) { + for (OutputState& output : outputs) { + if (!output.creation_started) { + continue; + } + db_->DeleteObsoleteFileImpl(job_id, output.temporary_path, + output_directory, kTempFile, + output.file_number); + if (output.rename_attempted) { + db_->DeleteObsoleteFileImpl(job_id, output.path, output_directory, + kBlobFile, output.file_number); + } + output.renamed = false; + } + } + + readers.clear(); + for (PhysicalInput& input : inputs) { + if (input.live_set != nullptr) { + Status release_status = input.live_set->ReleaseWorkingMemory(); + if (status.ok() && !release_status.ok()) { + status = release_status; + } else { + release_status.PermitUncheckedError(); + } + input.live_set.reset(); + } + } + + read_input_count = eligible_inputs.size(); + bins.clear(); + source_table_options.reset(); +} + +// Builds the MANIFEST edit of the job from the census snapshot and the +// routes of the input Version (see "Install protocol" in Install()). Called +// without the DB mutex. +Status DBImpl::IndirectBlobGCJob::BuildEdit() { + if (census == nullptr) { + return Status::Corruption("Blob GC census snapshot is unavailable"); + } + output_needed.fill(false); + published_output_count = 0; + published_relocated_count = 0; + published_relocated_value_bytes = 0; + + // A generation number is live if an SST holds it in the census snapshot, + // which shows the input Version. A running compaction can write only + // generation numbers that its input SSTs hold, so this is enough (see + // "Install protocol" in Install()). + const auto generation_is_live = [&](uint64_t generation) { + return census->GetSstCount(generation) != 0; + }; + const auto output_of = [&](const PhysicalInput& input) -> size_t { + return input.output_index < outputs.size() + ? input.output_index + : std::numeric_limits::max(); + }; + for (size_t input_index : eligible_inputs) { + const PhysicalInput& input = inputs[input_index]; + if (std::none_of(input.generations.begin(), input.generations.end(), + generation_is_live)) { + continue; + } + const size_t output_index = output_of(input); + if (output_index == std::numeric_limits::max()) { + // Invariant: the same census snapshot decides liveness here and the + // outputs in DecideInputs(), which gives an output to every input + // with a live generation number. + return invariant_failed("Indirect blob GC live input has no output"); + } + output_needed[output_index] = true; + } + + edit = std::make_unique(); + edit->SetColumnFamily(column_family.GetID()); + for (size_t i = 0; i < outputs.size(); ++i) { + if (!output_needed[i]) { + continue; + } + const OutputState& output = outputs[i]; + edit->AddBlobRelocationFile(BlobRelocationFileAddition( + output.file_number, output.file_size, output.checksum_method, + output.checksum_value, output.metadata.encoded_directory, + output.metadata.tail_size)); + ++published_output_count; + if (output.live_count > + std::numeric_limits::max() - published_relocated_count) { + return Status::Corruption("Published blob GC count overflow"); + } + published_relocated_count += output.live_count; + if (output.value_bytes > std::numeric_limits::max() - + published_relocated_value_bytes) { + return Status::Corruption("Published blob GC byte count overflow"); + } + published_relocated_value_bytes += output.value_bytes; + } + + // Each live generation number of an input is routed to the input's output + // (for a relocation input this re-points the routes into it); a routed + // generation number that no SST uses loses its route. The input leaves the + // Version. + for (size_t input_index : eligible_inputs) { + const PhysicalInput& input = inputs[input_index]; + const size_t output_index = output_of(input); + const uint64_t output_file_number = + output_index != std::numeric_limits::max() && + output_needed[output_index] + ? outputs[output_index].file_number + : kInvalidBlobFileNumber; + for (uint64_t generation : input.generations) { + const bool routed = generation != input.file_number; + if (generation_is_live(generation)) { + if (output_file_number == kInvalidBlobFileNumber) { + // Invariant: the same census snapshot decides liveness here and + // the outputs in DecideInputs() (see the check above). + return invariant_failed( + "Indirect blob GC live generation number has no output"); + } + edit->SetBlobRelocationRoute( + BlobRelocationRoute(generation, output_file_number)); + } else if (routed) { + edit->DeleteBlobRelocationRoute(generation); + } + } + if (input.normal_metadata != nullptr) { + edit->RetireBlobFile(input.file_number); + } else { + edit->DeleteBlobRelocationFile(input.file_number); + } + } + return Status::OK(); +} + +// At the MANIFEST queue head, check that each input is still valid: +// - A normal input is still an indirect origin with the same shared +// metadata. A compaction that adds garbage to it keeps its shared +// metadata, and only an indirect blob GC install retires an indirect +// origin. So this check is defensive. A change makes a normal stale job. +// - No input became pinned by a coarse SST (see the selection above for +// what pins an input). This is a normal stale job too. Example: a flush +// takes blob file number 25, indirect blob GC selects normal input 28, +// and then the flush installs its coarse SST with frontier 25 (see +// "Coarse SST" in db/blob/blob_reference_metadata.h), which pins file 28. +// - Invariant check (invariant_failed()), which cannot fail: a relocation +// input has the same metadata and the same number of routes into it. +// Only indirect blob GC adds or removes routes and relocation files, +// and only one GC job runs per CF (indirect_blob_gc_cfs_in_progress_), +// so an equal route count means the same routes. +Status DBImpl::IndirectBlobGCJob::ValidatePhysicalInputs() { + db_->mutex_.AssertHeld(); + Version* const current = column_family.current(); + if (column_family.IsDropped() || current == nullptr) { + return stale_input("Indirect blob GC column family changed"); + } + VersionStorageInfo* const current_storage = current->storage_info(); + for (size_t input_index : eligible_inputs) { + const PhysicalInput& input = inputs[input_index]; + const bool coarse_pinned = + input.normal_metadata != nullptr + ? current_storage->IsCoarseIndirectOriginBlobPinned( + input.file_number) + : std::any_of( + input.routed_generations.begin(), + input.routed_generations.end(), [&](uint64_t generation) { + return current_storage->IsCoarseIndirectOriginBlobPinned( + generation); + }); + if (coarse_pinned) { + return stale_input( + "Indirect blob GC input generation number became coarse"); + } + if (input.normal_metadata != nullptr) { + const std::shared_ptr current_metadata = + current_storage->GetBlobFileMetaData(input.file_number); + if (current_metadata == nullptr || + !current_metadata->IsIndirectOrigin() || + current_metadata->GetSharedMeta() != + input.normal_metadata->GetSharedMeta()) { + return stale_input("Indirect blob GC indirect-origin input changed"); + } + continue; + } + if (current_storage->GetBlobRelocationFileMetaData(input.file_number) != + input.relocation_metadata || + current_storage->GetBlobRelocationRouteTargetCount(input.file_number) != + input.routed_generations.size()) { + return invariant_failed("Indirect blob GC relocation input changed"); + } + } + return Status::OK(); +} + +void DBImpl::IndirectBlobGCJob::Install() { + db_->mutex_.Lock(); + + if (!status.ok()) { + Version* const current = cfd->current(); + if (status.IsCompactionTooLarge() && !cfd->IsDropped() && + current != nullptr && + current->GetVersionNumber() == input_version_number) { + current->storage_info()->DeferBlobFilesForIndirectBlobGC( + capacity_deferred_files); + } + delete_renamed_outputs.fill(true); + return; + } + Version* current = cfd->current(); + if (cfd->IsDropped() || current == nullptr) { + status = stale_input("Indirect blob GC column family changed"); + delete_renamed_outputs.fill(true); + return; + } + status = interrupted(); + if (!status.ok()) { + delete_renamed_outputs.fill(true); + return; + } + if (read_input_count == 0) { + VersionStorageInfo* const current_storage = current->storage_info(); + for (size_t i = 0; i < suppressed_files.size(); ++i) { + current_storage->SuppressBlobFileForIndirectBlobGC(suppressed_files[i]); + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Indirect blob GC skipped file #%" PRIu64 ": %s", + cfd->GetName().c_str(), suppressed_files[i], + suppression_reasons[i].c_str()); + } + if (deferred_input || + current_storage->BlobPhysicalFileForIndirectBlobGC() != + kInvalidBlobFileNumber) { + db_->EnqueuePendingCompaction(cfd); + } + // `status` is OK here, so the job returns OK. + return; + } + + db_->mutex_.Unlock(); + status = BuildEdit(); + db_->mutex_.Lock(); + status = locked_census_status(std::move(status)); + if (!status.ok()) { + delete_renamed_outputs.fill(true); + return; + } + + SuperVersionContext superversion_context(/*create_superversion=*/true); + // Install protocol: the job installs its edit with one LogAndApply() call. + // The edit comes from the census snapshot and the routes of the input + // Version. The job does not follow the Version changes after its snapshot. + // + // Why the snapshot is enough: the edit removes the route of a generation + // number that the snapshot shows as unused, and it removes every input. This + // is safe only if no SST that the snapshot does not count holds a + // generation number of an input. No job adds such an SST after the + // snapshot: + // - A compaction keeps or drops the references of its input SSTs and never + // adds a reference to an existing blob file. So every SST added after the + // snapshot got its references from SSTs of the input Version, directly or + // through earlier compactions. + // - A flush references only the new blob files that it writes. They are not + // inputs of this job. + // - Ingestion and import are assumed to add no SST with kIndirectBlob + // references. This code does not check it. + // So the snapshot can only count a holder too many (an SST that a later + // compaction removed). The edit then keeps a route that no SST needs, which + // is safe. The routes of the input Version are also still current: only + // indirect blob GC changes routes, and this job holds the only GC job + // registration of the census. ValidatePhysicalInputs() also checks that the + // number of routes into each relocation input is unchanged. + // + // The precondition of the call runs under the DB mutex at the MANIFEST queue + // head, and again after the unlocked apply of the edit (see the pre_cb + // paragraph of VersionSet::LogAndApply()). Each run does these steps: + // 1. It returns the interrupted() status at shutdown or compaction abort. + // 2. It checks that the column family is still alive and that the job is + // still registered (IsGcJobActive()). + // 3. It records in other_install_during_job whether the current Version is + // still the input Version. + // 4. It checks the inputs again (ValidatePhysicalInputs()). + // 5. On its first run, it starts the metadata commit + // (BeginMetadataCommit()). Until EndMetadataCommit() after the MANIFEST + // write, no other GC job or metadata commit can start. When the write + // result is unknown, BeginGcJob() returns Busy until recovery. + // Flushes and compactions do not wait for the GC job or its commit. A + // compaction that runs during the job writes only generation numbers that + // its input SSTs hold, and those SSTs are in the census snapshot. + // + // metadata_commit_intent is true after BeginMetadataCommit() succeeded, so + // that EndMetadataCommit() runs after LogAndApply(). other_install_during_job + // stays true when the precondition never runs. + bool metadata_commit_intent = false; + bool other_install_during_job = true; + status = db_->versions_->LogAndApply( + cfd, read_options, manifest_write_options, edit.get(), &db_->mutex_, + db_->directories_.GetDbDir(), + /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, + /*manifest_wcb=*/{}, [&]() { + Status precondition_status = interrupted(); + if (!precondition_status.ok()) { + return precondition_status; + } + Version* const precondition_current = column_family.current(); + if (column_family.IsDropped() || precondition_current == nullptr || + census == nullptr) { + return stale_input("Indirect blob GC column family changed"); + } + if (!blob_reference_census->IsGcJobActive(gc_job_token)) { + return stale_input("Blob GC job registration ended"); + } + other_install_during_job = + precondition_current->GetVersionNumber() != input_version_number; + precondition_status = ValidatePhysicalInputs(); + bool installable = precondition_status.ok(); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::RunIndirectBlobGC:InstallPrecondition", &installable); + if (precondition_status.ok() && !installable) { + precondition_status = stale_input("Indirect blob GC input changed"); + } + if (precondition_status.ok() && !metadata_commit_intent) { + precondition_status = blob_reference_census->BeginMetadataCommit(); + metadata_commit_intent = precondition_status.ok(); + } + return precondition_status; + }); + if (metadata_commit_intent) { + // If the MANIFEST write fails with an I/O error, the edit may still be in + // the MANIFEST. Then indirect blob GC and metadata commits stay blocked + // until recovery shows the result. A rejection before the append is a clear + // failure, and the block is released. + blob_reference_census->EndMetadataCommit(status.ok() || + db_->versions_->io_status().ok()); + } + if (status.ok()) { + db_->InstallSuperVersionAndScheduleWork(cfd, &superversion_context, + /*defer_snapshot_refresh=*/false); + TEST_SYNC_POINT("DBImpl::RunIndirectBlobGC:Installed"); + *made_progress = true; + installed = true; + published_input_count = read_input_count; + current = cfd->current(); + TEST_SYNC_POINT_CALLBACK("DBImpl::RunIndirectBlobGC:AfterMetadataInstall", + cfd); + if (current == nullptr) { + status = Status::Corruption( + "Indirect blob GC installed an unavailable Version"); + } + } + if (status.ok() && current != nullptr) { + VersionStorageInfo* const current_storage = current->storage_info(); + // The job decided its suppressions from the input Version. A newer Version + // can hold SST metadata that makes a suppressed input eligible, so the + // suppressions apply to the current Version only when no other Version was + // installed during the job. + // other_install_during_job comes from the last precondition run, which + // LogAndApply() makes right before this install. + const bool starting_suppressions_are_current = !other_install_during_job; + if (starting_suppressions_are_current) { + for (uint64_t file_number : suppressed_files) { + current_storage->SuppressBlobFileForIndirectBlobGC(file_number); + } + } + // A new output has no garbage yet. Suppress it in this Version, so that the + // next pick in this Version takes other candidates instead. + for (size_t i = 0; i < outputs.size(); ++i) { + if (output_needed[i]) { + current_storage->SuppressBlobFileForIndirectBlobGC( + outputs[i].file_number); + } else { + delete_renamed_outputs[i] = true; + } + } + if (!starting_suppressions_are_current || deferred_input || + current_storage->BlobPhysicalFileForIndirectBlobGC() != + kInvalidBlobFileNumber) { + db_->EnqueuePendingCompaction(cfd); + } + RecordTick(db_->stats_, BLOB_DB_GC_NUM_FILES, published_input_count); + RecordTick(db_->stats_, BLOB_DB_GC_NUM_NEW_FILES, published_output_count); + RecordTick(db_->stats_, BLOB_DB_GC_NUM_KEYS_RELOCATED, + published_relocated_count); + RecordTick(db_->stats_, BLOB_DB_GC_BYTES_RELOCATED, + published_relocated_value_bytes); + RecordTick(db_->stats_, INDIRECT_BLOB_GC_KEYS_RELOCATED, + published_relocated_count); + RecordTick(db_->stats_, INDIRECT_BLOB_GC_BYTES_RELOCATED, + published_relocated_value_bytes); + uint64_t removed_bytes = 0; + for (size_t input_index : eligible_inputs) { + removed_bytes += inputs[input_index].physical_bytes; + } + for (size_t i = 0; i < outputs.size(); ++i) { + if (output_needed[i]) { + bytes_written += outputs[i].file_size; + } + } + RecordTick(db_->stats_, INDIRECT_BLOB_GC_BYTES_RECLAIMED, + removed_bytes - std::min(removed_bytes, bytes_written)); + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Indirect blob GC packed %" ROCKSDB_PRIszt + " physical inputs into %" ROCKSDB_PRIszt + " relocation files (%" ROCKSDB_PRIszt " live blobs)", + cfd->GetName().c_str(), published_input_count, + published_output_count, published_relocated_count); + } + + if (!status.ok()) { + const autovector& quarantined = + db_->error_handler_.GetFilesToQuarantine(); + for (size_t i = 0; i < outputs.size(); ++i) { + if (outputs[i].renamed) { + delete_renamed_outputs[i] = + std::find(quarantined.begin(), quarantined.end(), + outputs[i].file_number) == quarantined.end(); + } + } + } + db_->mutex_.Unlock(); + superversion_context.Clean(); + db_->mutex_.Lock(); +} + +void DBImpl::IndirectBlobGCJob::RecordOutcome(const Status& result, + size_t skipped_count) { + db_->mutex_.AssertHeld(); + Statistics* const stats = db_->stats_; + const char* outcome = nullptr; + if (installed) { + outcome = "installed"; + RecordTick(stats, INDIRECT_BLOB_GC_JOBS_INSTALLED); + } else if (result.IsTryAgain() && stale_input_is_retryable) { + outcome = "stale"; + RecordTick(stats, INDIRECT_BLOB_GC_JOBS_STALE); + } else if (result.ok() || result.IsCompactionTooLarge() || + result.IsIncomplete() || result.IsShutdownInProgress() || + result.IsColumnFamilyDropped()) { + outcome = "nothing installed"; + RecordTick(stats, INDIRECT_BLOB_GC_JOBS_NOTHING_INSTALLED); + } else { + outcome = "failed"; + RecordTick(stats, INDIRECT_BLOB_GC_JOBS_FAILED); + RecordTick(stats, BLOB_DB_GC_FAILURES); + } + RecordTick(stats, INDIRECT_BLOB_GC_BYTES_READ, input_bytes_read); + const uint64_t now_micros = db_->immutable_db_options_.clock->NowMicros(); + RecordInHistogram(stats, INDIRECT_BLOB_GC_JOB_MICROS, + now_micros - std::min(now_micros, start_micros)); + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] [JOB %d] Indirect blob GC job ended: %s; %" ROCKSDB_PRIszt + " candidates, %" ROCKSDB_PRIszt " inputs read, %" ROCKSDB_PRIszt + " installed, %" ROCKSDB_PRIszt " skipped, %" ROCKSDB_PRIszt + " outputs, %" ROCKSDB_PRIszt " values relocated, %" PRIu64 + " bytes read, %" PRIu64 " bytes written; status: %s", + cfd->GetName().c_str(), job_id, outcome, + candidate_physical_file_numbers.size(), read_input_count, + published_input_count, skipped_count, published_output_count, + published_relocated_count, input_bytes_read, bytes_written, + result.ToString().c_str()); + // Candidates that remain start the idle time again (see + // ColumnFamilyData::NoteIndirectBlobGcCandidates()). + if (!cfd->IsDropped() && + cfd->current()->storage_info()->BlobPhysicalFileForIndirectBlobGC() != + kInvalidBlobFileNumber) { + cfd->NoteIndirectBlobGcCandidates(now_micros); + } +} + +Status DBImpl::IndirectBlobGCJob::Finish() { + db_->mutex_.AssertHeld(); + blob_reference_census->EndGcJob(gc_job_token); + gc_job_token = 0; + input_version->UnrefAndFreeBlobRelocationState(&db_->mutex_); + db_->mutex_.Unlock(); + if (status.IsCompactionTooLarge()) { + for (uint64_t file_number : capacity_deferred_files) { + suppressed_files.push_back(file_number); + // Keeps suppression_reasons parallel to suppressed_files. + suppression_reasons.emplace_back("deferred: outputs or disk space"); + transient_suppressions.insert(file_number); + } + } + // Install() logs the skipped inputs only when it installs nothing. Log them + // here too when other inputs were installed. A corrupt input is an ERROR. + for (size_t i = 0; i < suppressed_files.size(); ++i) { + const uint64_t file_number = suppressed_files[i]; + const auto corrupt = corrupt_inputs.find(file_number); + if (corrupt != corrupt_inputs.end()) { + RecordTick(db_->stats_, INDIRECT_BLOB_GC_CORRUPT_INPUTS); + if (!corrupt->second.logged) { + corrupt->second.logged = true; + ROCKS_LOG_ERROR(db_->immutable_db_options_.info_log, + "[%s] [JOB %d] Indirect blob GC found corrupt input " + "#%" PRIu64 + ": %s; the file is skipped until it changes or the " + "census is rebuilt", + cfd->GetName().c_str(), job_id, file_number, + corrupt->second.detail.c_str()); + } + } else if (published_input_count > 0 && i < suppression_reasons.size()) { + ROCKS_LOG_BUFFER( + log_buffer, "[%s] Indirect blob GC skipped file #%" PRIu64 ": %s", + cfd->GetName().c_str(), file_number, suppression_reasons[i].c_str()); + } + } + blob_reference_census->SuppressWorkerCandidates( + candidate_stamp, suppressed_files, transient_suppressions, + db_->immutable_db_options_.clock->NowMicros()); + for (size_t i = 0; i < outputs.size(); ++i) { + OutputState& output = outputs[i]; + if (output.renamed && delete_renamed_outputs[i]) { + db_->DeleteObsoleteFileImpl(job_id, output.path, output_directory, + kBlobFile, output.file_number); + output.renamed = false; + } + } + edit.reset(); + outputs.clear(); + eligible_inputs.clear(); + inputs.clear(); + // Release the SST reader charge (selected_reader_bytes) here, without the DB + // mutex, not in the return-scope cleanup after the caller takes the DB mutex + // again. + memory_tracker->SetReservation(&selected_reader_bytes, 0) + .PermitUncheckedError(); + memory_tracker.reset(); + census.reset(); + suppressed_files.clear(); + suppression_reasons.clear(); + db_->mutex_.Lock(); + db_->ScheduleBlobReferenceCensus(cfd); + return status; +} + +// Marks the job as a stale job that a later job can retry, and returns TryAgain +// with `message`. +Status DBImpl::IndirectBlobGCJob::stale_input(const char* message) { + stale_input_is_retryable = true; + return Status::TryAgain(message); +} + +// An invariant check that should never fail: `message` names a state that the +// rules of indirect blob GC exclude (see "Install protocol" in Install()). +// Debug builds assert. Release builds log a WARN line and end the job as a +// stale job, which installs nothing; a later job starts from the current +// Version. It is not Corruption, because the job installs nothing. +Status DBImpl::IndirectBlobGCJob::invariant_failed(const char* message) { + assert(false); + ROCKS_LOG_WARN(db_->immutable_db_options_.info_log, + "[%s] Indirect blob GC invariant check failed: %s", + cfd->GetName().c_str(), message); + return stale_input(message); +} + +// Maps a census status without the DB mutex. Start() uses it for the status of +// SnapshotSelected(). An Aborted status means that the job registration ended +// (CF drop, census cancel). It makes the job stale, so a later job retries it. +// At shutdown the job returns ShutdownInProgress. Neither is a background +// error. With the DB mutex held, use locked_census_status() instead. +Status DBImpl::IndirectBlobGCJob::unlocked_census_status(Status census_status) { + if (census_status.IsAborted()) { + if (db_->shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress("DB shutdown during indirect blob GC"); + } + return stale_input("Blob GC job registration ended"); + } + return census_status; +} + +// Maps a census status with the DB mutex held (Start() and Install()). When the +// status is Aborted (the job registration ended), it returns +// ColumnFamilyDropped if the column family was dropped, ShutdownInProgress if +// the DB is shutting down, or else a stale job. Other statuses stay unchanged. +Status DBImpl::IndirectBlobGCJob::locked_census_status(Status census_status) { + db_->mutex_.AssertHeld(); + if (!census_status.IsAborted()) { + return census_status; + } + if (column_family.IsDropped()) { + return Status::ColumnFamilyDropped( + "Column family dropped during indirect blob GC"); + } + if (db_->shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress("DB shutdown during indirect blob GC"); + } + return stale_input("Blob GC job registration ended"); +} + +// Returns ShutdownInProgress at DB shutdown, Incomplete when compactions are +// aborted for the whole DB or for this column family, and OK otherwise. +Status DBImpl::IndirectBlobGCJob::interrupted() { + if (db_->shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress(); + } + if (db_->IsCompactionAborted(cfd)) { + return Status::Incomplete(Status::SubCode::kCompactionAborted); + } + return Status::OK(); +} + +// Cache pressure is transient: back off every candidate of this job, so the +// retried pick can run other candidates instead of the same ones again. The +// census retries them later; the delay doubles after each failure, from 1 s up +// to 32 s (fixed choices, not measured). +void DBImpl::IndirectBlobGCJob::suppress_candidates_transiently( + const char* reason) { + for (uint64_t file_number : candidate_physical_file_numbers) { + if (transient_suppressions.insert(file_number).second) { + suppressed_files.push_back(file_number); + suppression_reasons.emplace_back(reason); + } + } +} + +// Marks the input at `input_index` as failed in its plan. Only the first reason +// is kept. +void DBImpl::IndirectBlobGCJob::fail_input(size_t input_index, + const char* reason) { + InputPlan& plan = plans[input_index]; + if (!plan.failed) { + plan.failed = true; + plan.reason = reason; + } +} + bool DBImpl::EnoughRoomForCompaction( ColumnFamilyData* cfd, const std::vector& inputs, bool* sfm_reserved_compact_space, LogBuffer* log_buffer) { @@ -1119,11 +3901,13 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta, if (shutting_down_.load(std::memory_order_acquire)) { return; } + // The stall decision counts L0 sorted runs, where an exact epoch run counts + // once (see VersionStorageInfo::NumL0SortedRuns()), so these flags do too. bool triggered_writes_slowdown = - (cfd->current()->storage_info()->NumLevelFiles(0) >= + (cfd->current()->storage_info()->NumL0SortedRuns() >= mutable_cf_options.level0_slowdown_writes_trigger); bool triggered_writes_stop = - (cfd->current()->storage_info()->NumLevelFiles(0) >= + (cfd->current()->storage_info()->NumL0SortedRuns() >= mutable_cf_options.level0_stop_writes_trigger); // release lock while notifying events mutex_.Unlock(); @@ -1164,11 +3948,13 @@ void DBImpl::NotifyOnFlushCompleted( if (shutting_down_.load(std::memory_order_acquire)) { return; } + // The stall decision counts L0 sorted runs, where an exact epoch run counts + // once (see VersionStorageInfo::NumL0SortedRuns()), so these flags do too. bool triggered_writes_slowdown = - (cfd->current()->storage_info()->NumLevelFiles(0) >= + (cfd->current()->storage_info()->NumL0SortedRuns() >= mutable_cf_options.level0_slowdown_writes_trigger); bool triggered_writes_stop = - (cfd->current()->storage_info()->NumLevelFiles(0) >= + (cfd->current()->storage_info()->NumL0SortedRuns() >= mutable_cf_options.level0_stop_writes_trigger); // release lock while notifying events mutex_.Unlock(); @@ -2055,8 +4841,12 @@ Status DBImpl::CompactFilesImpl( Status DBImpl::PauseBackgroundWork() { InstrumentedMutexLock guard_lock(&mutex_); bg_compaction_paused_++; + // The blob reference census job can write the MANIFEST too, so wait for it. + // ScheduleBlobReferenceCensus() does not start it again while paused. while (CompactionsScheduled() > 0 || remote_compaction_pool_jobs_ > 0 || - bg_flush_scheduled_ > 0 || bg_wbm_flush_scheduled_ > 0) { + bg_flush_scheduled_ > 0 || bg_wbm_flush_scheduled_ > 0 || + bg_blob_reference_census_state_ == + BlobReferenceCensusJobState::kScheduled) { TEST_SYNC_POINT("DBImpl::PauseBackgroundWork:Wait"); bg_cv_.Wait(); } @@ -2077,6 +4867,9 @@ Status DBImpl::ContinueBackgroundWork() { // bg_work_paused_ is always no greater than bg_compaction_paused_ if (bg_work_paused_ == 0) { MaybeScheduleFlushOrCompaction(); + // Start the census job again for the column families that wait in its + // queue (see ScheduleBlobReferenceCensus()). + ScheduleBlobReferenceCensus(); } return Status::OK(); } @@ -4200,6 +6993,10 @@ IOStatus DBImpl::CopyTrivialCompactionFiles( out_file_metadata.db_ownership_start_time = copy_time; out_file_metadata.data_file_provenance = in_file->data_file_provenance | kProvenanceTemperatureChangedByCopy; + out_file_metadata.blob_reference_descriptor = + in_file->blob_reference_descriptor; + out_file_metadata.blob_reference_legacy_metadata = + in_file->blob_reference_legacy_metadata; if (out_table_properties) { c->SetOutputTableProperties(out_fname, out_table_properties); @@ -4211,7 +7008,8 @@ IOStatus DBImpl::CopyTrivialCompactionFiles( out_table_properties ? *out_table_properties : TableProperties(), TableFileCreationReason::kCompaction, io_s, out_file_metadata.file_checksum, - out_file_metadata.file_checksum_func_name); + out_file_metadata.file_checksum_func_name, + /*blob_references=*/nullptr); unfinished_out_fname.clear(); out_files->push_back(std::move(out_file_metadata)); @@ -4223,7 +7021,8 @@ IOStatus DBImpl::CopyTrivialCompactionFiles( &event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(), unfinished_out_fname, job_id, FileDescriptor(), kInvalidBlobFileNumber, TableProperties(), TableFileCreationReason::kCompaction, io_s, - kUnknownFileChecksum, kUnknownFileChecksumFuncName); + kUnknownFileChecksum, kUnknownFileChecksumFuncName, + /*blob_references=*/nullptr); } return io_s; } @@ -4334,8 +7133,9 @@ BackgroundJobPressure DBImpl::CaptureBackgroundJobPressure() const { // Uses the same inputs as RecalculateWriteStallConditions() in // column_family.cc: // - l0_delay_trigger_count(): sorted run count used for write stall - // decisions. Equals NumLevelFiles(0) for level compaction; for universal - // compaction, also counts non-empty levels from L1+ as sorted runs. + // decisions. Equals the L0 sorted run count for level compaction (see + // VersionStorageInfo::NumL0SortedRuns()); for universal compaction, also + // counts non-empty levels from L1+ as sorted runs. // - estimated_compaction_needed_bytes(): pending compaction bytes. // Prefers slowdown/soft triggers (gradual); falls back to stop/hard triggers. // level0_slowdown_writes_trigger is sanitized to always be > 0; @@ -4869,7 +7669,8 @@ void DBImpl::BackgroundCallWBMFlush() { Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context, LogBuffer* log_buffer, FlushReason* reason, bool* flush_rescheduled_to_retain_udt, - Env::Priority thread_pri) { + Env::Priority thread_pri, + autovector* census_cfds) { mutex_.AssertHeld(); Status status; @@ -5001,7 +7802,14 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context, *reason = bg_flush_args[0].flush_reason_; for (auto& arg : bg_flush_args) { ColumnFamilyData* cfd = arg.cfd_; - if (cfd->UnrefAndTryDelete()) { + if (cfd->blob_reference_census() != nullptr && !cfd->IsDropped()) { + // The caller applies the census of `cfd` and then drops this + // reference. A dropped column family is released here as before, so + // FindObsoleteFiles() in the caller still sees its last reference go. + census_cfds->push_back(cfd); + mutex_.AssertHeld(); + ++blob_reference_census_pending_applies_; + } else if (cfd->UnrefAndTryDelete()) { arg.cfd_ = nullptr; } } @@ -5036,9 +7844,12 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) { CaptureListenerPendingOutput(**pending_outputs_inserted_elem); FlushReason reason; bool flush_rescheduled_to_retain_udt = false; - Status s = - BackgroundFlush(&made_progress, &job_context, &log_buffer, &reason, - &flush_rescheduled_to_retain_udt, thread_pri); + // The column families of this flush that have a census, each with a + // reference (see ApplyBlobReferenceCensusAfterInstall()). + autovector census_cfds; + Status s = BackgroundFlush(&made_progress, &job_context, &log_buffer, + &reason, &flush_rescheduled_to_retain_udt, + thread_pri, &census_cfds); if (s.IsTryAgain() && flush_rescheduled_to_retain_udt) { bg_cv_.SignalAll(); // In case a waiter can proceed despite the error mutex_.Unlock(); @@ -5097,6 +7908,7 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) { mutex_.Lock(); } TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:ContextCleanedUp"); + ApplyBlobReferenceCensusAfterInstall(census_cfds); assert(num_running_flushes_ > 0); num_running_flushes_--; @@ -5153,10 +7965,13 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction, : (bg_thread_pri == Env::Priority::BOTTOM ? bg_bottom_compaction_scheduled_ > 0 : bg_compaction_scheduled_ > 0)); - Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer, - prepicked_compaction, bg_thread_pri, - separate_remote_pool, &deferred_compactions, - **pending_outputs_inserted_elem); + // The column family of this compaction if it has a census, with a + // reference (see ApplyBlobReferenceCensusAfterInstall()). + autovector census_cfds; + Status s = BackgroundCompaction( + &made_progress, &job_context, &log_buffer, prepicked_compaction, + bg_thread_pri, separate_remote_pool, &deferred_compactions, + **pending_outputs_inserted_elem, &census_cfds); TEST_SYNC_POINT("BackgroundCallCompaction:1"); if (s.IsBusy()) { bg_cv_.SignalAll(); // In case a waiter can proceed despite the error @@ -5224,6 +8039,7 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction, job_context.Clean(); mutex_.Lock(); } + ApplyBlobReferenceCensusAfterInstall(census_cfds); assert(num_running_compactions_ > 0); num_running_compactions_--; @@ -5280,7 +8096,8 @@ Status DBImpl::BackgroundCompaction( bool* made_progress, JobContext* job_context, LogBuffer* log_buffer, PrepickedCompaction* prepicked_compaction, Env::Priority thread_pri, bool separate_remote_pool, int* deferred_compactions, - uint64_t min_output_file_number) { + uint64_t min_output_file_number, + autovector* census_cfds) { ManualCompactionState* manual_compaction = prepicked_compaction == nullptr ? nullptr @@ -5326,6 +8143,16 @@ Status DBImpl::BackgroundCompaction( ColumnFamilyData* in_flight_cfd = nullptr; Defer end_in_flight_compaction([&]() { if (in_flight_cfd != nullptr) { + if (in_flight_cfd->blob_reference_census() != nullptr && + !in_flight_cfd->IsDropped()) { + // The caller applies the census of this column family and then drops + // this reference. A dropped column family gets no extra reference, so + // the comment above stays true for it. + in_flight_cfd->Ref(); + census_cfds->push_back(in_flight_cfd); + mutex_.AssertHeld(); + ++blob_reference_census_pending_applies_; + } EndInFlightCompaction(in_flight_cfd); } }); @@ -5417,6 +8244,14 @@ Status DBImpl::BackgroundCompaction( std::unique_ptr task_token; bool sfm_reserved_compact_space = false; + uint64_t sfm_reserved_blob_gc_bytes = 0; + bool ran_indirect_blob_gc = false; + bool indirect_blob_gc_stale_input = false; + ColumnFamilyData* indirect_blob_gc_cfd = nullptr; + IOStatus io_s; + // Some early paths return before any I/O runs, so mark the default status as + // checked. Each later assignment replaces it and is checked before return. + io_s.PermitUncheckedError(); if (is_manual) { ManualCompactionState* m = manual_compaction; assert(m->in_progress); @@ -5596,15 +8431,87 @@ Status DBImpl::BackgroundCompaction( "but upon re-evaluation, no compaction was found necessary \n", cfd->GetName().c_str()); } + + // Normal LSM compaction comes first. Indirect blob GC runs only on a + // LOW-pool worker that has no pre-picked compaction and found no LSM + // compaction. Blob files, blob GC and blob indirection must be on, and + // the census must be ready. GC takes up to kIndirectBlobGCMaxInputs (8) + // candidates in the Version's rank order + // (GetBlobPhysicalFilesForIndirectBlobGC()), and RunIndirectBlobGCBatch() + // checks them again in full. Only one GC job runs per column family at a + // time (indirect_blob_gc_cfs_in_progress_). + BlobReferenceCensus* const blob_reference_census = + cfd == nullptr ? nullptr : cfd->blob_reference_census(); + Version* const current_version = + cfd == nullptr ? nullptr : cfd->current(); + if (status.ok() && c == nullptr && !is_prepicked && cfd != nullptr && + job_context != nullptr && thread_pri == Env::Priority::LOW && + mutable_cf_options.enable_blob_files && + mutable_cf_options.enable_blob_garbage_collection && + mutable_cf_options.enable_blob_indirection && + blob_reference_census != nullptr && current_version != nullptr && + blob_reference_census->IsReady(current_version->GetVersionNumber())) { + VersionStorageInfo* const storage = current_version->storage_info(); + std::vector candidates; + storage->GetBlobPhysicalFilesForIndirectBlobGC(kIndirectBlobGCMaxInputs, + &candidates); + if (!candidates.empty() && + indirect_blob_gc_cfs_in_progress_.insert(cfd->GetID()).second) { + indirect_blob_gc_cfd = cfd; + ran_indirect_blob_gc = true; + if (in_flight_cfd == nullptr) { + in_flight_cfd = cfd; + BeginInFlightCompaction(in_flight_cfd); + } + // With a separate remote compaction pool this thread is a remote + // coordinator. Indirect blob GC then runs as local work: it waits for + // a local compaction credit, is counted in bg_compaction_scheduled_, + // and runs in the LOW pool. If RunLocalCompaction returns without + // running the job, its status is returned and GC did not run. + const auto run_indirect_blob_gc_job = + [&](const std::function& gc) { + if (!separate_remote_pool) { + return gc(); + } + const std::atomic never_canceled{false}; + bool gc_ran = false; + Status gc_status; + mutex_.Unlock(); + Status local_status = RunLocalCompaction( + [&]() { + InstrumentedMutexLock gc_lock(&mutex_); + gc_ran = true; + gc_status = gc(); + }, + Env::Priority::LOW, cfd, never_canceled, + /*is_manual=*/false); + mutex_.Lock(); + if (gc_ran) { + local_status.PermitUncheckedError(); + return gc_status; + } + return local_status; + }; + status = run_indirect_blob_gc_job([&]() { + return RunIndirectBlobGCBatch( + cfd, candidates, made_progress, &sfm_reserved_blob_gc_bytes, + &indirect_blob_gc_stale_input, log_buffer, job_context->job_id); + }); + io_s = versions_->io_status(); + indirect_blob_gc_cfs_in_progress_.erase(cfd->GetID()); + } + } } } - IOStatus io_s; bool compaction_released = false; if (c) { c->SetSeparateRemoteCompactionPoolSelected(separate_remote_pool); } - if (!c) { + if (ran_indirect_blob_gc) { + // The indirect blob GC job (RunIndirectBlobGCBatch()) completed the work + // and returned with mutex_ held. + } else if (!c) { // Nothing to do ROCKS_LOG_BUFFER(log_buffer, "Compaction nothing to do"); } else if (c->deletion_compaction()) { @@ -5975,13 +8882,51 @@ Status DBImpl::BackgroundCompaction( compaction_job_stats, job_context->job_id); } + if (sfm_reserved_blob_gc_bytes != 0) { + auto* const sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + assert(sfm != nullptr); + sfm->OnCompactionCompletion(sfm_reserved_blob_gc_bytes); + } + + // Puts the column family that indirect blob GC ran on back in the compaction + // queue, so that a later worker picks GC again. With a separate remote pool, + // the new queue entry is counted in *deferred_compactions instead of + // unscheduled_compactions_, the same way as the compaction retry below. + // BackgroundCallCompaction() adds that count back to unscheduled_compactions_ + // when this job ends. + const auto retry_indirect_blob_gc = [&]() { + if (indirect_blob_gc_cfd != nullptr && !indirect_blob_gc_cfd->IsDropped() && + !error_handler_.IsBGWorkStopped()) { + const int unscheduled_before_retry = unscheduled_compactions_; + EnqueuePendingCompaction(indirect_blob_gc_cfd); + if (separate_remote_pool && + unscheduled_compactions_ > unscheduled_before_retry) { + --unscheduled_compactions_; + restore_compaction_credit(); + } + } + }; + if (status.ok() || status.IsCompactionTooLarge() || status.IsManualCompactionPaused() || status.IsCompactionAborted()) { + if (status.IsCompactionTooLarge() || status.IsCompactionAborted()) { + retry_indirect_blob_gc(); + } // Done + } else if (status.IsTryAgain() && ran_indirect_blob_gc && + indirect_blob_gc_stale_input) { + // A stale input is not an error, so it becomes Busy. Busy sets no + // background error and does not count as one. BackgroundCallCompaction() + // then waits only 10 ms (to prevent a hot loop) instead of the 1 s wait + // after an error, and it skips the full obsolete-file scan. + retry_indirect_blob_gc(); + status = Status::Busy("Indirect blob GC input changed"); } else if (status.IsColumnFamilyDropped() || status.IsShutdownInProgress()) { // Ignore compaction errors found during shutting down } else { - ROCKS_LOG_WARN(immutable_db_options_.info_log, "Compaction error: %s", + ROCKS_LOG_WARN(immutable_db_options_.info_log, "%s error: %s", + ran_indirect_blob_gc ? "Indirect blob GC" : "Compaction", status.ToString().c_str()); if (!io_s.ok()) { // Error while writing to MANIFEST. @@ -6016,6 +8961,12 @@ Status DBImpl::BackgroundCompaction( --unscheduled_compactions_; restore_compaction_credit(); } + } else if (ran_indirect_blob_gc) { + // Indirect blob GC hit a background error. If the error did not stop + // background work (for example, a retryable I/O error), put the CF back + // in the queue, the same as the compaction retry above. + // retry_indirect_blob_gc() does nothing if background work stopped. + retry_indirect_blob_gc(); } } if (c != nullptr && !status.ok() && @@ -6358,6 +9309,9 @@ void DBImpl::InstallSuperVersionAndScheduleWork( } cfd->InstallSuperVersion(sv_context, &mutex_, std::move(new_seqno_to_time_mapping)); + // Queue the census of this CF if it needs a run, for example when the CF got + // the format fence after open. The call is O(1). + ScheduleBlobReferenceCensus(cfd); // Refresh the seqno->time preserve-window bound used to gate bottommost file // marking, so a file whose largest seqno is still within the preserve window diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc index 7858d27e0756..8ef23da2fc00 100644 --- a/db/db_impl/db_impl_debug.cc +++ b/db/db_impl/db_impl_debug.cc @@ -16,6 +16,7 @@ #include "db/db_impl/db_impl.h" #include "db/error_handler.h" #include "db/periodic_task_scheduler.h" +#include "file/sst_file_manager_impl.h" #include "monitoring/thread_status_updater.h" #include "util/cast_util.h" #include "util/hash_containers.h" @@ -246,6 +247,30 @@ Status DBImpl::TEST_WaitForCompact( return WaitForCompact(wait_for_compact_options); } +Status DBImpl::TEST_RunIndirectBlobGCBatch( + ColumnFamilyData* cfd, + const std::vector& candidate_physical_file_numbers) { + bool made_progress = false; + uint64_t sfm_reserved_bytes = 0; + bool retryable_stale_input = false; + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, + immutable_db_options_.info_log.get()); + InstrumentedMutexLock l(&mutex_); + Status status = RunIndirectBlobGCBatch(cfd, candidate_physical_file_numbers, + &made_progress, &sfm_reserved_bytes, + &retryable_stale_input, &log_buffer, + /*job_id=*/0); + if (sfm_reserved_bytes != 0) { + auto* const sfm = static_cast( + immutable_db_options_.sst_file_manager.get()); + assert(sfm != nullptr); + sfm->OnCompactionCompletion(sfm_reserved_bytes); + } + // OK without progress is possible: indirect blob GC ends quietly while an + // earlier metadata commit has an unknown outcome. + return status; +} + Status DBImpl::TEST_WaitForPurge() { InstrumentedMutexLock l(&mutex_); while (bg_purge_scheduled_ && error_handler_.GetBGError().ok()) { diff --git a/db/db_impl/db_impl_experimental.cc b/db/db_impl/db_impl_experimental.cc index c68f2aab2339..9df0a0ee44d2 100644 --- a/db/db_impl/db_impl_experimental.cc +++ b/db/db_impl/db_impl_experimental.cc @@ -11,6 +11,7 @@ #include #include "db/column_family.h" +#include "db/compaction/compaction.h" #include "db/db_impl/db_impl.h" #include "db/job_context.h" #include "db/version_set.h" @@ -106,6 +107,20 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) { return status; } + // A coarse file must be rewritten with exact metadata when it leaves L0 + // (see Compaction::MustRewriteCoarseL0File()). + if (Compaction::MustRewriteCoarseL0File(*f)) { + ROCKS_LOG_INFO(immutable_db_options_.info_log, + "PromoteL0 FAILED. File %" PRIu64 + " has coarse blob reference metadata\n", + f->fd.GetNumber()); + job_context.Clean(); + status = Status::NotSupported( + "PromoteL0 cannot move an L0 file with coarse blob reference " + "metadata"); + return status; + } + if (i == 0) { continue; } diff --git a/db/db_impl/db_impl_files.cc b/db/db_impl/db_impl_files.cc index 52b8a20202d9..6ef16ddd12ef 100644 --- a/db/db_impl/db_impl_files.cc +++ b/db/db_impl/db_impl_files.cc @@ -13,6 +13,7 @@ #include "db/blob/blob_file_partition_manager.h" #include "db/blob/blob_log_format.h" +#include "db/blob/blob_relocation_file.h" #include "db/db_impl/db_impl.h" #include "db/event_helpers.h" #include "db/memtable_list.h" @@ -86,10 +87,6 @@ bool ShouldKeepFooterlessBlobFile(FileSystem* fs, return !io_s.IsPathNotFound(); } - if (file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) { - return true; - } - FileOptions read_file_options = file_options; read_file_options.use_direct_reads = false; @@ -98,21 +95,48 @@ bool ShouldKeepFooterlessBlobFile(FileSystem* fs, if (!io_s.ok()) { return !io_s.IsPathNotFound(); } - - std::array scratch{}; - Slice footer_slice; - io_s = file->Read(file_size - BlobLogFooter::kSize, BlobLogFooter::kSize, - io_options, &footer_slice, scratch.data(), dbg); - if (!io_s.ok()) { - return !io_s.IsPathNotFound(); + if (file == nullptr) { + return true; } - if (footer_slice.size() != BlobLogFooter::kSize) { - return true; + if (file_size >= BlobLogHeader::kSize + BlobLogFooter::kSize) { + std::array scratch{}; + Slice footer_slice; + io_s = file->Read(file_size - BlobLogFooter::kSize, BlobLogFooter::kSize, + io_options, &footer_slice, scratch.data(), dbg); + if (!io_s.ok()) { + return !io_s.IsPathNotFound(); + } + BlobLogFooter footer; + if (footer_slice.size() == BlobLogFooter::kSize && + footer.DecodeFrom(footer_slice).ok()) { + return false; + } } - BlobLogFooter footer; - return !footer.DecodeFrom(footer_slice).ok(); + // The file has no valid blob log footer. Only such a file gets the + // relocation-prefix read; a blob file with a valid footer costs no extra + // read. + if (file_size >= kBlobRelocationFilePrefixSize) { + std::array prefix_scratch{}; + Slice prefix_slice; + io_s = file->Read(/*offset=*/0, kBlobRelocationFilePrefixSize, io_options, + &prefix_slice, prefix_scratch.data(), dbg); + // Lets a test fail the prefix read. + TEST_SYNC_POINT_CALLBACK("ShouldKeepFooterlessBlobFile:PrefixRead", &io_s); + if (!io_s.ok()) { + return !io_s.IsPathNotFound(); + } + if (prefix_slice == + Slice(kBlobRelocationFilePrefix, kBlobRelocationFilePrefixSize)) { + // Relocation files are always built under a temp name and protected by + // pending_outputs_ across rename and publication. They are never written + // by blob direct write, whose in-flight files have no footer yet, so an + // obsolete one is safe to delete although it has no blob log footer. + return false; + } + } + return true; } } // namespace @@ -228,6 +252,12 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, bool no_full_scan) { mutex_.AssertHeld(); + // Move the relocation state of destroyed Versions to the JobContext; + // job_context->Clean() frees it without the DB mutex. This runs before the + // check below, so the state is freed also while file deletion is disabled. + versions_->TakeObsoleteBlobRelocationStates( + &job_context->blob_relocation_states_to_free); + // if deletion is disabled, do nothing if (disable_delete_obsolete_files_ > 0) { return; @@ -604,6 +634,14 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { } for (const auto& blob_file : state.blob_delete_files) { + // The file is obsolete, so evict its cached reader now, before the keep + // decision below. The decision can keep the file (for example after an I/O + // error while it reads the file), but the file has then left the obsolete + // list, so a reader left here would stay cached with its file descriptor + // and its pinned location blocks until a full-scan purge. An eviction is + // always safe: a later read opens the file again. It takes no DB mutex and + // no stripe mutex (TableCache::Evict() of the shared cache). + TableCache::Evict(table_cache_.get(), blob_file.GetBlobFileNumber()); candidate_files.emplace_back(BlobFileName(blob_file.GetBlobFileNumber()), blob_file.GetPath()); } @@ -757,7 +795,12 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) { // // TODO(yhchiang): carefully modify the third condition to safely // remove the temp options files. - keep = (sst_live_set.find(number) != sst_live_set.end()) || + // + // Indirect blob GC writes its output as a temp file and renames it to + // its blob file name before the install. Like other in-progress + // outputs, pending_outputs_ protects its number. + keep = number >= state.min_pending_output || + (sst_live_set.find(number) != sst_live_set.end()) || (blob_live_set.find(number) != blob_live_set.end()) || (number == state.pending_manifest_file_number) || (to_delete.find(kOptionsFileNamePrefix) != std::string::npos); diff --git a/db/db_impl/db_impl_follower.cc b/db/db_impl/db_impl_follower.cc index 5bd535ca7daa..e3d8be73a206 100644 --- a/db/db_impl/db_impl_follower.cc +++ b/db/db_impl/db_impl_follower.cc @@ -90,6 +90,8 @@ Status DBImplFollower::Recover( // 1. Cleanup obsolete files afterward // 2. Add some error notifications and statistics Status DBImplFollower::TryCatchUpWithLeader() { + TEST_SYNC_POINT("DBImplFollower::TryCatchUpWithLeader:BeforeCatchUpLock"); + std::lock_guard catch_up_lock(catch_up_mutex_); assert(versions_.get() != nullptr); assert(manifest_reader_.get() != nullptr); Status s; diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index fb324ca4d732..534119977e77 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -8,9 +8,12 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include #include +#include #include +#include #include "db/blob/blob_file_partition_manager.h" +#include "db/blob/blob_reference_census.h" #include "db/builder.h" #include "db/db_impl/db_impl.h" #include "db/error_handler.h" @@ -29,7 +32,10 @@ #include "rocksdb/options.h" #include "rocksdb/table.h" #include "rocksdb/wal_filter.h" +#include "table/adaptive/adaptive_table_factory.h" #include "test_util/sync_point.h" +#include "util/defer.h" +#include "util/random.h" #include "util/rate_limiter_impl.h" #include "util/string_util.h" #include "util/udt_util.h" @@ -472,6 +478,15 @@ Status DBImpl::Recover( bool& is_new_db = recovery_ctx ? recovery_ctx->is_new_db_ : tmp_is_new_db; assert(db_lock_ == nullptr); std::vector files_in_dbname; + if (read_only) { + // Read-only opens (DBImplReadOnly and CompactedDBImpl) do not call + // ValidateOptions(), so the blob indirection FIFO check runs here, once per + // CF. Secondary opens run it in OpenAsSecondaryImpl(). + Status s = CheckBlobIndirectionCompactionStyle(column_families); + if (!s.ok()) { + return s; + } + } if (!read_only) { Status s = directories_.SetDirectories(fs_.get(), dbname_, immutable_db_options_.wal_dir, @@ -705,7 +720,8 @@ Status DBImpl::Recover( for (const FileMetaData* f : level_files) { edit.DeleteFile(from_level, f->fd.GetNumber()); // Keeps f->temperature, which can be different from - // `last_level_temperature` + // `last_level_temperature`, and all other persisted metadata, + // so exact blob references remain attached to the same SST. edit.AddExistingFile(to_level, *f, kProvenanceMovedBetweenLevels); ROCKS_LOG_WARN(immutable_db_options_.info_log, "[%s] Moving #%" PRIu64 @@ -2178,6 +2194,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, // Note that here we treat flush as level 0 compaction in internal stats InternalStats::CompactionStats flush_stats(CompactionReason::kFlush, 1 /* count */); + bool output_built_with_indirection = false; { ScopedArenaPtr iter( logical_strip_timestamp @@ -2195,6 +2212,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, // Get the latest mutable cf options while the mutex is still locked const MutableCFOptions mutable_cf_options_copy = cfd->GetLatestMutableCFOptions(); + output_built_with_indirection = + mutable_cf_options_copy.enable_blob_indirection; bool paranoid_file_checks = cfd->GetLatestMutableCFOptions().paranoid_file_checks; @@ -2255,6 +2274,9 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, dbname_ /* db_name */); Version* version = cfd->current(); version->Ref(); + tboptions.track_blob_reference_classification = + UsesBlobIndirection(mutable_cf_options_copy, version->storage_info()); + tboptions.collect_blob_reference_metadata = false; TableProperties temp_table_proerties; s = BuildTable( dbname_, versions_.get(), immutable_db_options_, tboptions, @@ -2338,6 +2360,10 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd, if (s.ok() && has_output) { edit->AddFile(level, meta); + // The WAL recovery flush can be the first job that writes indirect + // BlobIndexes. So, like a normal flush, it sets the per-CF format fence + // when its SST was built with enable_blob_indirection on. + edit->MaybeSetBlobIndirectionFormatFence(output_built_with_indirection); for (const auto& blob : blob_file_additions) { edit->AddBlobFile(blob); @@ -3060,6 +3086,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, if (impl->immutable_db_options_.open_files_async) { impl->ScheduleAsyncFileOpening(); } + if (impl->read_only_) { + impl->bg_blob_reference_census_state_ = + DBImpl::BlobReferenceCensusJobState::kComplete; + } impl->MaybeScheduleAsyncWALPrecreate( impl->GetWalPreallocateBlockSize(max_write_buffer_size)); impl->mutex_.Unlock(); @@ -3097,6 +3127,13 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname, if (s.ok()) { s = impl->StartPeriodicTaskScheduler(); } + if (s.ok() && !impl->read_only_) { + // The census job may start now (see + // DBImpl::blob_reference_census_job_allowed_). + InstrumentedMutexLock lock(&impl->mutex_); + impl->blob_reference_census_job_allowed_ = true; + impl->ScheduleBlobReferenceCensus(); + } if (s.ok()) { s = impl->RegisterRecordSeqnoTimeWorker(); } @@ -3260,4 +3297,912 @@ void DBImpl::BGWorkAsyncFileOpen(void* arg) { TEST_SYNC_POINT("DBImpl::BGWorkAsyncFileOpen:Done"); } +void DBImpl::ScheduleBlobReferenceCensus() { + ScheduleBlobReferenceCensus(/*requested_cfd=*/nullptr); +} + +void DBImpl::ScheduleBlobReferenceCensus(ColumnFamilyData* requested_cfd) { + mutex_.AssertHeld(); + if (read_only_ || shutting_down_.load(std::memory_order_acquire)) { + return; + } + + const auto needs_census = [](ColumnFamilyData* cfd) { + if (cfd == nullptr || cfd->IsDropped()) { + return false; + } + Version* const current = cfd->current(); + if (current == nullptr || + !current->storage_info()->HasBlobIndirectionFormat()) { + return false; + } + // A fenced CF recovered from the MANIFEST gets its census here. + return cfd->GetOrCreateBlobReferenceCensus()->NeedsWorkerRun( + current->GetVersionNumber()); + }; + + const auto enqueue = [this, &needs_census](ColumnFamilyData* cfd) { + if (!needs_census(cfd) || cfd->queued_for_blob_reference_census_) { + return; + } + cfd->blob_reference_census()->SetBuildWakeup( + [](void* db, void* owner) { + static_cast(db)->ScheduleBlobReferenceCensus( + static_cast(owner)); + }, + this, cfd); + cfd->Ref(); + cfd->queued_for_blob_reference_census_ = true; + cfd->next_blob_reference_census_ = nullptr; + if (blob_reference_census_queue_tail_ == nullptr) { + assert(blob_reference_census_queue_head_ == nullptr); + blob_reference_census_queue_head_ = cfd; + } else { + blob_reference_census_queue_tail_->next_blob_reference_census_ = cfd; + } + blob_reference_census_queue_tail_ = cfd; + }; + + if (requested_cfd != nullptr) { + enqueue(requested_cfd); + } else { + TEST_SYNC_POINT("DBImpl::ScheduleBlobReferenceCensus:AllColumnFamilies"); + for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) { + enqueue(cfd); + } + } + // While PauseBackgroundWork() is in effect, the queue keeps the column + // families, and ContinueBackgroundWork() schedules the job again. + if (!blob_reference_census_job_allowed_ || + bg_blob_reference_census_state_ == + BlobReferenceCensusJobState::kScheduled || + bg_compaction_paused_ > 0) { + return; + } + if (blob_reference_census_queue_head_ == nullptr) { + bg_blob_reference_census_state_ = BlobReferenceCensusJobState::kComplete; + return; + } + + bg_blob_reference_census_state_ = BlobReferenceCensusJobState::kScheduled; + env_->Schedule(&DBImpl::BGWorkBlobReferenceCensus, this, Env::Priority::LOW, + &bg_blob_reference_census_state_, + &DBImpl::UnscheduleBlobReferenceCensus); +} + +void DBImpl::UnscheduleBlobReferenceCensus(void* arg) { + auto* db = static_cast(arg); + db->mutex_.AssertHeld(); + + ColumnFamilyData* pending = db->blob_reference_census_queue_head_; + if (db->blob_reference_census_queue_tail_ != nullptr) { + db->blob_reference_census_queue_tail_->next_blob_reference_census_ = + db->blob_reference_census_retry_head_; + } else { + pending = db->blob_reference_census_retry_head_; + } + db->blob_reference_census_queue_head_ = nullptr; + db->blob_reference_census_queue_tail_ = nullptr; + db->blob_reference_census_retry_head_ = nullptr; + db->blob_reference_census_retry_tail_ = nullptr; + db->blob_reference_census_retry_count_ = 0; + // Release each retained CF in a separate critical section. A dropped CF can + // run nontrivial destruction from its final UnrefAndTryDelete(). + db->mutex_.Unlock(); + while (pending != nullptr) { + db->mutex_.Lock(); + ColumnFamilyData* const next = pending->next_blob_reference_census_; + pending->next_blob_reference_census_ = nullptr; + pending->queued_for_blob_reference_census_ = false; + BlobReferenceCensusGarbage garbage = pending->CancelBlobReferenceCensus(); + db->mutex_.Unlock(); + // An initial build that waits between slices pins its CFD: the deleter of + // its starting Version releases a CFD reference. This makes a cycle CFD -> + // census -> build context -> CFD. Free the census garbage, which breaks the + // cycle, before dropping the queue's pin. This also covers dropped CFs, + // which ColumnFamilySet iteration no longer visits. + garbage = {}; + db->ReleaseColumnFamilyDataRef(pending); + pending = next; + } + db->mutex_.Lock(); + db->bg_blob_reference_census_state_ = BlobReferenceCensusJobState::kComplete; + db->bg_cv_.SignalAll(); +} + +Status DBImpl::BuildBlobReferenceCensus(ColumnFamilyData* cfd, bool* incomplete, + bool* overflowed) { + assert(cfd != nullptr && incomplete != nullptr && overflowed != nullptr); + *incomplete = false; + *overflowed = false; + if (!blob_reference_census_retry_registered_.exchange( + true, std::memory_order_acq_rel)) { + const Status registration = periodic_task_scheduler_.Register( + PeriodicTaskType::kBlobReferenceCensus, + [this]() { RetryBlobReferenceCensus(); }, + /*run_immediately=*/false); + if (!registration.ok()) { + blob_reference_census_retry_registered_.store(false, + std::memory_order_release); + return registration; + } + } + BlobReferenceCensus& census = *cfd->blob_reference_census(); + if (census.TakeMismatchReport()) { + // Indirect blob GC reported a census mismatch. Fail this build. + // ReportMismatch() also set RebuildRequested(), so a new build starts at + // once from the current Version. This status fails only the build slice. It + // is not a background error. Any non-OK status would do: the slice caller + // (BGWorkBlobReferenceCensus) normally sees RebuildRequested() and restarts + // at once. Corruption names the cause: the census does not match the SSTs. + return Status::Corruption("Blob GC found a census mismatch"); + } + std::shared_ptr context = + census.GetBuildContext(); + Status status; + if (context == nullptr) { + Version* starting_version = nullptr; + uint64_t token = 0; + { + InstrumentedMutexLock lock(&mutex_); + if (shutting_down_.load(std::memory_order_acquire) || cfd->IsDropped()) { + status = Status::ShutdownInProgress(); + return status; + } + starting_version = cfd->current(); + starting_version->Ref(); + cfd->Ref(); + token = census.BeginBuild(); + const uint64_t now_micros = immutable_db_options_.clock->NowMicros(); + census.NoteUnready(now_micros); + census.NoteBuildStart(now_micros); + } + context = std::make_shared(); + context->token = token; + for (std::shared_ptr& seed : + census.TakeRebuildSeeds()) { + const uint64_t file_number = seed->file_number; + context->seeds.emplace(file_number, std::move(seed)); + } + context->starting_version = std::shared_ptr( + starting_version, [this, cfd](Version* version) { + { + InstrumentedMutexLock lock(&mutex_); + version->UnrefAndFreeBlobRelocationState(&mutex_); + } + ReleaseColumnFamilyDataRef(cfd); + }); + const BlockBasedTableOptions* table_options = + GetBlockBasedTableWriterOptions( + starting_version->GetMutableCFOptions().table_factory.get()); + std::shared_ptr block_cache; + if (table_options != nullptr && table_options->block_cache != nullptr) { + const CacheEntryRoleOptions::Decision charged = + table_options->cache_usage_options.options_overrides + .at(CacheEntryRole::kBlobReferenceCensus) + .charged; + if (charged != CacheEntryRoleOptions::Decision::kDisabled) { + block_cache = table_options->block_cache; + context->reservation_manager = std::make_shared< + CacheReservationManagerImpl>( + block_cache); + context->summary_reservation_manager = + std::make_shared( + std::make_shared>(block_cache)); + } + } + size_t starting_file_count = 0; + for (int level = 0; level < starting_version->storage_info()->num_levels(); + ++level) { + starting_file_count += + starting_version->storage_info()->LevelFiles(level).size(); + } + status = context->ReserveInitialFiles(starting_file_count); + if (status.ok()) { + status = census.ResetWorkerState( + token, starting_version->GetVersionNumber(), + starting_version->storage_info()->GetBlobRelocationState(), + std::move(block_cache)); + } + if (status.ok()) { + std::vector physical_files; + physical_files.reserve( + starting_version->storage_info()->GetBlobFiles().size()); + for (const std::shared_ptr& file : + starting_version->storage_info()->GetBlobFiles()) { + if (file->IsIndirectOrigin()) { + physical_files.push_back({file->GetBlobFileNumber(), + file->GetTotalBlobBytes(), + /*is_indirect_origin=*/true}); + } + } + status = census.SetWorkerPhysicalFiles(token, physical_files); + if (status.ok()) { + status = census.SetWorkerCoarseIndirectOriginFrontier( + token, starting_version->storage_info() + ->OldestCoarseIndirectOriginFileNumber()); + } + } + if (!census.TrySetBuildContext(context) && status.ok()) { + status = Status::Aborted("Blob reference census build became stale"); + } + if (!status.ok()) { + return status; + } + // Lets tests stop a new build (for example a restart after an overflow) + // after its build cursor opened and before it applies any delta. + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:NewBuildReady"); + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:NewBuildWait"); + } + + const auto catch_up = [&]() -> Status { + // A stored failure may have changed the worker: rebuild instead of applying + // that delta again. + if (!context->failure.ok()) { + return context->failure; + } + bool log_overflowed = false; + if (context->pending_deltas.empty()) { + Status take_status = census.TakeBuildDeltas( + context->token, &context->pending_deltas, &log_overflowed); + if (!take_status.ok()) { + return take_status; + } + if (log_overflowed) { + *overflowed = true; + return Status::Aborted("Blob reference census build cursor overflowed"); + } + } + while (!context->pending_deltas.empty()) { + const std::shared_ptr& delta = + context->pending_deltas.front(); + if (delta == nullptr) { + return Status::Corruption("Null blob reference census delta"); + } + Status apply_status = census.ApplyWorkerDelta(context->token, *delta); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::BuildBlobReferenceCensus:CatchUpApplyStatus", &apply_status); + if (!apply_status.ok()) { + return apply_status; + } + if (!context->scan_complete) { + for (uint64_t file_number : delta->removed_files) { + if (context->starting_version->storage_info() + ->GetFileMetaDataByNumber(file_number) != nullptr) { + context->deleted_starting_files.insert(file_number); + } + } + } + // The delta is applied, so it leaves the queue now, whatever the + // acknowledgment returns. + const size_t weight = delta->QueueWeight(); + context->pending_deltas.pop_front(); + apply_status = census.AcknowledgeBuildDeltas(context->token, weight); + if (!apply_status.ok()) { + if (census.BuildCursorOverflowed(context->token)) { + *overflowed = true; + } + return apply_status; + } + } + return Status::OK(); + }; + + // The initial build reads the summary of every exact SST (an SST with a + // reference descriptor) in the starting Version. It runs in slices, so one + // LOW-priority task never runs long. A slice visits at most kMaxFilesPerSlice + // (16, an estimate) SSTs, and SSTs without a descriptor count too. The + // starting Version stays pinned across slices. Between reads, the build + // applies and acknowledges the edits installed in the meantime, so they do + // not pile up until the scan ends. + constexpr size_t kMaxFilesPerSlice = 16; + size_t visited = 0; + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:BeforeCatchUpReady"); + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:BeforeCatchUpWait"); + status = catch_up(); + if (status.IsTryAgain()) { + *incomplete = true; + return Status::OK(); + } + if (!context->scan_complete && status.ok()) { + Version& version = *context->starting_version; + VersionStorageInfo* const storage = version.storage_info(); + const MutableCFOptions& options = version.GetMutableCFOptions(); + ReadOptions read_options(Env::IOActivity::kCompaction); + read_options.fill_cache = false; + read_options.rate_limiter_priority = Env::IO_LOW; + while (status.ok() && context->level < storage->num_levels() && + visited < kMaxFilesPerSlice) { + const std::vector& files = + storage->LevelFiles(context->level); + if (context->next_file == files.size()) { + ++context->level; + context->next_file = 0; + continue; + } + FileMetaData* const file = files[context->next_file++]; + ++visited; + if (shutting_down_.load(std::memory_order_acquire)) { + return Status::ShutdownInProgress(); + } + if (file->blob_reference_descriptor.empty()) { + continue; + } + const uint64_t file_number = file->fd.GetNumber(); + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:BeforeReadFileReady"); + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:BeforeReadFileWait"); + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:BeforeReadFile"); + status = catch_up(); + if (status.IsTryAgain()) { + // An indirect blob GC job copies its snapshot. Visit this SST again in + // the next run. + --context->next_file; + *incomplete = true; + return Status::OK(); + } + if (!status.ok()) { + break; + } + if (context->deleted_starting_files.count(file_number) != 0) { + continue; + } + std::shared_ptr summary; + const auto seed = context->seeds.find(file_number); + if (seed != context->seeds.end()) { + summary = seed->second; + context->seeds.erase(seed); + } else { + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:ReadSst"); + status = cfd->table_cache()->ReadBlobReferenceSstSummary( + read_options, versions_->file_options(), cfd->internal_comparator(), + *file, options, &summary, context->summary_reservation_manager); + } + TEST_SYNC_POINT_CALLBACK( + "DBImpl::BuildBlobReferenceCensus:ReadFileStatus", &status); + if (!status.ok()) { + break; + } + TEST_SYNC_POINT("DBImpl::BuildBlobReferenceCensus:AfterReadFile"); + status = catch_up(); + if (status.IsTryAgain()) { + // Keep the summary, so the next run adds this SST without reading it + // again. + context->seeds[file_number] = std::move(summary); + --context->next_file; + *incomplete = true; + return Status::OK(); + } + if (status.ok() && + context->deleted_starting_files.count(file_number) == 0) { + status = census.AddStartingFile(context->token, std::move(summary)); + } + } + if (status.ok() && context->level == storage->num_levels()) { + context->scan_complete = true; + context->starting_version.reset(); + std::set().swap(context->deleted_starting_files); + context->seeds.clear(); + status = context->ReserveInitialFiles(0); + } + } + if (!status.ok()) { + return status; + } + if (!context->scan_complete) { + *incomplete = true; + return Status::OK(); + } + status = catch_up(); + if (status.IsTryAgain()) { + *incomplete = true; + return Status::OK(); + } + if (!status.ok()) { + return status; + } + status = census.FinalizeWorkerBuild(context->token); + if (!status.ok()) { + return status; + } + return PublishBlobReferenceCensusWorker(cfd, context.get(), incomplete); +} + +Status DBImpl::PublishBlobReferenceCensusWorker( + ColumnFamilyData* cfd, BlobReferenceCensusBuildContext* context, + bool* incomplete) { + BlobReferenceCensus& census = *cfd->blob_reference_census(); + const uint64_t applied_version = census.WorkerVersionNumber(); + // The census job reads candidate_retry_micros under the DB mutex, so it is + // stored below, after the DB mutex is taken. + const uint64_t candidate_retry_micros = census.RetryWorkerCandidates( + context->token, immutable_db_options_.clock->NowMicros()); + const BlobGcCandidateHeap::Selection candidates = + census.SelectWorkerCandidates(context->token); + const BlobGcCandidateHeap::Selection empty_candidates = + census.SelectWorkerEmptyCandidates(context->token); + const uint64_t relocation_garbage = + census.WorkerRelocationGarbageBytes(context->token); + const BlobGcCandidateHeap::Totals candidate_totals = + census.WorkerCandidateTotals(context->token); + InstrumentedMutexLock lock(&mutex_); + context->candidate_retry_micros = candidate_retry_micros; + if (shutting_down_.load(std::memory_order_acquire) || cfd->IsDropped()) { + return Status::ShutdownInProgress(); + } + const BlobReferenceCensus::PublishResult result = census.TryPublishWorker( + context->token, applied_version, cfd->current()->GetVersionNumber()); + if (result == BlobReferenceCensus::PublishResult::kPublished) { + census.NoteReady(); + census.SetPublishedTotals(candidate_totals, relocation_garbage); + if (candidates[0] != kInvalidBlobFileNumber) { + // A running GC job keeps the idle time at 0. The job's end starts it + // again if candidates remain. + if (!census.HasGcJob()) { + cfd->NoteIndirectBlobGcCandidates( + immutable_db_options_.clock->NowMicros()); + } + } else { + cfd->ClearIndirectBlobGcCandidates(); + } + const uint64_t build_start_micros = census.TakeBuildStartMicros(); + if (build_start_micros != 0) { + const uint64_t now_micros = immutable_db_options_.clock->NowMicros(); + ROCKS_LOG_INFO( + immutable_db_options_.info_log, + "[%s] Blob reference census is ready at Version %" PRIu64 + " after %" PRIu64 " ms", + cfd->GetName().c_str(), applied_version, + (now_micros - std::min(now_micros, build_start_micros)) / 1000); + } + cfd->current()->storage_info()->SetBlobGcCandidates(candidates); + cfd->current()->storage_info()->SetEmptyBlobGcCandidates(empty_candidates); + cfd->current()->storage_info()->SetBlobRelocationGarbageBytes( + relocation_garbage); + EnqueuePendingCompaction(cfd); + MaybeScheduleFlushOrCompaction(); + } else if (result == BlobReferenceCensus::PublishResult::kStale) { + return Status::Aborted( + "Blob reference census build is stale (replaced, canceled or " + "overflowed)"); + } else { + *incomplete = true; + } + return Status::OK(); +} + +void DBImpl::ApplyBlobReferenceCensusAfterInstall( + const autovector& cfds) { + mutex_.AssertHeld(); + // `cfds` holds only the column families with a census that this job flushed + // or compacted (also when the job failed). A column family dropped before + // the job ended is not in it. So the work is O(the job's column families), + // and a DB without blob indirection has none. Each one is visited in any + // scheduling state (queued, running or in the retry list). + bool any_live = false; + for (ColumnFamilyData* cfd : cfds) { + any_live = any_live || !cfd->IsDropped(); + } + if (any_live) { + mutex_.Unlock(); + for (ColumnFamilyData* cfd : cfds) { + // A drop after the check above only makes the apply useless work: the + // reference keeps the column family alive. + ApplyBlobReferenceCensusDeltas(cfd); + } + mutex_.Lock(); + } + for (ColumnFamilyData* cfd : cfds) { + cfd->UnrefAndTryDelete(); + } + assert(blob_reference_census_pending_applies_ >= cfds.size()); + blob_reference_census_pending_applies_ -= cfds.size(); + if (!cfds.empty() && blob_reference_census_pending_applies_ == 0) { + bg_cv_.SignalAll(); + } + TEST_SYNC_POINT("DBImpl::ApplyBlobReferenceCensusAfterInstall:Done"); +} + +void DBImpl::ApplyBlobReferenceCensusDeltas(ColumnFamilyData* cfd) { + TEST_SYNC_POINT_CALLBACK("DBImpl::ApplyBlobReferenceCensusDeltas:Start", cfd); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + if (census == nullptr || shutting_down_.load(std::memory_order_acquire)) { + return; + } + std::unique_lock run_lock(census->worker_run_mutex(), + std::try_to_lock); + if (!run_lock.owns_lock()) { + return; + } + const std::shared_ptr context = + census->GetBuildContext(); + if (context == nullptr || !context->scan_complete) { + return; + } + // An earlier failure (of an apply after an install, or of a census job run) + // may have changed the worker part way. The census job fails the build with + // it, so this thread applies nothing more. + if (!context->failure.ok()) { + return; + } + bool applied = false; + Status status; + while (status.ok()) { + if (context->pending_deltas.empty()) { + bool overflowed = false; + status = census->TakeBuildDeltas(context->token, &context->pending_deltas, + &overflowed); + if (!status.ok() || overflowed || context->pending_deltas.empty()) { + // The census job handles an overflow (a restart, or a failure on a + // second overflow) and an error (a rebuild). No delta is taken on an + // overflow. + status = overflowed ? Status::Incomplete() : status; + break; + } + } + while (!context->pending_deltas.empty()) { + const std::shared_ptr delta = + context->pending_deltas.front(); + if (delta == nullptr) { + status = Status::Corruption("Null blob reference census delta"); + break; + } + // TryAgain (an indirect blob GC job copies its snapshot) changes nothing, + // so the delta stays at the front for the census job. After any other + // failure the delta may be partly applied, so it must never be applied + // again (see below). + status = census->ApplyWorkerDelta(context->token, *delta); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::ApplyBlobReferenceCensusDeltas:ApplyStatus", &status); + if (!status.ok()) { + break; + } + context->pending_deltas.pop_front(); + applied = true; + status = + census->AcknowledgeBuildDeltas(context->token, delta->QueueWeight()); + if (!status.ok()) { + break; + } + } + } + if (!status.ok() && !status.IsTryAgain() && !status.IsIncomplete() && + census->GetBuildContext() == context) { + // A context that a new build replaced or that Cancel() detached is + // dropped anyway, so it stores nothing and logs nothing. + if (census->BuildCursorOverflowed(context->token)) { + // The census job restarts or fails the build after the overflow. Drop + // the queued deltas, so the failed one is never applied again. + context->pending_deltas.clear(); + } else { + // Later applies after an install skip this build, and the census job + // fails the build with this failure. + context->failure = status; + ROCKS_LOG_WARN(immutable_db_options_.info_log, + "[%s] Blob reference census apply after an install " + "failed; the census job rebuilds the census: %s", + cfd->GetName().c_str(), status.ToString().c_str()); + } + } + if (!status.ok() || !applied) { + return; + } + // A publish that fails or must wait is left to the census job. The install + // queued it, or, if the column family is in the retry list, the periodic task + // runs it. + if (census->FinalizeWorkerBuild(context->token).ok()) { + bool incomplete = false; + PublishBlobReferenceCensusWorker(cfd, context.get(), &incomplete) + .PermitUncheckedError(); + } +} + +void DBImpl::CheckBlobReferenceCensus(ColumnFamilyData* cfd) { + bool force = false; + TEST_SYNC_POINT_CALLBACK("DBImpl::CheckBlobReferenceCensus:Force", &force); + if (!force && !Random::GetTLSInstance()->OneIn(16)) { + return; + } + Version* version = nullptr; + { + InstrumentedMutexLock lock(&mutex_); + if (shutting_down_.load(std::memory_order_acquire) || cfd->IsDropped()) { + return; + } + version = cfd->current(); + version->Ref(); + } + Defer release_version([this, version]() { + InstrumentedMutexLock lock(&mutex_); + version->UnrefAndFreeBlobRelocationState(&mutex_); + }); + const std::shared_ptr actual = + cfd->blob_reference_census()->TEST_GetReadyState( + version->GetVersionNumber()); + if (actual == nullptr) { + // The census moved on to a later Version. + return; + } + ReadOptions read_options(Env::IOActivity::kCompaction); + read_options.fill_cache = false; + std::vector> files; + VersionStorageInfo* const storage = version->storage_info(); + for (int level = 0; level < storage->num_levels(); ++level) { + for (FileMetaData* file : storage->LevelFiles(level)) { + if (file->blob_reference_descriptor.empty()) { + continue; + } + std::shared_ptr summary; + const Status status = cfd->table_cache()->ReadBlobReferenceSstSummary( + read_options, versions_->file_options(), cfd->internal_comparator(), + *file, version->GetMutableCFOptions(), &summary, + /*reservation_manager=*/nullptr); + if (!status.ok()) { + // The check is best effort, for example when a file was deleted. + return; + } + files.push_back(std::move(summary)); + } + } + std::shared_ptr expected; + Status status = BlobReferenceCensus::BuildState(version->GetVersionNumber(), + files, &expected); + std::string difference; + if (status.ok() && expected->SameMemberships(*actual, &difference)) { + return; + } + ROCKS_LOG_ERROR( + immutable_db_options_.info_log, + "Blob reference census of CF %s does not match Version %" PRIu64 + ": %s %s", + cfd->GetName().c_str(), version->GetVersionNumber(), + status.ToString().c_str(), difference.c_str()); + assert(false); +} + +void DBImpl::RetryBlobReferenceCensus() { + InstrumentedMutexLock lock(&mutex_); + if (shutting_down_.load(std::memory_order_acquire) || read_only_) { + return; + } + const uint64_t now = immutable_db_options_.clock->NowMicros(); + // Check at most 32 CFs of the retry list (an estimate), in round-robin order, + // so the DB mutex is never held for a walk over all CFs. + const size_t count = std::min(32, blob_reference_census_retry_count_); + for (size_t index = 0; index < count; ++index) { + ColumnFamilyData* const cfd = blob_reference_census_retry_head_; + blob_reference_census_retry_head_ = cfd->next_blob_reference_census_; + --blob_reference_census_retry_count_; + if (blob_reference_census_retry_head_ == nullptr) { + blob_reference_census_retry_tail_ = nullptr; + } + cfd->next_blob_reference_census_ = nullptr; + // Move the CF to the run queue when it is dropped, when its retry time has + // come, or when it has no failures and NeedsWorkerRun() is true. + // NeedsWorkerRun() is also true after a failed build, so a CF with failures + // waits for its exponential backoff. Only a CF without failures (it waits + // for candidates or for a snapshot to publish) can run early for new work. + if (cfd->IsDropped() || cfd->blob_reference_census_retry_micros_ <= now || + (cfd->blob_reference_census_failures_ == 0 && + cfd->blob_reference_census()->NeedsWorkerRun( + cfd->current()->GetVersionNumber()))) { + if (blob_reference_census_queue_tail_ == nullptr) { + blob_reference_census_queue_head_ = cfd; + } else { + blob_reference_census_queue_tail_->next_blob_reference_census_ = cfd; + } + blob_reference_census_queue_tail_ = cfd; + } else { + if (blob_reference_census_retry_tail_ == nullptr) { + blob_reference_census_retry_head_ = cfd; + } else { + blob_reference_census_retry_tail_->next_blob_reference_census_ = cfd; + } + blob_reference_census_retry_tail_ = cfd; + ++blob_reference_census_retry_count_; + } + } + if (blob_reference_census_queue_head_ != nullptr) { + // Passing an already queued CF dispatches the existing queue without + // adding a reference or scanning the ColumnFamilySet. + ScheduleBlobReferenceCensus(blob_reference_census_queue_head_); + } +} + +void DBImpl::BGWorkBlobReferenceCensus(void* arg) { + TEST_SYNC_POINT("DBImpl::BGWorkBlobReferenceCensus:Start"); + DBImpl* const db = static_cast(arg); + ColumnFamilyData* cfd = nullptr; + bool should_build = false; + { + InstrumentedMutexLock lock(&db->mutex_); + cfd = db->blob_reference_census_queue_head_; + assert(cfd != nullptr); + db->blob_reference_census_queue_head_ = cfd->next_blob_reference_census_; + if (db->blob_reference_census_queue_head_ == nullptr) { + db->blob_reference_census_queue_tail_ = nullptr; + } + cfd->next_blob_reference_census_ = nullptr; + // Keep queued=true while running: concurrent installs must not start a + // second worker or duplicate this CF's intrusive queue entry. + should_build = !db->shutting_down_.load(std::memory_order_acquire) && + !cfd->IsDropped(); + } + bool incomplete = false; + bool overflowed = false; + TEST_SYNC_POINT("DBImpl::BGWorkBlobReferenceCensus:BeforeBuildReady"); + TEST_SYNC_POINT("DBImpl::BGWorkBlobReferenceCensus:BeforeBuildWait"); + Status status; + if (should_build) { + std::lock_guard run_lock( + cfd->blob_reference_census()->worker_run_mutex()); + status = db->BuildBlobReferenceCensus(cfd, &incomplete, &overflowed); + if (!status.ok()) { + // FailBuild() below runs after the run lock is released. Store the + // failure first, so an apply after an install that takes the run lock + // in between applies nothing (a failed delta may still be queued). + const std::shared_ptr context = + cfd->blob_reference_census()->GetBuildContext(); + if (context != nullptr && context->failure.ok()) { + context->failure = status; + } + } + } else { + status = Status::ShutdownInProgress(); + } + if (should_build && !status.ok()) { + TEST_SYNC_POINT("DBImpl::BGWorkBlobReferenceCensus:FailedRunReady"); + TEST_SYNC_POINT("DBImpl::BGWorkBlobReferenceCensus:FailedRunWait"); + } + // A change-log overflow restarts the build at once, one time only, on the + // latest Version, with the larger restart limit (see + // BlobReferenceCensus::UseOverflowRestart()). A second overflow before a + // publish fails the build, which then backs off like any other failure. + const bool restart_after_overflow = + overflowed && cfd->blob_reference_census()->UseOverflowRestart(); + // A rebuild install (see BlobReferenceCensusDelta::rebuild) invalidated this + // build. The next build starts at once. + const bool restart_for_rebuild = + !status.ok() && !overflowed && + cfd->blob_reference_census()->RebuildRequested(); + if (restart_for_rebuild) { + ROCKS_LOG_INFO(db->immutable_db_options_.info_log, + "Blob reference census of CF %s is rebuilt on the latest " + "Version", + cfd->GetName().c_str()); + } else if (restart_after_overflow) { + ROCKS_LOG_INFO(db->immutable_db_options_.info_log, + "Blob reference census of CF %s restarts after its change " + "log overflowed", + cfd->GetName().c_str()); + } else if (overflowed) { + RecordTick(db->stats_, BLOB_DB_CENSUS_RESTART_OVERFLOW_COUNT); + ROCKS_LOG_WARN(db->immutable_db_options_.info_log, + "Blob reference census of CF %s overflowed its change log " + "again before it published; it retries after a backoff " + "(GC waits until the census is ready)", + cfd->GetName().c_str()); + } else if (status.IsCorruption()) { + ROCKS_LOG_ERROR(db->immutable_db_options_.info_log, + "Blob reference census of CF %s does not match its " + "SSTs and is rebuilt: %s", + cfd->GetName().c_str(), status.ToString().c_str()); + } else if (!status.ok() && !status.IsShutdownInProgress()) { + ROCKS_LOG_WARN(db->immutable_db_options_.info_log, + "Blob reference census unavailable for CF %s: %s", + cfd->GetName().c_str(), status.ToString().c_str()); + } +#ifndef NDEBUG + if (status.ok() && !incomplete && should_build) { + db->CheckBlobReferenceCensus(cfd); + } +#endif // NDEBUG + BlobReferenceCensusGarbage garbage; + bool retained_for_retry = false; + { + InstrumentedMutexLock lock(&db->mutex_); + cfd->queued_for_blob_reference_census_ = false; + if (!status.ok()) { + const std::shared_ptr context = + cfd->blob_reference_census()->GetBuildContext(); + if (context != nullptr) { + garbage = cfd->blob_reference_census()->FailBuild(context->token); + } + cfd->blob_reference_census()->NoteUnready( + db->immutable_db_options_.clock->NowMicros()); + } + // What happens next, by the result of this run: + // - Rebuild install (restart_for_rebuild): schedule again at once. + // - Change log overflow (restart_after_overflow): restart at once, one + // time only, with the restart limit. + // - Other failure, also a second overflow before a publish: retry list, + // after 1 s, doubled after each failure, at most 32 s. + // - OK, but an indirect blob GC snapshot is pending (snapshot_paused): + // retry list after 1 ms (1000 microseconds). + // - OK and waiting for candidates (waiting_for_candidates): retry list at + // candidate_retry_micros. + // - Any other case: ScheduleBlobReferenceCensus() below. It puts the + // column family at the back of the run queue when it still has work. A + // run that waits for an install (kInstallPending) is not queued; + // Install() or CancelInstallPreparation() wakes it. + // The retry list is checked by the periodic task + // (PeriodicTaskType::kBlobReferenceCensus), which runs once per second. A + // column family with failures runs at the first tick after its retry time. + // A column family without failures can run at an earlier tick when it has + // new work (NeedsWorkerRun()). + const bool snapshot_paused = + status.ok() && incomplete && + cfd->blob_reference_census()->IsSnapshotPending(); + const std::shared_ptr retry_context = + cfd->blob_reference_census()->GetBuildContext(); + const bool waiting_for_candidates = + status.ok() && !incomplete && retry_context != nullptr && + retry_context->candidate_retry_micros != 0 && + !cfd->blob_reference_census()->NeedsWorkerRun( + cfd->current()->GetVersionNumber()); + if (status.ok()) { + cfd->blob_reference_census_failures_ = 0; + } + if ((!status.ok() || snapshot_paused || waiting_for_candidates) && + !restart_after_overflow && !restart_for_rebuild && + !db->shutting_down_.load(std::memory_order_acquire) && + !cfd->IsDropped()) { + // The retry delays are fixed choices, not measured: a short 1 ms wait + // while an indirect blob GC snapshot is pending, and for failures 1 s + // that doubles after each failure, up to 32 s from the sixth failure on. + constexpr uint64_t kSnapshotRetryMicros = 1000; + constexpr uint64_t kFirstFailureRetryMicros = 1000 * 1000; + constexpr uint32_t kMaxCountedFailures = 6; + uint64_t delay = kSnapshotRetryMicros; + if (!status.ok()) { + cfd->blob_reference_census_failures_ = std::min( + cfd->blob_reference_census_failures_ + 1, kMaxCountedFailures); + delay = kFirstFailureRetryMicros + << (cfd->blob_reference_census_failures_ - 1); + TEST_SYNC_POINT_CALLBACK( + "DBImpl::BGWorkBlobReferenceCensus:FailureRetryDelay", &delay); + } + cfd->blob_reference_census_retry_micros_ = + waiting_for_candidates + ? retry_context->candidate_retry_micros + : db->immutable_db_options_.clock->NowMicros() + delay; + cfd->queued_for_blob_reference_census_ = true; + if (db->blob_reference_census_retry_tail_ == nullptr) { + db->blob_reference_census_retry_head_ = cfd; + } else { + db->blob_reference_census_retry_tail_->next_blob_reference_census_ = + cfd; + } + db->blob_reference_census_retry_tail_ = cfd; + ++db->blob_reference_census_retry_count_; + retained_for_retry = true; + } else { + cfd->blob_reference_census_failures_ = 0; + } + if (!db->shutting_down_.load(std::memory_order_acquire)) { + // An incomplete CF goes to the back. Every invocation returns to LOW, + // allowing ordinary compactions and other CFs to run between slices. + db->ScheduleBlobReferenceCensus(cfd); + } + } + garbage = {}; + if (!retained_for_retry) { + db->ReleaseColumnFamilyDataRef(cfd); + } + TEST_SYNC_POINT("DBImpl::BGWorkBlobReferenceCensus:Done"); + // Mark the job complete only after the cleanup without the mutex (Version, + // CFD and change log) is done, not after the last census change. Close and + // the next worker wait for this state. Otherwise Close could free this DB + // while the deleters above still lock its mutex. + { + InstrumentedMutexLock lock(&db->mutex_); + db->bg_blob_reference_census_state_ = + BlobReferenceCensusJobState::kComplete; + if (!db->shutting_down_.load(std::memory_order_acquire) && + db->blob_reference_census_queue_head_ != nullptr) { + db->ScheduleBlobReferenceCensus(db->blob_reference_census_queue_head_); + } + db->bg_cv_.SignalAll(); + } +} } // namespace ROCKSDB_NAMESPACE diff --git a/db/db_impl/db_impl_secondary.cc b/db/db_impl/db_impl_secondary.cc index a2ebdd55ffbf..a16a2bbee73d 100644 --- a/db/db_impl/db_impl_secondary.cc +++ b/db/db_impl/db_impl_secondary.cc @@ -1069,6 +1069,8 @@ Status DBImplSecondary::NewIteratorsImpl( } Status DBImplSecondary::TryCatchUpWithPrimary() { + TEST_SYNC_POINT("DBImplSecondary::TryCatchUpWithPrimary:BeforeCatchUpLock"); + std::lock_guard catch_up_lock(catch_up_mutex_); assert(versions_.get() != nullptr); Status s; // read the manifest and apply new changes to the secondary instance @@ -1234,6 +1236,10 @@ Status DBImplSecondary::OpenAsSecondaryImpl( std::vector* handles, std::unique_ptr* dbptr, bool recover_wal, bool trust_manifest_recovery) { *dbptr = nullptr; + Status style_status = CheckBlobIndirectionCompactionStyle(column_families); + if (!style_status.ok()) { + return style_status; + } // Only normal DB::OpenAsSecondary recovery can defer SST opening. // `recover_wal` excludes OpenAndCompact even when no snapshot floor is // supplied, `open_files_async` is the explicit opt-in, and trust-MANIFEST diff --git a/db/db_impl/db_impl_secondary.h b/db/db_impl/db_impl_secondary.h index f6630d8a1654..f12158041dff 100644 --- a/db/db_impl/db_impl_secondary.h +++ b/db/db_impl/db_impl_secondary.h @@ -346,6 +346,13 @@ class DBImplSecondary : public DBImpl { JobContext* job_context); std::unique_ptr manifest_reader_; + // Serializes catch-ups. ReactiveVersionSet::ReadAndApply() releases mutex_ + // while it decodes MANIFEST records, checks files and builds Versions. No + // other catch-up may run meanwhile. Lock order: this mutex first, then + // mutex_. TryCatchUpWithPrimary() and the follower's TryCatchUpWithLeader() + // both hold it, so a second caller waits for the first one and then does its + // own catch-up, instead of returning TryAgain. + std::mutex catch_up_mutex_; std::unique_ptr manifest_reporter_; std::unique_ptr manifest_reader_status_; diff --git a/db/db_options_test.cc b/db/db_options_test.cc index db869cd07d90..a38c351a5d4f 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -8,11 +8,14 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include #include +#include #include #include #include #include +#include #include +#include #include #include #include @@ -1909,10 +1912,10 @@ TEST_F(DBOptionsTest, SetOptionsNoManifestWrite) { ASSERT_OK(Put("x", "x")); ASSERT_OK(Flush()); - // In addition to checking manifest file, we want to ensure that SetOptions - // is essentially atomic, without releasing the DB mutex between applying - // the options to the cfd and installing new Version and SuperVersion. We - // probabilistically verify that by attempting to catch an inconsistency. + // Besides the manifest file, check what readers see. SetOptions builds the + // new Version with the DB mutex released. Until it publishes, a reader must + // see the old options with the old Version. After that, it must see the new + // options with the new Version. auto* const cfd = static_cast(db_->DefaultColumnFamily())->cfd(); SyncPoint::GetInstance()->DisableProcessing(); @@ -1929,14 +1932,15 @@ TEST_F(DBOptionsTest, SetOptionsNoManifestWrite) { cfd->GetCurrentMutableCFOptions().disable_auto_compactions); t = std::thread([mu, cfd]() { InstrumentedMutexLock l(mu); - // Assuming above correctness, we can only acquire the mutex after - // options fully installed. - ASSERT_TRUE( - cfd->GetLatestMutableCFOptions().disable_auto_compactions); - ASSERT_TRUE( + // Depending on thread timing, this reader runs before or after the + // publication, but it never sees a mix of old and new state. + const bool disabled = + cfd->GetLatestMutableCFOptions().disable_auto_compactions; + ASSERT_EQ( + disabled, cfd->current()->GetMutableCFOptions().disable_auto_compactions); - ASSERT_TRUE( - cfd->GetCurrentMutableCFOptions().disable_auto_compactions); + ASSERT_EQ(disabled, + cfd->GetCurrentMutableCFOptions().disable_auto_compactions); }); }); SyncPoint::GetInstance()->EnableProcessing(); @@ -1955,6 +1959,9 @@ TEST_F(DBOptionsTest, SetOptionsNoManifestWrite) { // Verify that our above check was activated and completed ASSERT_TRUE(t.has_value()); t->join(); + ASSERT_TRUE(cfd->GetLatestMutableCFOptions().disable_auto_compactions); + ASSERT_TRUE(cfd->current()->GetMutableCFOptions().disable_auto_compactions); + ASSERT_TRUE(cfd->GetCurrentMutableCFOptions().disable_auto_compactions); SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); @@ -1968,6 +1975,188 @@ TEST_F(DBOptionsTest, SetOptionsNoManifestWrite) { ASSERT_EQ(Get("x"), "x"); } +TEST_F(DBOptionsTest, SetOptionsForcedManifestPublishesPreparedState) { + auto* const cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + ASSERT_FALSE(cfd->GetLatestMutableCFOptions().disable_auto_compactions); + + bool forced_manifest_write = false; + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SetOptions:dummy_edit", [&](void* arg) { + auto* dummy_edit = static_cast(arg); + dummy_edit->Clear(); + forced_manifest_write = true; + }); + SyncPoint::GetInstance()->EnableProcessing(); + const Status status = db_->SetOptions({{"disable_auto_compactions", "true"}}); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_OK(status); + ASSERT_TRUE(forced_manifest_write); + ASSERT_TRUE(cfd->GetLatestMutableCFOptions().disable_auto_compactions); + ASSERT_TRUE(cfd->current()->GetMutableCFOptions().disable_auto_compactions); + ASSERT_TRUE(cfd->GetCurrentMutableCFOptions().disable_auto_compactions); +} + +TEST_F(DBOptionsTest, SetOptionsForcedManifestWriterIsBatchBoundary) { + // SetOptions normally uses dummy MANIFEST writers, one per column family, + // that are never grouped with another caller's writers. A test can clear the + // dummy flag (DBImpl::SetOptions:dummy_edit), and then the SetOptions writer + // is a normal writer that carries prepared options. Such a writer must never + // be grouped with another caller's writers, before or after, because the + // batch publishes only the options of its leader's caller. + // + // Writer W0 (an empty edit) blocks inside its MANIFEST write. Behind it + // queue the callers of one round in order: N is a caller with an empty edit, + // S is the SetOptions caller. When W0 is done, the next caller leads the + // next batch. The rounds: + // - "NS": S comes right after the leader's own writers. + // - "NNS": S comes after a caller that is not the leader's. + // - "SN": S leads; the N after it must lead its own batch. + // In each round SetOptions publishes its options. A writer that is woken + // already done was grouped into an earlier batch + // (VersionSet::LogAndApply:WakeUpAndDone); only the second N of "NNS" may + // be. + auto* const cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + VersionSet* const versions = dbfull()->GetVersionSet(); + for (const std::string round : {"NS", "NNS", "SN"}) { + SCOPED_TRACE(round); + // Each round flips disable_auto_compactions. + const bool new_value = + !cfd->GetLatestMutableCFOptions().disable_auto_compactions; + std::mutex state_mutex; + std::condition_variable state_changed; + int queued_callers = 0; + bool first_write_blocked = false; + bool release_first_write = false; + std::atomic first_write_seen{false}; + std::atomic woken_done{0}; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) { + std::lock_guard lock(state_mutex); + ++queued_callers; + state_changed.notify_all(); + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifestStart", [&](void*) { + if (first_write_seen.exchange(true)) { + return; + } + std::unique_lock lock(state_mutex); + first_write_blocked = true; + state_changed.notify_all(); + state_changed.wait(lock, [&] { return release_first_write; }); + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WakeUpAndDone", [&](void*) { ++woken_done; }); + SyncPoint::GetInstance()->SetCallBack( + "DBImpl::SetOptions:dummy_edit", + [&](void* arg) { static_cast(arg)->Clear(); }); + SyncPoint::GetInstance()->EnableProcessing(); + const auto wait_for = [&](const std::function& done) { + std::unique_lock lock(state_mutex); + state_changed.wait(lock, done); + }; + const auto apply_empty_edit = [&](Status* status) { + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + dbfull()->TEST_LockMutex(); + *status = versions->LogAndApply(cfd, ReadOptions(), WriteOptions(), &edit, + dbfull()->TEST_Mutex(), + /*dir_contains_current_file=*/nullptr); + dbfull()->TEST_UnlockMutex(); + }; + + // W0, then the callers of the round in order. + std::vector statuses(1 + round.size()); + std::vector callers; + callers.emplace_back([&] { apply_empty_edit(&statuses[0]); }); + wait_for([&] { return first_write_blocked; }); + for (size_t k = 0; k < round.size(); ++k) { + Status* const status = &statuses[1 + k]; + if (round[k] == 'S') { + callers.emplace_back([&, status] { + *status = db_->SetOptions( + {{"disable_auto_compactions", new_value ? "true" : "false"}}); + }); + } else { + callers.emplace_back([&, status] { apply_empty_edit(status); }); + } + wait_for([&] { return queued_callers >= static_cast(2 + k); }); + } + { + std::lock_guard lock(state_mutex); + release_first_write = true; + state_changed.notify_all(); + } + for (auto& caller : callers) { + caller.join(); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + for (const Status& status : statuses) { + ASSERT_OK(status); + } + ASSERT_EQ(cfd->GetLatestMutableCFOptions().disable_auto_compactions, + new_value); + ASSERT_EQ(cfd->current()->GetMutableCFOptions().disable_auto_compactions, + new_value); + EXPECT_EQ(woken_done.load(), round == "NNS" ? 1 : 0); + } +} + +TEST_F(DBOptionsTest, SetOptionsPublishesCompactionPhaseAtomically) { + env_->SetMockSleep(); + Options options = CurrentOptions(); + options.env = env_; + options.disable_auto_compactions = true; + options.periodic_compaction_phase_recovery_percent = 25; + Reopen(options); + + auto* const cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + VersionSet* const versions = dbfull()->GetVersionSet(); + const PeriodicCompactionPhaseParams before = + versions->GetPeriodicCompactionPhaseParams(cfd->GetID()); + ASSERT_EQ(cfd->current() + ->storage_info() + ->TEST_GetPeriodicCompactionPhaseParams() + .anchor_time, + before.anchor_time); + + env_->MockSleepForSeconds(60); + int callbacks = 0; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeBlobFinalization", [&](void*) { + ++callbacks; + const PeriodicCompactionPhaseParams during = + versions->GetPeriodicCompactionPhaseParams(cfd->GetID()); + ASSERT_EQ(during.anchor_time, before.anchor_time); + ASSERT_EQ(cfd->current() + ->storage_info() + ->TEST_GetPeriodicCompactionPhaseParams() + .anchor_time, + before.anchor_time); + }); + SyncPoint::GetInstance()->EnableProcessing(); + ASSERT_OK(db_->SetOptions({{"periodic_compaction_seconds", "3600"}})); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_EQ(callbacks, 1); + const PeriodicCompactionPhaseParams after = + versions->GetPeriodicCompactionPhaseParams(cfd->GetID()); + ASSERT_GT(after.anchor_time, before.anchor_time); + ASSERT_EQ(cfd->current() + ->storage_info() + ->TEST_GetPeriodicCompactionPhaseParams() + .anchor_time, + after.anchor_time); +} + TEST_F(DBOptionsTest, SetOptionsMultipleColumnFamilies) { Options options; options.create_if_missing = true; @@ -1975,37 +2164,175 @@ TEST_F(DBOptionsTest, SetOptionsMultipleColumnFamilies) { options.disable_auto_compactions = true; Reopen(options); - // Create two additional column families - CreateColumnFamilies({"cf1", "cf2"}, options); - ReopenWithColumnFamilies({"default", "cf1", "cf2"}, options); + const std::vector column_family_names = { + "cf1", "cf2", "cf3", "cf4", "cf5", "cf6", "cf7", "cf8"}; + CreateColumnFamilies(column_family_names, options); + std::vector reopen_column_family_names = {"default"}; + reopen_column_family_names.insert(reopen_column_family_names.end(), + column_family_names.begin(), + column_family_names.end()); + ReopenWithColumnFamilies(reopen_column_family_names, options); // Verify initial state - auto compaction should be disabled - ASSERT_TRUE(dbfull()->GetOptions(handles_[0]).disable_auto_compactions); - ASSERT_TRUE(dbfull()->GetOptions(handles_[1]).disable_auto_compactions); - ASSERT_TRUE(dbfull()->GetOptions(handles_[2]).disable_auto_compactions); + for (auto* handle : handles_) { + ASSERT_TRUE(dbfull()->GetOptions(handle).disable_auto_compactions); + } // Set options on multiple column families at once - ASSERT_OK(dbfull()->SetOptions({handles_[1], handles_[2]}, + std::vector changed_handles(handles_.begin() + 1, + handles_.end()); + ASSERT_OK(dbfull()->SetOptions(changed_handles, {{"disable_auto_compactions", "false"}})); ASSERT_TRUE( dbfull()->GetOptions(handles_[0]).disable_auto_compactions); // unchanged - ASSERT_FALSE( - dbfull()->GetOptions(handles_[1]).disable_auto_compactions); // changed - ASSERT_FALSE( - dbfull()->GetOptions(handles_[2]).disable_auto_compactions); // changed + for (size_t i = 1; i < handles_.size(); ++i) { + ASSERT_FALSE( + dbfull()->GetOptions(handles_[i]).disable_auto_compactions); // changed + } std::unordered_map> options_map; + for (auto* handle : handles_) { + options_map[handle] = {{"disable_auto_compactions", "true"}}; + } options_map[handles_[0]] = {{"disable_auto_compactions", "false"}}; - options_map[handles_[1]] = {{"disable_auto_compactions", "true"}}; - options_map[handles_[2]] = {{"disable_auto_compactions", "true"}}; ASSERT_OK(dbfull()->SetOptions(options_map)); ASSERT_FALSE(dbfull()->GetOptions(handles_[0]).disable_auto_compactions); - ASSERT_TRUE(dbfull()->GetOptions(handles_[1]).disable_auto_compactions); - ASSERT_TRUE(dbfull()->GetOptions(handles_[2]).disable_auto_compactions); + for (size_t i = 1; i < handles_.size(); ++i) { + ASSERT_TRUE(dbfull()->GetOptions(handles_[i]).disable_auto_compactions); + } + + // Two different handles can point to one column family. SetOptions must apply + // the maps of both handles in one update of that column family. Two writers + // for one column family would share one Version in ProcessManifestWrites, and + // the options of one writer would be lost. + auto alias = dbfull()->GetColumnFamilyHandleUnlocked(handles_[1]->GetID()); + ASSERT_NE(alias, nullptr); + options_map.clear(); + options_map[handles_[1]] = {{"write_buffer_size", "1048576"}}; + options_map[alias.get()] = {{"max_write_buffer_number", "4"}}; + ASSERT_OK(dbfull()->SetOptions(options_map)); + const ColumnFamilyOptions aliased_options = dbfull()->GetOptions(handles_[1]); + ASSERT_EQ(aliased_options.write_buffer_size, 1048576U); + ASSERT_EQ(aliased_options.max_write_buffer_number, 4); +} + +TEST_F(DBOptionsTest, SetOptionsMultipleColumnFamiliesIsAllOrNothing) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.track_options_file_number_in_manifest = true; + CreateAndReopenWithCF({"cf1", "cf2"}, options); + ASSERT_EQ(handles_.size(), 3U); + + std::vector write_buffer_sizes; + for (auto* handle : handles_) { + write_buffer_sizes.push_back( + dbfull()->GetOptions(handle).write_buffer_size); + } + OptionsFileManifestState before; + ASSERT_OK(VersionSet::GetOptionsFileManifestState( + dbname_, env_->GetFileSystem().get(), &before)); + ASSERT_TRUE(before.has_committed_options_file_number); + + // Each round sets two valid maps and one invalid map, and the invalid map + // must fail the whole call. Each round puts the invalid map on a different + // column family. So, whatever the map iteration order is, at least one round + // handles a valid column family before the invalid one. + for (size_t invalid = 0; invalid < handles_.size(); ++invalid) { + SCOPED_TRACE("invalid column family " + std::to_string(invalid)); + std::unordered_map> + options_map; + for (size_t i = 0; i < handles_.size(); ++i) { + options_map[handles_[i]] = + i == invalid + ? std::unordered_map{{"not_an_option", + "1"}} + : std::unordered_map{ + {"write_buffer_size", "1048576"}}; + } + ASSERT_NOK(dbfull()->SetOptions(options_map)); + + for (size_t i = 0; i < handles_.size(); ++i) { + ASSERT_EQ(dbfull()->GetOptions(handles_[i]).write_buffer_size, + write_buffer_sizes[i]); + auto* cfd = static_cast(handles_[i])->cfd(); + ASSERT_EQ(cfd->current()->GetMutableCFOptions().write_buffer_size, + write_buffer_sizes[i]); + } + } + OptionsFileManifestState after; + ASSERT_OK(VersionSet::GetOptionsFileManifestState( + dbname_, env_->GetFileSystem().get(), &after)); + ASSERT_TRUE(after.has_committed_options_file_number); + ASSERT_EQ(after.committed_options_file_number, + before.committed_options_file_number); +} + +TEST_F(DBOptionsTest, SetOptionsMultipleColumnFamiliesCommitsOneOptionsFile) { + Options options = CurrentOptions(); + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.track_options_file_number_in_manifest = true; + CreateAndReopenWithCF({"cf1", "cf2"}, options); + ASSERT_EQ(handles_.size(), 3U); + + OptionsFileManifestState before; + ASSERT_OK(VersionSet::GetOptionsFileManifestState( + dbname_, env_->GetFileSystem().get(), &before)); + ASSERT_TRUE(before.has_committed_options_file_number); + + std::unordered_map> + options_map; + options_map[handles_[1]] = {{"write_buffer_size", "1048576"}}; + options_map[handles_[2]] = {{"max_write_buffer_number", "5"}}; + ASSERT_OK(dbfull()->SetOptions(options_map)); + + // One SetOptions call writes one OPTIONS file and commits its number once. + OptionsFileManifestState after; + ASSERT_OK(VersionSet::GetOptionsFileManifestState( + dbname_, env_->GetFileSystem().get(), &after)); + ASSERT_TRUE(after.has_committed_options_file_number); + ASSERT_GT(after.committed_options_file_number, + before.committed_options_file_number); + ASSERT_EQ(after.committed_options_file_number, + dbfull()->GetVersionSet()->options_file_number()); + std::vector children; + ASSERT_OK(env_->GetChildren(dbname_, &children)); + int new_options_files = 0; + for (const auto& child : children) { + uint64_t number = 0; + FileType type; + if (ParseFileName(child, &number, &type) && type == kOptionsFile && + number > before.committed_options_file_number) { + ++new_options_files; + } + } + ASSERT_EQ(new_options_files, 1); + + // The committed OPTIONS file holds the new values of every column family. + DBOptions loaded_db_options; + std::vector loaded_cfs; + ConfigOptions config_options; + config_options.env = env_; + ASSERT_OK(LoadLatestOptions(config_options, dbname_, &loaded_db_options, + &loaded_cfs, nullptr)); + ASSERT_EQ(loaded_cfs.size(), 3U); + for (const auto& cf : loaded_cfs) { + if (cf.name == "cf1") { + ASSERT_EQ(cf.options.write_buffer_size, 1048576U); + } else if (cf.name == "cf2") { + ASSERT_EQ(cf.options.max_write_buffer_number, 5); + } else { + ASSERT_EQ(cf.name, kDefaultColumnFamilyName); + ASSERT_EQ(cf.options.write_buffer_size, options.write_buffer_size); + } + } } TEST_F(DBOptionsTest, SetOptionsSkipsUnchangedValues) { @@ -2111,12 +2438,12 @@ TEST_F(DBOptionsTest, SetDBOptionsRewritesStaleOptionsFile) { SetOptionsWritesOptionsFile(db_->DefaultColumnFamily(), cf_opts)); } -TEST_F(DBOptionsTest, SetOptionsFinishesUpdateLeftIncompleteByFailure) { +TEST_F(DBOptionsTest, SetOptionsFailureLeavesNoIncompleteUpdate) { Options options = CurrentOptions(); CreateAndReopenWithCF({"one", "two"}, options); - // DB::SetOptions() changes the column families in the iteration order of - // the map and stops at the first error. + // One valid and one invalid map. SetOptions() validates every map before it + // changes any column family, so the valid map must not be applied either. std::unordered_map> opts_map = {{handles_[1], {}}, {handles_[2], {}}}; @@ -2126,11 +2453,11 @@ TEST_F(DBOptionsTest, SetOptionsFinishesUpdateLeftIncompleteByFailure) { (++it)->second = {{"no_such_option", "1"}}; ASSERT_EQ(db_->SetOptions(opts_map).code(), Status::Code::kInvalidArgument); - // The first column family has the new value, but its SuperVersion, the - // OPTIONS file and the seqno-to-time worker do not. + // The options, the SuperVersion, the OPTIONS file and the seqno-to-time + // worker all keep the old state. ColumnFamilyData* const changed_cfd = static_cast_with_check(changed_cfh)->cfd(); - ASSERT_EQ(db_->GetOptions(changed_cfh).preserve_internal_time_seconds, 3600); + ASSERT_EQ(db_->GetOptions(changed_cfh).preserve_internal_time_seconds, 0); { InstrumentedMutexLock l(dbfull()->mutex()); ASSERT_EQ(changed_cfd->GetCurrentMutableCFOptions() @@ -2144,18 +2471,14 @@ TEST_F(DBOptionsTest, SetOptionsFinishesUpdateLeftIncompleteByFailure) { dbfull()->TEST_GetPeriodicTaskScheduler(); ASSERT_FALSE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime)); - // A request on another column family that changes nothing is applied, and - // it updates the OPTIONS file and the worker. - ASSERT_TRUE(SetOptionsWritesOptionsFile( + // The failed call changed no column family, no OPTIONS file and no + // seqno-to-time worker, so a request that changes nothing is skipped. + ASSERT_FALSE(SetOptionsWritesOptionsFile( handles_[0], {{"write_buffer_size", std::to_string(options.write_buffer_size)}})); - ASSERT_EQ( - PersistedCFOptions(changed_cfh->GetName()).preserve_internal_time_seconds, - 3600); - ASSERT_TRUE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime)); - // The changed column family still needs a new SuperVersion, so the same - // request on it is applied once more before it is skipped. + // The valid map on its own is applied in full. The same request again is then + // skipped. const std::unordered_map opts = { {"preserve_internal_time_seconds", "3600"}}; ASSERT_TRUE(SetOptionsWritesOptionsFile(changed_cfh, opts)); @@ -2165,6 +2488,10 @@ TEST_F(DBOptionsTest, SetOptionsFinishesUpdateLeftIncompleteByFailure) { .preserve_internal_time_seconds, 3600); } + ASSERT_EQ( + PersistedCFOptions(changed_cfh->GetName()).preserve_internal_time_seconds, + 3600); + ASSERT_TRUE(scheduler.TEST_HasTask(PeriodicTaskType::kRecordSeqnoTime)); ASSERT_FALSE(SetOptionsWritesOptionsFile(changed_cfh, opts)); } diff --git a/db/db_secondary_test.cc b/db/db_secondary_test.cc index 16deb9d221c1..1b56b3337197 100644 --- a/db/db_secondary_test.cc +++ b/db/db_secondary_test.cc @@ -2381,6 +2381,69 @@ TEST_F(DBSecondaryTest, CatchUpKeepsUnflushedWalData) { VerifySecondaryValue(handles_secondary_[1], "bar", "v1"); } +TEST_F(DBSecondaryTest, ConcurrentCatchUpWaitsAcrossUnlockedManifestWork) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + Reopen(options); + ASSERT_OK(Put("key", "old")); + ASSERT_OK(Flush()); + options.max_open_files = -1; + OpenSecondary(options); + ASSERT_OK(Put("key", "new")); + ASSERT_OK(Flush()); + + std::mutex gate_mutex; + std::condition_variable gate_cv; + bool paused = false; + bool release = false; + unsigned attempts = 0; + std::atomic second_completed{false}; + SyncPoint* sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack( + "DBImplSecondary::TryCatchUpWithPrimary:BeforeCatchUpLock", [&](void*) { + std::lock_guard lock(gate_mutex); + ++attempts; + gate_cv.notify_all(); + }); + sync_point->SetCallBack("ManifestTailer::DecodeVersionEdit:Unlocked", + [&](void*) { + std::unique_lock lock(gate_mutex); + if (paused) { + return; + } + paused = true; + gate_cv.notify_all(); + gate_cv.wait(lock, [&] { return release; }); + }); + sync_point->EnableProcessing(); + Status first_status; + Status second_status; + port::Thread first( + [&] { first_status = db_secondary_->TryCatchUpWithPrimary(); }); + { + std::unique_lock lock(gate_mutex); + gate_cv.wait(lock, [&] { return paused; }); + } + port::Thread second([&] { + second_status = db_secondary_->TryCatchUpWithPrimary(); + second_completed.store(true); + }); + { + std::unique_lock lock(gate_mutex); + gate_cv.wait(lock, [&] { return attempts == 2; }); + EXPECT_FALSE(second_completed.load()); + release = true; + gate_cv.notify_all(); + } + first.join(); + second.join(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + ASSERT_OK(first_status); + ASSERT_OK(second_status); + VerifySecondaryValue("key", "new"); +} + TEST_F(DBSecondaryTest, NewIteratorsConsistentViewDuringCatchUp) { Options options; options.env = env_; diff --git a/db/db_tailing_iter_test.cc b/db/db_tailing_iter_test.cc index 5b811fd48600..8c3448d7f8cf 100644 --- a/db/db_tailing_iter_test.cc +++ b/db/db_tailing_iter_test.cc @@ -14,6 +14,7 @@ #include "db/db_test_util.h" #include "db/forward_iterator.h" #include "port/stack_trace.h" +#include "rocksdb/lazy_wide_columns.h" namespace { static bool enable_io_uring = true; @@ -32,6 +33,163 @@ class DBTestTailingIterator : public DBTestBase, INSTANTIATE_TEST_CASE_P(DBTestTailingIterator, DBTestTailingIterator, ::testing::Bool()); +TEST_P(DBTestTailingIterator, DroppedColumnFamilyPinsLazyBlobFiles) { + // A tailing iterator of a dropped column family whose handle is destroyed + // keeps reading: the SST and blob files that it opens lazily stay live + // and readable. The base fix "Fix use-after-free in + // ForwardIterator::RebuildIterators on a dropped column family" + // (ForwardIterator keeps its current SuperVersion when it rebuilds its child + // iterators) makes this pass. This test covers the lazy blob file case, which + // the base test does not. + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.enable_blob_indirection = false; + options.disable_auto_compactions = true; + CreateAndReopenWithCF({"dropped"}, options); + const std::string value(1024, 'v'); + for (int i = 0; i < 64; ++i) { + ASSERT_OK(Put(1, Key(i), value)); + } + ASSERT_OK(Flush(1)); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)); + ASSERT_GT(NumTableFilesAtLevel(1, 1), 0); + + ReadOptions read_options; + read_options.tailing = true; + read_options.async_io = GetParam(); + std::unique_ptr single(db_->NewIterator(read_options, handles_[1])); + std::vector multiple; + ASSERT_OK(db_->NewIterators(read_options, {handles_[1]}, &multiple)); + ASSERT_EQ(multiple.size(), 1U); + std::unique_ptr batch(multiple[0]); + const uint32_t cf_id = handles_[1]->GetID(); + const auto is_linked = [&] { + InstrumentedMutexLock lock(dbfull()->mutex()); + for (ColumnFamilyData* cfd : + *dbfull()->GetVersionSet()->GetColumnFamilySet()) { + if (cfd->GetID() == cf_id) { + return true; + } + } + return false; + }; + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[1])); + handles_.pop_back(); + + // Neither iterator has done a seek yet. So their level iterators open the SST + // and blob files later, after this full purge has scanned the files. + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_TRUE(is_linked()); + for (Iterator* iter : {single.get(), batch.get()}) { + iter->SeekToFirst(); + for (int i = 0; i < 64; ++i) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->key().ToString(), Key(i)); + ASSERT_EQ(iter->value().ToString(), value); + iter->Next(); + } + ASSERT_FALSE(iter->Valid()); + ASSERT_OK(iter->status()); + } + single.reset(); + batch.reset(); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_FALSE(is_linked()); +} + +TEST_P(DBTestTailingIterator, DroppedColumnFamilyOutlivedBySuperVersionPins) { + // A GetEntityLazy result and a pinned GetMergeOperands result each hold a + // SuperVersion, and a SuperVersion holds a column family reference. That + // reference alone must keep a dropped column family alive, with its files + // live, until the last result is released. An ordinary iterator is released + // first. + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + options.merge_operator = MergeOperators::CreateStringAppendOperator(); + options.max_open_files = -1; // Required by GetEntityLazy. + CreateAndReopenWithCF({"dropped"}, options); + const std::string value(1024, 'v'); + for (int i = 0; i < 16; ++i) { + ASSERT_OK(Put(1, Key(i), value)); + } + ASSERT_OK(db_->PutEntity(WriteOptions(), handles_[1], "entity", + {{kDefaultWideColumnName, value}})); + // Large operands, so GetMergeOperands pins the SuperVersion. + const std::string operand_a(20000, 'a'); + const std::string operand_b(20000, 'b'); + ASSERT_OK(db_->Merge(WriteOptions(), handles_[1], "merge", operand_a)); + ASSERT_OK(Flush(1)); + ASSERT_OK(db_->Merge(WriteOptions(), handles_[1], "merge", operand_b)); + ASSERT_OK(Flush(1)); + + ReadOptions read_options; + read_options.async_io = GetParam(); + std::unique_ptr iter(db_->NewIterator(read_options, handles_[1])); + auto lazy = std::make_unique(); + ASSERT_OK( + db_->GetEntityLazy(read_options, handles_[1], "entity", lazy.get())); + ASSERT_EQ(lazy->size(), 1U); + std::vector operands(2); + GetMergeOperandsOptions merge_options; + merge_options.expected_max_number_of_operands = 2; + int num_operands = 0; + ASSERT_OK(db_->GetMergeOperands(read_options, handles_[1], "merge", + operands.data(), &merge_options, + &num_operands)); + ASSERT_EQ(num_operands, 2); + + const uint32_t cf_id = handles_[1]->GetID(); + const auto is_linked = [&] { + InstrumentedMutexLock lock(dbfull()->mutex()); + for (ColumnFamilyData* cfd : + *dbfull()->GetVersionSet()->GetColumnFamilySet()) { + if (cfd->GetID() == cf_id) { + return true; + } + } + return false; + }; + ASSERT_OK(db_->DropColumnFamily(handles_[1])); + ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[1])); + handles_.pop_back(); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_TRUE(is_linked()); + + // Release the iterator and the results one by one. The column family stays in + // the column family list until the last one is released. + iter->SeekToFirst(); + for (int i = 0; i < 16; ++i) { + ASSERT_TRUE(iter->Valid()); + ASSERT_EQ(iter->value().ToString(), value); + iter->Next(); + } + ASSERT_OK(iter->status()); + iter.reset(); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_TRUE(is_linked()); + + PinnableSlice resolved; + ASSERT_OK(lazy->ResolveColumn((*lazy)[0], &resolved)); + ASSERT_EQ(resolved.ToString(), value); + resolved.Reset(); + lazy.reset(); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_TRUE(is_linked()); + + ASSERT_EQ(operands[0].ToString(), operand_a); + ASSERT_EQ(operands[1].ToString(), operand_b); + operands.clear(); + dbfull()->TEST_DeleteObsoleteFiles(); + ASSERT_FALSE(is_linked()); +} + TEST_P(DBTestTailingIterator, TailingIteratorSingle) { ReadOptions read_options; read_options.tailing = true; diff --git a/db/db_test_util.cc b/db/db_test_util.cc index 87da764defe7..d639dedd91a7 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -96,6 +96,7 @@ DBTestBase::DBTestBase(const std::string& path, bool env_do_fsync) env_->SetBackgroundThreads(1, Env::LOW); env_->SetBackgroundThreads(1, Env::HIGH); env_->skip_fsync_ = !env_do_fsync; + ColumnFamilyData::TEST_SetAllowBlobIndirection(true); dbname_ = test::PerThreadDBPath(env_, path); alternative_wal_dir_ = dbname_ + "/wal"; alternative_db_log_dir_ = dbname_ + "/db_log_dir"; @@ -2009,6 +2010,8 @@ template class TargetCacheChargeTrackingCache< CacheEntryRole::kBlockBasedTableReader>; template class TargetCacheChargeTrackingCache; template class TargetCacheChargeTrackingCache; +template class TargetCacheChargeTrackingCache< + CacheEntryRole::kBlobReferenceCensus>; const std::vector kKnownTemperatures = { Temperature::kHot, Temperature::kWarm, Temperature::kCool, diff --git a/db/event_helpers.cc b/db/event_helpers.cc index 854b9c576dac..28671eff5c24 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -76,7 +76,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( uint64_t oldest_blob_file_number, const TableProperties& table_properties, TableFileCreationReason reason, const Status& s, const std::string& file_checksum, - const std::string& file_checksum_func_name) { + const std::string& file_checksum_func_name, + const TableFileBlobReferenceInfo* blob_references) { if (!event_logger && listeners.empty()) { s.PermitUncheckedError(); return; @@ -171,6 +172,13 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( jwriter << "oldest_blob_file_number" << oldest_blob_file_number; } + if (blob_references != nullptr) { + jwriter << "blob_references" << blob_references->count; + if (blob_references->closed_by_limit) { + jwriter << "close_reason" << "blob_reference_limit"; + } + } + jwriter << "status" << s.ToString(); jwriter.EndObject(); diff --git a/db/event_helpers.h b/db/event_helpers.h index c141071b70bd..d1d8ca70d9a9 100644 --- a/db/event_helpers.h +++ b/db/event_helpers.h @@ -16,6 +16,15 @@ namespace ROCKSDB_NAMESPACE { +// Blob reference fields of a table_file_creation event, for an SST with exact +// blob reference metadata. +struct TableFileBlobReferenceInfo { + uint64_t count = 0; + // True when the blob reference cap closed the output (see + // CompactionOutputs::Output::closed_by_blob_reference_limit). + bool closed_by_limit = false; +}; + class EventHelpers { public: static void AppendCurrentTime(JSONWriter* json_writer); @@ -35,7 +44,8 @@ class EventHelpers { uint64_t oldest_blob_file_number, const TableProperties& table_properties, TableFileCreationReason reason, const Status& s, const std::string& file_checksum, - const std::string& file_checksum_func_name); + const std::string& file_checksum_func_name, + const TableFileBlobReferenceInfo* blob_references); static void LogAndNotifyTableFileDeletion( EventLogger* event_logger, int job_id, uint64_t file_number, const std::string& file_path, const Status& status, diff --git a/db/flush_job.cc b/db/flush_job.cc index 39d72c4e7e11..9c828c7b73e8 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -1069,6 +1069,17 @@ Status FlushJob::WriteLevel0Table() { ? preclude_last_level_min_seqno_ : std::min(earliest_snapshot_, preclude_last_level_min_seqno_), dbname_ /*db_name*/); + tboptions.track_blob_reference_classification = + UsesBlobIndirection(mutable_cf_options_, base_->storage_info()); + // Flush output gets only the coarse classification. A flushed SST holds + // no flattened reference (the builder returns Corruption for one), so one + // frontier file number is enough: the oldest indirect-origin file number + // the SST may reference (see "Coarse SST" in + // db/blob/blob_reference_metadata.h). The flush needs no collector memory + // and stays one output file. + tboptions.collect_blob_reference_metadata = false; + TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:TableBuilderOptions", + &tboptions); s = BuildTable( dbname_, versions_, db_options_, tboptions, file_options_, cfd_->table_cache(), iter.get(), std::move(range_del_iters), &meta_, @@ -1167,6 +1178,8 @@ Status FlushJob::WriteLevel0Table() { // Add file to L0 TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", &meta_); edit_->AddFile(0 /* level */, meta_); + edit_->MaybeSetBlobIndirectionFormatFence( + mutable_cf_options_.enable_blob_indirection); } edit_->SetBlobFileAdditions(std::move(blob_file_additions)); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index ce291186c3c5..89e2f2293fa2 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -22,6 +22,7 @@ #include "cache/cache_entry_roles.h" #include "cache/cache_entry_stats.h" +#include "db/blob/blob_reference_census.h" #include "db/column_family.h" #include "db/db_impl/db_impl.h" #include "db/write_stall_stats.h" @@ -338,6 +339,13 @@ static const std::string live_blob_file_garbage_size = static const std::string blob_cache_capacity = "blob-cache-capacity"; static const std::string blob_cache_usage = "blob-cache-usage"; static const std::string blob_cache_pinned_usage = "blob-cache-pinned-usage"; +static const std::string indirect_blob_gc_idle_micros_with_candidates = + "indirect-blob-gc-idle-micros-with-candidates"; +static const std::string indirect_blob_gc_stats = "indirect-blob-gc-stats"; +static const std::string blob_reference_census_unready_micros = + "blob-reference-census-unready-micros"; +static const std::string blob_indirection_format_version = + "blob-indirection-format-version"; const std::string DB::Properties::kNumFilesAtLevelPrefix = rocksdb_prefix + num_files_at_level_prefix; @@ -466,6 +474,14 @@ const std::string DB::Properties::kBlobCacheUsage = rocksdb_prefix + blob_cache_usage; const std::string DB::Properties::kBlobCachePinnedUsage = rocksdb_prefix + blob_cache_pinned_usage; +const std::string DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates = + rocksdb_prefix + indirect_blob_gc_idle_micros_with_candidates; +const std::string DB::Properties::kIndirectBlobGcStats = + rocksdb_prefix + indirect_blob_gc_stats; +const std::string DB::Properties::kBlobReferenceCensusUnreadyMicros = + rocksdb_prefix + blob_reference_census_unready_micros; +const std::string DB::Properties::kBlobIndirectionFormatVersion = + rocksdb_prefix + blob_indirection_format_version; const std::string InternalStats::kPeriodicCFStats = DB::Properties::kCFStats + ".periodic"; @@ -673,6 +689,20 @@ const UnorderedMap {DB::Properties::kBlobCachePinnedUsage, {false, nullptr, &InternalStats::HandleBlobCachePinnedUsage, nullptr, nullptr}}, + {DB::Properties::kIndirectBlobGcIdleMicrosWithCandidates, + {false, nullptr, + &InternalStats::HandleIndirectBlobGcIdleMicrosWithCandidates, nullptr, + nullptr}}, + {DB::Properties::kIndirectBlobGcStats, + {false, &InternalStats::HandleIndirectBlobGcStatsString, nullptr, + &InternalStats::HandleIndirectBlobGcStats, nullptr}}, + {DB::Properties::kBlobReferenceCensusUnreadyMicros, + {false, nullptr, + &InternalStats::HandleBlobReferenceCensusUnreadyMicros, nullptr, + nullptr}}, + {DB::Properties::kBlobIndirectionFormatVersion, + {false, nullptr, &InternalStats::HandleBlobIndirectionFormatVersion, + nullptr, nullptr}}, }; InternalStats::InternalStats(int num_levels, SystemClock* clock, @@ -920,8 +950,15 @@ bool InternalStats::HandleBlobStats(std::string* value, Slice /*suffix*/) { std::ostringstream oss; - oss << "Number of blob files: " << vstorage->GetBlobFiles().size() - << "\nTotal size of blob files: " << blob_st.total_file_size + // The relocation file count is printed only when there are relocation + // files, so the text stays the same as in base RocksDB otherwise. + const size_t relocation_file_count = + vstorage->GetBlobRelocationFiles().size(); + oss << "Number of blob files: " << vstorage->GetBlobFiles().size(); + if (relocation_file_count != 0) { + oss << " (+" << relocation_file_count << " relocation files)"; + } + oss << "\nTotal size of blob files: " << blob_st.total_file_size << "\nTotal size of garbage in blob files: " << blob_st.total_garbage_size << "\nBlob file space amplification: " << blob_st.space_amp << '\n'; @@ -997,6 +1034,125 @@ bool InternalStats::HandleBlobCacheUsage(uint64_t* value, DBImpl* /*db*/, return false; } +bool InternalStats::HandleBlobIndirectionFormatVersion(uint64_t* value, + DBImpl* /*db*/, + Version* /*version*/) { + assert(value); + assert(cfd_); + const Version* const current = cfd_->current(); + assert(current); + *value = current->storage_info()->GetBlobIndirectionFormatVersion(); + return true; +} + +uint64_t InternalStats::BlobReferenceCensusUnreadyMicros() const { + assert(cfd_); + const BlobReferenceCensus* const census = cfd_->blob_reference_census(); + if (census == nullptr || + census->IsReady(cfd_->current()->GetVersionNumber())) { + return 0; + } + const uint64_t since = census->UnreadySinceMicros(); + const uint64_t now = clock_->NowMicros(); + return since == 0 || now < since ? 0 : now - since; +} + +bool InternalStats::HandleBlobReferenceCensusUnreadyMicros( + uint64_t* value, DBImpl* /*db*/, Version* /*version*/) { + assert(value); + *value = BlobReferenceCensusUnreadyMicros(); + return true; +} + +// DB::GetProperty() needs a string handler for every property (db_stress, for +// example, reads every property with it). The string form has one "key=value" +// line per map key, in key order. +bool InternalStats::HandleIndirectBlobGcStatsString(std::string* value, + Slice suffix) { + assert(value); + std::map values; + if (!HandleIndirectBlobGcStats(&values, suffix)) { + return false; + } + value->clear(); + for (const auto& [key, val] : values) { + value->append(key); + value->append("="); + value->append(val); + value->append("\n"); + } + return true; +} + +bool InternalStats::HandleIndirectBlobGcStats( + std::map* values, Slice /*suffix*/) { + assert(values); + assert(cfd_); + const Version* const current = cfd_->current(); + assert(current); + const VersionStorageInfo* const vstorage = current->storage_info(); + const BlobReferenceCensus* const census = cfd_->blob_reference_census(); + const char* state = "none"; + if (census != nullptr) { + if (census->IsReady(current->GetVersionNumber())) { + state = "ready"; + } else if (cfd_->blob_reference_census_failures() > 0) { + state = "failed"; + } else { + state = "building"; + } + } + (*values)["census-state"] = state; + (*values)["census-unready-micros"] = + std::to_string(BlobReferenceCensusUnreadyMicros()); + (*values)["census-consecutive-failures"] = + std::to_string(cfd_->blob_reference_census_failures()); + (*values)["census-builds"] = + std::to_string(census != nullptr ? census->build_count() : 0); + (*values)["census-bytes"] = + std::to_string(census != nullptr ? census->StayingMemoryBytes() : 0); + (*values)["route-count"] = + std::to_string(vstorage->GetBlobRelocationRouteCount()); + (*values)["relocation-file-count"] = + std::to_string(vstorage->GetBlobRelocationFiles().size()); + (*values)["relocation-file-bytes"] = + std::to_string(vstorage->GetBlobRelocationFileBytes()); + (*values)["coarse-sst-count"] = std::to_string(vstorage->CoarseSstCount()); + (*values)["lowest-coarse-frontier"] = + std::to_string(vstorage->OldestCoarseIndirectOriginFileNumber()); + const BlobGcCandidateHeap::Totals totals = + census != nullptr ? census->PublishedTotals() + : BlobGcCandidateHeap::Totals{}; + (*values)["tracked-files"] = std::to_string(totals.candidate_count); + (*values)["idle-micros-with-candidates"] = + std::to_string(IndirectBlobGcIdleMicrosWithCandidates()); + (*values)["garbage-bytes-collectable"] = + std::to_string(totals.collectable_garbage_bytes); + (*values)["garbage-bytes-suppressed"] = + std::to_string(totals.suppressed_garbage_bytes); + (*values)["coarse-pinned-files"] = std::to_string(totals.coarse_pinned_files); + (*values)["coarse-pinned-bytes"] = std::to_string(totals.coarse_pinned_bytes); + (*values)["coarse-pinned-garbage-bytes"] = + std::to_string(totals.coarse_pinned_garbage_bytes); + (*values)["relocation-garbage-bytes"] = std::to_string( + census != nullptr ? census->PublishedRelocationGarbageBytes() : 0); + return true; +} + +uint64_t InternalStats::IndirectBlobGcIdleMicrosWithCandidates() const { + assert(cfd_); + const uint64_t since = cfd_->IndirectBlobGcCandidatesSinceMicros(); + const uint64_t now = clock_->NowMicros(); + return since == 0 || now < since ? 0 : now - since; +} + +bool InternalStats::HandleIndirectBlobGcIdleMicrosWithCandidates( + uint64_t* value, DBImpl* /*db*/, Version* /*version*/) { + assert(value); + *value = IndirectBlobGcIdleMicrosWithCandidates(); + return true; +} + bool InternalStats::HandleBlobCachePinnedUsage(uint64_t* value, DBImpl* /*db*/, Version* /*version*/) { Cache* blob_cache = GetBlobCacheForStats(); diff --git a/db/internal_stats.h b/db/internal_stats.h index a319835dabf9..61bdd7d09143 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -653,6 +653,19 @@ class InternalStats { // This should only be called while NOT holding the DB mutex. void CollectCacheEntryStats(bool foreground); + // How long the blob reference census of this column family has been + // unready, in microseconds; 0 when it is ready or there is none. Both + // rocksdb.blob-reference-census-unready-micros and the census-unready-micros + // key of rocksdb.indirect-blob-gc-stats read it. REQUIRES: DB mutex held. + uint64_t BlobReferenceCensusUnreadyMicros() const; + + // How long this column family has had indirect blob GC candidates while no + // GC job began, in microseconds; 0 when it has none or a job began since. + // Both + // rocksdb.indirect-blob-gc-idle-micros-with-candidates and the + // idle-micros-with-candidates key of rocksdb.indirect-blob-gc-stats read it. + uint64_t IndirectBlobGcIdleMicrosWithCandidates() const; + const uint64_t* TEST_GetCFStatsValue() const { return cf_stats_value_; } const std::vector& TEST_GetCompactionStats() const { @@ -930,6 +943,15 @@ class InternalStats { Version* version); bool HandleBlobCacheCapacity(uint64_t* value, DBImpl* db, Version* version); bool HandleBlobCacheUsage(uint64_t* value, DBImpl* db, Version* version); + bool HandleBlobIndirectionFormatVersion(uint64_t* value, DBImpl* db, + Version* version); + bool HandleBlobReferenceCensusUnreadyMicros(uint64_t* value, DBImpl* db, + Version* version); + bool HandleIndirectBlobGcStatsString(std::string* value, Slice suffix); + bool HandleIndirectBlobGcStats(std::map* values, + Slice suffix); + bool HandleIndirectBlobGcIdleMicrosWithCandidates(uint64_t* value, DBImpl* db, + Version* version); bool HandleBlobCachePinnedUsage(uint64_t* value, DBImpl* db, Version* version); diff --git a/db/job_context.h b/db/job_context.h index 93279376fb97..9d53fdfd3a12 100644 --- a/db/job_context.h +++ b/db/job_context.h @@ -16,6 +16,7 @@ #include "db/column_family.h" #include "db/log_writer.h" #include "db/version_set.h" +#include "test_util/sync_point.h" #include "util/autovector.h" #include "util/hash_containers.h" @@ -127,7 +128,8 @@ struct JobContext { } } return memtables_to_free.size() > 0 || wals_to_free.size() > 0 || - job_snapshot != nullptr || sv_have_sth; + !blob_relocation_states_to_free.empty() || job_snapshot != nullptr || + sv_have_sth; } SequenceNumber GetJobSnapshotSequence() const { @@ -198,6 +200,11 @@ struct JobContext { // the list of blob files that we need to delete std::vector blob_delete_files; + // File tables of destroyed Versions (see + // VersionSet::AddObsoleteBlobRelocationState()), freed by Clean() without the + // DB mutex. + std::vector> blob_relocation_states_to_free; + // a list of log files that we need to delete std::vector log_delete_files; @@ -305,12 +312,17 @@ struct JobContext { memtables_to_free.clear(); wals_to_free.clear(); + if (!blob_relocation_states_to_free.empty()) { + TEST_SYNC_POINT("JobContext::Clean:FreeBlobRelocationStates"); + blob_relocation_states_to_free.clear(); + } job_snapshot.reset(); } ~JobContext() { assert(memtables_to_free.size() == 0); assert(wals_to_free.size() == 0); + assert(blob_relocation_states_to_free.empty()); } }; diff --git a/db/memtable_list.cc b/db/memtable_list.cc index 41ac798f3d89..02c6aa0240d3 100644 --- a/db/memtable_list.cc +++ b/db/memtable_list.cc @@ -1051,80 +1051,117 @@ Status InstallMemtableAtomicFlushResults( assert(0 == num_entries); } - // this can release and reacquire the mutex. - s = vset->LogAndApply(cfds, read_options, write_options, edit_lists, mu, - db_directory); - - for (size_t k = 0; k != cfds.size(); ++k) { - auto* imm = (imm_lists == nullptr) ? cfds[k]->imm() : imm_lists->at(k); - imm->InstallNewVersion(); - } + // An atomic flush installs the results of several column families in one + // LogAndApply call, with one writer per column family. The callback of the + // first writer (publish_atomic_flush) updates the memtable lists of the whole + // group under the DB mutex. On success, it runs in the same mutex hold as the + // install of the new Versions (see the callback contract in + // VersionSet::LogAndApply()). The callbacks of the other writers are empty, + // so each memtable list changes exactly once. If LogAndApply() returns before + // the batch starts, no callback runs, and the code right after the call runs + // publish_atomic_flush itself. For an atomic flush this happens only when + // every column family was dropped while its writers waited: the flush always + // has edits and passes no pre_cb. + bool published = false; + const auto publish_atomic_flush = [&](const Status& status) { + published = true; + for (size_t k = 0; k != cfds.size(); ++k) { + auto* imm = (imm_lists == nullptr) ? cfds[k]->imm() : imm_lists->at(k); + imm->InstallNewVersion(); + } - if (s.ok() || s.IsColumnFamilyDropped()) { - for (size_t i = 0; i != cfds.size(); ++i) { - if (cfds[i]->IsDropped()) { - continue; + if (status.ok() || status.IsColumnFamilyDropped()) { + for (size_t i = 0; i != cfds.size(); ++i) { + if (cfds[i]->IsDropped()) { + continue; + } + auto* imm = (imm_lists == nullptr) ? cfds[i]->imm() : imm_lists->at(i); + for (auto m : *mems_list[i]) { + assert(m->GetFileNumber() > 0); + uint64_t mem_id = m->GetID(); + + const VersionEdit* const edit = m->GetEdits(); + assert(edit); + + if (edit->GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + ": memtable #%" PRIu64 " done", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + mem_id); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + " (+%zu blob files)" + ": memtable #%" PRIu64 " done", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + edit->GetBlobFileAdditions().size(), mem_id); + } + + imm->current_->Remove(m, to_delete); + imm->UpdateCachedValuesFromMemTableListVersion(); + imm->ResetTrimHistoryNeeded(); + } } - auto* imm = (imm_lists == nullptr) ? cfds[i]->imm() : imm_lists->at(i); - for (auto m : *mems_list[i]) { - assert(m->GetFileNumber() > 0); - uint64_t mem_id = m->GetID(); - - const VersionEdit* const edit = m->GetEdits(); - assert(edit); - - if (edit->GetBlobFileAdditions().empty()) { - ROCKS_LOG_BUFFER(log_buffer, - "[%s] Level-0 commit table #%" PRIu64 - ": memtable #%" PRIu64 " done", - cfds[i]->GetName().c_str(), m->GetFileNumber(), - mem_id); - } else { - ROCKS_LOG_BUFFER(log_buffer, - "[%s] Level-0 commit table #%" PRIu64 - " (+%zu blob files)" - ": memtable #%" PRIu64 " done", - cfds[i]->GetName().c_str(), m->GetFileNumber(), - edit->GetBlobFileAdditions().size(), mem_id); + } else { + for (size_t i = 0; i != cfds.size(); ++i) { + auto* imm = (imm_lists == nullptr) ? cfds[i]->imm() : imm_lists->at(i); + for (auto m : *mems_list[i]) { + uint64_t mem_id = m->GetID(); + + const VersionEdit* const edit = m->GetEdits(); + assert(edit); + + if (edit->GetBlobFileAdditions().empty()) { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + ": memtable #%" PRIu64 " failed", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + mem_id); + } else { + ROCKS_LOG_BUFFER(log_buffer, + "[%s] Level-0 commit table #%" PRIu64 + " (+%zu blob files)" + ": memtable #%" PRIu64 " failed", + cfds[i]->GetName().c_str(), m->GetFileNumber(), + edit->GetBlobFileAdditions().size(), mem_id); + } + + m->SetFlushCompleted(false); + m->SetFlushInProgress(false); + m->GetEdits()->Clear(); + m->SetFileNumber(0); + imm->num_flush_not_started_++; } - - imm->current_->Remove(m, to_delete); - imm->UpdateCachedValuesFromMemTableListVersion(); - imm->ResetTrimHistoryNeeded(); + imm->imm_flush_needed.store(true, std::memory_order_release); } } - } else { - for (size_t i = 0; i != cfds.size(); ++i) { - auto* imm = (imm_lists == nullptr) ? cfds[i]->imm() : imm_lists->at(i); - for (auto m : *mems_list[i]) { - uint64_t mem_id = m->GetID(); + }; + std::vector> manifest_wcbs(cfds.size()); + manifest_wcbs.front() = publish_atomic_flush; - const VersionEdit* const edit = m->GetEdits(); - assert(edit); - - if (edit->GetBlobFileAdditions().empty()) { - ROCKS_LOG_BUFFER(log_buffer, - "[%s] Level-0 commit table #%" PRIu64 - ": memtable #%" PRIu64 " failed", - cfds[i]->GetName().c_str(), m->GetFileNumber(), - mem_id); - } else { - ROCKS_LOG_BUFFER(log_buffer, - "[%s] Level-0 commit table #%" PRIu64 - " (+%zu blob files)" - ": memtable #%" PRIu64 " failed", - cfds[i]->GetName().c_str(), m->GetFileNumber(), - edit->GetBlobFileAdditions().size(), mem_id); - } - - m->SetFlushCompleted(false); - m->SetFlushInProgress(false); - m->GetEdits()->Clear(); - m->SetFileNumber(0); - imm->num_flush_not_started_++; - } - imm->imm_flush_needed.store(true, std::memory_order_release); + // LogAndApply can release the DB mutex and take it again. + s = vset->LogAndApply(cfds, read_options, write_options, edit_lists, mu, + db_directory, + /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, manifest_wcbs); + // LogAndApply() returns without calling any callback when every column + // family of the group was dropped while its writers waited in the MANIFEST + // queue. Update the memtable lists here in that case, with the same result. + if (!published) { + // That early return always has an error. An OK status here would remove + // the flushed memtables although nothing was committed, so treat it as + // Corruption. + assert(!s.ok()); + if (s.ok()) { + s = Status::Corruption( + "Atomic flush install returned OK without running its callback"); } + TEST_SYNC_POINT_CALLBACK( + "MemTableList::InstallMemtableAtomicFlushResults:" + "PublishWithoutCallback", + &s); + publish_atomic_flush(s); } return s; diff --git a/db/periodic_compaction_phaser.cc b/db/periodic_compaction_phaser.cc index 81c767ad7b43..52b07a0e4540 100644 --- a/db/periodic_compaction_phaser.cc +++ b/db/periodic_compaction_phaser.cc @@ -40,6 +40,15 @@ PeriodicCompactionPhaseParams PeriodicCompactionPhaser::ParamsForCf( return params; } +PeriodicCompactionPhaseParams PeriodicCompactionPhaser::ParamsForCfAtAnchor( + uint32_t cf_id, uint64_t anchor_time) const { + PeriodicCompactionPhaseParams params = ParamsForCf(cf_id); + if (params.recovery_percent > 0) { + params.anchor_time = anchor_time; + } + return params; +} + uint64_t PeriodicCompactionPhaser::TriggerTime( uint64_t file_modification_time, uint64_t periodic_compaction_seconds, const PeriodicCompactionPhaseParams& params) { diff --git a/db/periodic_compaction_phaser.h b/db/periodic_compaction_phaser.h index 99b69d520aeb..2a6ea1e17f47 100644 --- a/db/periodic_compaction_phaser.h +++ b/db/periodic_compaction_phaser.h @@ -62,6 +62,14 @@ class PeriodicCompactionPhaser { // recovery percent. Returns disabled params when phasing is off. PeriodicCompactionPhaseParams ParamsForCf(uint32_t cf_id) const; + // Returns the parameters that column family `cf_id` would get after + // Reanchor(anchor_time), without changing this phaser. SetOptions() uses this + // to build a new Version before it publishes the new anchor. When phasing is + // off (recovery percent 0), it returns the same disabled params as + // ParamsForCf() and ignores `anchor_time`. + PeriodicCompactionPhaseParams ParamsForCfAtAnchor(uint32_t cf_id, + uint64_t anchor_time) const; + // --- Stateless helpers (static; unit-tested directly) --- // Wall-clock time (unix seconds) at which a file with the given modification diff --git a/db/periodic_task_scheduler.cc b/db/periodic_task_scheduler.cc index 848096d98e11..2e58b5887bc8 100644 --- a/db/periodic_task_scheduler.cc +++ b/db/periodic_task_scheduler.cc @@ -28,6 +28,7 @@ static const std::map kDefaultPeriodSeconds = { {PeriodicTaskType::kFlushInfoLog, 10}, {PeriodicTaskType::kRecordSeqnoTime, kInvalidPeriodSec}, {PeriodicTaskType::kTriggerCompaction, kInvalidPeriodSec}, + {PeriodicTaskType::kBlobReferenceCensus, 1}, }; static const std::map kPeriodicTaskTypeNames = { @@ -36,6 +37,7 @@ static const std::map kPeriodicTaskTypeNames = { {PeriodicTaskType::kFlushInfoLog, "flush_info_log"}, {PeriodicTaskType::kRecordSeqnoTime, "record_seq_time"}, {PeriodicTaskType::kTriggerCompaction, "trigger_compaction"}, + {PeriodicTaskType::kBlobReferenceCensus, "blob_reference_census"}, }; Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type, diff --git a/db/periodic_task_scheduler.h b/db/periodic_task_scheduler.h index 428762a21f47..410da0a9012d 100644 --- a/db/periodic_task_scheduler.h +++ b/db/periodic_task_scheduler.h @@ -22,6 +22,7 @@ enum class PeriodicTaskType : uint8_t { kFlushInfoLog, kRecordSeqnoTime, kTriggerCompaction, + kBlobReferenceCensus, kMax, }; diff --git a/db/periodic_task_scheduler_test.cc b/db/periodic_task_scheduler_test.cc index 73d4928eeadd..540ef7bc34be 100644 --- a/db/periodic_task_scheduler_test.cc +++ b/db/periodic_task_scheduler_test.cc @@ -6,12 +6,17 @@ #include "db/periodic_task_scheduler.h" +#include +#include +#include #include +#include "db/blob/blob_reference_census.h" #include "db/column_family.h" #include "db/db_test_util.h" #include "env/composite_env_wrapper.h" #include "test_util/mock_time_env.h" +#include "util/defer.h" namespace ROCKSDB_NAMESPACE { @@ -79,7 +84,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) { const PeriodicTaskScheduler& scheduler = dbfull()->TEST_GetPeriodicTaskScheduler(); - ASSERT_EQ((int)PeriodicTaskType::kMax - 1, scheduler.TEST_GetValidTaskNum()); + ASSERT_EQ((int)PeriodicTaskType::kMax - 2, scheduler.TEST_GetValidTaskNum()); ASSERT_EQ(1, dump_st_counter); ASSERT_EQ(1, pst_st_counter); @@ -144,6 +149,216 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) { Close(); } +TEST_F(PeriodicTaskSchedulerTest, CensusFailureHonorsExponentialBackoff) { + Options options = CurrentOptions(); + options.env = mock_env_.get(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->EnableProcessing(); + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + Close(); + + std::mutex gate_mutex; + std::condition_variable gate_cv; + size_t completed_workers = 0; + bool inject_failure = true; + sync_point->SetCallBack("DBImpl::BuildBlobReferenceCensus:ReadFileStatus", + [&](void* arg) { + std::lock_guard lock(gate_mutex); + if (inject_failure) { + *static_cast(arg) = + Status::IOError("Injected census failure"); + } + }); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Done", [&](void*) { + std::lock_guard lock(gate_mutex); + ++completed_workers; + gate_cv.notify_all(); + }); + auto cleanup = Defer([&] { + // Close first, so every census worker stops before the variables that the + // callbacks capture and the mock Env are destroyed. + Close(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Reopen(options); + { + std::unique_lock lock(gate_mutex); + ASSERT_TRUE(gate_cv.wait_for(lock, std::chrono::seconds(30), + [&] { return completed_workers == 1; })); + } + for (int retry = 0; retry != 6; ++retry) { + const int delay_seconds = 1 << retry; + if (delay_seconds > 1) { + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(delay_seconds - 1); }); + std::lock_guard lock(gate_mutex); + // The census is not ready after the failure, but the retry must still + // wait until the backoff delay ends. + ASSERT_EQ(completed_workers, static_cast(retry + 1)); + } + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(1); }); + std::unique_lock lock(gate_mutex); + ASSERT_TRUE(gate_cv.wait_for(lock, std::chrono::seconds(30), [&] { + return completed_workers == static_cast(retry + 2); + })); + } + { + std::lock_guard lock(gate_mutex); + inject_failure = false; + } + dbfull()->TEST_WaitForPeriodicTaskRun( + [&] { mock_clock_->MockSleepForSeconds(32); }); + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + +TEST_F(PeriodicTaskSchedulerTest, CensusPendingInstallReturnsLowWorker) { + Options options = CurrentOptions(); + options.env = mock_env_.get(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.enable_blob_garbage_collection = false; + options.disable_auto_compactions = true; + options.min_blob_size = 0; + Reopen(options); + ASSERT_OK(Put("key", std::string(4096, 'v'))); + ASSERT_OK(Flush()); + CompactRangeOptions compact_options; + compact_options.change_level = true; + compact_options.target_level = 1; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr)); + dbfull()->TEST_WaitForBlobReferenceCensus(); + Close(); + + std::mutex gate_mutex; + std::condition_variable gate_cv; + bool read_started = false; + bool allow_read = false; + std::atomic started_workers{0}; + bool manifest_started = false; + bool allow_manifest = false; + bool inject_manifest_failure = false; + std::unique_ptr install_thread; + Status install_status; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->SetCallBack("DBImpl::BGWorkBlobReferenceCensus:Start", + [&](void*) { ++started_workers; }); + sync_point->SetCallBack("DBImpl::BuildBlobReferenceCensus:ReadFileStatus", + [&](void*) { + std::unique_lock lock(gate_mutex); + read_started = true; + gate_cv.notify_all(); + gate_cv.wait(lock, [&] { return allow_read; }); + }); + sync_point->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeBlobFinalization", [&](void*) { + std::unique_lock lock(gate_mutex); + if (!inject_manifest_failure) { + return; + } + manifest_started = true; + gate_cv.notify_all(); + gate_cv.wait(lock, [&] { return allow_manifest; }); + }); + sync_point->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) { + std::lock_guard lock(gate_mutex); + if (inject_manifest_failure) { + *static_cast(arg) = IOStatus::IOError("injected failure"); + } + }); + sync_point->EnableProcessing(); + auto cleanup = Defer([&] { + { + std::lock_guard lock(gate_mutex); + allow_read = true; + allow_manifest = true; + inject_manifest_failure = false; + gate_cv.notify_all(); + } + if (install_thread != nullptr) { + install_thread->join(); + } + // If an assertion fails early, the check of install_status below does not + // run, so mark the status as checked here. + install_status.PermitUncheckedError(); + Close(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + }); + Reopen(options); + { + std::unique_lock lock(gate_mutex); + ASSERT_TRUE(gate_cv.wait_for(lock, std::chrono::seconds(30), + [&] { return read_started; })); + } + ColumnFamilyData* cfd = + dbfull()->GetVersionSet()->GetColumnFamilySet()->GetDefault(); + BlobReferenceCensus* const census = cfd->blob_reference_census(); + { + std::lock_guard lock(gate_mutex); + inject_manifest_failure = true; + } + install_thread = std::make_unique([&] { + InstrumentedMutexLock lock(dbfull()->TEST_Mutex()); + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + edit.SetLogNumber(cfd->GetLogNumber()); + install_status = dbfull()->GetVersionSet()->LogAndApply( + cfd, ReadOptions(), WriteOptions(), &edit, dbfull()->TEST_Mutex(), + /*dir_contains_current_file=*/nullptr); + }); + { + std::unique_lock lock(gate_mutex); + ASSERT_TRUE(gate_cv.wait_for(lock, std::chrono::seconds(30), + [&] { return manifest_started; })); + } + { + std::lock_guard lock(gate_mutex); + allow_read = true; + gate_cv.notify_all(); + } + // The install is paused while it holds its census install reservation. The + // census worker must return and give its LOW pool thread back, and the census + // stays not ready. If the worker kept retrying here, this wait would hang. + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(started_workers.load(), 1U); + ASSERT_FALSE(census->IsReady(cfd->current()->GetVersionNumber())); + { + std::lock_guard lock(gate_mutex); + allow_manifest = true; + gate_cv.notify_all(); + } + install_thread->join(); + install_thread.reset(); + ASSERT_TRUE(install_status.IsIOError()); + // The failed MANIFEST write cancels its install reservation under the DB + // mutex. That cleanup must schedule the waiting CF again by itself, without a + // timer, another write or a sleep. + dbfull()->TEST_WaitForBlobReferenceCensus(); + ASSERT_EQ(started_workers.load(), 2U); + ASSERT_TRUE(census->IsReady(cfd->current()->GetVersionNumber())); + ASSERT_EQ(Get("key"), std::string(4096, 'v')); +} + TEST_F(PeriodicTaskSchedulerTest, TriggerCompactionPeriodComputation) { // Test that ComputeTriggerCompactionPeriod() returns the expected values // for different configurations. @@ -481,8 +696,9 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) { auto dbi = static_cast_with_check(dbs[kInstanceNum - 1].get()); const PeriodicTaskScheduler& scheduler = dbi->TEST_GetPeriodicTaskScheduler(); - // kRecordSeqnoTime is not registered since the feature is not enabled - ASSERT_EQ(kInstanceNum * ((int)PeriodicTaskType::kMax - 1), + // kRecordSeqnoTime and kBlobReferenceCensus are not registered, because these + // DBs enable neither feature. + ASSERT_EQ(kInstanceNum * ((int)PeriodicTaskType::kMax - 2), scheduler.TEST_GetValidTaskNum()); int expected_run = kInstanceNum; diff --git a/db/repair.cc b/db/repair.cc index be34a9f55952..1224e5f8cfa0 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -61,6 +61,8 @@ #include +#include "db/blob/blob_reference_metadata.h" +#include "db/blob/blob_relocation_file.h" #include "db/builder.h" #include "db/db_impl/db_impl.h" #include "db/dbformat.h" @@ -72,6 +74,7 @@ #include "db/version_edit.h" #include "db/write_batch_internal.h" #include "file/filename.h" +#include "file/random_access_file_reader.h" #include "file/writable_file_writer.h" #include "logging/logging.h" #include "options/cf_options.h" @@ -80,7 +83,9 @@ #include "rocksdb/env.h" #include "rocksdb/options.h" #include "rocksdb/write_buffer_manager.h" +#include "table/meta_blocks.h" #include "table/unique_id_impl.h" +#include "test_util/sync_point.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { @@ -206,6 +211,9 @@ class Repairer { return status; } status = FindFiles(); + if (status.ok()) { + status = CheckBlobIndirectionRepairAdmission(); + } DBImpl* db_impl = nullptr; if (status.ok()) { // Discard older manifests and start a fresh one @@ -280,6 +288,7 @@ class Repairer { InstrumentedMutex mutex_; std::vector manifests_; + std::vector blob_feature_probe_paths_; std::vector table_fds_; std::vector logs_; std::vector tables_; @@ -289,6 +298,184 @@ class Repairer { FileLock* db_lock_; bool closed_; + Status UnsupportedBlobIndirectionRepair() { + Status status; + TEST_SYNC_POINT_CALLBACK("Repairer::BlobIndirectionDetected", &status); + return status.ok() ? Status::NotSupported( + "Blob indirection RepairDB requires complete " + "repair support") + : status; + } + + Status CheckBlobIndirectionRepairAdmission() { + // Background: RepairDB cannot rebuild relocation state. It writes a new + // MANIFEST from the files it finds, so on a DB that uses blob indirection + // it would lose the relocation files and routes that the old MANIFEST + // records. This check runs before RepairDB changes any file, and returns + // NotSupported if it finds feature evidence: + // - in any MANIFEST: a format fence; a relocation file, route, + // accepted-loss or blob file retirement record; a blob file addition + // marked as an indirect origin; or a new SST record whose exact + // reference metadata descriptor decodes, or that has a coarse frontier + // (the oldest indirect-origin file number a coarse SST may reference). + // - in any SST: an exact reference metadata descriptor that decodes. + // The search is best effort: it skips a file or record that it cannot read. + // So when it finds nothing, that does not prove that the DB never used the + // feature. But once it finds evidence, RepairDB never runs the legacy + // repair, which changes files. + class Reporter : public log::Reader::Reporter { + public: + void Corruption(size_t, const Status& status, uint64_t) override { + status.PermitUncheckedError(); + } + }; + for (const std::string& manifest : manifests_) { + std::unique_ptr file; + Status status = SequentialFileReader::Create( + env_->GetFileSystem(), dbname_ + "/" + manifest, file_options_, &file, + nullptr, nullptr); + TEST_SYNC_POINT_CALLBACK("Repairer::FeatureProbe", &status); + if (!status.ok()) { + continue; + } + Reporter reporter; + log::Reader reader(nullptr, std::move(file), &reporter, true, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch)) { + VersionEdit edit; + status = edit.DecodeFrom(record); + if (!status.ok()) { + continue; + } + bool feature = edit.HasBlobIndirectionFormatVersion() || + edit.HasDbBlobIndirectionFormatVersion() || + !edit.GetBlobRelocationFileAdditions().empty() || + !edit.GetBlobRelocationFileDeletions().empty() || + !edit.GetBlobRelocationRoutes().empty() || + !edit.GetBlobRelocationRouteDeletions().empty() || + !edit.GetBlobPhysicalFileLosses().empty() || + !edit.GetBlobFileRetirements().empty(); + for (const BlobFileAddition& addition : edit.GetBlobFileAdditions()) { + feature |= addition.IsIndirectOrigin(); + } + for (const std::pair& addition : + edit.GetNewFiles()) { + if (!addition.second.blob_reference_descriptor.empty()) { + BlobReferenceMetadataDescriptor descriptor; + status = BlobReferenceMetadataDescriptor::DecodeFrom( + addition.second.blob_reference_descriptor, &descriptor); + feature |= status.ok(); + } + feature |= + addition.second.oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber; + } + if (feature) { + return UnsupportedBlobIndirectionRepair(); + } + } + } + for (const FileDescriptor& fd : table_fds_) { + FileMetaData metadata; + uint64_t size = 0; + Status status = env_->GetFileSize( + TableFileName(db_options_.db_paths, fd.GetNumber(), fd.GetPathId()), + &size); + if (!status.ok()) { + continue; + } + metadata.fd = FileDescriptor(fd.GetNumber(), fd.GetPathId(), size); + std::shared_ptr properties; + ReadOptions read_options; + read_options.fill_cache = false; + status = table_cache_->GetTableProperties( + file_options_, read_options, icmp_, metadata, &properties, + default_mopts_, /*no_io=*/false, /*level=*/-1, + /*file_read_hist=*/nullptr, + /*prefetch_index_and_filter_in_cache=*/true); + if (!status.ok()) { + continue; + } + const UserCollectedProperties& collected = + properties->user_collected_properties; + const UserCollectedProperties::const_iterator descriptor = + collected.find(kBlobReferenceDescriptorPropertyName); + if (descriptor != collected.end()) { + BlobReferenceMetadataDescriptor decoded; + status = BlobReferenceMetadataDescriptor::DecodeFrom(descriptor->second, + &decoded); + if (status.ok()) { + return UnsupportedBlobIndirectionRepair(); + } + } + const UserCollectedProperties::const_iterator classification = + collected.find(kBlobReferenceClassificationPropertyName); + if (classification != collected.end()) { + uint64_t frontier = 0; + status = DecodeBlobReferenceClassification(classification->second, + &frontier); + // Any valid classification, also frontier 0, shows that the DB used + // enable_blob_indirection. RepairDB cannot rebuild that state, so it + // refuses the DB instead of writing a MANIFEST without the blob + // indirection format marker. + if (status.ok()) { + return UnsupportedBlobIndirectionRepair(); + } + } + } + for (const std::string& path : blob_feature_probe_paths_) { + uint64_t size = 0; + Status status = env_->GetFileSize(path, &size); + if (!status.ok()) { + continue; + } + std::unique_ptr reader; + status = RandomAccessFileReader::Create(env_->GetFileSystem(), path, + file_options_, &reader, nullptr); + if (!status.ok()) { + continue; + } + char scratch[kBlobRelocationFilePrefixSize]; + Slice prefix; + status = reader->Read(0, sizeof(scratch), IOOptions(), &prefix, scratch, + nullptr); + if (!status.ok() || prefix != Slice(kBlobRelocationFilePrefix, + kBlobRelocationFilePrefixSize)) { + continue; + } + // The file starts with kBlobRelocationFilePrefix. The .blob name + // and the prefix alone do not prove that it is a relocation file. Treat + // it as one only if its checksummed table properties hold a relocation + // directory that decodes and is valid for the file size. RepairDB does + // not support relocation files, so it then stops with + // UnsupportedBlobIndirectionRepair(). + std::unique_ptr properties; + ReadOptions read_options; + read_options.fill_cache = false; + status = ReadTableProperties(reader.get(), size, + kBlockBasedTableMagicNumber, default_iopts_, + read_options, &properties, nullptr, nullptr); + if (!status.ok()) { + continue; + } + const auto directory = properties->user_collected_properties.find( + kBlobRelocationDirectoryPropertyName); + if (directory == properties->user_collected_properties.end()) { + continue; + } + BlobRelocationDirectory decoded; + status = BlobRelocationDirectory::DecodeFrom(directory->second, &decoded); + if (status.ok()) { + status = decoded.ValidateForFileSize(size); + } + if (status.ok()) { + return UnsupportedBlobIndirectionRepair(); + } + } + return Status::OK(); + } + Status FindFiles() { std::vector filenames; bool found_file = false; @@ -331,6 +518,9 @@ class Repairer { } else if (type == kTableFile) { table_fds_.emplace_back(number, static_cast(path_id), 0); + } else if (type == kBlobFile) { + blob_feature_probe_paths_.push_back(to_search_paths[path_id] + + "/" + filenames[i]); } else { // Ignore other files } @@ -481,8 +671,16 @@ class Repairer { // TODO: plumb Env::IOActivity, Env::IOPriority const ReadOptions read_options; const WriteOptions write_option(Env::IO_HIGH); + // RepairDB runs only the legacy repair (see + // CheckBlobIndirectionRepairAdmission()), so build the converted SSTs + // with enable_blob_indirection off. An SST with exact reference metadata + // but no format fence would look like feature evidence to a later + // RepairDB, which would then refuse to run. + MutableCFOptions legacy_mutable_cf_options = + cfd->GetLatestMutableCFOptions(); + legacy_mutable_cf_options.enable_blob_indirection = false; TableBuilderOptions tboptions( - cfd->ioptions(), cfd->GetLatestMutableCFOptions(), read_options, + cfd->ioptions(), legacy_mutable_cf_options, read_options, write_option, cfd->internal_comparator(), cfd->internal_tbl_prop_coll_factories(), kNoCompression, default_compression, cfd->GetID(), cfd->GetName(), -1 /* level */, diff --git a/db/repair_test.cc b/db/repair_test.cc index 1ea7a7e84eb4..614636e0b7cb 100644 --- a/db/repair_test.cc +++ b/db/repair_test.cc @@ -7,6 +7,7 @@ #include #include +#include "db/blob/blob_reference_metadata.h" #include "db/db_impl/db_impl.h" #include "db/db_test_util.h" #include "db/db_with_timestamp_test_util.h" @@ -62,6 +63,212 @@ class RepairTest : public DBTestBase { } }; +TEST_F(RepairTest, PrefixRepairAdmissionPreservesPositiveEvidence) { + for (int scenario = 0; scenario != 3; ++scenario) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + Close(); + std::vector names; + ASSERT_OK(env_->GetChildren(dbname_, &names)); + std::vector> original; + for (const std::string& name : names) { + uint64_t number = 0; + FileType type; + if (!ParseFileName(name, &number, &type) || + (type != kCurrentFile && type != kDescriptorFile && + type != kTableFile && type != kBlobFile && type != kWalFile)) { + continue; + } + std::string bytes; + ASSERT_OK(ReadFileToString(env_, dbname_ + "/" + name, &bytes)); + original.emplace_back(name, std::move(bytes)); + } + ASSERT_FALSE(original.empty()); + if (scenario == 1) { + // The MANIFEST read fails. RepairDB must still find the evidence in the + // SST (its exact reference metadata) and refuse to run. + SyncPoint::GetInstance()->SetCallBack( + "Repairer::FeatureProbe", [](void* arg) { + *static_cast(arg) = Status::IOError("optional probe"); + }); + } else if (scenario == 2) { + SyncPoint::GetInstance()->SetCallBack( + "Repairer::BlobIndirectionDetected", [](void* arg) { + *static_cast(arg) = + Status::IOError("after positive evidence"); + }); + } + SyncPoint::GetInstance()->EnableProcessing(); + // Only durable evidence decides if the DB uses the feature, never the + // current option value. Here the option is off, but the files hold feature + // data, so RepairDB must still refuse and change no file. + options.enable_blob_indirection = false; + const Status repaired = RepairDB(dbname_, options); + if (scenario == 2) { + ASSERT_TRUE(repaired.IsIOError()); + } else { + ASSERT_TRUE(repaired.IsNotSupported()) << repaired.ToString(); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + for (const auto& file : original) { + std::string bytes; + ASSERT_OK(ReadFileToString(env_, dbname_ + "/" + file.first, &bytes)); + ASSERT_EQ(file.second, bytes) << file.first; + } + } +} + +TEST_F(RepairTest, PrefixUnknownFeatureProbeRetainsLegacySalvage) { + Options options = CurrentOptions(); + DestroyAndReopen(options); + ASSERT_OK(Put("key", "value")); + ASSERT_OK(Flush()); + Close(); + SyncPoint::GetInstance()->SetCallBack( + "Repairer::FeatureProbe", [](void* arg) { + *static_cast(arg) = Status::IOError("optional probe"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + // The MANIFEST read fails, and no SST holds feature evidence, so RepairDB + // runs the legacy repair and keeps the data. This is not a proof that the DB + // never used the feature: RepairDB could not read the MANIFEST. + options.enable_blob_files = true; + options.enable_blob_indirection = true; + ASSERT_OK(RepairDB(dbname_, options)); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Reopen(options); + ASSERT_EQ("value", Get("key")); +} + +TEST_F(RepairTest, NeverActivatedRepairWithPolicyEnabledKeepsLegacyOutput) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = false; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 128; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + ASSERT_OK(Put("legacy", "sst")); + ASSERT_OK(Flush()); + // GetAllDataFiles() takes a std::unordered_map. + std::unordered_map legacy_ssts; + ASSERT_OK(GetAllDataFiles(kTableFile, &legacy_ssts)); + ASSERT_EQ(legacy_ssts.size(), 1U); + + // The option is on, but no file holds feature evidence yet: the only SST was + // built with the option off, and the new write stays in the WAL. + options.enable_blob_indirection = true; + Reopen(options); + ASSERT_OK(Put("wal", "unflushed")); + Close(); + + ASSERT_OK(RepairDB(dbname_, options)); + // The first repair converts the WAL to an SST without exact reference + // metadata, so a second repair finds no feature evidence and succeeds. + ASSERT_OK(RepairDB(dbname_, options)); + Reopen(options); + EXPECT_EQ(Get("legacy"), "sst"); + EXPECT_EQ(Get("wal"), "unflushed"); + for (const auto& sst : legacy_ssts) { + ASSERT_OK(env_->FileExists(sst.first)); + } + const VersionStorageInfo* const vstorage = dbfull() + ->GetVersionSet() + ->GetColumnFamilySet() + ->GetDefault() + ->current() + ->storage_info(); + size_t files = 0; + for (int level = 0; level < vstorage->num_levels(); ++level) { + for (const FileMetaData* file : vstorage->LevelFiles(level)) { + EXPECT_TRUE(file->blob_reference_descriptor.empty()); + ++files; + } + } + EXPECT_EQ(files, 2U); +} + +TEST_F(RepairTest, PrefixRepairRejectsLegacyClassification) { + Options options = CurrentOptions(); + options.enable_blob_files = true; + options.enable_blob_indirection = false; + options.enable_blob_garbage_collection = false; + options.min_blob_size = 1 << 20; + options.disable_auto_compactions = true; + DestroyAndReopen(options); + // A legacy L1 SST (written before blob indirection is on) with the same keys + // makes the L0 to L0 compaction below not bottommost. A bottommost compaction + // writes exact output. + for (const char* key : {"a", "b"}) { + ASSERT_OK(Put(key, "low")); + } + ASSERT_OK(Flush()); + MoveFilesToLevel(1); + for (const char* key : {"a", "b"}) { + ASSERT_OK(Put(key, "value")); + ASSERT_OK(Flush()); + } + ASSERT_OK(db_->SetOptions(db_->DefaultColumnFamily(), + {{"enable_blob_indirection", "true"}})); + // The values are inline (min_blob_size is 1 MB). An L0 compaction output with + // no exact input that is not bottommost gets only the classification + // property, with frontier 0 (no indirect references; see "Coarse SST" in + // db/blob/blob_reference_metadata.h), and no exact metadata descriptor. + ColumnFamilyMetaData cf_meta; + db_->GetColumnFamilyMetaData(&cf_meta); + std::vector inputs; + for (const SstFileMetaData& file : cf_meta.levels[0].files) { + inputs.push_back(file.name); + } + ASSERT_EQ(inputs.size(), 2U); + ASSERT_EQ(cf_meta.levels[1].files.size(), 1U); + ASSERT_OK(db_->CompactFiles(CompactionOptions(), inputs, 0)); + TablePropertiesCollection properties; + ASSERT_OK(db_->GetPropertiesOfAllTables(&properties)); + ASSERT_EQ(properties.size(), 2U); + // Only the L0 output has the classification; the legacy L1 SST has no + // blob reference properties. + std::string sst_path; + for (const auto& entry : properties) { + const UserCollectedProperties& collected = + entry.second->user_collected_properties; + ASSERT_EQ(collected.count(kBlobReferenceDescriptorPropertyName), 0U); + if (collected.count(kBlobReferenceClassificationPropertyName) == 1U) { + ASSERT_TRUE(sst_path.empty()); + sst_path = entry.first; + } + } + ASSERT_FALSE(sst_path.empty()); + const std::string manifest_path = + DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); + Close(); + // Delete the MANIFEST, so the SST's classification is the only sign that the + // DB used enable_blob_indirection. + ASSERT_OK(env_->DeleteFile(manifest_path)); + + // An earlier test in this binary can leave a lost/ directory, so check that + // the failed repair moved nothing into it. + const auto lost_files = [&](std::vector* files) { + const Status status = env_->GetChildren(dbname_ + "/lost", files); + return status.IsNotFound() ? Status::OK() : status; + }; + std::vector lost_before; + ASSERT_OK(lost_files(&lost_before)); + ASSERT_TRUE(RepairDB(dbname_, options).IsNotSupported()); + ASSERT_OK(env_->FileExists(sst_path)); + std::vector lost_after; + ASSERT_OK(lost_files(&lost_after)); + ASSERT_EQ(lost_after, lost_before); +} + TEST_F(RepairTest, SortRepairedDBL0ByEpochNumber) { Options options = CurrentOptions(); DestroyAndReopen(options); diff --git a/db/table_cache.cc b/db/table_cache.cc index 19cdea07ac05..f8163e90897f 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -12,6 +12,8 @@ #include #include +#include "cache/cache_reservation_manager.h" +#include "db/blob/blob_reference_metadata.h" #include "db/dbformat.h" #include "db/range_tombstone_fragmenter.h" #include "db/snapshot_impl.h" @@ -25,6 +27,7 @@ #include "options/options_helper.h" #include "rocksdb/advanced_options.h" #include "rocksdb/statistics.h" +#include "table/adaptive/adaptive_table_factory.h" #include "table/block_based/block_based_table_reader.h" #include "table/get_context.h" #include "table/internal_iterator.h" @@ -616,8 +619,35 @@ bool TableCache::GetFromRowCache(const Slice& user_key, IterKey& row_cache_key, return found; } +bool TableCache::RangeTombstonesMayCoverUserKey( + const Comparator* user_comparator, const FileMetaData& file_meta, + const Slice& user_key) { + assert(user_comparator != nullptr); + if (file_meta.smallest.size() == 0 || file_meta.largest.size() == 0) { + return true; + } + if (user_comparator->CompareWithoutTimestamp( + user_key, file_meta.smallest.user_key()) < 0) { + return false; + } + const int cmp_largest = user_comparator->CompareWithoutTimestamp( + user_key, file_meta.largest.user_key()); + if (cmp_largest != 0) { + return cmp_largest < 0; + } + SequenceNumber largest_seq = 0; + ValueType largest_type = kTypeDeletion; + UnPackSequenceAndType(ExtractInternalKeyFooter(file_meta.largest.Encode()), + &largest_seq, &largest_type); + // The exclusive end of a cut range tombstone: the file holds nothing for + // this user key. + return !(largest_seq == kMaxSequenceNumber && + largest_type == kTypeRangeDeletion); +} + void TableCache::UpdateRangeTombstoneSeqnums( const ReadOptions& options, TableReader* t, + const Comparator* user_comparator, const FileMetaData* l0_file, MultiGetContext::Range& table_range) { std::unique_ptr latest_range_del_iter; bool track_newer_versions = false; @@ -652,6 +682,11 @@ void TableCache::UpdateRangeTombstoneSeqnums( } if (range_del_iter != nullptr || latest_range_del_iter != nullptr) { for (auto iter = table_range.begin(); iter != table_range.end(); ++iter) { + if (l0_file != nullptr && + !RangeTombstonesMayCoverUserKey(user_comparator, *l0_file, + iter->ukey_with_ts)) { + continue; + } if (latest_range_del_iter != nullptr && iter->get_context->NeedToTrackNewerVersions()) { const SequenceNumber covering_seq = @@ -728,7 +763,9 @@ Status TableCache::MultiGetFilter( // Update the range tombstone sequence numbers for the keys here // as TableCache::MultiGet may or may not be called, and even if it // is, it may be called with fewer keys in the rangedue to filtering. - UpdateRangeTombstoneSeqnums(options, t, tombstone_range); + UpdateRangeTombstoneSeqnums( + options, t, internal_comparator.user_comparator(), + level == 0 ? &file_meta : nullptr, tombstone_range); } if (mget_range->empty() && *handle) { cache_.Release(*handle); @@ -762,6 +799,183 @@ Status TableCache::GetTableProperties( return s; } +Status TableCache::OpenEphemeralBlobReferenceReader( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::unique_ptr* reader) { + assert(reader != nullptr); + assert(*reader == nullptr); + if (file_meta.blob_reference_descriptor.empty()) { + return Status::InvalidArgument( + "SST does not require exact blob reference metadata"); + } + + TableCacheOpenOptions open_options; + open_options.open_ephemeral_table_reader = true; + open_options.avoid_shared_metadata_cache = true; + open_options.skip_filters = true; + TableReader* raw_reader = nullptr; + TypedHandle* handle = nullptr; + Status status = + FindTable(read_options, file_options, internal_comparator, file_meta, + &handle, mutable_cf_options, &raw_reader, false /* no_io */, + nullptr /* histogram */, true /* skip_filters */, + -1 /* level */, false /* prefetch_index_and_filter_in_cache */, + 0 /* max_file_size_for_l0_meta_pin */, file_meta.temperature, + false /* pin_table_handle */, nullptr /* file_open_metadata */, + reader, open_options); + assert(handle == nullptr); + if (!status.ok()) { + return status; + } + assert(raw_reader != nullptr); + assert(reader->get() == raw_reader); + return Status::OK(); +} + +void TableCache::BlobReferenceReader::Reset() { + if (handle_ != nullptr) { + cache_->Release(handle_); + handle_ = nullptr; + } + owned_.reset(); + reader_ = nullptr; +} + +Status TableCache::GetBlobReferenceReader( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + BlobReferenceReader* reader) { + assert(reader != nullptr); + reader->Reset(); + if (file_meta.blob_reference_descriptor.empty()) { + return Status::InvalidArgument( + "SST does not require exact blob reference metadata"); + } + TableReader* cached = nullptr; + TypedHandle* handle = nullptr; + Status status = + FindTable(read_options, file_options, internal_comparator, file_meta, + &handle, mutable_cf_options, &cached, /*no_io=*/true); + TEST_SYNC_POINT_CALLBACK("TableCache::GetBlobReferenceReader:Lookup", + &status); + if (status.ok()) { + assert(cached != nullptr); + reader->cache_ = &cache_; + reader->handle_ = handle; + reader->reader_ = cached; + return status; + } + if (handle != nullptr) { + cache_.Release(handle); + } + if (!status.IsIncomplete()) { + return status; + } + status = OpenEphemeralBlobReferenceReader( + read_options, file_options, internal_comparator, file_meta, + mutable_cf_options, &reader->owned_); + if (status.ok()) { + reader->reader_ = reader->owned_.get(); + } + return status; +} + +Status TableCache::ReadBlobReferenceSstSummary( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::shared_ptr* summary, + const std::shared_ptr& reservation_manager) { + if (summary == nullptr) { + return Status::InvalidArgument("Null blob reference summary output"); + } + if (file_meta.blob_reference_summary != nullptr) { + *summary = file_meta.blob_reference_summary; + return Status::OK(); + } + BlobReferenceReader reader; + Status status = + GetBlobReferenceReader(read_options, file_options, internal_comparator, + file_meta, mutable_cf_options, &reader); + return status.ok() ? ReadBlobReferenceSstSummary( + read_options, *reader.get(), file_meta, + mutable_cf_options, summary, reservation_manager) + : status; +} + +Status TableCache::ReadBlobReferenceSstSummary( + const ReadOptions& read_options, const TableReader& reader, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::shared_ptr* summary, + const std::shared_ptr& reservation_manager) { + if (summary == nullptr) { + return Status::InvalidArgument("Null blob reference summary output"); + } + if (file_meta.blob_reference_summary != nullptr) { + *summary = file_meta.blob_reference_summary; + return Status::OK(); + } + std::shared_ptr result = + std::make_shared(); + result->file_number = file_meta.fd.GetNumber(); + Status status = BlobReferenceMetadataDescriptor::DecodeFrom( + file_meta.blob_reference_descriptor, &result->descriptor); + if (!status.ok()) { + return status; + } + result->reservation_manager = + reservation_manager != nullptr + ? reservation_manager + : GetBlobReferenceCacheReservationManager( + mutable_cf_options.table_factory.get()); + if (result->reservation_manager != nullptr) { + status = result->reservation_manager->MakeCacheReservation( + sizeof(*result), &result->reservation_handle); + if (!status.ok()) { + return status; + } + } + // Charge the group buffer before it is allocated. + status = reader.ReadBlobReferenceGroups( + read_options, file_meta.blob_reference_descriptor, + [&result](size_t buffer_bytes) -> Status { + return result->reservation_handle == nullptr + ? Status::OK() + : result->reservation_handle->Resize(sizeof(*result) + + buffer_bytes); + }, + &result->groups); + if (!status.ok()) { + return status; + } + if (result->reservation_handle != nullptr) { + status = result->reservation_handle->Resize(result->MemoryUsage()); + if (!status.ok()) { + return status; + } + } + *summary = std::move(result); + return Status::OK(); +} + +Status TableCache::ReadBlobReferences( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::vector* requests) { + BlobReferenceReader reader; + Status status = + GetBlobReferenceReader(read_options, file_options, internal_comparator, + file_meta, mutable_cf_options, &reader); + return status.ok() ? reader->ReadBlobReferences( + read_options, file_meta.blob_reference_descriptor, + /*after_index_read=*/nullptr, requests) + : status; +} + Status TableCache::ApproximateKeyAnchors( const ReadOptions& ro, const InternalKeyComparator& internal_comparator, const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, diff --git a/db/table_cache.h b/db/table_cache.h index f18bd29c8e20..9070175c7d97 100644 --- a/db/table_cache.h +++ b/db/table_cache.h @@ -29,6 +29,11 @@ namespace ROCKSDB_NAMESPACE { +struct BlobReferenceGroup; +struct BlobReferenceSstSummary; +struct BlobReference; +class CacheReservationManager; + class Env; class Arena; struct FileDescriptor; @@ -85,6 +90,32 @@ class TableCache { BasicTypedCacheInterface; using TypedHandle = CacheInterface::TypedHandle; + // Holds the table reader that reads the exact blob reference metadata of one + // SST (see GetBlobReferenceReader()). The reader is either the SST's reader + // in the table cache or an ephemeral reader that this object owns. The + // destructor releases the table cache handle or frees the ephemeral reader. + // A pinned reader (the SST's reader held by its FileMetaData) has no cache + // handle, so the caller must keep `file_meta` alive (for example, hold a + // Version that contains it) while it uses the reader. + class BlobReferenceReader { + public: + BlobReferenceReader() = default; + ~BlobReferenceReader() { Reset(); } + BlobReferenceReader(const BlobReferenceReader&) = delete; + BlobReferenceReader& operator=(const BlobReferenceReader&) = delete; + + const TableReader* get() const { return reader_; } + const TableReader* operator->() const { return reader_; } + void Reset(); + + private: + friend class TableCache; + CacheInterface* cache_ = nullptr; + TypedHandle* handle_ = nullptr; + std::unique_ptr owned_; + const TableReader* reader_ = nullptr; + }; + // Cache interface for row cache using RowCacheInterface = BasicTypedCacheInterface; @@ -273,6 +304,43 @@ class TableCache { HistogramImpl* file_read_hist, bool prefetch_index_and_filter_in_cache); + // Returns the summary of the exact blob references of an SST. If `file_meta` + // already holds a summary (a new output of BuildTable() or compaction), it + // returns that summary. Otherwise it reads and validates the groups from + // the SST through GetBlobReferenceReader(). + // Background work calls it, for example the census build and compaction for + // its input SSTs. Do not call it in the MANIFEST write path, because its I/O + // would delay every queued MANIFEST writer. The memory of a summary read from + // the SST is charged through `reservation_manager`. When + // `reservation_manager` is null, the table factory's blob reference + // reservation manager is used. + Status ReadBlobReferenceSstSummary( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::shared_ptr* summary, + const std::shared_ptr& reservation_manager); + + // Same as above, but reads the groups through `reader`, a table reader that + // the caller already has open for the SST. + Status ReadBlobReferenceSstSummary( + const ReadOptions& read_options, const TableReader& reader, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::shared_ptr* summary, + const std::shared_ptr& reservation_manager); + + // Reads, from one SST, the exact references of the sorted groups in + // `requests` (see TableReader::ReadBlobReferences()), through a reader from + // GetBlobReferenceReader(). Returns InvalidArgument for an SST without exact + // blob reference metadata. Only tests call it. Indirect blob GC gets its + // reader with GetBlobReferenceReader() and reads its batch itself. + Status ReadBlobReferences(const ReadOptions& read_options, + const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, + const MutableCFOptions& mutable_cf_options, + std::vector* requests); + Status ApproximateKeyAnchors(const ReadOptions& ro, const InternalKeyComparator& internal_comparator, const FileMetaData& file_meta, @@ -331,6 +399,35 @@ class TableCache { } private: + // DBImpl gets per-job SST readers for indirect blob GC, outside the DB mutex. + friend class DBImpl; + + // Gets a reader for the exact blob reference metadata of `file_meta`. It + // first looks for the SST's reader in the table cache without I/O (the + // pinned reader of the SST, or a cache lookup). That reader already read the + // table tail, so the caller reads only the blocks it needs. On a miss, it + // opens an ephemeral reader (OpenEphemeralBlobReferenceReader()), which is + // not inserted into the table cache, so this call never evicts the reader of + // another SST. Both kinds read the metadata blocks with fill_cache=false, + // checksum checks and no persistent cache (see + // ReadBlobReferenceMetadataBlock() in block_based_table_reader.cc). The + // caller must keep `file_meta` alive (for example, hold a Version that + // contains it) while it uses the reader. Returns InvalidArgument for an SST + // without exact blob reference metadata. + Status GetBlobReferenceReader( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + BlobReferenceReader* reader); + + // Opens a reader that is not kept in the table cache. Its open does not + // insert metadata blocks into the shared block cache. + Status OpenEphemeralBlobReferenceReader( + const ReadOptions& read_options, const FileOptions& file_options, + const InternalKeyComparator& internal_comparator, + const FileMetaData& file_meta, const MutableCFOptions& mutable_cf_options, + std::unique_ptr* reader); + // Build a table reader Status GetTableReader(const ReadOptions& ro, const FileOptions& file_options, const InternalKeyComparator& internal_comparator, @@ -346,10 +443,35 @@ class TableCache { bool avoid_shared_metadata_cache = false); // Update the max_covering_tombstone_seq in the GetContext for each key based - // on the range deletions in the table + // on the range deletions in the table. `l0_file` is the file's metadata if + // it is in L0, else nullptr (see RangeTombstonesMayCoverUserKey()). + // Version::MultiGet() does not end a key's search on that seqno today, but + // the same check keeps the seqno correct for both Get and MultiGet. void UpdateRangeTombstoneSeqnums(const ReadOptions& options, TableReader* t, + const Comparator* user_comparator, + const FileMetaData* l0_file, MultiGetContext::Range& table_range); + // Returns false if `user_key` (with timestamp, if any) is outside the key + // range of L0 file `file_meta`, so no range tombstone of the file may cover + // it. A point lookup reads a file's range tombstones untruncated. With blob + // indirection, the reference cap can cut one output into an L0 epoch run (see + // "Reference cap and epoch runs" in db/blob/blob_reference_metadata.h), and a + // cut inside a range tombstone leaves the whole tombstone in both files. + // Version::Get() can ask the left file about the cut key: FilePicker skips + // its key range filter when all files are in L0 and there are at most 3 of + // them (version_set.cc, "Do key range filtering"), and its filter compares + // user keys, so it keeps the left file for a lookup at the cut key. Without + // this check, that file's tombstone could end the search before the neighbor + // file that holds the key. The callers run it only for L0 files that have + // range tombstones. Base L0 files are never cut and their bounds include + // their tombstones, so for them the check never skips a tombstone that covers + // the key. L1+ needs no check: FindFile() compares internal keys, so it never + // asks a file about a key past the file's exclusive end. + static bool RangeTombstonesMayCoverUserKey(const Comparator* user_comparator, + const FileMetaData& file_meta, + const Slice& user_key); + // Create a key prefix for looking up the row cache. The prefix is of the // format row_cache_id + fd_number + seq_no. Later, the user key can be // appended to form the full key diff --git a/db/table_cache_sync_and_async.h b/db/table_cache_sync_and_async.h index c598c4de0294..893006ada3a0 100644 --- a/db/table_cache_sync_and_async.h +++ b/db/table_cache_sync_and_async.h @@ -54,12 +54,19 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::Get) if (s.ok() && ((max_covering_tombstone_seq != nullptr && !options.ignore_range_deletions) || track_newer_versions)) { + // Runs only for an L0 file with range tombstones (see + // RangeTombstonesMayCoverUserKey()). + const auto may_cover = [&]() { + return level != 0 || RangeTombstonesMayCoverUserKey( + internal_comparator.user_comparator(), + file_meta, ExtractUserKey(k)); + }; if (track_newer_versions) { std::unique_ptr latest_range_del_iter( t->NewRangeTombstoneIterator(GetInternalKeySeqno(k), options.timestamp)); const SequenceNumber covering_seq = - latest_range_del_iter != nullptr + latest_range_del_iter != nullptr && may_cover() ? latest_range_del_iter->MaxCoveringTombstoneSeqnum( ExtractUserKey(k), get_context->read_callback()) : 0; @@ -72,7 +79,7 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::Get) !options.ignore_range_deletions) { std::unique_ptr range_del_iter( t->NewRangeTombstoneIterator(options)); - if (range_del_iter != nullptr) { + if (range_del_iter != nullptr && may_cover()) { SequenceNumber seq = range_del_iter->MaxCoveringTombstoneSeqnum(ExtractUserKey(k)); if (seq > *max_covering_tombstone_seq) { @@ -201,7 +208,9 @@ DEFINE_SYNC_AND_ASYNC(Status, TableCache::MultiGet) } if (s.ok() && !skip_range_deletions && (!options.ignore_range_deletions || track_newer_versions)) { - UpdateRangeTombstoneSeqnums(options, t, table_range); + UpdateRangeTombstoneSeqnums( + options, t, internal_comparator.user_comparator(), + level == 0 ? &file_meta : nullptr, table_range); } if (s.ok()) { CO_AWAIT(t->MultiGet, options, &table_range, diff --git a/db/version_builder.cc b/db/version_builder.cc index 50c225363875..b1eb5eff322e 100644 --- a/db/version_builder.cc +++ b/db/version_builder.cc @@ -27,6 +27,7 @@ #include "cache/cache_reservation_manager.h" #include "db/blob/blob_file_cache.h" #include "db/blob/blob_file_meta.h" +#include "db/blob/blob_reference_metadata.h" #include "db/dbformat.h" #include "db/internal_stats.h" #include "db/table_cache.h" @@ -43,6 +44,27 @@ namespace ROCKSDB_NAMESPACE { +namespace { + +Status ApplyBlobRelocationCountDelta(size_t current, int64_t signed_delta, + size_t* result) { + assert(result != nullptr); + // A count is the number of routes to one relocation file. It is far below the + // size_t and int64_t limits, so the addition below cannot overflow. + if (signed_delta >= 0) { + *result = current + static_cast(signed_delta); + return Status::OK(); + } + const uint64_t magnitude = static_cast(-signed_delta); + if (magnitude > current) { + return Status::Corruption("Invalid blob relocation route count"); + } + *result = current - static_cast(magnitude); + return Status::OK(); +} + +} // namespace + // Scalar/flag state that a VersionEdit can mutate while being applied. Factored // into a small struct so the single-edit undo (VersionBuilder::Rep::EditUndo) // can snapshot and restore all of it with a single copy/swap, and so the field @@ -74,6 +96,12 @@ struct MutableScalars { // cached `valid_version_available_` must be recomputed on the next check // rather than reused. bool version_updated_since_last_check_ = false; + + // The per-CF blob indirection format fence: 0 when it is not set, else + // kBlobIndirectionManifestFormatVersion (2). Records for indirect-origin blob + // files, relocation files and routes need it. Once set, it is never cleared, + // and every Version this builder saves inherits it. + uint32_t blob_indirection_format_version_ = 0; }; class VersionBuilder::Rep : public MutableScalars { @@ -316,12 +344,70 @@ class VersionBuilder::Rep : public MutableScalars { // Dropped blob files (see VersionStorageInfo::GetDroppedBlobFiles()) // recorded by the series of version edits. std::unordered_set dropped_blob_files_; + // The smallest oldest_blob_file_number among the SSTs that the last + // SaveTableFilesTo() call saved. SaveBlobFilesAndCheckTo() drops every blob + // file with a smaller number, because no saved SST references it. If no saved + // SST references a blob file, the value is kInvalidBlobFileNumber and + // SaveBlobFilesAndCheckTo() drops all blob files. The two steps are separate + // public calls, and ProcessManifestWrites() releases the DB mutex between + // them. SaveTableFilesTo() is const, so it keeps the value in a mutable + // member. + mutable uint64_t saved_min_oldest_blob_file_num_ = kInvalidBlobFileNumber; +#ifndef NDEBUG + // Debug check of the call order. SaveTableFilesTo() sets it, and the next + // SaveBlobFilesAndCheckTo() checks and clears it. So + // saved_min_oldest_blob_file_num_ is never used before a save, and never used + // twice for one save. + mutable bool table_files_saved_ = false; +#endif // NDEBUG + + // The relocation file and route changes this builder made on top of the base + // Version. The base Version is immutable, so only the changes are kept here. + // A deletion of a base entry is a null pointer in the relocation file map and + // std::nullopt in the route map. + // + // How a relocation change reaches the new Version: + // 1. VersionBuilder::Apply() calls ApplyBlobRelocationState(), which + // checks one VersionEdit in local staged maps. Only if every check + // passes does it copy the staged changes into the change maps below + // (the *_changes_ fields; they hold the changes of all VersionEdits this + // builder applied). + // 2. SaveBlobFilesAndCheckTo() starts from the base Version's shared + // relocation state. It syncs the blob file entries of the file table, + // then replays the change maps through + // VersionStorageInfo::AddBlobRelocationFile(), + // DeleteBlobRelocationRoute(), DeleteBlobRelocationFile() and + // AddBlobRelocationRoute(), in the order explained in + // SaveBlobFilesAndCheckTo(). + // 3. BlobRelocationState buffers these table changes in its pending map + // (the first change copies the state; the base stays unchanged). + // 4. VersionStorageInfo::SealBlobRelocationState() calls + // BlobRelocationState::Seal(), which turns the buffer into one + // BlobFileTable::Apply() call. BlobFileTable::Apply() rebuilds only the + // changed shards. + // + // Relocation files, keyed by file number. A null pointer marks a deletion. + // (A std::optional here would make the undo map below hold an optional of an + // optional of a shared_ptr, which gcc 13 wrongly reports as maybe + // uninitialized.) + std::map> + blob_relocation_file_changes_; + // Routes, keyed by the route's old file number. + std::map> + blob_relocation_route_changes_; + // Per relocation file, the change in the number of routes to it. + std::map blob_relocation_target_count_deltas_; std::shared_ptr file_metadata_cache_res_mgr_; ColumnFamilyData* cfd_; VersionEditHandler* version_edit_handler_; bool track_found_and_missing_files_; + // True once the consistency check of base_vstorage_ has passed. The base + // Version never changes during the builder's life, so Apply() checks it only + // until the first check passes. A failed check leaves this false, so the next + // Apply() reports the problem again. + bool base_checked_ = false; // If false, only a complete Version with all files consisting it found is // considered valid. If true, besides complete Version, if the Version is // never edited in an atomic group, an incomplete Version with only a suffix @@ -343,6 +429,17 @@ class VersionBuilder::Rep : public MutableScalars { // Missing blob files, useful to check if only the missing L0 files' // associated blob files are missing. std::unordered_set missing_blob_files_; + // Relocation files that recovery could not find or verify. Best-efforts + // recovery can accept a Version that misses only a suffix of L0 files and the + // blob files that only those L0 files use. A missing relocation file never + // qualifies: unlike a normal blob file, it has no linked-SST metadata to + // prove that only the missing L0 files use it. + UnorderedSet missing_blob_relocation_files_; + // Indirect-origin blob files that a blob file retirement record removed from + // the Version. Their metadata stays in mutable_blob_file_metas_, but + // AddBlobFileIfNeeded() does not save them. A route may use a retired file's + // number as its old file number. + UnorderedSet retired_blob_files_; // Note: additional scalar/flag state mutated while applying edits // (has_invalid_levels_, missing_blob_files_high_, valid_version_available_, @@ -399,6 +496,17 @@ class VersionBuilder::Rep : public MutableScalars { // kept alive here rather than destroyed (which would spuriously mark the // blob obsolete). UnorderedMap> blob_metas; + // The three maps below hold blob_relocation_file_changes_, + // blob_relocation_route_changes_ and blob_relocation_target_count_deltas_ + // entries. The outer optional is empty when the edit map had no entry for + // the key on the non-live side. A deletion entry is a null pointer in the + // first map and an empty inner optional in the second. + std::map>> + blob_relocation_files; + std::map>> + blob_relocation_routes; + std::map> blob_relocation_target_counts; // Membership on the non-live side for each tracking set, keyed by file // number (blob file number for `missing_blob_files`). UnorderedMap found_files; @@ -406,6 +514,8 @@ class VersionBuilder::Rep : public MutableScalars { UnorderedMap non_l0_missing_files; UnorderedMap missing_blob_files; UnorderedMap dropped_blob_files; + UnorderedMap missing_blob_relocation_files; + UnorderedMap retired_blob_files; // `intermediate_files_` is append-only within an `Apply()`. Restore by // truncating to `intermediate_files_size` (rollback) / re-appending @@ -446,6 +556,9 @@ class VersionBuilder::Rep : public MutableScalars { allow_incomplete_valid_version_(allow_incomplete_valid_version) { assert(ioptions_); + blob_indirection_format_version_ = + base_vstorage_->GetBlobIndirectionFormatVersion(); + levels_ = new LevelState[num_levels_]; if (track_found_and_missing_files_) { assert(cfd_); @@ -537,9 +650,13 @@ class VersionBuilder::Rep : public MutableScalars { const auto& key = elem.first; auto& other = elem.second; auto it = map.find(key); - std::optional live = - it != map.end() ? std::optional(it->second) - : std::nullopt; + // Built with emplace, not a conditional expression: GCC release builds + // report a false maybe-uninitialized warning for the latter when the + // mapped type is itself an optional. + std::optional live; + if (it != map.end()) { + live.emplace(it->second); + } if (other.has_value()) { // insert_or_assign avoids requiring a default-constructible mapped_type // (e.g. MutableBlobFileMetaData has no default constructor). @@ -602,7 +719,7 @@ class VersionBuilder::Rep : public MutableScalars { // Snapshots the pre-edit value of every piece of state that `edit` will // mutate, so the edit can later be rolled back / redone. Must be called // before the edit is applied and only in tracking mode. - void CaptureUndo(const VersionEdit* edit) { + void CaptureUndo(const VersionEdit& edit) { assert(track_found_and_missing_files_); assert(!undo_); undo_ = std::make_unique(); @@ -633,17 +750,67 @@ class VersionBuilder::Rep : public MutableScalars { blob_file_number); }; - for (const auto& ba : edit->GetBlobFileAdditions()) { + for (const auto& ba : edit.GetBlobFileAdditions()) { touch_blob_file(ba.GetBlobFileNumber()); } - for (const auto& bg : edit->GetBlobFileGarbages()) { + for (const auto& bg : edit.GetBlobFileGarbages()) { touch_blob_file(bg.GetBlobFileNumber()); } - for (const uint64_t blob_file_number : edit->GetDroppedBlobFiles()) { + for (const uint64_t blob_file_number : edit.GetDroppedBlobFiles()) { RecordSetMembership(dropped_blob_files_, undo_->dropped_blob_files, blob_file_number); } - for (const auto& deleted_file : edit->GetDeletedFiles()) { + for (uint64_t file_number : edit.GetBlobFileRetirements()) { + touch_blob_file(file_number); + RecordSetMembership(retired_blob_files_, undo_->retired_blob_files, + file_number); + } + for (const BlobRelocationFileAddition& addition : + edit.GetBlobRelocationFileAdditions()) { + RecordMapEntry(blob_relocation_file_changes_, + undo_->blob_relocation_files, addition.GetFileNumber()); + RecordSetMembership(missing_blob_files_, undo_->missing_blob_files, + addition.GetFileNumber()); + RecordSetMembership(missing_blob_relocation_files_, + undo_->missing_blob_relocation_files, + addition.GetFileNumber()); + } + for (uint64_t file_number : edit.GetBlobRelocationFileDeletions()) { + RecordMapEntry(blob_relocation_file_changes_, + undo_->blob_relocation_files, file_number); + RecordSetMembership(missing_blob_relocation_files_, + undo_->missing_blob_relocation_files, file_number); + } + for (uint64_t file_number : edit.GetBlobPhysicalFileLosses()) { + RecordMapEntry(blob_relocation_file_changes_, + undo_->blob_relocation_files, file_number); + RecordSetMembership(missing_blob_relocation_files_, + undo_->missing_blob_relocation_files, file_number); + } + const auto touch_route = [&](uint64_t old_file_number, + const BlobRelocationRoute* replacement) { + RecordMapEntry(blob_relocation_route_changes_, + undo_->blob_relocation_routes, old_file_number); + const auto touch_count = [&](uint64_t target) { + RecordMapEntry(blob_relocation_target_count_deltas_, + undo_->blob_relocation_target_counts, target); + }; + const uint64_t current = GetBlobRelocationRouteTarget(old_file_number); + if (current != kInvalidBlobFileNumber) { + touch_count(current); + } + if (replacement != nullptr) { + touch_count(replacement->GetTargetFileNumber()); + } + }; + for (const BlobRelocationRoute& route : edit.GetBlobRelocationRoutes()) { + touch_route(route.GetOldFileNumber(), &route); + } + for (uint64_t old_file_number : edit.GetBlobRelocationRouteDeletions()) { + // A null replacement distinguishes deletion from route replacement. + touch_route(old_file_number, nullptr); + } + for (const auto& deleted_file : edit.GetDeletedFiles()) { const int level = deleted_file.first; const uint64_t fn = deleted_file.second; touch_table_file(level, fn); @@ -653,13 +820,13 @@ class VersionBuilder::Rep : public MutableScalars { touch_blob_file(GetOldestBlobFileNumberForTableFile(level, fn)); } } - for (const auto& new_file : edit->GetNewFiles()) { + for (const auto& new_file : edit.GetNewFiles()) { const int level = new_file.first; const FileMetaData& meta = new_file.second; touch_table_file(level, meta.fd.GetNumber()); touch_blob_file(meta.oldest_blob_file_number); } - for (const auto& cursor : edit->GetCompactCursors()) { + for (const auto& cursor : edit.GetCompactCursors()) { const int level = cursor.first; if (level >= 0 && level < num_levels_) { RecordMapEntry(updated_compact_cursors_, undo_->compact_cursors, level); @@ -706,11 +873,20 @@ class VersionBuilder::Rep : public MutableScalars { ToggleMapEntries(invalid_level_sizes_, undo_->invalid_level_sizes); ToggleMapEntries(updated_compact_cursors_, undo_->compact_cursors); ToggleMapEntries(mutable_blob_file_metas_, undo_->blob_metas); + ToggleMapEntries(blob_relocation_file_changes_, + undo_->blob_relocation_files); + ToggleMapEntries(blob_relocation_route_changes_, + undo_->blob_relocation_routes); + ToggleMapEntries(blob_relocation_target_count_deltas_, + undo_->blob_relocation_target_counts); ToggleSetMembership(found_files_, undo_->found_files); ToggleSetMembership(l0_missing_files_, undo_->l0_missing_files); ToggleSetMembership(non_l0_missing_files_, undo_->non_l0_missing_files); ToggleSetMembership(missing_blob_files_, undo_->missing_blob_files); ToggleSetMembership(dropped_blob_files_, undo_->dropped_blob_files); + ToggleSetMembership(missing_blob_relocation_files_, + undo_->missing_blob_relocation_files); + ToggleSetMembership(retired_blob_files_, undo_->retired_blob_files); if (!undo_->rolled_back) { // Forward -> pre-edit: peel off the entries appended by this edit. @@ -952,8 +1128,9 @@ class VersionBuilder::Rep : public MutableScalars { const uint64_t blob_file_number = blob_file_meta->GetBlobFileNumber(); - if (blob_file_meta->GetGarbageBlobCount() >= - blob_file_meta->GetTotalBlobCount()) { + if (!blob_file_meta->IsIndirectOrigin() && + blob_file_meta->GetGarbageBlobCount() >= + blob_file_meta->GetTotalBlobCount()) { std::ostringstream oss; oss << "Blob file #" << blob_file_number << " consists entirely of garbage"; @@ -1054,6 +1231,22 @@ class VersionBuilder::Rep : public MutableScalars { Status ApplyBlobFileAddition(const BlobFileAddition& blob_file_addition) { const uint64_t blob_file_number = blob_file_addition.GetBlobFileNumber(); + if (blob_file_addition.IsIndirectOrigin() && + blob_indirection_format_version_ != + kBlobIndirectionManifestFormatVersion) { + return Status::Corruption( + "Indirect-origin blob file requires the format fence"); + } + + if (HasBlobRelocationFileNumber(blob_file_number)) { + return Status::Corruption( + "Normal blob file conflicts with a relocation file"); + } + if (GetBlobRelocationRouteTarget(blob_file_number) != + kInvalidBlobFileNumber) { + return Status::Corruption("Normal blob file conflicts with a route"); + } + if (IsBlobFileInVersion(blob_file_number)) { std::ostringstream oss; oss << "Blob file #" << blob_file_number << " already added"; @@ -1083,7 +1276,8 @@ class VersionBuilder::Rep : public MutableScalars { blob_file_number, blob_file_addition.GetTotalBlobCount(), blob_file_addition.GetTotalBlobBytes(), blob_file_addition.GetChecksumMethod(), - blob_file_addition.GetChecksumValue(), std::move(deleter)); + blob_file_addition.GetChecksumValue(), + blob_file_addition.IsIndirectOrigin(), std::move(deleter)); mutable_blob_file_metas_.emplace( blob_file_number, MutableBlobFileMetaData(std::move(shared_meta))); @@ -1113,6 +1307,15 @@ class VersionBuilder::Rep : public MutableScalars { GetOrCreateMutableBlobFileMetaData(blob_file_number); if (!mutable_meta) { + // The file is not in the Version. If it has a route, an indirect blob GC + // install edit already moved its live values and wrote the blob file + // retirement record of the file. A compaction that started before that + // edit can still report garbage for the file. A garbage count has no use + // for a removed file, so ignore it. + if (GetBlobRelocationRouteTarget(blob_file_number) != + kInvalidBlobFileNumber) { + return Status::OK(); + } std::ostringstream oss; oss << "Blob file #" << blob_file_number << " not found"; @@ -1129,6 +1332,514 @@ class VersionBuilder::Rep : public MutableScalars { return Status::OK(); } + std::shared_ptr GetBlobRelocationFile( + uint64_t file_number) const { + const auto edit = blob_relocation_file_changes_.find(file_number); + if (edit != blob_relocation_file_changes_.end()) { + return edit->second; + } + return base_vstorage_->GetBlobRelocationFileMetaData(file_number); + } + + bool HasBlobRelocationFileNumber(uint64_t file_number) const { + return blob_relocation_file_changes_.find(file_number) != + blob_relocation_file_changes_.end() || + base_vstorage_->GetBlobRelocationFileMetaData(file_number) != + nullptr; + } + + // True if an earlier edit applied to this builder deleted relocation file + // `file_number`. It is not about blob file retirement. + bool IsBlobRelocationFileDeletedInBuilder(uint64_t file_number) const { + const auto edit = blob_relocation_file_changes_.find(file_number); + if (edit != blob_relocation_file_changes_.end()) { + return edit->second == nullptr; + } + // The base Version keeps no record of deleted files. This builder sees the + // deletions applied to it: the whole MANIFEST at replay, or one LogAndApply + // batch of version edits at runtime. At runtime, LogAndApply also checks + // the earlier batches of the current MANIFEST + // (VersionSet::RejectReaddedBlobRelocationState()). + return false; + } + + // Returns the relocation file that the route of old file `old_file_number` + // points to, or kInvalidBlobFileNumber when `old_file_number` has no route. + uint64_t GetBlobRelocationRouteTarget(uint64_t old_file_number) const { + const auto edit = blob_relocation_route_changes_.find(old_file_number); + if (edit != blob_relocation_route_changes_.end()) { + return edit->second.has_value() ? edit->second->GetTargetFileNumber() + : kInvalidBlobFileNumber; + } + return base_vstorage_->GetBlobRelocationRouteTarget(old_file_number); + } + + // True if an earlier edit applied to this builder deleted the route of old + // file `old_file_number` (a route deletion record). + bool IsBlobRelocationRouteDeletedInBuilder(uint64_t old_file_number) const { + const auto edit = blob_relocation_route_changes_.find(old_file_number); + if (edit != blob_relocation_route_changes_.end()) { + return !edit->second.has_value(); + } + // The base Version keeps no record of deleted routes. This builder sees the + // deletions applied to it: the whole MANIFEST at replay, or one LogAndApply + // batch of version edits at runtime. At runtime, LogAndApply also checks + // the earlier batches of the current MANIFEST + // (VersionSet::RejectReaddedBlobRelocationState()). + return false; + } + + Status GetBlobRelocationTargetCount(uint64_t physical_file_number, + int64_t additional_delta, + size_t* count) const { + assert(count != nullptr); + const size_t base_count = + base_vstorage_->GetBlobRelocationRouteTargetCount(physical_file_number); + const auto delta = + blob_relocation_target_count_deltas_.find(physical_file_number); + const int64_t existing_delta = + delta == blob_relocation_target_count_deltas_.end() ? 0 : delta->second; + size_t current = 0; + Status status = + ApplyBlobRelocationCountDelta(base_count, existing_delta, ¤t); + if (!status.ok()) { + return status; + } + return ApplyBlobRelocationCountDelta(current, additional_delta, count); + } + + std::shared_ptr + CreateVersionOwnedBlobRelocationFile( + BlobRelocationFileAddition addition) const { + auto deleter = [vs = version_set_, ioptions = ioptions_, + bc = cfd_ ? cfd_->blob_file_cache() + : nullptr](BlobRelocationFileMetaData* metadata) { + if (vs != nullptr) { + assert(ioptions != nullptr); + assert(!ioptions->cf_paths.empty()); + assert(metadata != nullptr); + vs->AddObsoleteBlobFile(metadata->GetFileNumber(), + ioptions->cf_paths.front().path); + } + // This runs under the DB mutex when the last Version that owns the file + // is destroyed. It runs without the DB mutex when a failed LogAndApply + // batch or ManifestTailer frees relocation state (see "Freeing + // relocation state" at VersionSet::FreeBlobRelocationState()). Because + // it can run under the DB mutex, it must not take the stripe mutex of + // the file, which a relocation file open can hold while it reads its + // location blocks. So it only drops a remembered failed open, and + // PurgeObsoleteFiles() evicts the cached reader later (see + // BlobFileCache::Evict()). While file deletions are disabled, no purge + // runs, so the reader of an obsolete relocation file stays cached until + // they are enabled again. + if (bc != nullptr) { + bc->ForgetFailedRelocationOpen(metadata->GetFileNumber()); + } + delete metadata; + }; + return BlobRelocationFileMetaData::Create(std::move(addition), + std::move(deleter)); + } + + Status ApplyBlobFileRetirement(uint64_t file_number) { + if (retired_blob_files_.count(file_number) != 0) { + // A retirement record holds only the file number. A second record for a + // file that an earlier edit of this builder already retired (for example, + // in a replay) changes nothing, so accept it. + return Status::OK(); + } + MutableBlobFileMetaData* const meta = + GetOrCreateMutableBlobFileMetaData(file_number); + if (meta == nullptr || !meta->GetSharedMeta()->IsIndirectOrigin()) { + return Status::Corruption( + "Blob file retirement requires a live indirect-origin blob file"); + } + retired_blob_files_.insert(file_number); + if (track_found_and_missing_files_ && + missing_blob_files_.erase(file_number) != 0) { + // Recovery applies the file's addition first. The file can already be + // deleted from disk, so the addition put it in missing_blob_files_. The + // retirement shows that the file left on purpose, so remove it from the + // missing set and recompute missing_blob_files_high_. Otherwise the + // Version looks incomplete. CaptureUndo() saved the set membership and + // missing_blob_files_high_ for rollback. + missing_blob_files_high_ = kInvalidBlobFileNumber; + for (uint64_t missing_file_number : missing_blob_files_) { + missing_blob_files_high_ = + std::max(missing_blob_files_high_, missing_file_number); + } + } + return Status::OK(); + } + + // Checks and applies the format fence and the relocation records of one + // edit. Steps: + // 1. Check the per-CF fence (records need it; the compaction style must + // allow it). + // 2. Stage relocation file additions, deletions and accepted losses in + // local maps. A lost file stays in the Version, marked lost. + // 3. Stage route sets and deletions, and the change in the route count of + // each target. + // 4. Check the final state: no route to a deleted or missing file, no route + // from a live file, no chains, and at most 2^32 - 1 routes per target + // (the file table keeps a 32-bit route count). + // 5. Only then commit the staged changes to the builder's change maps, so a + // bad edit changes nothing. + Status ApplyBlobRelocationState(const VersionEdit& edit) { + const bool has_records = !edit.GetBlobRelocationFileAdditions().empty() || + !edit.GetBlobRelocationFileDeletions().empty() || + !edit.GetBlobPhysicalFileLosses().empty() || + !edit.GetBlobRelocationRoutes().empty() || + !edit.GetBlobRelocationRouteDeletions().empty() || + !edit.GetBlobFileRetirements().empty(); + if (!has_records && !edit.HasBlobIndirectionFormatVersion()) { + return Status::OK(); + } + // Step 1: check the per-CF fence. + uint32_t final_format_version = blob_indirection_format_version_; + if (edit.HasBlobIndirectionFormatVersion()) { + const uint32_t requested = edit.GetBlobIndirectionFormatVersion(); + if (requested != kBlobIndirectionManifestFormatVersion || + (final_format_version != 0 && final_format_version != requested)) { + return Status::Corruption("Invalid blob indirection format fence"); + } + final_format_version = requested; + } + if (has_records && + final_format_version != kBlobIndirectionManifestFormatVersion) { + return Status::Corruption( + "Blob relocation records require the format fence"); + } + if (ioptions_ != nullptr) { + const Status s = CheckBlobIndirectionCompactionStyle( + ioptions_->compaction_style, final_format_version != 0); + if (!s.ok()) { + return s; + } + } + if (!has_records) { + blob_indirection_format_version_ = final_format_version; + return Status::OK(); + } + + // Step 2: stage relocation file additions, deletions and accepted losses. + using StagedFile = + std::optional>; + using StagedRoute = std::optional; + std::map staged_files; + std::map staged_routes; + std::map staged_target_count_deltas; + std::set seen_file_additions; + std::set newly_created_files; + const std::set file_losses( + edit.GetBlobPhysicalFileLosses().begin(), + edit.GetBlobPhysicalFileLosses().end()); + if (file_losses.size() != edit.GetBlobPhysicalFileLosses().size() || + file_losses.count(kInvalidBlobFileNumber) != 0) { + return Status::Corruption("Invalid or duplicate blob relocation loss"); + } + std::set seen_route_additions; + std::set new_table_file_numbers; + std::set new_blob_file_numbers; + for (const auto& new_file : edit.GetNewFiles()) { + new_table_file_numbers.insert(new_file.second.fd.GetNumber()); + } + for (const BlobFileAddition& blob_addition : edit.GetBlobFileAdditions()) { + new_blob_file_numbers.insert(blob_addition.GetBlobFileNumber()); + } + + const auto get_final_file = [&](uint64_t file_number) { + const auto staged = staged_files.find(file_number); + if (staged != staged_files.end()) { + return staged->second.has_value() ? *staged->second : nullptr; + } + return GetBlobRelocationFile(file_number); + }; + const auto get_final_route_target = [&](uint64_t old_file_number) { + const auto staged = staged_routes.find(old_file_number); + if (staged != staged_routes.end()) { + return staged->second.has_value() + ? staged->second->GetTargetFileNumber() + : kInvalidBlobFileNumber; + } + return GetBlobRelocationRouteTarget(old_file_number); + }; + + for (const BlobRelocationFileAddition& addition : + edit.GetBlobRelocationFileAdditions()) { + if (!seen_file_additions.insert(addition.GetFileNumber()).second || + staged_files.find(addition.GetFileNumber()) != staged_files.end()) { + return Status::Corruption("Duplicate blob relocation file edit"); + } + Status status = addition.Validate(); + if (!status.ok()) { + return status; + } + if (IsBlobFileInVersion(addition.GetFileNumber())) { + return Status::Corruption( + "Blob relocation file conflicts with a normal blob file"); + } + if (GetCurrentLevelForTableFile(addition.GetFileNumber()) != + VersionStorageInfo::FileLocation::Invalid().GetLevel()) { + return Status::Corruption( + "Blob relocation file conflicts with a table file"); + } + if (new_table_file_numbers.count(addition.GetFileNumber()) != 0) { + return Status::Corruption( + "Blob relocation file conflicts with a table file addition"); + } + if (new_blob_file_numbers.count(addition.GetFileNumber()) != 0) { + return Status::Corruption( + "Blob relocation file conflicts with a normal blob addition"); + } + if (IsBlobRelocationFileDeletedInBuilder(addition.GetFileNumber())) { + return Status::Corruption("Blob relocation file number was deleted"); + } + const std::shared_ptr current = + GetBlobRelocationFile(addition.GetFileNumber()); + if (current != nullptr) { + // A repeated record (MANIFEST replay or catch-up) must equal the file + // the builder has. The record holds only small fields (numbers and the + // file checksum strings), so the compare is cheap enough to run here, + // under the DB mutex. + if (current->GetAddition() != addition) { + return Status::Corruption( + "Conflicting blob relocation file addition"); + } + continue; + } + bool marked_lost = file_losses.count(addition.GetFileNumber()) != 0; + if (track_found_and_missing_files_ && !marked_lost) { + assert(version_edit_handler_ != nullptr); + status = + version_edit_handler_->VerifyBlobRelocationFile(cfd_, addition); + if (status.IsPathNotFound() || status.IsNotFound() || + status.IsCorruption()) { + missing_blob_relocation_files_.insert(addition.GetFileNumber()); + } else if (!status.ok()) { + return status; + } + } + std::shared_ptr added = + BlobRelocationFileMetaData::Create(addition); + newly_created_files.insert(addition.GetFileNumber()); + if (marked_lost) { + added = BlobRelocationFileMetaData::CreateMarkedLost(std::move(added)); + } + staged_files.emplace(addition.GetFileNumber(), std::move(added)); + } + + for (uint64_t file_number : file_losses) { + const std::shared_ptr current = + get_final_file(file_number); + if (current == nullptr || + std::find(edit.GetBlobRelocationFileDeletions().begin(), + edit.GetBlobRelocationFileDeletions().end(), file_number) != + edit.GetBlobRelocationFileDeletions().end()) { + return Status::Corruption("Blob relocation loss names no live target"); + } + if (!current->IsMarkedLost()) { + staged_files[file_number] = + BlobRelocationFileMetaData::CreateMarkedLost(current); + } + if (track_found_and_missing_files_) { + missing_blob_relocation_files_.erase(file_number); + } + } + + for (uint64_t file_number : edit.GetBlobRelocationFileDeletions()) { + if (file_number == kInvalidBlobFileNumber || + seen_file_additions.count(file_number) != 0 || + staged_files.find(file_number) != staged_files.end()) { + return Status::Corruption("Invalid blob relocation file deletion"); + } + if (IsBlobRelocationFileDeletedInBuilder(file_number)) { + // A repeated deletion record (MANIFEST replay or catch-up) changes + // nothing, so accept it. + if (track_found_and_missing_files_) { + missing_blob_relocation_files_.erase(file_number); + } + continue; + } + // As for an SST deletion, the file must be live in this column family. + // A deletion of another column family's file is a bad record. + if (GetBlobRelocationFile(file_number) == nullptr) { + return Status::Corruption( + "Blob relocation file deletion names no live file"); + } + staged_files.emplace(file_number, std::nullopt); + if (track_found_and_missing_files_) { + missing_blob_relocation_files_.erase(file_number); + } + } + + // Step 3: stage route sets and deletions. + const auto stage_route = [&](uint64_t old_file_number, + StagedRoute replacement) -> Status { + if (staged_routes.find(old_file_number) != staged_routes.end()) { + return Status::Corruption("Duplicate blob relocation route edit"); + } + const uint64_t current = GetBlobRelocationRouteTarget(old_file_number); + if (current != kInvalidBlobFileNumber) { + staged_target_count_deltas[current] -= 1; + } + if (replacement.has_value()) { + staged_target_count_deltas[replacement->GetTargetFileNumber()] += 1; + } + staged_routes.emplace(old_file_number, std::move(replacement)); + return Status::OK(); + }; + + for (const BlobRelocationRoute& route : edit.GetBlobRelocationRoutes()) { + const uint64_t old_file_number = route.GetOldFileNumber(); + if (!seen_route_additions.insert(old_file_number).second) { + return Status::Corruption("Duplicate blob relocation route edit"); + } + Status status = route.Validate(); + if (!status.ok()) { + return status; + } + if (IsBlobRelocationRouteDeletedInBuilder(old_file_number)) { + return Status::Corruption("Blob relocation route was deleted"); + } + if (GetBlobRelocationRouteTarget(old_file_number) == + route.GetTargetFileNumber()) { + // The route already exists with this target (a repeated record), so + // there is nothing to stage. + continue; + } + status = stage_route(old_file_number, route); + if (!status.ok()) { + return status; + } + } + + for (uint64_t old_file_number : edit.GetBlobRelocationRouteDeletions()) { + if (old_file_number == kInvalidBlobFileNumber || + seen_route_additions.count(old_file_number) != 0 || + staged_routes.find(old_file_number) != staged_routes.end()) { + return Status::Corruption("Invalid blob relocation route deletion"); + } + if (IsBlobRelocationRouteDeletedInBuilder(old_file_number)) { + // A repeated deletion record (MANIFEST replay or catch-up) changes + // nothing, so accept it. + continue; + } + Status status = stage_route(old_file_number, std::nullopt); + if (!status.ok()) { + return status; + } + } + + const auto get_final_target_count = [&](uint64_t physical_file_number, + size_t* count) { + const auto delta = staged_target_count_deltas.find(physical_file_number); + return GetBlobRelocationTargetCount( + physical_file_number, + delta == staged_target_count_deltas.end() ? 0 : delta->second, count); + }; + + // Step 4: check the final state after this edit: no route points to a + // deleted relocation file, and no added relocation file number is the old + // file number of a route. + for (const auto& file : staged_files) { + if (file.second.has_value()) { + if (get_final_route_target(file.first) != kInvalidBlobFileNumber) { + return Status::Corruption( + "Blob relocation file conflicts with a route"); + } + continue; + } + size_t target_count = 0; + Status status = get_final_target_count(file.first, &target_count); + if (!status.ok()) { + return status; + } + if (target_count != 0) { + return Status::Corruption( + "Deleted blob relocation file is still a route target"); + } + } + + for (const auto& route_edit : staged_routes) { + if (!route_edit.second.has_value()) { + continue; + } + const BlobRelocationRoute& route = *route_edit.second; + // The target must be a live relocation file and must not have a route of + // its own: one hop, no chains. + if (get_final_file(route.GetTargetFileNumber()) == nullptr) { + return Status::Corruption("Invalid blob relocation route target"); + } + if (get_final_route_target(route.GetTargetFileNumber()) != + kInvalidBlobFileNumber) { + return Status::Corruption("Blob relocation route would form a chain"); + } + size_t target_count = 0; + Status count_status = + get_final_target_count(route.GetTargetFileNumber(), &target_count); + if (!count_status.ok()) { + return count_status; + } + if (target_count > std::numeric_limits::max()) { + return Status::Corruption("Too many routes to one relocation file"); + } + // The old file number must not be a live file: not a live relocation + // file, not a blob file in the Version, and not a blob file this edit + // adds. A blob file removed by blob file retirement in this or an earlier + // edit is not live. + const uint64_t old_file_number = route.GetOldFileNumber(); + if (get_final_file(old_file_number) != nullptr || + (IsBlobFileInVersion(old_file_number) && + retired_blob_files_.count(old_file_number) == 0) || + new_blob_file_numbers.count(old_file_number) != 0) { + return Status::Corruption( + "Blob relocation route old file is a live file"); + } + size_t incoming_count = 0; + Status status = get_final_target_count(old_file_number, &incoming_count); + if (!status.ok()) { + return status; + } + if (incoming_count != 0) { + return Status::Corruption("Blob relocation route would form a chain"); + } + } + + // Step 5: every record passed the checks against the final state. Commit + // them only now, so a bad record cannot leave part of the edit's file and + // route changes in the builder. + blob_indirection_format_version_ = final_format_version; + for (auto& file : staged_files) { + if (!file.second.has_value()) { + blob_relocation_file_changes_[file.first] = nullptr; + continue; + } + const std::shared_ptr& staged = *file.second; + if (newly_created_files.count(file.first) == 0) { + blob_relocation_file_changes_[file.first] = staged; + } else { + std::shared_ptr owned = + CreateVersionOwnedBlobRelocationFile(staged->GetAddition()); + if (staged->IsMarkedLost()) { + owned = + BlobRelocationFileMetaData::CreateMarkedLost(std::move(owned)); + } + blob_relocation_file_changes_[file.first] = std::move(owned); + } + } + for (auto& route : staged_routes) { + blob_relocation_route_changes_[route.first] = std::move(route.second); + } + for (const auto& delta : staged_target_count_deltas) { + blob_relocation_target_count_deltas_[delta.first] += delta.second; + if (blob_relocation_target_count_deltas_[delta.first] == 0) { + blob_relocation_target_count_deltas_.erase(delta.first); + } + } + return Status::OK(); + } + int GetCurrentLevelForTableFile(uint64_t file_number) const { auto it = table_file_levels_.find(file_number); if (it != table_file_levels_.end()) { @@ -1257,6 +1968,11 @@ class VersionBuilder::Rep : public MutableScalars { const uint64_t file_number = meta.fd.GetNumber(); + if (HasBlobRelocationFileNumber(file_number)) { + return Status::Corruption( + "Table file conflicts with a blob relocation file"); + } + const int current_level = GetCurrentLevelForTableFile(file_number); if (current_level != @@ -1288,6 +2004,26 @@ class VersionBuilder::Rep : public MutableScalars { FileMetaData* const f = new FileMetaData(meta); f->refs = 1; + // A Version does not keep the in-memory blob reference summary of an SST + // alive. The census delta takes the summary from the edit (see + // VersionSet::ProcessManifestWrites()). Later readers read the reference + // metadata from the SST file. + f->blob_reference_summary.reset(); + if (blob_indirection_format_version_ == + kBlobIndirectionManifestFormatVersion && + f->blob_reference_descriptor.empty()) { + // An empty descriptor means the SST has no exact reference metadata. + // ValidateBlobReferenceMetadata() allows this only for a coarse SST + // (frontier set: the SST records the oldest indirect-origin file number + // it may reference; see "Coarse SST" in + // db/blob/blob_reference_metadata.h), a level move of an SST already in + // the Version, or an SST with the legacy marker. Mark every such SST + // without a frontier as legacy, so an edit that copies its metadata (for + // example a trivial move) carries the marker. + f->blob_reference_legacy_metadata = + f->oldest_coarse_indirect_origin_file_number == + kInvalidBlobFileNumber; + } if (file_metadata_cache_res_mgr_) { Status s = file_metadata_cache_res_mgr_->UpdateCacheReservation( @@ -1364,14 +2100,90 @@ class VersionBuilder::Rep : public MutableScalars { return Status::OK(); } + Status ValidateBlobReferenceMetadata(const VersionEdit& edit) const { + uint32_t final_format_version = blob_indirection_format_version_; + if (edit.HasBlobIndirectionFormatVersion()) { + final_format_version = edit.GetBlobIndirectionFormatVersion(); + } + + for (const auto& new_file : edit.GetNewFiles()) { + const FileMetaData& meta = new_file.second; + if (meta.oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber) { + if (final_format_version != kBlobIndirectionManifestFormatVersion || + meta.oldest_coarse_indirect_origin_file_number > kFileNumberMask || + meta.blob_reference_legacy_metadata || + !meta.blob_reference_descriptor.empty()) { + return Status::Corruption("Invalid coarse blob reference metadata"); + } + continue; + } + if (!meta.blob_reference_descriptor.empty()) { + if (meta.blob_reference_legacy_metadata) { + return Status::Corruption("Conflicting blob reference metadata"); + } + if (final_format_version != kBlobIndirectionManifestFormatVersion) { + return Status::Corruption( + "Exact blob reference metadata requires the format fence"); + } + BlobReferenceMetadataDescriptor descriptor; + Status status = BlobReferenceMetadataDescriptor::DecodeFrom( + meta.blob_reference_descriptor, &descriptor); + if (!status.ok()) { + return status; + } + continue; + } + if (meta.blob_reference_legacy_metadata) { + if (final_format_version != kBlobIndirectionManifestFormatVersion) { + return Status::Corruption( + "Legacy blob reference metadata requires the format fence"); + } + continue; + } + if (final_format_version != kBlobIndirectionManifestFormatVersion) { + continue; + } + + // The SST has no descriptor, no legacy marker and no coarse frontier. In + // a fenced CF this is allowed only for a level move: the SST is already + // in this Version and the same edit deletes it from its current level. + // Such an SST can be older than the fence. Every SST built after the + // fence has a descriptor, a coarse frontier, or the legacy marker + // (blob_reference_legacy_metadata), also when it keeps no blob reference. + const uint64_t file_number = meta.fd.GetNumber(); + const int current_level = GetCurrentLevelForTableFile(file_number); + if (current_level == + VersionStorageInfo::FileLocation::Invalid().GetLevel() || + edit.GetDeletedFiles().find({current_level, file_number}) == + edit.GetDeletedFiles().end()) { + return Status::Corruption( + "An SST added after the format fence has no exact blob reference " + "metadata"); + } + } + return Status::OK(); + } + // Apply all of the edits in *edit to the current state. Status Apply(const VersionEdit* edit) { + if (edit == nullptr) { + return Status::InvalidArgument("VersionBuilder edit is null"); + } bool version_updated = false; - { + if (!base_checked_) { + TEST_SYNC_POINT("VersionBuilder::Apply:CheckBaseConsistency"); const Status s = CheckConsistency(base_vstorage_); if (!s.ok()) { return s; } + base_checked_ = true; + } + { + const Status s = ValidateBlobReferenceMetadata(*edit); + if (!s.ok()) { + return s; + } } // In tracking mode, snapshot how to undo this edit before mutating, so @@ -1379,13 +2191,48 @@ class VersionBuilder::Rep : public MutableScalars { // (on a valid->invalid negative edge) via RollbackLastApply() instead of // copying the whole Rep on every edit. if (track_found_and_missing_files_) { - CaptureUndo(edit); + CaptureUndo(*edit); } // Note: we process the blob file related changes first because the // table file addition/deletion logic depends on the blob files // already being there. + // Apply blob file retirement records before the relocation records. For + // example, an indirect blob GC install edit writes the blob file + // retirement record of origin file 10 and adds a route from file 10 in the + // same edit. The route check requires that file 10 is no longer live. + { + const std::vector& retirements = edit->GetBlobFileRetirements(); + if (std::set(retirements.begin(), retirements.end()).size() != + retirements.size()) { + return Status::Corruption("Duplicate blob file retirement"); + } + } + for (uint64_t file_number : edit->GetBlobFileRetirements()) { + const Status s = ApplyBlobFileRetirement(file_number); + if (!s.ok()) { + return s; + } + version_updated = true; + } + + { + const Status s = ApplyBlobRelocationState(*edit); + if (!s.ok()) { + return s; + } + if (edit->HasBlobIndirectionFormatVersion() || + !edit->GetBlobRelocationFileAdditions().empty() || + !edit->GetBlobRelocationFileDeletions().empty() || + !edit->GetBlobPhysicalFileLosses().empty() || + !edit->GetBlobRelocationRoutes().empty() || + !edit->GetBlobRelocationRouteDeletions().empty() || + !edit->GetBlobFileRetirements().empty()) { + version_updated = true; + } + } + // Add new blob files for (const auto& blob_file_addition : edit->GetBlobFileAdditions()) { const Status s = ApplyBlobFileAddition(blob_file_addition); @@ -1606,19 +2453,41 @@ class VersionBuilder::Rep : public MutableScalars { template void AddBlobFileIfNeeded(VersionStorageInfo* vstorage, Meta&& meta, uint64_t blob_file_number, + uint64_t oldest_linked_blob_file, std::vector* dropped_blob_files) const { assert(vstorage); assert(meta); assert(dropped_blob_files); + if (retired_blob_files_.count(blob_file_number) != 0) { + return; + } const auto& linked_ssts = meta->GetLinkedSsts(); + // Legacy rule: a normal blob file below the oldest linked blob file + // (`oldest_linked_blob_file`) has no reader left, so it is not saved. The + // rule does not apply to an indirect origin. After it is relocated, an SST + // can still use its file number as a generation number without linking it + // through oldest_blob_file_number. An indirect origin leaves the Version + // only through a blob file retirement record. + if (!meta->IsIndirectOrigin() && + (oldest_linked_blob_file == kInvalidBlobFileNumber || + blob_file_number < oldest_linked_blob_file)) { + return; + } if (track_found_and_missing_files_) { if (missing_blob_files_.find(blob_file_number) != missing_blob_files_.end()) { - // A blob file without linked SST files or live blobs was deleted as - // obsolete, so no SST file references it. + // A normal blob file without linked SST files or live blobs was deleted + // as obsolete, so no SST file references it. When missing blob files + // are recorded as dropped, a missing indirect origin is always + // recorded, even when all its values count as garbage. This rule is + // defensive. Until a blob file retirement record removes an indirect + // origin, only unflattened references read values from it. So when all + // its values count as garbage, no SST reads from it, unless the garbage + // count is wrong. When the garbage count is right, no read sees it as + // dropped. if (record_missing_blob_files_as_dropped_ && - (!linked_ssts.empty() || + (meta->IsIndirectOrigin() || !linked_ssts.empty() || meta->GetGarbageBlobCount() < meta->GetTotalBlobCount())) { dropped_blob_files->push_back(blob_file_number); } @@ -1635,7 +2504,13 @@ class VersionBuilder::Rep : public MutableScalars { } } - if (linked_ssts.empty() && + // Legacy rule: a normal blob file that no SST links and whose values all + // count as garbage is dropped from the Version. This rule does not apply + // to an indirect origin. This is a change from base RocksDB behavior. An + // indirect origin stays in the Version, even when all its values count as + // garbage, until a blob file retirement record removes it. So the life of + // an indirect origin never depends on its garbage count. + if (!meta->IsIndirectOrigin() && linked_ssts.empty() && meta->GetGarbageBlobCount() >= meta->GetTotalBlobCount()) { return; } @@ -1656,31 +2531,47 @@ class VersionBuilder::Rep : public MutableScalars { vstorage->ReserveBlob(base_vstorage_->GetBlobFiles().size() + mutable_blob_file_metas_.size()); - if (min_oldest_blob_file_num == kInvalidBlobFileNumber) { + // The oldest linked blob file is the smallest oldest_blob_file_number of + // the saved SSTs. It comes from the SSTs, not from the blob files that list + // linked SSTs. In a fenced CF an SST can have, as its oldest blob file, an + // indirect origin that blob file retirement removed. That SST links no blob + // file in this Version, but it can still reference newer normal blob files. + const uint64_t oldest_linked_blob_file = min_oldest_blob_file_num; + + // In a legacy CF (format version 0), an invalid oldest linked blob file + // means that no SST references a blob file, so no blob file is saved. A + // fenced CF cannot take this shortcut: its indirect origins leave only + // through blob file retirement records, whatever the oldest linked blob + // file is. + if (blob_indirection_format_version_ == 0 && + oldest_linked_blob_file == kInvalidBlobFileNumber) { return; } auto process_base = - [this, vstorage, dropped_blob_files]( + [this, vstorage, oldest_linked_blob_file, dropped_blob_files]( const std::shared_ptr& base_meta) { assert(base_meta); AddBlobFileIfNeeded(vstorage, base_meta, base_meta->GetBlobFileNumber(), - dropped_blob_files); + oldest_linked_blob_file, dropped_blob_files); return true; }; - auto process_mutable = [this, vstorage, dropped_blob_files]( - const MutableBlobFileMetaData& mutable_meta) { - AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta), - mutable_meta.GetBlobFileNumber(), dropped_blob_files); + auto process_mutable = + [this, vstorage, oldest_linked_blob_file, + dropped_blob_files](const MutableBlobFileMetaData& mutable_meta) { + AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta), + mutable_meta.GetBlobFileNumber(), + oldest_linked_blob_file, dropped_blob_files); - return true; - }; + return true; + }; - auto process_both = [this, vstorage, dropped_blob_files]( + auto process_both = [this, vstorage, oldest_linked_blob_file, + dropped_blob_files]( const std::shared_ptr& base_meta, const MutableBlobFileMetaData& mutable_meta) { assert(base_meta); @@ -1694,19 +2585,22 @@ class VersionBuilder::Rep : public MutableScalars { assert(base_meta->GetLinkedSsts() == mutable_meta.GetLinkedSsts()); AddBlobFileIfNeeded(vstorage, base_meta, base_meta->GetBlobFileNumber(), - dropped_blob_files); + oldest_linked_blob_file, dropped_blob_files); return true; } AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta), - mutable_meta.GetBlobFileNumber(), dropped_blob_files); + mutable_meta.GetBlobFileNumber(), + oldest_linked_blob_file, dropped_blob_files); return true; }; - MergeBlobFileMetas(min_oldest_blob_file_num, process_base, process_mutable, - process_both); + MergeBlobFileMetas(blob_indirection_format_version_ == 0 + ? oldest_linked_blob_file + : kInvalidBlobFileNumber, + process_base, process_mutable, process_both); } // Save into *vstorage the dropped blob files of the base Version and of the @@ -1786,6 +2680,7 @@ class VersionBuilder::Rep : public MutableScalars { bool ContainsCompleteVersion() const { assert(track_found_and_missing_files_); return l0_missing_files_.empty() && non_l0_missing_files_.empty() && + missing_blob_relocation_files_.empty() && (record_missing_blob_files_as_dropped_ || missing_blob_files_high_ == kInvalidBlobFileNumber || missing_blob_files_high_ < GetMinOldestBlobFileNumber()); @@ -1799,10 +2694,33 @@ class VersionBuilder::Rep : public MutableScalars { record_missing_blob_files_as_dropped_ = true; } + std::shared_ptr ReleaseBlobRelocationChanges() { + if (blob_relocation_file_changes_.empty() && + blob_relocation_route_changes_.empty()) { + return nullptr; + } + struct Edits { + decltype(blob_relocation_file_changes_) files; + decltype(blob_relocation_route_changes_) routes; + decltype(blob_relocation_target_count_deltas_) target_counts; + }; + auto edits = std::make_shared(); + edits->files.swap(blob_relocation_file_changes_); + edits->routes.swap(blob_relocation_route_changes_); + edits->target_counts.swap(blob_relocation_target_count_deltas_); + return edits; + } + bool HasMissingFiles() const { assert(track_found_and_missing_files_); return !l0_missing_files_.empty() || !non_l0_missing_files_.empty() || - missing_blob_files_high_ != kInvalidBlobFileNumber; + missing_blob_files_high_ != kInvalidBlobFileNumber || + !missing_blob_relocation_files_.empty(); + } + + bool HasMissingBlobRelocationFiles() const { + assert(track_found_and_missing_files_); + return !missing_blob_relocation_files_.empty(); } std::vector GetAndClearIntermediateFiles() { @@ -1935,7 +2853,8 @@ class VersionBuilder::Rep : public MutableScalars { } bool OnlyMissingL0Suffix() const { - if (!non_l0_missing_files_.empty()) { + if (!non_l0_missing_files_.empty() || + !missing_blob_relocation_files_.empty()) { return false; } assert(!(l0_missing_files_.empty() && missing_blob_files_.empty())); @@ -2024,8 +2943,7 @@ class VersionBuilder::Rep : public MutableScalars { return true; } - // Save the current state in *vstorage. - Status SaveTo(VersionStorageInfo* vstorage) const { + Status SaveTableFilesTo(VersionStorageInfo* vstorage) const { assert(!track_found_and_missing_files_ || valid_version_available_); Status s; @@ -2042,18 +2960,172 @@ class VersionBuilder::Rep : public MutableScalars { return s; } - const uint64_t min_oldest_blob_file_num = SaveSSTFilesTo(vstorage); + saved_min_oldest_blob_file_num_ = SaveSSTFilesTo(vstorage); +#ifndef NDEBUG + table_files_saved_ = true; +#endif // NDEBUG + SaveCompactCursorsTo(vstorage); + + return Status::OK(); + } - std::vector dropped_blob_files; - SaveBlobFilesTo(vstorage, min_oldest_blob_file_num, &dropped_blob_files); + Status SaveBlobFilesAndCheckTo(VersionStorageInfo* vstorage) { + assert(!track_found_and_missing_files_ || valid_version_available_); + if (vstorage == nullptr) { + return Status::InvalidArgument("VersionStorageInfo is null"); + } - SaveDroppedBlobFilesTo(vstorage, min_oldest_blob_file_num, +#ifndef NDEBUG + assert(table_files_saved_); + table_files_saved_ = false; +#endif // NDEBUG + std::vector dropped_blob_files; + SaveBlobFilesTo(vstorage, saved_min_oldest_blob_file_num_, + &dropped_blob_files); + SaveDroppedBlobFilesTo(vstorage, saved_min_oldest_blob_file_num_, std::move(dropped_blob_files)); + if (blob_indirection_format_version_ != 0) { + // SaveBlobFilesTo() can drop a normal blob file just because the oldest + // linked blob file moved, even when no edit names the file. So compare + // the base and the new blob file lists here, outside the DB mutex, and + // record every base file that is gone. The census delta removes the same + // files from its candidate heap, so the heap and the Version agree on the + // live blob files. + const VersionStorageInfo::BlobFiles& next_files = + vstorage->GetBlobFiles(); + size_t next = 0; + for (const std::shared_ptr& file : + base_vstorage_->GetBlobFiles()) { + const uint64_t file_number = file->GetBlobFileNumber(); + while (next < next_files.size() && + next_files[next]->GetBlobFileNumber() < file_number) { + ++next; + } + if (next == next_files.size() || + next_files[next]->GetBlobFileNumber() != file_number) { + vstorage->RecordRemovedBlobFile(file_number); + } + } + } + vstorage->SetBlobIndirectionFormatVersion(blob_indirection_format_version_); + + Status base_state_status = base_vstorage_->ValidateBlobRelocationState(); + if (!base_state_status.ok()) { + return base_state_status; + } + // Start from the base Version's relocation state, which is immutable and + // shared. A Version keeps sharing all of it only when it changes no + // relocation file, no route and (with the fence set) no blob file. The + // first change copies the state, and the file table copies only the shards + // it changes (BlobFileTable is copy-on-write). At runtime (LogAndApply, + // ManifestTailer) this runs without the DB mutex. + vstorage->ShareBlobRelocationState(*base_vstorage_); + if (blob_indirection_format_version_ != 0) { + // The file table also lists the live blob files. The first fenced Version + // adds all of them. Otherwise, merge the sorted blob file vectors of the + // base and new Version, and update only the files that were added, + // removed or given new metadata. + const bool first_fenced = !base_vstorage_->HasBlobIndirectionFormat(); + std::vector changed_blob_files; + if (!first_fenced) { + const VersionStorageInfo::BlobFiles& before = + base_vstorage_->GetBlobFiles(); + const VersionStorageInfo::BlobFiles& after = vstorage->GetBlobFiles(); + size_t b = 0; + size_t a = 0; + while (b < before.size() || a < after.size()) { + if (a == after.size() || + (b < before.size() && before[b]->GetBlobFileNumber() < + after[a]->GetBlobFileNumber())) { + changed_blob_files.push_back(before[b++]->GetBlobFileNumber()); + } else if (b == before.size() || after[a]->GetBlobFileNumber() < + before[b]->GetBlobFileNumber()) { + changed_blob_files.push_back(after[a++]->GetBlobFileNumber()); + } else { + if (before[b] != after[a]) { + changed_blob_files.push_back(after[a]->GetBlobFileNumber()); + } + ++a; + ++b; + } + } + } + vstorage->SyncBlobFileTable(changed_blob_files, first_fenced, + file_metadata_cache_res_mgr_); + } + // Apply the changes in this order: + // 1. Add relocation files, so a new route's target is in the table. + // 2. Delete routes while their targets are still in the table, so the route + // count of each target is updated. + // 3. Delete relocation files. DeleteBlobRelocationFile() finds the file + // through its table entry, so it must run before step 4 replaces that + // entry with a route of the same number. + // 4. Set routes. A deleted relocation file's number can become a route's + // old file number in the same edit (indirect blob GC routes a + // collected relocation file to its output). + using FileEdits = decltype(blob_relocation_file_changes_); + using RouteEdits = decltype(blob_relocation_route_changes_); + for (const FileEdits::value_type& file_edit : + blob_relocation_file_changes_) { + if (file_edit.second != nullptr) { + vstorage->AddBlobRelocationFile(file_edit.second); + } + } + for (const RouteEdits::value_type& route_edit : + blob_relocation_route_changes_) { + if (!route_edit.second.has_value()) { + vstorage->DeleteBlobRelocationRoute(route_edit.first); + } + } + for (const FileEdits::value_type& file_edit : + blob_relocation_file_changes_) { + if (file_edit.second == nullptr) { + vstorage->DeleteBlobRelocationFile(file_edit.first); + } + } + for (const RouteEdits::value_type& route_edit : + blob_relocation_route_changes_) { + if (route_edit.second.has_value()) { + vstorage->AddBlobRelocationRoute(*route_edit.second); + } + } + // Apply the buffered file table changes above in one step. + vstorage->SealBlobRelocationState(); + + // Apply() already checked, in all builds, every changed file and route + // against the final state. The base state was validated above, so the + // unchanged entries are valid too. Debug builds still run the full check as + // a test. Release builds mark the new state as validated, so + // ValidateBlobRelocationState() checks only the format fence and the + // byte-total overflow flag, and does not scan every entry of each new + // Version (for example, on each flush). + Status status; +#ifndef NDEBUG + status = vstorage->ValidateBlobRelocationState(); +#else + vstorage->MarkBlobRelocationChangesValidated(); + status = vstorage->ValidateBlobRelocationState(); +#endif + if (!status.ok()) { + return status; + } + if (!blob_relocation_file_changes_.empty() || + !blob_relocation_route_changes_.empty() || + blob_indirection_format_version_ != + base_vstorage_->GetBlobIndirectionFormatVersion()) { + vstorage->ComputeBlobRelocationStateFingerprint(); + } + return CheckConsistency(vstorage); + } - SaveCompactCursorsTo(vstorage); + // Save the current state in *vstorage. + Status SaveTo(VersionStorageInfo* vstorage) { + Status s = SaveTableFilesTo(vstorage); + if (!s.ok()) { + return s; + } - s = CheckConsistency(vstorage); - return s; + return SaveBlobFilesAndCheckTo(vstorage); } Status LoadTableHandlers(InternalStats* internal_stats, int max_threads, @@ -2150,10 +3222,18 @@ Status VersionBuilder::Apply(const VersionEdit* edit) { return rep_->Apply(edit); } -Status VersionBuilder::SaveTo(VersionStorageInfo* vstorage) const { +Status VersionBuilder::SaveTo(VersionStorageInfo* vstorage) { return rep_->SaveTo(vstorage); } +Status VersionBuilder::SaveTableFilesTo(VersionStorageInfo* vstorage) const { + return rep_->SaveTableFilesTo(vstorage); +} + +Status VersionBuilder::SaveBlobFilesAndCheckTo(VersionStorageInfo* vstorage) { + return rep_->SaveBlobFilesAndCheckTo(vstorage); +} + Status VersionBuilder::LoadTableHandlers( InternalStats* internal_stats, int max_threads, bool prefetch_index_and_filter_in_cache, bool is_initial_load, @@ -2181,6 +3261,14 @@ void VersionBuilder::RecordMissingBlobFilesAsDropped() { rep_->RecordMissingBlobFilesAsDropped(); } +std::shared_ptr VersionBuilder::ReleaseBlobRelocationChanges() { + return rep_->ReleaseBlobRelocationChanges(); +} + +bool VersionBuilder::HasMissingBlobRelocationFiles() const { + return rep_->HasMissingBlobRelocationFiles(); +} + std::vector VersionBuilder::GetAndClearIntermediateFiles() { return rep_->GetAndClearIntermediateFiles(); } diff --git a/db/version_builder.h b/db/version_builder.h index 6b1db7eef9f7..08468747258d 100644 --- a/db/version_builder.h +++ b/db/version_builder.h @@ -51,7 +51,39 @@ class VersionBuilder { Status Apply(const VersionEdit* edit); // Save the current Version to the provided `vstorage`. - Status SaveTo(VersionStorageInfo* vstorage) const; + Status SaveTo(VersionStorageInfo* vstorage); + + // Saving a Version can also be done in two steps, so that step 2 (the blob + // files and the consistency check) can run with the DB mutex released (see + // VersionSet::ProcessManifestWrites()). Step 1 saves the SST metadata and + // the compact cursors. The caller must hold the DB mutex, because this step + // increments FileMetaData reference counts, which are not atomic. + Status SaveTableFilesTo(VersionStorageInfo* vstorage) const; + + // Step 2, after SaveTableFilesTo(), does two things: it saves the blob file + // metadata and the list of dropped blob files (see + // VersionStorageInfo::GetDroppedBlobFiles()), and then it checks the + // consistency of the whole new Version, SST files and blob files. When + // force_consistency_checks is on (the default) or in debug builds, the check + // walks every SST. We expect, without a measurement, that this check is the + // main cost of this step. This step is different from + // VersionStorageInfo::PrepareForVersionAppend(), which builds derived data + // such as file indexes. VersionSet::AppendVersion() marks the + // VersionStorageInfo as finalized later. + // + // Step 2 does not need the DB mutex, because no other thread changes the + // state it reads. The builder and the new Version are not shared yet. The + // parts of the base Version that this step reads (its blob file list, and + // the file numbers, keys, sequence numbers, epoch numbers and oldest blob + // file numbers of its SSTs) do not change while the base Version is + // referenced. So the caller must keep the base Version of the builder + // referenced until this returns. + // + // The step copies shared_ptr (the reference count is + // atomic) and creates new BlobFileMetaData objects for the blob files that + // the edits add or change. It does not touch FileMetaData reference + // counts. + Status SaveBlobFilesAndCheckTo(VersionStorageInfo* vstorage); // Load table handlers for newly added files in the builder. This does not // load any files in the base storage. @@ -88,6 +120,19 @@ class VersionBuilder { // are true, and no edit has been applied yet. void RecordMissingBlobFilesAsDropped(); + // Moves out the staged relocation file and route changes, which can be large + // after a long replay, so that they can be freed without the DB mutex while + // the builder itself is destroyed under it (see + // VersionSet::FreeBlobRelocationState()). Returns null when there is nothing + // to move. The builder must not be used afterwards. + std::shared_ptr ReleaseBlobRelocationChanges(); + + // True if a relocation file that the current replay state still uses is + // missing on disk or fails verification. A file that an old edit added and a + // later edit deleted does not count. Requires tracking of found and missing + // files. + bool HasMissingBlobRelocationFiles() const; + // When applying a sequence of VersionEdits, intermediate files are the ones // that are added and then deleted. Returns the tracked files and clears them // so that subsequent VersionEdits start with a clean state. @@ -127,6 +172,7 @@ class BaseReferencedVersionBuilder { bool allow_incomplete_valid_version = false); ~BaseReferencedVersionBuilder(); VersionBuilder* version_builder() const { return version_builder_.get(); } + Version* base_version() const { return version_; } private: std::unique_ptr version_builder_; diff --git a/db/version_builder_test.cc b/db/version_builder_test.cc index 0e8fefb4def8..9e27de958b85 100644 --- a/db/version_builder_test.cc +++ b/db/version_builder_test.cc @@ -5,12 +5,18 @@ #include "db/version_builder.h" +#include #include +#include #include +#include +#include #include #include #include +#include "cache/cache_reservation_manager.h" +#include "db/blob/blob_reference_metadata.h" #include "db/db_test_util.h" #include "db/version_edit.h" #include "db/version_set.h" @@ -19,6 +25,7 @@ #include "table/unique_id_impl.h" #include "test_util/testharness.h" #include "test_util/testutil.h" +#include "util/coding.h" #include "util/defer.h" #include "util/string_util.h" #include "utilities/fault_injection_fs.h" @@ -110,6 +117,59 @@ class VersionBuilderTest : public testing::Test { vstorage_.AddBlobFile(std::move(meta)); } + // Builds the addition record of relocation file `file_number`. The file holds + // one value from each origin in `origin_file_numbers`. Its served files (the + // old file numbers that may be routed to it) are those origins plus + // `extra_generations`. + Status MakeRelocationAddition( + uint64_t file_number, const std::vector& origin_file_numbers, + BlobRelocationFileAddition* addition, + const std::vector& extra_generations = {}) { + if (addition == nullptr) { + return Status::InvalidArgument("Null relocation addition output"); + } + BlobRelocationDirectoryBuilder builder; + for (uint64_t origin_file_number : origin_file_numbers) { + Status status = builder.Add(origin_file_number, + /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression); + if (status.ok()) { + status = builder.AddServedFile(origin_file_number); + } + for (uint64_t generation : extra_generations) { + if (status.ok()) { + status = builder.AddServedFile(generation); + } + } + if (!status.ok()) { + return status; + } + } + BlobRelocationDirectory directory; + Status status = builder.Finish(&directory); + if (!status.ok()) { + return status; + } + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + *addition = BlobRelocationFileAddition( + file_number, /*file_size=*/4096, "crc32c", "checksum", + std::move(encoded_directory), /*tail_size=*/1); + return addition->Validate(); + } + + Status AddRelocationToBase(const BlobRelocationFileAddition& addition) { + Status status = addition.Validate(); + if (!status.ok()) { + return status; + } + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + vstorage_.AddBlobRelocationFile( + BlobRelocationFileMetaData::Create(addition)); + return Status::OK(); + } + void AddDummyFile(uint64_t table_file_number, uint64_t blob_file_number, uint64_t epoch_number) { constexpr int level = 0; @@ -150,6 +210,18 @@ class VersionBuilderTest : public testing::Test { 0, 0, /* user_defined_timestamps_persisted */ true); } + void AddBlobReferenceDescriptorToLastFile(VersionEdit* edit) { + assert(edit != nullptr); + assert(!edit->GetMutableNewFiles().empty()); + BlobReferenceMetadataDescriptor descriptor; + descriptor.column_family_id = edit->GetColumnFamily(); + descriptor.sst_file_number = + edit->GetMutableNewFiles().back().second.fd.GetNumber(); + descriptor.index_handle = BlockHandle(/*offset=*/10, /*size=*/20); + descriptor.EncodeTo( + &edit->GetMutableNewFiles().back().second.blob_reference_descriptor); + } + void UpdateVersionStorageInfo(VersionStorageInfo* vstorage) { assert(vstorage); @@ -510,6 +582,64 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) { UnrefFilesInVersion(&new_vstorage); } +TEST_F(VersionBuilderTest, ApplyChecksBaseConsistencyOnce) { + // The base Version does not change during a builder's life. So once the check + // passes, the builder does not check it again, even when it applies many + // edits. + Add(2, 1U, "150", "200", 100U); + UpdateVersionStorageInfo(); + std::atomic base_checks{0}; + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::Apply:CheckBaseConsistency", + [&](void*) { ++base_checks; }); + SyncPoint::GetInstance()->EnableProcessing(); + + EnvOptions env_options; + VersionBuilder version_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + for (uint64_t i = 0; i < 5; ++i) { + VersionEdit edit; + const std::string smallest = std::to_string(300 + 100 * i); + const std::string largest = std::to_string(350 + 100 * i); + edit.AddFile(2, 10 + i, 0, 100U, GetInternalKey(smallest.c_str()), + GetInternalKey(largest.c_str()), 200, 200, false, + Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncestorTime, kUnknownFileCreationTime, + kUnknownEpochNumber, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, + /* user_defined_timestamps_persisted */ true); + ASSERT_OK(version_builder.Apply(&edit)); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + EXPECT_EQ(base_checks.load(), 1); + + VersionStorageInfo new_vstorage( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, nullptr, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(version_builder.SaveTo(&new_vstorage)); + UpdateVersionStorageInfo(&new_vstorage); + EXPECT_EQ(new_vstorage.NumLevelFiles(2), 6); + UnrefFilesInVersion(&new_vstorage); +} + +TEST_F(VersionBuilderTest, ApplyReportsInconsistentBaseOnEveryApply) { + // Overlapping L1 files make the base inconsistent. Every Apply reports it, + // because a failed check is not remembered as passed. + Add(1, 1U, "100", "300", 100U); + Add(1, 2U, "200", "400", 100U); + EnvOptions env_options; + VersionBuilder version_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + VersionEdit edit; + ASSERT_TRUE(version_builder.Apply(&edit).IsCorruption()); + ASSERT_TRUE(version_builder.Apply(&edit).IsCorruption()); +} + TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) { UpdateVersionStorageInfo(); @@ -911,6 +1041,1483 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) { UnrefFilesInVersion(&new_vstorage); } +TEST_F(VersionBuilderTest, IndirectOriginBlobFileRequiresFormatFence) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + VersionEdit addition; + addition.AddBlobFile(/*blob_file_number=*/100, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + AddDummyFileToEdit(&addition, /*table_file_number=*/101, + /*blob_file_number=*/100, /*epoch_number=*/1); + + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_TRUE(builder.Apply(&addition).IsCorruption()); + } + + addition.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + AddBlobReferenceDescriptorToLastFile(&addition); + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_OK(builder.Apply(&addition)); + + VersionStorageInfo output( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&output)); + const std::shared_ptr meta = + output.GetBlobFileMetaData(/*blob_file_number=*/100); + ASSERT_NE(meta, nullptr); + ASSERT_TRUE(meta->IsIndirectOrigin()); + ASSERT_OK(output.ValidateBlobRelocationState()); + UnrefFilesInVersion(&output); +} + +TEST_F(VersionBuilderTest, PhysicalFileLossRejectsNormalBlobFile) { + // A loss record names only a relocation file. A normal or indirect-origin + // blob file that is missing uses the base dropped-file record instead + // (best_efforts_blob_db_repair), so a loss record that names one is + // Corruption: in the edit that adds the file, and in a later edit. A later + // loss record that names a relocation file (file 200 in this test) is + // accepted. + UpdateVersionStorageInfo(); + EnvOptions env_options; + const auto make_edit = [&](bool is_indirect_origin) { + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobFile(/*blob_file_number=*/100, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}, is_indirect_origin); + AddDummyFileToEdit( + &edit, /*table_file_number=*/101, + /*blob_file_number=*/is_indirect_origin ? kInvalidBlobFileNumber : 100, + /*epoch_number=*/1); + if (is_indirect_origin) { + AddBlobReferenceDescriptorToLastFile(&edit); + } else { + edit.GetMutableNewFiles().back().second.blob_reference_legacy_metadata = + true; + } + return edit; + }; + for (const bool is_indirect_origin : {true, false}) { + SCOPED_TRACE("is_indirect_origin=" + std::to_string(is_indirect_origin)); + VersionEdit lost = make_edit(is_indirect_origin); + lost.MarkBlobPhysicalFileLost(100); + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + const Status status = builder.Apply(&lost); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("names no live target"), + std::string::npos); + } + + VersionEdit added = make_edit(/*is_indirect_origin=*/true); + BlobRelocationFileAddition relocation; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{90}, &relocation)); + added.AddBlobRelocationFile(relocation); + const auto make_later = [](uint64_t lost_file_number) { + VersionEdit later; + later.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + later.MarkBlobPhysicalFileLost(lost_file_number); + return later; + }; + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_OK(builder.Apply(&added)); + VersionEdit later_origin = make_later(100); + const Status status = builder.Apply(&later_origin); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("names no live target"), std::string::npos); + VersionEdit later_relocation = make_later(200); + ASSERT_OK(builder.Apply(&later_relocation)); +} + +TEST_F(VersionBuilderTest, ImplicitNormalBlobPruningReportsPhysicalDeletion) { + constexpr uint64_t kBlob = 100; + constexpr uint64_t kSst = 101; + AddDummyFile(kSst, kBlob, /*epoch_number=*/1); + vstorage_.LevelFiles(0).front()->blob_reference_legacy_metadata = true; + AddBlob(kBlob, /*total_blob_count=*/1, /*total_blob_bytes=*/100, + /*checksum_method=*/{}, /*checksum_value=*/{}, {kSst}, + /*garbage_blob_count=*/0, /*garbage_blob_bytes=*/0); + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + // A fenced Version's file table holds its blob files. + vstorage_.SyncBlobFileTable({}, /*rebuild=*/true, /*cache_res_mgr=*/nullptr); + UpdateVersionStorageInfo(); + VersionEdit edit; + edit.DeleteFile(0, kSst); + // No BlobFileGarbage or blob file retirement record accompanies this SST + // deletion. + ASSERT_TRUE(edit.GetBlobFileGarbages().empty()); + EnvOptions env_options; + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_OK(builder.Apply(&edit)); + VersionStorageInfo output( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&output)); + ASSERT_TRUE(output.GetBlobFiles().empty()); + ASSERT_EQ(output.RemovedBlobFiles(), std::vector{kBlob}); + UnrefFilesInVersion(&output); +} + +TEST_F(VersionBuilderTest, FormatFenceRequiresExactSstMetadata) { + AddDummyFile(/*table_file_number=*/200, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/2); + UpdateVersionStorageInfo(); + EnvOptions env_options; + + VersionEdit missing; + missing.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + AddDummyFileToEdit(&missing, /*table_file_number=*/101, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/1); + VersionBuilder missing_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + ASSERT_TRUE(missing_builder.Apply(&missing).IsCorruption()); + + missing.GetMutableNewFiles().back().second.blob_reference_legacy_metadata = + true; + VersionBuilder legacy_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + ASSERT_OK(legacy_builder.Apply(&missing)); + + missing.GetMutableNewFiles().back().second.blob_reference_legacy_metadata = + false; + AddBlobReferenceDescriptorToLastFile(&missing); + VersionBuilder complete_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + ASSERT_OK(complete_builder.Apply(&missing)); + + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + VersionEdit legacy_move; + legacy_move.DeleteFile(/*level=*/0, /*file_number=*/200); + AddDummyFileToEdit(&legacy_move, /*table_file_number=*/200, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/2); + VersionBuilder move_builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_OK(move_builder.Apply(&legacy_move)); +} + +TEST_F(VersionBuilderTest, ApplyBlobRelocationStateAtomically) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + edit.SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/200)); + ASSERT_OK(builder.Apply(&edit)); + // Applying the same edit again is accepted and changes nothing, as in a + // MANIFEST replay. + ASSERT_OK(builder.Apply(&edit)); + + VersionStorageInfo output( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&output)); + ASSERT_EQ(output.GetBlobIndirectionFormatVersion(), + kBlobIndirectionManifestFormatVersion); + ASSERT_NE(output.GetBlobRelocationFileMetaData(200), nullptr); + ASSERT_EQ(output.GetBlobRelocationRouteTarget(/*source_file_number=*/100), + 200U); + ASSERT_EQ(output.GetBlobRelocationRouteTargetCount(200), 1U); + ASSERT_OK(output.ValidateBlobRelocationState()); +} + +TEST_F(VersionBuilderTest, RetiredAnchorRootDoesNotDropNormalBlobFiles) { + // SST #101 names indirect origin #100 as its oldest blob file and also holds + // values in normal blob file #150, which no SST links. Normal file #160 + // becomes fully garbage. The blob file retirement of #100 leaves no blob file + // with linked SSTs. Both the Version that installs that blob file retirement + // and a later Version must keep #150 and drop #160. + constexpr uint64_t kSourceFileNumber = 100; + constexpr uint64_t kTableFileNumber = 101; + constexpr uint64_t kLiveNormalFileNumber = 150; + constexpr uint64_t kGarbageNormalFileNumber = 160; + constexpr uint64_t kRelocationFileNumber = 200; + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + AddDummyFile(kTableFileNumber, kSourceFileNumber, /*epoch_number=*/1); + auto shared = SharedBlobFileMetaData::Create( + kSourceFileNumber, /*total_blob_count=*/2, + /*total_blob_bytes=*/100, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + vstorage_.AddBlobFile(BlobFileMetaData::Create( + std::move(shared), BlobFileMetaData::LinkedSsts{kTableFileNumber}, + /*garbage_blob_count=*/1, /*garbage_blob_bytes=*/40)); + AddBlob(kLiveNormalFileNumber, /*total_blob_count=*/10, + /*total_blob_bytes=*/1000, /*checksum_method=*/{}, + /*checksum_value=*/{}, BlobFileMetaData::LinkedSsts{}, + /*garbage_blob_count=*/0, /*garbage_blob_bytes=*/0); + AddBlob(kGarbageNormalFileNumber, /*total_blob_count=*/5, + /*total_blob_bytes=*/500, /*checksum_method=*/{}, + /*checksum_value=*/{}, BlobFileMetaData::LinkedSsts{}, + /*garbage_blob_count=*/4, /*garbage_blob_bytes=*/400); + // A fenced Version's file table holds its blob files. + vstorage_.SyncBlobFileTable({}, /*rebuild=*/true, /*cache_res_mgr=*/nullptr); + UpdateVersionStorageInfo(); + + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(kRelocationFileNumber, {kSourceFileNumber}, + &addition)); + + VersionEdit retire; + retire.AddBlobRelocationFile(addition); + retire.SetBlobRelocationRoute( + BlobRelocationRoute(kSourceFileNumber, kRelocationFileNumber)); + retire.RetireBlobFile(kSourceFileNumber); + retire.AddBlobFileGarbage(kGarbageNormalFileNumber, + /*garbage_blob_count=*/1, + /*garbage_blob_bytes=*/100); + + const auto make_output = [&](VersionStorageInfo* base) { + return std::make_unique( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, base, + /*force_consistency_checks=*/false, + EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + }; + + EnvOptions env_options; + std::unique_ptr retired = make_output(&vstorage_); + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_OK(builder.Apply(&retire)); + ASSERT_OK(builder.SaveTo(retired.get())); + } + UpdateVersionStorageInfo(retired.get()); + + // Any later Version is built from a base where nothing links a blob file. + std::unique_ptr next = make_output(retired.get()); + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + retired.get(), /*version_set=*/nullptr); + VersionEdit unrelated; + ASSERT_OK(builder.Apply(&unrelated)); + ASSERT_OK(builder.SaveTo(next.get())); + } + UpdateVersionStorageInfo(next.get()); + + for (VersionStorageInfo* storage : {retired.get(), next.get()}) { + ASSERT_EQ(storage->GetBlobFileMetaData(kSourceFileNumber), nullptr); + ASSERT_NE(storage->GetBlobFileMetaData(kLiveNormalFileNumber), nullptr); + ASSERT_EQ(storage->GetBlobFileMetaData(kGarbageNormalFileNumber), nullptr); + } + UnrefFilesInVersion(next.get()); + UnrefFilesInVersion(retired.get()); +} + +TEST_F(VersionBuilderTest, + IndirectOriginApproximateGarbageDoesNotAuthorizeRetirement) { + constexpr uint64_t kSourceFileNumber = 100; + constexpr uint64_t kTableFileNumber = 101; + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + AddDummyFile(kTableFileNumber, kSourceFileNumber, /*epoch_number=*/1); + auto shared = SharedBlobFileMetaData::Create( + kSourceFileNumber, /*total_blob_count=*/1, + /*total_blob_bytes=*/100, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + vstorage_.AddBlobFile(BlobFileMetaData::Create( + std::move(shared), BlobFileMetaData::LinkedSsts{kTableFileNumber}, + /*garbage_blob_count=*/0, /*garbage_blob_bytes=*/0)); + // A fenced Version's file table holds its blob files. + vstorage_.SyncBlobFileTable({}, /*rebuild=*/true, /*cache_res_mgr=*/nullptr); + UpdateVersionStorageInfo(); + + EnvOptions env_options; + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit approximate; + approximate.AddBlobFileGarbage(kSourceFileNumber, + /*garbage_blob_count=*/1, + /*garbage_blob_bytes=*/100); + ASSERT_OK(builder.Apply(&approximate)); + + VersionStorageInfo output( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/true, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&output)); + const std::shared_ptr retained = + output.GetBlobFileMetaData(kSourceFileNumber); + ASSERT_NE(retained, nullptr); + EXPECT_TRUE(retained->IsIndirectOrigin()); + EXPECT_EQ(retained->GetGarbageBlobCount(), 1U); + EXPECT_EQ(retained->GetGarbageBlobBytes(), 100U); + UnrefFilesInVersion(&output); +} + +TEST_F(VersionBuilderTest, IndirectBlobGCSuppressionAdvancesCandidate) { + auto first_shared = SharedBlobFileMetaData::Create( + /*blob_file_number=*/100, /*total_blob_count=*/10, + /*total_blob_bytes=*/1000, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + auto second_shared = SharedBlobFileMetaData::Create( + /*blob_file_number=*/200, /*total_blob_count=*/10, + /*total_blob_bytes=*/1000, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + vstorage_.AddBlobFile(BlobFileMetaData::Create( + std::move(first_shared), BlobFileMetaData::LinkedSsts{}, + /*garbage_blob_count=*/8, /*garbage_blob_bytes=*/800)); + vstorage_.AddBlobFile(BlobFileMetaData::Create( + std::move(second_shared), BlobFileMetaData::LinkedSsts{}, + /*garbage_blob_count=*/6, /*garbage_blob_bytes=*/600)); + BlobRelocationFileAddition relocation_addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/300, + /*origin_file_numbers=*/{50}, + &relocation_addition)); + vstorage_.AddBlobRelocationFile( + BlobRelocationFileMetaData::Create(relocation_addition)); + vstorage_.AddBlobRelocationRoute( + BlobRelocationRoute(/*source_file_number=*/50, + /*physical_file_number=*/300)); + + vstorage_.SetBlobGcCandidates({100, 200, 300}); + EXPECT_EQ(vstorage_.BlobPhysicalFileForIndirectBlobGC(), 100U); + std::vector candidates; + vstorage_.GetBlobPhysicalFilesForIndirectBlobGC(/*max_files=*/3, &candidates); + EXPECT_EQ(candidates, (std::vector{100, 200, 300})); + EXPECT_EQ(vstorage_.GetBlobRelocationRouteTargetCount(300), 1U); + + // A job that tried some candidates and could not finish them defers them. + // DeferBlobFilesForIndirectBlobGC() moves them to the end of the list, so the + // next job looks at the other candidates first. + vstorage_.DeferBlobFilesForIndirectBlobGC({100, 200}); + vstorage_.GetBlobPhysicalFilesForIndirectBlobGC(/*max_files=*/3, &candidates); + EXPECT_EQ(candidates, (std::vector{300, 100, 200})); + vstorage_.SetBlobGcCandidates({100, 200, 300}); + + // Suppress a candidate that is not first in the list. + // SuppressBlobFileForIndirectBlobGC() removes it and shifts the later + // candidates forward, so the list never has holes. + vstorage_.SuppressBlobFileForIndirectBlobGC(200); + vstorage_.GetBlobPhysicalFilesForIndirectBlobGC(/*max_files=*/3, &candidates); + EXPECT_EQ(candidates, (std::vector{100, 300})); + EXPECT_EQ(vstorage_.BlobPhysicalFileForIndirectBlobGC(), 100U); + + vstorage_.SuppressBlobFileForIndirectBlobGC(100); + EXPECT_EQ(vstorage_.BlobPhysicalFileForIndirectBlobGC(), 300U); + + vstorage_.SuppressBlobFileForIndirectBlobGC(300); + EXPECT_EQ(vstorage_.BlobPhysicalFileForIndirectBlobGC(), + kInvalidBlobFileNumber); +} + +TEST_F(VersionBuilderTest, BlobRelocationMetadataPinsPhysicalFile) { + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + int destruction_count = 0; + std::shared_ptr metadata = + BlobRelocationFileMetaData::Create( + addition, [&destruction_count](BlobRelocationFileMetaData* value) { + ++destruction_count; + delete value; + }); + std::unique_ptr first = + std::make_unique( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, + /*src_vstorage=*/nullptr, /*force_consistency_checks=*/false, + EpochNumberRequirement::kMightMissing, /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + first->SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + first->AddBlobRelocationFile(metadata); + + std::unique_ptr second = + std::make_unique( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, + /*src_vstorage=*/nullptr, /*force_consistency_checks=*/false, + EpochNumberRequirement::kMightMissing, /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + second->SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + second->AddBlobRelocationFile( + first->GetBlobRelocationFileMetaData(/*file_number=*/200)); + + metadata.reset(); + first.reset(); + ASSERT_EQ(destruction_count, 0); + second.reset(); + ASSERT_EQ(destruction_count, 1); +} + +TEST_F(VersionBuilderTest, InvalidBlobRelocationEditHasNoPartialEffect) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + edit.SetBlobRelocationRoute( + BlobRelocationRoute(/*source_file_number=*/100, + /*physical_file_number=*/999)); + ASSERT_TRUE(builder.Apply(&edit).IsCorruption()); + + VersionStorageInfo output( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&output)); + ASSERT_FALSE(output.HasBlobIndirectionFormat()); + ASSERT_TRUE(output.GetBlobRelocationFiles().empty()); + ASSERT_TRUE(output.GetBlobRelocationRouteCount() == 0); +} + +TEST_F(VersionBuilderTest, BlobRelocationRejectsDuplicateRecords) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit duplicate; + duplicate.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + duplicate.AddBlobRelocationFile(addition); + duplicate.AddBlobRelocationFile(addition); + ASSERT_TRUE(builder.Apply(&duplicate).IsCorruption()); + } + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit duplicate_route; + duplicate_route.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + duplicate_route.AddBlobRelocationFile(addition); + const BlobRelocationRoute route(/*source_file_number=*/100, + /*physical_file_number=*/200); + duplicate_route.SetBlobRelocationRoute(route); + duplicate_route.SetBlobRelocationRoute(route); + ASSERT_TRUE(builder.Apply(&duplicate_route).IsCorruption()); + } +} + +TEST_F(VersionBuilderTest, BlobRelocationRejectsPhysicalFileNumberCollisions) { + UpdateVersionStorageInfo(); + + EnvOptions env_options; + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + AddDummyFileToEdit(&edit, /*table_file_number=*/200, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/1); + ASSERT_TRUE(builder.Apply(&edit).IsCorruption()); + } + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + edit.AddBlobFile(/*blob_file_number=*/200, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}); + ASSERT_TRUE(builder.Apply(&edit).IsCorruption()); + } + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit install; + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(addition); + ASSERT_OK(builder.Apply(&install)); + + VersionEdit deletion; + deletion.DeleteBlobRelocationFile(/*file_number=*/200); + ASSERT_OK(builder.Apply(&deletion)); + + VersionEdit blob_reuse; + blob_reuse.AddBlobFile(/*blob_file_number=*/200, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}); + ASSERT_TRUE(builder.Apply(&blob_reuse).IsCorruption()); + + VersionEdit table_reuse; + AddDummyFileToEdit(&table_reuse, /*table_file_number=*/200, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/1); + ASSERT_TRUE(builder.Apply(&table_reuse).IsCorruption()); + } +} + +TEST_F(VersionBuilderTest, BlobRelocationFileDeletionNeedsLiveFile) { + // As for an SST deletion, a relocation file deletion must name a live + // relocation file of the edit's column family. The base Version here has no + // relocation file 200, so a deletion of file 200 is Corruption. A deletion + // after file 200 is added is accepted. A repeated deletion of a file that + // the builder already deleted is accepted too, because MANIFEST replay and + // catch-up can repeat a record. + UpdateVersionStorageInfo(); + + EnvOptions env_options; + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit deletion; + deletion.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + deletion.DeleteBlobRelocationFile(/*file_number=*/200); + const Status status = builder.Apply(&deletion); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << status.ToString(); + } + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit install; + install.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + install.AddBlobRelocationFile(addition); + ASSERT_OK(builder.Apply(&install)); + VersionEdit deletion; + deletion.DeleteBlobRelocationFile(/*file_number=*/200); + ASSERT_OK(builder.Apply(&deletion)); + VersionEdit repeated_deletion; + repeated_deletion.DeleteBlobRelocationFile(/*file_number=*/200); + ASSERT_OK(builder.Apply(&repeated_deletion)); + } +} + +TEST_F(VersionBuilderTest, BlobRelocationFileNumberChecksPerColumnFamily) { + // VersionBuilder checks file numbers only against the state of the edit's + // own column family (see "File protection" in + // db/blob/blob_relocation_manifest.h). This test lists each check of a + // relocation file number, a new SST number, a new blob file number and a + // route target, with its error text. + // + // The example base Version is fenced and has SST 10, normal blob file 50, + // relocation file 200, and a route from old file 100 to file 200. + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + AddDummyFile(/*table_file_number=*/10, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/1); + AddBlob(/*blob_file_number=*/50, /*total_blob_count=*/1, + /*total_blob_bytes=*/100, /*checksum_method=*/{}, + /*checksum_value=*/{}, BlobFileMetaData::LinkedSsts{}, + /*garbage_blob_count=*/0, /*garbage_blob_bytes=*/0); + BlobRelocationFileAddition relocation; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &relocation)); + ASSERT_OK(AddRelocationToBase(relocation)); + vstorage_.AddBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/200)); + UpdateVersionStorageInfo(); + + const auto add_relocation = [&](uint64_t file_number, + const std::vector& origins, + VersionEdit* edit) { + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(file_number, origins, &addition)); + edit->AddBlobRelocationFile(addition); + }; + const auto add_blob_file = [](uint64_t file_number, VersionEdit* edit) { + edit->AddBlobFile(file_number, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}); + }; + struct Case { + const char* name; + std::function make_edit; + // An empty text means that the edit is accepted. + const char* error; + }; + const std::vector cases = { + {"relocation file 10 is an SST", + [&](VersionEdit* edit) { add_relocation(10, {5}, edit); }, + "Blob relocation file conflicts with a table file"}, + {"relocation file 50 is a normal blob file", + [&](VersionEdit* edit) { add_relocation(50, {5}, edit); }, + "Blob relocation file conflicts with a normal blob file"}, + {"relocation file 200 again with other contents", + [&](VersionEdit* edit) { add_relocation(200, {90}, edit); }, + "Conflicting blob relocation file addition"}, + {"relocation file 200 again with the same contents (as at replay)", + [&](VersionEdit* edit) { edit->AddBlobRelocationFile(relocation); }, ""}, + {"new SST 200 is a relocation file", + [&](VersionEdit* edit) { + AddDummyFileToEdit(edit, /*table_file_number=*/200, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/2); + // A new SST of a fenced column family needs exact blob reference + // metadata. + AddBlobReferenceDescriptorToLastFile(edit); + }, + "Table file conflicts with a blob relocation file"}, + {"new blob file 200 is a relocation file", + [&](VersionEdit* edit) { add_blob_file(200, edit); }, + "Normal blob file conflicts with a relocation file"}, + {"new blob file 100 has a route", + [&](VersionEdit* edit) { add_blob_file(100, edit); }, + "Normal blob file conflicts with a route"}, + {"route target 999 does not exist", + [&](VersionEdit* edit) { + edit->SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/150, /*physical_file_number=*/999)); + }, + "Invalid blob relocation route target"}, + {"route target 50 is a normal blob file", + [&](VersionEdit* edit) { + edit->SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/150, /*physical_file_number=*/50)); + }, + "Invalid blob relocation route target"}, + }; + EnvOptions env_options; + for (const Case& test_case : cases) { + SCOPED_TRACE(test_case.name); + VersionEdit edit; + test_case.make_edit(&edit); + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + const Status status = builder.Apply(&edit); + if (std::string(test_case.error).empty()) { + ASSERT_OK(status); + } else { + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find(test_case.error), std::string::npos) + << status.ToString(); + } + } +} + +TEST_F(VersionBuilderTest, BlobRelocationRejectsSetAndDeleteSameKey) { + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &addition)); + ASSERT_OK(AddRelocationToBase(addition)); + vstorage_.AddBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/200)); + UpdateVersionStorageInfo(); + + EnvOptions env_options; + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit edit; + edit.AddBlobRelocationFile(addition); + edit.DeleteBlobRelocationFile(/*file_number=*/200); + ASSERT_TRUE(builder.Apply(&edit).IsCorruption()); + } + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit edit; + edit.SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/200)); + edit.DeleteBlobRelocationRoute(/*source_file_number=*/100); + ASSERT_TRUE(builder.Apply(&edit).IsCorruption()); + } +} + +TEST_F(VersionBuilderTest, BlobRelocationRouteReplacementAndDeletion) { + BlobRelocationFileAddition first_addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, + &first_addition)); + ASSERT_OK(AddRelocationToBase(first_addition)); + vstorage_.AddBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/200)); + UpdateVersionStorageInfo(); + + EnvOptions env_options; + VersionBuilder replacement_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + BlobRelocationFileAddition second_addition; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/300, + /*origin_file_numbers=*/{100}, + &second_addition)); + VersionEdit replacement; + replacement.AddBlobRelocationFile(second_addition); + replacement.DeleteBlobRelocationFile(/*file_number=*/200); + replacement.SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/300)); + ASSERT_OK(replacement_builder.Apply(&replacement)); + + VersionStorageInfo replaced( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(replacement_builder.SaveTo(&replaced)); + ASSERT_EQ(replaced.GetBlobRelocationFileMetaData(200), nullptr); + ASSERT_NE(replaced.GetBlobRelocationFileMetaData(300), nullptr); + ASSERT_EQ(replaced.GetBlobRelocationRouteTargetCount(200), 0U); + ASSERT_EQ(replaced.GetBlobRelocationRouteTargetCount(300), 1U); + + VersionBuilder deletion_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &replaced, + /*version_set=*/nullptr); + VersionEdit deletion; + deletion.DeleteBlobRelocationRoute(/*source_file_number=*/100); + deletion.DeleteBlobRelocationFile(/*file_number=*/300); + ASSERT_OK(deletion_builder.Apply(&deletion)); + VersionStorageInfo deleted( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &replaced, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(deletion_builder.SaveTo(&deleted)); + ASSERT_TRUE(deleted.GetBlobRelocationFiles().empty()); + ASSERT_TRUE(deleted.GetBlobRelocationRouteCount() == 0); + // A saved Version keeps only live files and routes, not a record of + // deletions. As for an SST, a new builder on top of it rejects a deletion of + // relocation file 300, because file 300 is not live there. + { + VersionBuilder fresh_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &deleted, + /*version_set=*/nullptr); + VersionEdit file_deletion; + file_deletion.DeleteBlobRelocationFile(/*file_number=*/300); + const Status status = fresh_builder.Apply(&file_deletion); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << status.ToString(); + } + // A builder that applied the deletions remembers them. It accepts the same + // deletions again (as in a replay), and it rejects any reuse of file number + // 300 and a new route from 100. + VersionBuilder replay_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &replaced, + /*version_set=*/nullptr); + ASSERT_OK(replay_builder.Apply(&deletion)); + ASSERT_OK(replay_builder.Apply(&deletion)); + + VersionEdit relocation_reuse; + relocation_reuse.AddBlobRelocationFile(second_addition); + ASSERT_TRUE(replay_builder.Apply(&relocation_reuse).IsCorruption()); + + VersionEdit blob_reuse; + blob_reuse.AddBlobFile(/*blob_file_number=*/300, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}); + ASSERT_TRUE(replay_builder.Apply(&blob_reuse).IsCorruption()); + + VersionEdit table_reuse; + AddDummyFileToEdit(&table_reuse, /*table_file_number=*/300, + /*blob_file_number=*/kInvalidBlobFileNumber, + /*epoch_number=*/1); + ASSERT_TRUE(replay_builder.Apply(&table_reuse).IsCorruption()); + + VersionEdit route_reuse; + route_reuse.SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/100, /*physical_file_number=*/300)); + ASSERT_TRUE(replay_builder.Apply(&route_reuse).IsCorruption()); +} + +TEST_F(VersionBuilderTest, BlobRelocationUnchangedVersionSharesLiveState) { + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(200, {100}, &addition)); + ASSERT_OK(AddRelocationToBase(addition)); + for (uint64_t generation = 1000; generation < 2000; ++generation) { + vstorage_.AddBlobRelocationRoute(BlobRelocationRoute(generation, 200)); + } + UpdateVersionStorageInfo(); + ASSERT_OK(vstorage_.ValidateBlobRelocationState()); + vstorage_.ComputeBlobRelocationStateFingerprint(); + + EnvOptions env_options; + VersionBuilder builder(env_options, &ioptions_, nullptr, &vstorage_, nullptr); + VersionStorageInfo unchanged( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + false, EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&unchanged)); + ASSERT_EQ(unchanged.GetBlobRelocationState(), + vstorage_.GetBlobRelocationState()); + ASSERT_EQ(unchanged.GetBlobRelocationStateFingerprintHigh(), + vstorage_.GetBlobRelocationStateFingerprintHigh()); + + VersionEdit deletion; + deletion.DeleteBlobRelocationRoute(1000); + VersionBuilder changed_builder(env_options, &ioptions_, nullptr, &unchanged, + nullptr); + ASSERT_OK(changed_builder.Apply(&deletion)); + VersionStorageInfo changed(&icmp_, ucmp_, options_.num_levels, + kCompactionStyleLevel, &unchanged, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(changed_builder.SaveTo(&changed)); + ASSERT_NE(changed.GetBlobRelocationState(), + unchanged.GetBlobRelocationState()); + ASSERT_EQ(changed.GetBlobRelocationRouteTarget(1000), kInvalidBlobFileNumber); + ASSERT_EQ(unchanged.GetBlobRelocationRouteTarget(1000), 200U); + ASSERT_NE(changed.GetBlobRelocationStateFingerprintHigh(), + unchanged.GetBlobRelocationStateFingerprintHigh()); + ASSERT_EQ(changed.GetBlobRelocationRouteTargetCount(200), 999U); + ASSERT_EQ(unchanged.GetBlobRelocationRouteTargetCount(200), 1000U); +} + +TEST_F(VersionBuilderTest, BlobRelocationFingerprintIsOrderFree) { + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(200, {100, 101}, &addition)); + const std::shared_ptr file = + BlobRelocationFileMetaData::Create(addition); + BlobRelocationState first; + first.SetFile(file); + first.SetRoute(BlobRelocationRoute(1000, 200)); + first.SetRoute(BlobRelocationRoute(1065, 200)); + first.Seal(); + uint64_t first_high; + uint64_t first_low; + first.ComputeFingerprint(1, &first_high, &first_low); + BlobRelocationState second; + second.SetRoute(BlobRelocationRoute(1065, 200)); + second.SetRoute(BlobRelocationRoute(1000, 300)); + second.SetFile(file); + // Re-pointing, adding and removing a route leave the same final digest. + second.SetRoute(BlobRelocationRoute(1000, 200)); + second.SetRoute(BlobRelocationRoute(1001, 200)); + second.DeleteRoute(1001); + second.Seal(); + uint64_t second_high; + uint64_t second_low; + second.ComputeFingerprint(1, &second_high, &second_low); + ASSERT_EQ(first_high, second_high); + ASSERT_EQ(first_low, second_low); + // The digest covers the route target and the loss marker. + BlobRelocationState copied(first); + copied.SetRoute(BlobRelocationRoute(1065, 300)); + copied.Seal(); + uint64_t copied_high; + uint64_t copied_low; + copied.ComputeFingerprint(1, &copied_high, &copied_low); + ASSERT_NE(copied_high, first_high); + BlobRelocationState lost(first); + lost.SetFile(BlobRelocationFileMetaData::CreateMarkedLost(file)); + lost.Seal(); + uint64_t lost_high; + uint64_t lost_low; + lost.ComputeFingerprint(1, &lost_high, &lost_low); + ASSERT_NE(lost_high, first_high); + ASSERT_TRUE(lost.HasLostFiles()); + ASSERT_TRUE(BlobFileTable::IsLost(lost.table().Find(200))); + ASSERT_FALSE(BlobFileTable::IsLost(first.table().Find(200))); + ASSERT_EQ(first.FindRouteTarget(1000), 200U); + ASSERT_EQ(first.RouteCount(200), 2U); +} + +TEST_F(VersionBuilderTest, BlobRelocationFingerprintCoversFileFields) { + // Each field of a relocation file that identifies it (size, checksum, + // directory digest) changes the fingerprint. + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(200, {100}, &addition)); + const auto fingerprint = [](const BlobRelocationFileAddition& file) { + BlobRelocationState state; + state.SetFile(BlobRelocationFileMetaData::Create(file)); + state.Seal(); + uint64_t high = 0; + uint64_t low = 0; + state.ComputeFingerprint(1, &high, &low); + return std::make_pair(high, low); + }; + const auto base = fingerprint(BlobRelocationFileAddition( + 200, addition.GetFileSize(), "crc32c", "aaaa", addition.GetTotalEntries(), + addition.GetTotalRecordBytes(), addition.GetDirectoryDigest(), + addition.GetTailSize())); + EXPECT_NE(base, + fingerprint(BlobRelocationFileAddition( + 200, addition.GetFileSize() + 1, "crc32c", "aaaa", + addition.GetTotalEntries(), addition.GetTotalRecordBytes(), + addition.GetDirectoryDigest(), addition.GetTailSize()))); + EXPECT_NE(base, + fingerprint(BlobRelocationFileAddition( + 200, addition.GetFileSize(), "crc32c", "aaab", + addition.GetTotalEntries(), addition.GetTotalRecordBytes(), + addition.GetDirectoryDigest(), addition.GetTailSize()))); + EXPECT_NE(base, + fingerprint(BlobRelocationFileAddition( + 200, addition.GetFileSize(), "crc32c", "aaaa", + addition.GetTotalEntries(), addition.GetTotalRecordBytes(), + addition.GetDirectoryDigest() ^ 1, addition.GetTailSize()))); +} + +TEST_F(VersionBuilderTest, BlobRelocationRetentionLeavesTableToItsCharge) { + // A census change log entry that holds a newer state is charged only for + // memory that no other reservation charges. The file table is charged by + // ownership to the file metadata reservation, so a route change on a large + // table adds no table bytes to the entry. + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(200, {100}, &addition)); + BlobRelocationState base; + base.SetFile(BlobRelocationFileMetaData::Create(addition)); + for (uint64_t source = 1000; source < 11000; ++source) { + base.SetRoute(BlobRelocationRoute(source, 200)); + } + base.Seal(); + BlobRelocationState changed(base); + changed.DeleteRoute(1000); + changed.Seal(); + ASSERT_GT(changed.table().ApproximateMemoryUsage(), 100000U); + EXPECT_LT(changed.IncrementalRetentionBytes(base, /*changed_files=*/{}), + 1024U); + + // A changed relocation file is counted, old and new side. + BlobRelocationFileAddition other; + ASSERT_OK(MakeRelocationAddition(300, {150}, &other)); + BlobRelocationState with_file(base); + with_file.SetFile(BlobRelocationFileMetaData::Create(other)); + with_file.Seal(); + EXPECT_GT(with_file.IncrementalRetentionBytes(base, {300}), + with_file.IncrementalRetentionBytes(base, {})); +} + +TEST_F(VersionBuilderTest, AcceptedLossMarksRelocationFile) { + // Relocation file #200 holds the values of old file #100, which is routed + // to it. + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(200, {100}, &addition)); + ASSERT_OK(AddRelocationToBase(addition)); + vstorage_.AddBlobRelocationRoute(BlobRelocationRoute(100, 200)); + UpdateVersionStorageInfo(); + ASSERT_OK(vstorage_.ValidateBlobRelocationState()); + vstorage_.ComputeBlobRelocationStateFingerprint(); + + EnvOptions env_options; + const auto apply = [&](VersionEdit* edit, VersionStorageInfo* base, + VersionStorageInfo* output) { + VersionBuilder builder(env_options, &ioptions_, nullptr, base, nullptr); + Status status = builder.Apply(edit); + if (status.ok() && output != nullptr) { + status = builder.SaveTo(output); + } + return status; + }; + const auto new_storage = [&](VersionStorageInfo* base) { + return std::make_unique( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, base, false, + EpochNumberRequirement::kMightMissing, nullptr, 0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + }; + { + // A loss record is rejected for a duplicate file number, file number 0, an + // unknown file, and a file that the same edit deletes. + for (const std::vector& losses : + std::vector>{{200, 200}, {0}, {999}}) { + VersionEdit edit; + for (uint64_t file_number : losses) { + edit.MarkBlobPhysicalFileLost(file_number); + } + ASSERT_TRUE(apply(&edit, &vstorage_, nullptr).IsCorruption()); + } + VersionEdit deleted; + deleted.DeleteBlobRelocationRoute(100); + deleted.DeleteBlobRelocationFile(200); + deleted.MarkBlobPhysicalFileLost(200); + ASSERT_TRUE(apply(&deleted, &vstorage_, nullptr).IsCorruption()); + } + + VersionEdit loss; + loss.MarkBlobPhysicalFileLost(200); + std::unique_ptr lost = new_storage(&vstorage_); + ASSERT_OK(apply(&loss, &vstorage_, lost.get())); + lost->ComputeBlobRelocationStateFingerprint(); + ASSERT_TRUE(lost->GetBlobRelocationFileMetaData(200)->IsMarkedLost()); + ASSERT_TRUE(lost->HasLostBlobRelocationFiles()); + ASSERT_TRUE(BlobFileTable::IsLost(lost->GetBlobFileTable().Find(200))); + ASSERT_NE(lost->GetBlobRelocationStateFingerprintHigh(), + vstorage_.GetBlobRelocationStateFingerprintHigh()); + // A reference whose generation number is the routed old file number and one + // flattened to the file are both lost; a reference with another generation + // number is not. + const auto index_of = [](uint64_t generation, uint64_t origin) { + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, generation, + {.origin_file_number = origin, + .original_value_offset = 11, + .stored_value_size = 17, + .compression = kNoCompression, + .stable_checksum = 1, + .stored_checksum = 2}); + BlobIndex index; + EXPECT_OK(index.DecodeFrom(encoded)); + return index; + }; + ASSERT_TRUE(lost->IsBlobReferenceLost(index_of(100, 100))); + ASSERT_TRUE(lost->IsBlobReferenceLost(index_of(200, 100))); + ASSERT_FALSE(lost->IsBlobReferenceLost(index_of(150, 150))); + UpdateVersionStorageInfo(lost.get()); + + // Deleting the lost file (after its route) clears the loss count. + VersionEdit deletion; + deletion.DeleteBlobRelocationRoute(100); + deletion.DeleteBlobRelocationFile(200); + std::unique_ptr deleted = new_storage(lost.get()); + ASSERT_OK(apply(&deletion, lost.get(), deleted.get())); + ASSERT_FALSE(deleted->HasLostBlobRelocationFiles()); +} + +TEST_F(VersionBuilderTest, BlobRelocationStateKeepsFileTotals) { + // BlobRelocationState keeps two totals over its relocation files: the file + // sizes and the record bytes from each file's directory. It updates them when + // files are set and deleted. When a sum would overflow, it sets + // AggregateOverflow() and stops changing the totals. + const auto make_file = [&](uint64_t file_number, + const std::vector& origins, + uint64_t file_size) { + BlobRelocationFileAddition addition; + EXPECT_OK(MakeRelocationAddition(file_number, origins, &addition)); + addition = BlobRelocationFileAddition( + file_number, file_size, "crc32c", "checksum", + addition.GetTotalEntries(), addition.GetTotalRecordBytes(), + addition.GetDirectoryDigest(), /*tail_size=*/1); + if (file_size != std::numeric_limits::max()) { + EXPECT_OK(addition.Validate()); + } + return BlobRelocationFileMetaData::Create(addition); + }; + BlobRelocationState state; + const auto first = make_file(200, {100}, /*file_size=*/4096); + const auto second = make_file(201, {101, 102}, /*file_size=*/1000); + state.SetFile(first); + state.SetFile(second); + EXPECT_EQ(state.PhysicalFileBytes(), 5096U); + EXPECT_EQ(state.RecordBytes(), + first->GetTotalRecordBytes() + second->GetTotalRecordBytes()); + state.DeleteFile(200); + EXPECT_EQ(state.PhysicalFileBytes(), 1000U); + EXPECT_EQ(state.RecordBytes(), second->GetTotalRecordBytes()); + EXPECT_FALSE(state.AggregateOverflow()); + state.SetFile(make_file(202, {103}, std::numeric_limits::max())); + EXPECT_TRUE(state.AggregateOverflow()); + EXPECT_EQ(state.PhysicalFileBytes(), 1000U); +} + +TEST_F(VersionBuilderTest, BlobRelocationSupportsMaximumFileNumber) { + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + UpdateVersionStorageInfo(); + + constexpr uint64_t kMaxFileNumber = kMaxBlobRelocationFileNumber; + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(kMaxFileNumber, + /*origin_file_numbers=*/{100}, &addition)); + // Numbers above kMaxBlobRelocationFileNumber are rejected, for relocation + // files and for both ends of a route. + BlobRelocationFileAddition too_large; + ASSERT_TRUE(MakeRelocationAddition(kMaxFileNumber + 1, + /*origin_file_numbers=*/{100}, &too_large) + .IsCorruption()); + ASSERT_TRUE( + BlobRelocationRoute(kMaxFileNumber + 1, 200).Validate().IsCorruption()); + ASSERT_TRUE( + BlobRelocationRoute(100, kMaxFileNumber + 1).Validate().IsCorruption()); + ASSERT_OK(BlobRelocationRoute(100, kMaxFileNumber).Validate()); + + EnvOptions env_options; + VersionBuilder addition_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + VersionEdit addition_edit; + addition_edit.AddBlobRelocationFile(addition); + ASSERT_OK(addition_builder.Apply(&addition_edit)); + VersionStorageInfo live( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(addition_builder.SaveTo(&live)); + ASSERT_NE(live.GetBlobRelocationFileMetaData(kMaxFileNumber), nullptr); + + VersionBuilder deletion_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &live, + /*version_set=*/nullptr); + VersionEdit deletion_edit; + deletion_edit.DeleteBlobRelocationFile(kMaxFileNumber); + ASSERT_OK(deletion_builder.Apply(&deletion_edit)); + VersionStorageInfo retired( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &live, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(deletion_builder.SaveTo(&retired)); + ASSERT_EQ(retired.GetBlobRelocationFileMetaData(kMaxFileNumber), nullptr); + ASSERT_TRUE(retired.GetBlobRelocationFiles().empty()); +} + +TEST_F(VersionBuilderTest, BlobFileRetirementRemovesCollectedFile) { + // This test models an indirect blob GC install edit. Indirect-origin blob + // file #100 still has live values, which go to relocation file #200. + // Indirect-origin blob file #110 has no live values left. + constexpr uint64_t kSourceFileNumber = 100; + constexpr uint64_t kEmptyFileNumber = 110; + constexpr uint64_t kRelocationFileNumber = 200; + constexpr uint64_t kTableFileNumber = 101; + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + AddDummyFile(kTableFileNumber, kSourceFileNumber, /*epoch_number=*/1); + for (uint64_t file_number : {kSourceFileNumber, kEmptyFileNumber}) { + auto shared = SharedBlobFileMetaData::Create( + file_number, /*total_blob_count=*/2, + /*total_blob_bytes=*/100, /*checksum_method=*/{}, + /*checksum_value=*/{}, /*is_indirect_origin=*/true); + vstorage_.AddBlobFile(BlobFileMetaData::Create( + std::move(shared), + file_number == kSourceFileNumber + ? BlobFileMetaData::LinkedSsts{kTableFileNumber} + : BlobFileMetaData::LinkedSsts{}, + /*garbage_blob_count=*/1, /*garbage_blob_bytes=*/40)); + } + // A normal blob file written with enable_blob_indirection off. + constexpr uint64_t kLegacyFileNumber = 120; + AddBlob(kLegacyFileNumber, /*total_blob_count=*/2, /*total_blob_bytes=*/100, + /*checksum_method=*/{}, /*checksum_value=*/{}, + BlobFileMetaData::LinkedSsts{}, /*garbage_blob_count=*/1, + /*garbage_blob_bytes=*/40); + // The format fence is set on this Version, so its file table must list its + // blob files. + vstorage_.SyncBlobFileTable({}, /*rebuild=*/true, /*cache_res_mgr=*/nullptr); + UpdateVersionStorageInfo(); + + BlobRelocationFileAddition addition; + ASSERT_OK(MakeRelocationAddition(kRelocationFileNumber, {kSourceFileNumber}, + &addition)); + EnvOptions env_options; + { + // A route from blob file #100 is rejected while #100 is still live (the + // edit has no blob file retirement for it). + VersionBuilder missing_retirement_builder( + env_options, &ioptions_, /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + VersionEdit missing_retirement; + missing_retirement.AddBlobRelocationFile(addition); + missing_retirement.SetBlobRelocationRoute( + BlobRelocationRoute(kSourceFileNumber, kRelocationFileNumber)); + ASSERT_TRUE( + missing_retirement_builder.Apply(&missing_retirement).IsCorruption()); + } + { + // A blob file retirement must name a live indirect-origin blob file. #105 + // does not exist. + VersionBuilder unknown_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + VersionEdit unknown; + unknown.RetireBlobFile(/*file_number=*/105); + ASSERT_TRUE(unknown_builder.Apply(&unknown).IsCorruption()); + } + { + // A blob file retirement of a blob file written with + // enable_blob_indirection off is rejected. + VersionBuilder legacy_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + VersionEdit legacy; + legacy.RetireBlobFile(kLegacyFileNumber); + ASSERT_TRUE(legacy_builder.Apply(&legacy).IsCorruption()); + } + { + // The same file twice in one edit is rejected. + VersionBuilder duplicate_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &vstorage_, + /*version_set=*/nullptr); + VersionEdit duplicate; + duplicate.RetireBlobFile(kEmptyFileNumber); + duplicate.RetireBlobFile(kEmptyFileNumber); + const Status status = duplicate_builder.Apply(&duplicate); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("Duplicate blob file retirement"), + std::string::npos) + << status.ToString(); + } + + // The GC install edit adds relocation file #200, routes #100 to it, and + // records the blob file retirement of #100 and #110. #110 has no live values, + // so it needs no route. + VersionEdit edit; + edit.AddBlobRelocationFile(addition); + edit.SetBlobRelocationRoute( + BlobRelocationRoute(kSourceFileNumber, kRelocationFileNumber)); + edit.RetireBlobFile(kSourceFileNumber); + edit.RetireBlobFile(kEmptyFileNumber); + + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + ASSERT_OK(builder.Apply(&edit)); + // A blob file retirement record holds only the file number. A second record + // for #100 in a later edit of the same builder (as in a replay) changes + // nothing. + VersionEdit replay; + replay.RetireBlobFile(kSourceFileNumber); + ASSERT_OK(builder.Apply(&replay)); + + VersionStorageInfo output( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, EpochNumberRequirement::kMightMissing, + /*clock=*/nullptr, /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&output)); + ASSERT_EQ(output.GetBlobFileMetaData(kSourceFileNumber), nullptr); + ASSERT_EQ(output.GetBlobFileMetaData(kEmptyFileNumber), nullptr); + ASSERT_NE(output.GetBlobRelocationFileMetaData(kRelocationFileNumber), + nullptr); + ASSERT_EQ(output.GetBlobRelocationRouteTarget(kSourceFileNumber), + kRelocationFileNumber); + ASSERT_EQ(output.GetBlobRelocationRouteTarget(kEmptyFileNumber), + kInvalidBlobFileNumber); + ASSERT_EQ(output.GetBlobFileTable().Find(kSourceFileNumber), + BlobFileTable::RouteValue(kRelocationFileNumber)); + ASSERT_EQ(output.GetBlobFileTable().Find(kEmptyFileNumber), 0U); + ASSERT_OK(output.ValidateBlobRelocationState()); + UpdateVersionStorageInfo(&output); + + { + // The saved Version no longer has #110, so a second blob file retirement of + // #110 is rejected. + VersionBuilder again_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &output, + /*version_set=*/nullptr); + VersionEdit again; + again.RetireBlobFile(kEmptyFileNumber); + ASSERT_TRUE(again_builder.Apply(&again).IsCorruption()); + } + { + // #100 is now the old file number of a route, so a new blob file or + // relocation file may not use that number. + VersionBuilder reuse_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &output, + /*version_set=*/nullptr); + VersionEdit blob_reuse; + blob_reuse.AddBlobFile(kSourceFileNumber, /*total_blob_count=*/1, + /*total_blob_bytes=*/10, /*checksum_method=*/{}, + /*checksum_value=*/{}, + /*is_indirect_origin=*/true); + ASSERT_TRUE(reuse_builder.Apply(&blob_reuse).IsCorruption()); + + BlobRelocationFileAddition relocation_reuse; + ASSERT_OK( + MakeRelocationAddition(kSourceFileNumber, {50}, &relocation_reuse)); + VersionBuilder relocation_reuse_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &output, + /*version_set=*/nullptr); + VersionEdit relocation_edit; + relocation_edit.AddBlobRelocationFile(relocation_reuse); + ASSERT_TRUE( + relocation_reuse_builder.Apply(&relocation_edit).IsCorruption()); + } + { + // Late approximate garbage for the routed file is a no-op. + VersionBuilder post_retirement_builder(env_options, &ioptions_, + /*table_cache=*/nullptr, &output, + /*version_set=*/nullptr); + VersionEdit post_retirement_garbage; + post_retirement_garbage.AddBlobFileGarbage(kSourceFileNumber, + /*garbage_blob_count=*/1, + /*garbage_blob_bytes=*/60); + ASSERT_OK(post_retirement_builder.Apply(&post_retirement_garbage)); + } + UnrefFilesInVersion(&output); +} + +TEST_F(VersionBuilderTest, BlobRelocationRouteEndpoints) { + // A route goes from an old file number that is not a live file to a live + // relocation file. + BlobRelocationFileAddition target; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &target)); + ASSERT_OK(AddRelocationToBase(target)); + BlobRelocationFileAddition other; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/300, + /*origin_file_numbers=*/{100}, &other)); + ASSERT_OK(AddRelocationToBase(other)); + AddBlob(/*blob_file_number=*/50, /*total_blob_count=*/1, + /*total_blob_bytes=*/100, /*checksum_method=*/"", + /*checksum_value=*/"", BlobFileMetaData::LinkedSsts(), + /*garbage_blob_count=*/0, /*garbage_blob_bytes=*/0); + UpdateVersionStorageInfo(); + EnvOptions env_options; + const struct { + uint64_t source; + uint64_t target; + bool ok; + } cases[] = { + {/*source=*/100, /*target=*/200, /*ok=*/true}, + // The target must be a live relocation file. + {100, 999, false}, + {100, 50, false}, + // The source must not be a live file. + {300, 200, false}, + {50, 200, false}, + }; + for (const auto& test_case : cases) { + SCOPED_TRACE(std::to_string(test_case.source) + "->" + + std::to_string(test_case.target)); + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit edit; + edit.SetBlobRelocationRoute( + BlobRelocationRoute(test_case.source, test_case.target)); + const Status status = builder.Apply(&edit); + if (test_case.ok) { + ASSERT_OK(status); + } else { + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + } + } +} + +TEST_F(VersionBuilderTest, BlobRelocationRejectsRouteChains) { + // The base Version has route 100 -> 200. No route may point to a file that + // has a route itself (one hop). A relocation file may not be deleted while + // routes point to it. + BlobRelocationFileAddition target; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/200, + /*origin_file_numbers=*/{100}, &target)); + ASSERT_OK(AddRelocationToBase(target)); + vstorage_.AddBlobRelocationRoute( + BlobRelocationRoute(/*source_file_number=*/100, + /*physical_file_number=*/200)); + UpdateVersionStorageInfo(); + ASSERT_OK(vstorage_.ValidateBlobRelocationState()); + + EnvOptions env_options; + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit to_route_key; + to_route_key.SetBlobRelocationRoute( + BlobRelocationRoute(/*source_file_number=*/400, + /*physical_file_number=*/100)); + ASSERT_TRUE(builder.Apply(&to_route_key).IsCorruption()); + } + { + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit delete_target; + delete_target.DeleteBlobRelocationFile(/*file_number=*/200); + ASSERT_TRUE(builder.Apply(&delete_target).IsCorruption()); + } + { + // An indirect blob GC job collects relocation file 200 into 300. The edit + // deletes 200, adds route 200 -> 300 and re-points route 100 to 300. Every + // route still has one hop. + BlobRelocationFileAddition later_target; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/300, + /*origin_file_numbers=*/{100}, + &later_target, {200})); + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit collect; + collect.AddBlobRelocationFile(later_target); + collect.DeleteBlobRelocationFile(/*file_number=*/200); + collect.SetBlobRelocationRoute(BlobRelocationRoute(200, 300)); + collect.SetBlobRelocationRoute(BlobRelocationRoute(100, 300)); + ASSERT_OK(builder.Apply(&collect)); + VersionStorageInfo collected( + &icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, &vstorage_, + /*force_consistency_checks=*/false, + EpochNumberRequirement::kMightMissing, /*clock=*/nullptr, + /*bottommost_file_compaction_delay=*/0, + OffpeakTimeOption(options_.daily_offpeak_time_utc), + PeriodicCompactionPhaseParams{}); + ASSERT_OK(builder.SaveTo(&collected)); + ASSERT_EQ(collected.GetBlobRelocationRouteTarget(100), 300U); + ASSERT_EQ(collected.GetBlobRelocationRouteTarget(200), 300U); + ASSERT_EQ(collected.GetBlobRelocationRouteTargetCount(300), 2U); + ASSERT_OK(collected.ValidateBlobRelocationState()); + } + { + // Adding route 200 -> 300 without re-pointing 100 leaves route + // 100 -> 200 pointing to deleted file 200, so the edit is rejected. + BlobRelocationFileAddition later_target; + ASSERT_OK(MakeRelocationAddition(/*file_number=*/300, + /*origin_file_numbers=*/{100}, + &later_target, {200})); + VersionBuilder builder(env_options, &ioptions_, /*table_cache=*/nullptr, + &vstorage_, /*version_set=*/nullptr); + VersionEdit partial; + partial.AddBlobRelocationFile(later_target); + partial.DeleteBlobRelocationFile(/*file_number=*/200); + partial.SetBlobRelocationRoute(BlobRelocationRoute(200, 300)); + ASSERT_TRUE(builder.Apply(&partial).IsCorruption()); + } +} + TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) { // Attempt to add a blob file that is already present in the base version. @@ -2183,6 +3790,285 @@ TEST_F(VersionBuilderTest, EstimatedActiveKeys) { (kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles); } +// Tests for BlobFileTable, the packed file table of a Version. Two tests apply +// random edits and compare the table with a std::map. The others check 64-bit +// keys, shard growth and sharing, tagged values and memory charging. The +// parameter selects the plain key scan (true) or the SSE2 scan when available +// (false). +class BlobFileTableTest : public testing::TestWithParam { + protected: + void SetUp() override { BlobFileTable::TEST_UsePlainScan(GetParam()); } + void TearDown() override { BlobFileTable::TEST_UsePlainScan(false); } + + static uint64_t RandomValue(Random64* rnd) { + switch (rnd->Uniform(3)) { + case 0: + return BlobFileTable::RouteValue(1 + rnd->Uniform(1 << 20)); + case 1: + // A fake aligned blob file pointer; the table never reads it. + return (1 + rnd->Uniform(1 << 20)) << 3; + default: + return ((1 + rnd->Uniform(1 << 20)) << 3) | 2 | (rnd->OneIn(2) ? 4 : 0); + } + } + + // file number -> (value, route count). + using Expected = std::map>; + + static void CheckEqual(const BlobFileTable& table, const Expected& expected, + const std::vector& probes) { + ASSERT_EQ(table.size(), expected.size()); + for (const auto& entry : expected) { + uint32_t count = 0; + ASSERT_EQ(table.Find(entry.first, &count), entry.second.first) + << entry.first; + ASSERT_EQ(count, entry.second.second) << entry.first; + } + for (uint64_t probe : probes) { + const auto it = expected.find(probe); + ASSERT_EQ(table.Find(probe), it != expected.end() ? it->second.first : 0) + << probe; + } + Expected visited; + table.ForEach([&](uint64_t file_number, uint64_t value, uint32_t count) { + ASSERT_TRUE( + visited.emplace(file_number, std::make_pair(value, count)).second); + }); + ASSERT_EQ(visited, expected); + ASSERT_EQ(table.Find(kInvalidBlobFileNumber), 0); + } + + // Applies random batches of sets and erases over keys below `key_limit`, plus + // keys above 2^32 when `wide_keys` is true. It keeps copies of some old + // tables ("pinned") and checks at the end that later edits did not change + // them (the table is copy-on-write). + static void RunRandomEdits(uint64_t seed, uint64_t key_limit, + bool wide_keys) { + SCOPED_TRACE("seed=" + std::to_string(seed)); + Random64 rnd(seed); + const auto random_key = [&]() { + if (wide_keys && rnd.OneIn(3)) { + return (uint64_t{1} << 32) + rnd.Uniform(key_limit); + } + return 1 + rnd.Uniform(key_limit); + }; + BlobFileTable table; + Expected expected; + std::vector> pinned; + for (int round = 0; round < 60; ++round) { + std::vector ops; + // Grow in the first rounds, then shrink, so shards split and merge. + const bool erase_heavy = round >= 40; + const size_t batch = 1 + rnd.Uniform(round % 7 == 0 ? 2000 : 100); + for (size_t i = 0; i < batch; ++i) { + uint64_t key = random_key(); + if (!expected.empty() && rnd.OneIn(erase_heavy ? 1 : 3)) { + // Erase or re-set an existing key. + auto it = expected.lower_bound(key); + if (it == expected.end()) { + it = expected.begin(); + } + key = it->first; + } + const bool erase = rnd.OneIn(erase_heavy ? 1 : 4); + const uint64_t value = erase ? 0 : RandomValue(&rnd); + // Only relocation files carry a route count; some counts are large. + const uint32_t count = + BlobFileTable::IsRelocationFile(value) + ? (rnd.OneIn(8) ? std::numeric_limits::max() - 1 + : static_cast(rnd.Uniform(1000))) + : 0; + ops.push_back({key, value, count}); + if (erase) { + expected.erase(key); + } else { + expected[key] = std::make_pair(value, count); + } + } + table.Apply(ops); + std::vector probes; + for (int i = 0; i < 64; ++i) { + probes.push_back(random_key()); + } + ASSERT_NO_FATAL_FAILURE(CheckEqual(table, expected, probes)); + if (rnd.OneIn(4)) { + pinned.emplace_back(table, expected); + } + } + for (const auto& old : pinned) { + ASSERT_NO_FATAL_FAILURE(CheckEqual(old.first, old.second, {})); + } + } +}; + +TEST_P(BlobFileTableTest, MatchesMapOverRandomEdits) { + for (uint64_t seed = 1; seed <= 8; ++seed) { + ASSERT_NO_FATAL_FAILURE(RunRandomEdits(seed, /*key_limit=*/5000, + /*wide_keys=*/false)); + } +} + +TEST_P(BlobFileTableTest, MatchesMapWithWideKeys) { + for (uint64_t seed = 101; seed <= 108; ++seed) { + ASSERT_NO_FATAL_FAILURE(RunRandomEdits(seed, /*key_limit=*/5000, + /*wide_keys=*/true)); + } +} + +TEST_P(BlobFileTableTest, WideShardKeysShareLow32Bits) { + // 5 and 2^32 + 5 have the same low 32 bits. Whichever shard holds 2^32 + 5 + // is wide, and a lookup must compare all 64 bits. + const uint64_t high = (uint64_t{1} << 32) + 5; + BlobFileTable table; + std::vector ops{{5, BlobFileTable::RouteValue(7)}, + {high, BlobFileTable::RouteValue(9)}}; + // Enough other keys that some share a shard with 5 or 2^32 + 5. + for (uint64_t key = 6; key < 200; ++key) { + ops.push_back({key, BlobFileTable::RouteValue(key + 1)}); + } + table.Apply(ops); + ASSERT_EQ(table.Find(5), BlobFileTable::RouteValue(7)); + ASSERT_EQ(table.Find(high), BlobFileTable::RouteValue(9)); + ASSERT_EQ(table.Find((uint64_t{2} << 32) + 5), 0); + table.Apply({{5, 0}}); + ASSERT_EQ(table.Find(5), 0); + ASSERT_EQ(table.Find(high), BlobFileTable::RouteValue(9)); + table.Apply({{5, BlobFileTable::RouteValue(11)}, {high, 0}}); + ASSERT_EQ(table.Find(5), BlobFileTable::RouteValue(11)); + ASSERT_EQ(table.Find(high), 0); +} + +TEST_P(BlobFileTableTest, ShardsGrowShrinkAndShare) { + BlobFileTable table; + std::vector ops; + for (uint64_t key = 1; key <= 10000; ++key) { + ops.push_back({key, BlobFileTable::RouteValue(key + 1)}); + } + table.Apply(ops); + ASSERT_EQ(table.size(), 10000); + const size_t full_bytes = table.ApproximateMemoryUsage(); + // 10000 entries make 512 shards: the smallest power of two with at most + // kTargetEntriesPerShard (32) entries per shard on average. An entry needs 16 + // bytes (4-byte key, 8-byte value, 4-byte route count). Shard headers, + // padding and the top array add more, so the estimate is about 20 bytes per + // entry. The bounds below allow 12 to 32 bytes. + ASSERT_GE(full_bytes, 10000 * (4 + 8)); + ASSERT_LE(full_bytes, 10000 * 32); + + // A copy shares every shard. An edit of the copy leaves the original + // unchanged. + BlobFileTable copy(table); + copy.Apply({{5000, 0}, {20000, BlobFileTable::RouteValue(7)}}); + ASSERT_EQ(table.Find(5000), BlobFileTable::RouteValue(5001)); + ASSERT_EQ(table.Find(20000), 0); + ASSERT_EQ(copy.Find(5000), 0); + ASSERT_EQ(copy.Find(20000), BlobFileTable::RouteValue(7)); + + // Erasing almost everything rebuilds with fewer shards. + ops.clear(); + for (uint64_t key = 1; key <= 10000; ++key) { + if (key % 1000 != 0) { + ops.push_back({key, 0}); + } + } + table.Apply(ops); + ASSERT_EQ(table.size(), 10); + ASSERT_LT(table.ApproximateMemoryUsage(), 1024); + for (uint64_t key = 1000; key <= 10000; key += 1000) { + ASSERT_EQ(table.Find(key), BlobFileTable::RouteValue(key + 1)); + } + ASSERT_EQ(copy.size(), 10000); + ASSERT_EQ(copy.Find(1), BlobFileTable::RouteValue(2)); +} + +TEST_P(BlobFileTableTest, TaggedValues) { + alignas(8) static const char fake[16] = {}; + const auto* blob = reinterpret_cast(fake); + const auto* relocation = + reinterpret_cast(fake + 8); + const uint64_t blob_value = BlobFileTable::BlobFileValue(blob); + const uint64_t relocation_value = + BlobFileTable::RelocationFileValue(relocation, /*lost=*/true); + const uint64_t route_value = BlobFileTable::RouteValue(uint64_t{1} << 40); + ASSERT_TRUE(BlobFileTable::IsBlobFile(blob_value)); + ASSERT_FALSE(BlobFileTable::IsRelocationFile(blob_value)); + ASSERT_EQ(BlobFileTable::AsBlobFile(blob_value), blob); + ASSERT_TRUE(BlobFileTable::IsRelocationFile(relocation_value)); + ASSERT_TRUE(BlobFileTable::IsLost(relocation_value)); + ASSERT_EQ(BlobFileTable::AsRelocationFile(relocation_value), relocation); + ASSERT_TRUE(BlobFileTable::IsRoute(route_value)); + ASSERT_FALSE(BlobFileTable::IsBlobFile(route_value)); + ASSERT_EQ(BlobFileTable::RouteTarget(route_value), uint64_t{1} << 40); +} + +TEST_P(BlobFileTableTest, ChargesMemoryOnce) { + std::shared_ptr manager = + std::make_shared( + std::make_shared< + CacheReservationManagerImpl>( + NewLRUCache(64 << 20))); + { + BlobFileTable table; + table.SetCacheReservationManager(manager); + std::vector ops; + for (uint64_t key = 1; key <= 5000; ++key) { + ops.push_back({key, BlobFileTable::RouteValue(key + 1)}); + } + table.Apply(ops); + ASSERT_EQ(manager->GetTotalMemoryUsed(), table.ApproximateMemoryUsage()); + { + // A copy charges its own top array but shares the shards. + BlobFileTable copy(table); + copy.Apply({{1, 0}}); + ASSERT_GT(manager->GetTotalMemoryUsed(), table.ApproximateMemoryUsage()); + ASSERT_LT(manager->GetTotalMemoryUsed(), + table.ApproximateMemoryUsage() * 3 / 2); + } + ASSERT_EQ(manager->GetTotalMemoryUsed(), table.ApproximateMemoryUsage()); + } + ASSERT_EQ(manager->GetTotalMemoryUsed(), 0); +} + +TEST_P(BlobFileTableTest, ChargeStaysUntilLastHolderFrees) { + // Each table copy (one per Version, or one held by a census change log entry) + // shares unchanged shards. Whichever copy is freed first, every shard that a + // remaining copy holds stays charged. When one copy is left, the charge + // equals its memory. + std::shared_ptr manager = + std::make_shared( + std::make_shared< + CacheReservationManagerImpl>( + NewLRUCache(64 << 20))); + auto first = std::make_unique(); + first->SetCacheReservationManager(manager); + std::vector ops; + for (uint64_t key = 1; key <= 5000; ++key) { + ops.push_back({key, BlobFileTable::RouteValue(key + 1)}); + } + first->Apply(ops); + auto second = std::make_unique(*first); + second->Apply({{1, 0}, {2, 0}}); + auto third = std::make_unique(*second); + third->Apply({{3, 0}}); + // Shared shards are charged once, so the total charge is below the sum of + // the memory of the three tables. + const size_t all_three = manager->GetTotalMemoryUsed(); + ASSERT_LT(all_three, first->ApproximateMemoryUsage() + + second->ApproximateMemoryUsage() + + third->ApproximateMemoryUsage()); + ASSERT_GE(all_three, first->ApproximateMemoryUsage()); + // Free the middle copy first, then the oldest, out of order. + second.reset(); + ASSERT_GE(manager->GetTotalMemoryUsed(), first->ApproximateMemoryUsage()); + ASSERT_GE(manager->GetTotalMemoryUsed(), third->ApproximateMemoryUsage()); + first.reset(); + ASSERT_EQ(manager->GetTotalMemoryUsed(), third->ApproximateMemoryUsage()); + third.reset(); + ASSERT_EQ(manager->GetTotalMemoryUsed(), 0); +} + +INSTANTIATE_TEST_CASE_P(BlobFileTableTest, BlobFileTableTest, testing::Bool()); + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/version_edit.cc b/db/version_edit.cc index 74d10172462c..de75fc093b35 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -10,6 +10,7 @@ #include "db/version_edit.h" #include "db/blob/blob_index.h" +#include "db/blob/blob_reference_metadata.h" #include "db/version_set.h" #include "db/wide/wide_column_serialization.h" #include "logging/event_logger.h" @@ -21,7 +22,38 @@ namespace ROCKSDB_NAMESPACE { +static_assert(kMaxBlobRelocationFileNumber == kFileNumberMask, + "Relocation file numbers use the file number limit"); + namespace { +// For DebugString() / DebugJSON(): the blob reference metadata kind of an SST +// written with blob indirection (see db/blob/blob_reference_metadata.h), or an +// empty string for an SST without it. Examples: "exact (3 references, 300 +// stored value bytes)", "coarse (frontier 12)", "legacy". The frontier is the +// oldest indirect-origin file number a coarse SST may reference (see +// "Coarse SST" in db/blob/blob_reference_metadata.h). +std::string BlobReferenceMetadataToString(const FileMetaData& f) { + if (!f.blob_reference_descriptor.empty()) { + BlobReferenceMetadataDescriptor descriptor; + const Status s = BlobReferenceMetadataDescriptor::DecodeFrom( + f.blob_reference_descriptor, &descriptor); + if (!s.ok()) { + return "exact (cannot decode the descriptor: " + s.ToString() + ")"; + } + return "exact (" + std::to_string(descriptor.reference_count) + + " references, " + std::to_string(descriptor.stored_value_bytes) + + " stored value bytes)"; + } + if (f.oldest_coarse_indirect_origin_file_number != kInvalidBlobFileNumber) { + return "coarse (frontier " + + std::to_string(f.oldest_coarse_indirect_origin_file_number) + ")"; + } + if (f.blob_reference_legacy_metadata) { + return "legacy"; + } + return std::string(); +} + // For DebugString() / DebugJSON(), e.g. "ingestion+moved+raw:0x1103" (the raw // value shows reserved bits that aren't decoded). std::string DataFileProvenanceToString(uint32_t encoded) { @@ -137,6 +169,14 @@ uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) { Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, SequenceNumber seqno, ValueType value_type) { + return UpdateBoundaries(key, value, seqno, value_type, + /*decoded_blob_index=*/nullptr); +} + +Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, + SequenceNumber seqno, + ValueType value_type, + const BlobIndex* decoded_blob_index) { // Helper: update oldest_blob_file_number from a single BlobIndex. auto update_oldest_blob = [&](const BlobIndex& blob_index) -> Status { if (!blob_index.IsInlined() && !blob_index.HasTTL()) { @@ -170,16 +210,20 @@ Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, }; if (value_type == kTypeBlobIndex) { - BlobIndex blob_index; - if (Status s = blob_index.DecodeFrom(value); !s.ok()) { - return s; + BlobIndex own_blob_index; + const BlobIndex* value_blob_index = decoded_blob_index; + if (value_blob_index == nullptr) { + if (Status s = own_blob_index.DecodeFrom(value); !s.ok()) { + return s; + } + value_blob_index = &own_blob_index; } - if (Status s = update_oldest_blob(blob_index); !s.ok()) { + if (Status s = update_oldest_blob(*value_blob_index); !s.ok()) { return s; } } else if (value_type == kTypeWideColumnEntity) { - if (Status s = WideColumnSerialization::ForEachBlobFileNumber( + if (Status s = WideColumnSerialization::ForEachBlobIndex( value, update_oldest_blob); !s.ok()) { return s; @@ -196,6 +240,15 @@ Status FileMetaData::UpdateBoundaries(const Slice& key, const Slice& value, return Status::OK(); } +size_t FileMetaData::BlobReferenceSummaryMemoryUsage() const { + // Only new files in a VersionEdit hold a summary; Version files drop it. The + // summary can be shared by several owners, so this may count it more than + // once. + return blob_reference_summary != nullptr + ? blob_reference_summary->MemoryUsage() + : 0; +} + void VersionEdit::Clear() { *this = VersionEdit(); } void VersionEdit::AddExistingFile(int level, const FileMetaData& f, @@ -212,6 +265,10 @@ void VersionEdit::AddExistingFile(int level, const FileMetaData& f, FileMetaData& added = new_files_.back().second; added.db_ownership_start_time = f.db_ownership_start_time; added.data_file_provenance = f.data_file_provenance | provenance_flags_to_set; + added.blob_reference_descriptor = f.blob_reference_descriptor; + added.blob_reference_legacy_metadata = f.blob_reference_legacy_metadata; + added.oldest_coarse_indirect_origin_file_number = + f.oldest_coarse_indirect_origin_file_number; } bool VersionEdit::ShouldEmitPerColumnFamilyRecoveryEdit( @@ -229,6 +286,10 @@ bool VersionEdit::ShouldEmitPerColumnFamilyRecoveryEdit( bool VersionEdit::EncodeTo(std::string* dst, std::optional ts_sz) const { assert(!IsNoManifestWriteDummy()); + if (has_blob_indirection_format_version_ && + has_db_blob_indirection_format_version_) { + return false; + } if (has_db_id_) { PutVarint32(dst, kDbId); PutLengthPrefixedSlice(dst, db_id_); @@ -292,6 +353,50 @@ bool VersionEdit::EncodeTo(std::string* dst, blob_file_garbage.EncodeTo(dst); } + if (has_blob_indirection_format_version_) { + PutVarint32(dst, kBlobIndirectionFormatVersion); + PutVarint32(dst, blob_indirection_format_version_); + } + if (has_db_blob_indirection_format_version_) { + PutVarint32(dst, kDbBlobIndirectionFormatVersion); + PutVarint32(dst, db_blob_indirection_format_version_); + } + + for (const BlobRelocationFileAddition& addition : + blob_relocation_file_additions_) { + PutVarint32(dst, kBlobRelocationFileAddition); + std::string encoded; + addition.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); + } + for (uint64_t file_number : blob_relocation_file_deletions_) { + PutVarint32Varint64(dst, kBlobRelocationFileDeletion, file_number); + } + for (uint64_t file_number : blob_physical_file_losses_) { + if (file_number == kInvalidBlobFileNumber) { + return false; + } + PutVarint32Varint64(dst, kBlobPhysicalFileLoss, file_number); + } + for (uint64_t file_number : blob_file_retirements_) { + if (file_number == kInvalidBlobFileNumber) { + return false; + } + PutVarint32Varint64(dst, kBlobFileRetirement, file_number); + } + for (const BlobRelocationRoute& route : blob_relocation_routes_) { + PutVarint32(dst, kBlobRelocationRoute); + std::string encoded; + route.EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); + } + for (uint64_t source : blob_relocation_route_deletions_) { + PutVarint32(dst, kBlobRelocationRouteDeletion); + std::string encoded; + BlobRelocationRouteDeletion(source).EncodeTo(&encoded); + PutLengthPrefixedSlice(dst, encoded); + } + for (const auto& wal_addition : wal_additions_) { PutVarint32(dst, kWalAddition2); std::string encoded; @@ -515,6 +620,20 @@ void VersionEdit::EncodeToNewFile4(const FileMetaData& f, int level, PutVarint32(&varint_data_file_provenance, f.data_file_provenance); PutLengthPrefixedSlice(dst, Slice(varint_data_file_provenance)); } + if (!f.blob_reference_descriptor.empty()) { + PutVarint32(dst, NewFileCustomTag::kBlobReferenceDescriptor); + PutLengthPrefixedSlice(dst, Slice(f.blob_reference_descriptor)); + } + if (f.blob_reference_legacy_metadata) { + PutVarint32(dst, NewFileCustomTag::kBlobReferenceLegacyMetadata); + PutLengthPrefixedSlice(dst, Slice()); + } + if (f.oldest_coarse_indirect_origin_file_number != kInvalidBlobFileNumber) { + PutVarint32(dst, kOldestCoarseIndirectOriginFileNumber); + std::string encoded; + PutVarint64(&encoded, f.oldest_coarse_indirect_origin_file_number); + PutLengthPrefixedSlice(dst, encoded); + } TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields", dst); @@ -673,6 +792,45 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input, int& max_level, return "invalid data file provenance"; } break; + case kBlobReferenceDescriptor: { + if (!f.blob_reference_descriptor.empty() || + f.blob_reference_legacy_metadata || + f.oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber) { + return "duplicate blob reference metadata"; + } + BlobReferenceMetadataDescriptor descriptor; + Status status = + BlobReferenceMetadataDescriptor::DecodeFrom(field, &descriptor); + if (!status.ok()) { + return "invalid blob reference descriptor"; + } + f.blob_reference_descriptor = field.ToString(); + break; + } + case kBlobReferenceLegacyMetadata: + if (!field.empty() || !f.blob_reference_descriptor.empty() || + f.blob_reference_legacy_metadata || + f.oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber) { + return "invalid legacy blob reference metadata"; + } + f.blob_reference_legacy_metadata = true; + break; + case kOldestCoarseIndirectOriginFileNumber: + if (f.blob_reference_legacy_metadata || + !f.blob_reference_descriptor.empty() || + f.oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber || + !GetVarint64(&field, + &f.oldest_coarse_indirect_origin_file_number) || + !field.empty() || + f.oldest_coarse_indirect_origin_file_number == + kInvalidBlobFileNumber || + f.oldest_coarse_indirect_origin_file_number > kFileNumberMask) { + return "invalid coarse blob reference metadata"; + } + break; default: if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) { // Should not proceed if cannot understand it @@ -913,6 +1071,125 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; } + case kBlobIndirectionFormatVersion: { + uint32_t format_version = 0; + if (has_blob_indirection_format_version_ || + has_db_blob_indirection_format_version_ || + !GetVarint32(&input, &format_version)) { + return Status::Corruption("VersionEdit", + "Malformed blob indirection format fence"); + } + if (format_version == 0) { + return Status::Corruption("VersionEdit", + "Invalid blob indirection format fence"); + } + if (format_version != kBlobIndirectionManifestFormatVersion) { + return Status::NotSupported( + "Unknown blob indirection manifest format"); + } + SetBlobIndirectionFormatVersion(format_version); + break; + } + + case kDbBlobIndirectionFormatVersion: { + uint32_t format_version = 0; + if (has_db_blob_indirection_format_version_ || + has_blob_indirection_format_version_ || + !GetVarint32(&input, &format_version)) { + return Status::Corruption( + "VersionEdit", + "Malformed DB-global blob indirection format fence"); + } + if (format_version == 0) { + return Status::Corruption( + "VersionEdit", "Invalid DB-global blob indirection format fence"); + } + if (format_version != kBlobIndirectionManifestFormatVersion) { + return Status::NotSupported( + "Unknown DB-global blob indirection manifest format"); + } + SetDbBlobIndirectionFormatVersion(format_version); + break; + } + + case kBlobRelocationFileAddition: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + return Status::Corruption("VersionEdit", + "Malformed blob relocation file addition"); + } + BlobRelocationFileAddition addition; + Status status = addition.DecodeFrom(&encoded); + if (!status.ok()) { + return status; + } + AddBlobRelocationFile(std::move(addition)); + break; + } + + case kBlobRelocationFileDeletion: { + uint64_t file_number = kInvalidBlobFileNumber; + if (!GetVarint64(&input, &file_number) || + file_number == kInvalidBlobFileNumber) { + return Status::Corruption("VersionEdit", + "Malformed blob relocation file deletion"); + } + DeleteBlobRelocationFile(file_number); + break; + } + + case kBlobRelocationRoute: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + return Status::Corruption("VersionEdit", + "Malformed blob relocation route"); + } + BlobRelocationRoute route; + Status status = route.DecodeFrom(&encoded); + if (!status.ok()) { + return status; + } + SetBlobRelocationRoute(std::move(route)); + break; + } + + case kBlobRelocationRouteDeletion: { + Slice encoded; + if (!GetLengthPrefixedSlice(&input, &encoded)) { + return Status::Corruption("VersionEdit", + "Malformed blob relocation route deletion"); + } + BlobRelocationRouteDeletion deletion; + Status status = deletion.DecodeFrom(&encoded); + if (!status.ok()) { + return status; + } + DeleteBlobRelocationRoute(deletion.GetOldFileNumber()); + break; + } + + case kBlobPhysicalFileLoss: { + uint64_t file_number = 0; + if (!GetVarint64(&input, &file_number) || + file_number == kInvalidBlobFileNumber) { + return Status::Corruption("VersionEdit", + "Malformed blob relocation loss"); + } + MarkBlobPhysicalFileLost(file_number); + break; + } + + case kBlobFileRetirement: { + uint64_t file_number = 0; + if (!GetVarint64(&input, &file_number) || + file_number == kInvalidBlobFileNumber) { + return Status::Corruption("VersionEdit", + "Malformed blob file retirement"); + } + RetireBlobFile(file_number); + break; + } + case kWalAddition: { WalAddition wal_addition; const Status s = wal_addition.DecodeFrom(&input); @@ -1213,6 +1490,11 @@ std::string VersionEdit::DebugString(bool hex_key) const { r.append(" file_open_metadata_size: "); AppendNumberTo(&r, f.file_open_metadata.size()); } + const std::string blob_references = BlobReferenceMetadataToString(f); + if (!blob_references.empty()) { + r.append(" blob_references: "); + r.append(blob_references); + } } for (const auto& blob_file_addition : blob_file_additions_) { @@ -1224,6 +1506,39 @@ std::string VersionEdit::DebugString(bool hex_key) const { r.append("\n BlobFileGarbage: "); r.append(blob_file_garbage.DebugString()); } + if (has_blob_indirection_format_version_) { + r.append("\n BlobIndirectionFormatVersion: "); + AppendNumberTo(&r, blob_indirection_format_version_); + } + if (has_db_blob_indirection_format_version_) { + r.append("\n DbBlobIndirectionFormatVersion: "); + AppendNumberTo(&r, db_blob_indirection_format_version_); + } + for (const BlobRelocationFileAddition& addition : + blob_relocation_file_additions_) { + r.append("\n BlobRelocationFileAddition: "); + r.append(addition.DebugString()); + } + for (uint64_t file_number : blob_relocation_file_deletions_) { + r.append("\n BlobRelocationFileDeletion: "); + AppendNumberTo(&r, file_number); + } + for (uint64_t file_number : blob_physical_file_losses_) { + r.append("\n BlobPhysicalFileLoss: "); + AppendNumberTo(&r, file_number); + } + for (uint64_t file_number : blob_file_retirements_) { + r.append("\n BlobFileRetirement: "); + AppendNumberTo(&r, file_number); + } + for (const BlobRelocationRoute& route : blob_relocation_routes_) { + r.append("\n BlobRelocationRoute: "); + r.append(route.DebugString()); + } + for (uint64_t old_file_number : blob_relocation_route_deletions_) { + r.append("\n BlobRelocationRouteDeletion: old="); + AppendNumberTo(&r, old_file_number); + } for (const auto& wal_addition : wal_additions_) { r.append("\n WalAddition: "); @@ -1358,6 +1673,10 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const { if (!f.file_open_metadata.empty()) { jw << "FileOpenMetadataSize" << f.file_open_metadata.size(); } + const std::string blob_references = BlobReferenceMetadataToString(f); + if (!blob_references.empty()) { + jw << "BlobReferences" << blob_references; + } jw.EndArrayedObject(); } @@ -1392,6 +1711,77 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const { jw.EndArray(); } + if (has_blob_indirection_format_version_) { + jw << "BlobIndirectionFormatVersion" << blob_indirection_format_version_; + } + if (has_db_blob_indirection_format_version_) { + jw << "DbBlobIndirectionFormatVersion" + << db_blob_indirection_format_version_; + } + + if (!blob_relocation_file_additions_.empty()) { + jw << "BlobRelocationFileAdditions"; + jw.StartArray(); + for (const BlobRelocationFileAddition& addition : + blob_relocation_file_additions_) { + jw.StartArrayedObject(); + jw << "FileNumber" << addition.GetFileNumber(); + jw << "FileSize" << addition.GetFileSize(); + jw.EndArrayedObject(); + } + jw.EndArray(); + } + + if (!blob_relocation_file_deletions_.empty()) { + jw << "BlobRelocationFileDeletions"; + jw.StartArray(); + for (uint64_t file_number : blob_relocation_file_deletions_) { + jw << file_number; + } + jw.EndArray(); + } + + if (!blob_relocation_routes_.empty()) { + jw << "BlobRelocationRoutes"; + jw.StartArray(); + for (const BlobRelocationRoute& route : blob_relocation_routes_) { + jw.StartArrayedObject(); + jw << "Old" << route.GetOldFileNumber(); + jw << "Target" << route.GetTargetFileNumber(); + jw.EndArrayedObject(); + } + jw.EndArray(); + } + + if (!blob_relocation_route_deletions_.empty()) { + jw << "BlobRelocationRouteDeletions"; + jw.StartArray(); + for (uint64_t old_file_number : blob_relocation_route_deletions_) { + jw.StartArrayedObject(); + jw << "Old" << old_file_number; + jw.EndArrayedObject(); + } + jw.EndArray(); + } + + if (!blob_physical_file_losses_.empty()) { + jw << "BlobPhysicalFileLosses"; + jw.StartArray(); + for (uint64_t file_number : blob_physical_file_losses_) { + jw << file_number; + } + jw.EndArray(); + } + + if (!blob_file_retirements_.empty()) { + jw << "BlobFileRetirements"; + jw.StartArray(); + for (uint64_t file_number : blob_file_retirements_) { + jw << file_number; + } + jw.EndArray(); + } + if (!wal_additions_.empty()) { jw << "WalAdditions"; diff --git a/db/version_edit.h b/db/version_edit.h index 9dc87369eea8..141a288a5142 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -17,6 +17,7 @@ #include "db/blob/blob_file_addition.h" #include "db/blob/blob_file_garbage.h" +#include "db/blob/blob_relocation_manifest.h" #include "db/dbformat.h" #include "db/wal_edit.h" #include "memory/arena.h" @@ -32,6 +33,10 @@ namespace ROCKSDB_NAMESPACE { +class BlobIndex; + +struct BlobReferenceSstSummary; + // Tag numbers for serialized VersionEdit. These numbers are written to // disk and should not be changed. The number should be forward compatible so // users can down-grade RocksDB safely. A future Tag is ignored by doing '&' @@ -61,6 +66,23 @@ enum Tag : uint32_t { kBlobFileAddition = 400, kBlobFileGarbage, + // Required tags for blob indirection (see "MANIFEST records" in + // db/blob/blob_relocation_manifest.h). kBlobIndirectionFormatVersion is the + // per-CF format fence and kDbBlobIndirectionFormatVersion is the DB-global + // format fence; both store kBlobIndirectionManifestFormatVersion (2). + kBlobIndirectionFormatVersion, + kBlobRelocationFileAddition, + kBlobRelocationFileDeletion, + kBlobRelocationRoute, + kBlobRelocationRouteDeletion, + kDbBlobIndirectionFormatVersion, + // Required: records the accepted loss of a relocation file. A binary that + // ignored it could return an older value or an old cache entry. + kBlobPhysicalFileLoss, + // Required: removes an indirect-origin blob file that indirect blob GC has + // collected. A binary that ignored it would keep a removed file in the + // Version. + kBlobFileRetirement, // Mask for an unidentified tag from the future which can be safely ignored. kTagSafeIgnoreMask = 1 << 13, @@ -128,6 +150,9 @@ enum NewFileCustomTag : uint32_t { // Forward incompatible (aka unignorable) fields kPathId, + kBlobReferenceDescriptor = kCustomTagNonSafeIgnoreMask | 18, + kBlobReferenceLegacyMetadata = kCustomTagNonSafeIgnoreMask | 19, + kOldestCoarseIndirectOriginFileNumber = kCustomTagNonSafeIgnoreMask | 20, }; class VersionSet; @@ -379,6 +404,48 @@ struct FileMetaData { // via FileOptions::file_metadata on subsequent opens. Empty if not available. std::string file_open_metadata; + // Encoded BlobReferenceMetadataDescriptor of the exact blob reference + // metadata in this SST. The encoding ends with a CRC32C. The MANIFEST stores + // a nonempty value as an unignorable NewFile field, so an older RocksDB + // cannot open the DB and drop it. + // - Empty: the SST has no exact metadata. It was created before the format + // fence, or it has a coarse frontier (the oldest indirect-origin file + // number a coarse SST may reference; see + // oldest_coarse_indirect_origin_file_number). + // - Nonempty: the SST has exact metadata. A descriptor with zero references + // means the SST keeps no blob reference, so recovery can tell an empty + // reference set from missing metadata. + std::string blob_reference_descriptor; + // Summary of the exact references of a newly built SST, taken from its table + // builder. Not persisted. BuildTable() (used by flush, WAL recovery at open + // and RepairDB) and local compaction set it on their output files before the + // edit is queued, so the census can add the SST without reopening it. + // VersionBuilder drops it, so Version files do not keep it; later readers + // read the SST's reference index. + std::shared_ptr blob_reference_summary; + // Marks an SST in a CF with the format fence that has no descriptor and no + // coarse frontier, so that recovery accepts the SST. There are two cases: an + // SST created before the format was activated, and a coarse SST with frontier + // 0 (it has no indirect references; see + // ExtractBlobReferenceClassificationFromTableProperties()). VersionBuilder + // sets it in memory for such an SST. A MANIFEST snapshot sets it for every + // SST of a fenced CF that has no descriptor and no coarse frontier, and + // writes it as an unignorable NewFile field. Edits that copy the file + // metadata (AddExistingFile()) carry it. Why the snapshot writes it: in a + // snapshot the format fence comes before (or in the same edit as) the SSTs of + // the CF, so replay checks every SST as one added after the fence. The marker + // lets replay accept an SST that has no descriptor and no coarse frontier. + bool blob_reference_legacy_metadata = false; + // Coarse frontier of a coarse SST: an SST that may hold kIndirectBlob + // references but has no exact reference metadata, so it does not list them. + // Instead it records only the oldest indirect-origin file number that it + // references (its coarse frontier). The SST may reference any indirect origin + // at or above this number, so indirect blob GC treats all of them as in use + // ("pinned"). kInvalidBlobFileNumber means the SST is not a coarse SST. This + // field is separate from oldest_blob_file_number, which covers normal blob + // references. See "Coarse SST" in db/blob/blob_reference_metadata.h. + uint64_t oldest_coarse_indirect_origin_file_number = kInvalidBlobFileNumber; + // Skips prefetching index and filter blocks into block cache on file open, // not persisted in MANIFEST. NOTE: false does not guarantee prefetching // either (e.g. when index and filter blocks are not cached in block cache). @@ -424,6 +491,12 @@ struct FileMetaData { Status UpdateBoundaries(const Slice& key, const Slice& value, SequenceNumber seqno, ValueType value_type); + // Same as above. `decoded_blob_index`, if not nullptr, is the caller's decode + // of a kTypeBlobIndex `value`. It saves a second decode. + Status UpdateBoundaries(const Slice& key, const Slice& value, + SequenceNumber seqno, ValueType value_type, + const BlobIndex* decoded_blob_index); + // Unlike UpdateBoundaries, ranges do not need to be presented in any // particular order. void UpdateBoundariesForRange(const InternalKey& start, @@ -497,6 +570,9 @@ struct FileMetaData { return kUnknownNewestKeyTime; } + // Memory used by blob_reference_summary, or 0 if it is null. + size_t BlobReferenceSummaryMemoryUsage() const; + // WARNING: manual update to this function is needed // whenever a new string property is added to FileMetaData // to reduce approximation error. @@ -512,7 +588,9 @@ struct FileMetaData { #endif // ROCKSDB_MALLOC_USABLE_SIZE usage += smallest.size() + largest.size() + file_checksum.size() + file_checksum_func_name.size() + min_timestamp.size() + - max_timestamp.size() + file_open_metadata.size(); + max_timestamp.size() + file_open_metadata.size() + + blob_reference_descriptor.size(); + usage += BlobReferenceSummaryMemoryUsage(); return usage; } @@ -963,6 +1041,16 @@ class VersionEdit { files_to_quarantine_.push_back(blob_file_number); } + void AddBlobFile(uint64_t blob_file_number, uint64_t total_blob_count, + uint64_t total_blob_bytes, std::string checksum_method, + std::string checksum_value, bool is_indirect_origin) { + blob_file_additions_.emplace_back( + blob_file_number, total_blob_count, total_blob_bytes, + std::move(checksum_method), std::move(checksum_value), + is_indirect_origin); + files_to_quarantine_.push_back(blob_file_number); + } + void AddBlobFile(BlobFileAddition blob_file_addition) { blob_file_additions_.emplace_back(std::move(blob_file_addition)); files_to_quarantine_.push_back( @@ -1020,6 +1108,125 @@ class VersionEdit { } bool HasDroppedBlobFiles() const { return !dropped_blob_files_.empty(); } + void SetBlobIndirectionFormatVersion(uint32_t format_version) { + has_blob_indirection_format_version_ = true; + blob_indirection_format_version_ = format_version; + } + bool HasBlobIndirectionFormatVersion() const { + return has_blob_indirection_format_version_; + } + uint32_t GetBlobIndirectionFormatVersion() const { + assert(HasBlobIndirectionFormatVersion()); + return blob_indirection_format_version_; + } + + // The DB-global fence marks the whole DB; the per-CF fence above marks one + // column family. They are separate records so that a MANIFEST snapshot does + // not activate the format for the default column family only because another + // column family uses blob indirection. + void SetDbBlobIndirectionFormatVersion(uint32_t format_version) { + has_db_blob_indirection_format_version_ = true; + db_blob_indirection_format_version_ = format_version; + } + bool HasDbBlobIndirectionFormatVersion() const { + return has_db_blob_indirection_format_version_; + } + uint32_t GetDbBlobIndirectionFormatVersion() const { + assert(HasDbBlobIndirectionFormatVersion()); + return db_blob_indirection_format_version_; + } + + // The one fence rule for flush, WAL-recovery flush and compaction: an edit + // that adds an SST built with enable_blob_indirection on sets the per-CF + // format fence. The first such edit activates the format. + void MaybeSetBlobIndirectionFormatFence(bool output_built_with_indirection) { + if (output_built_with_indirection && !new_files_.empty()) { + SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + } + } + + // True if this edit sets a nonzero per-CF or DB-global format fence, or has a + // relocation file, route or blob file retirement record (these records need + // the fence). An accepted-loss record does not count. + bool SetsBlobIndirectionState() const { + return (has_blob_indirection_format_version_ && + blob_indirection_format_version_ != 0) || + (has_db_blob_indirection_format_version_ && + db_blob_indirection_format_version_ != 0) || + !blob_relocation_file_additions_.empty() || + !blob_relocation_file_deletions_.empty() || + !blob_relocation_routes_.empty() || + !blob_relocation_route_deletions_.empty() || + !blob_file_retirements_.empty(); + } + + void AddBlobRelocationFile(BlobRelocationFileAddition addition) { + blob_relocation_file_additions_.emplace_back(std::move(addition)); + // Like SST and blob file additions: if the MANIFEST write fails, the new + // file is kept (quarantined) instead of deleted, because the write may + // have reached disk. + files_to_quarantine_.push_back( + blob_relocation_file_additions_.back().GetFileNumber()); + } + const std::vector& + GetBlobRelocationFileAdditions() const { + return blob_relocation_file_additions_; + } + + void DeleteBlobRelocationFile(uint64_t file_number) { + blob_relocation_file_deletions_.push_back(file_number); + } + const std::vector& GetBlobRelocationFileDeletions() const { + return blob_relocation_file_deletions_; + } + + // Records an accepted loss of physical blob file `file_number`. + void MarkBlobPhysicalFileLost(uint64_t file_number) { + blob_physical_file_losses_.push_back(file_number); + } + const std::vector& GetBlobPhysicalFileLosses() const { + return blob_physical_file_losses_; + } + + // Removes indirect-origin blob file `file_number`, which indirect blob GC has + // collected. A file with live values also gets route[file_number] in the + // same edit; an empty file leaves without a route. + void RetireBlobFile(uint64_t file_number) { + blob_file_retirements_.push_back(file_number); + } + const std::vector& GetBlobFileRetirements() const { + return blob_file_retirements_; + } + + void SetBlobRelocationRoute(BlobRelocationRoute route) { + blob_relocation_routes_.emplace_back(std::move(route)); + } + const std::vector& GetBlobRelocationRoutes() const { + return blob_relocation_routes_; + } + + // Deletes the route of old file `old_file_number`. + void DeleteBlobRelocationRoute(uint64_t old_file_number) { + blob_relocation_route_deletions_.push_back(old_file_number); + } + const std::vector& GetBlobRelocationRouteDeletions() const { + return blob_relocation_route_deletions_; + } + + // Reference summaries of the exact SSTs that this edit deletes (see + // GetDeletedFiles()). They are kept only in memory, never written to the + // MANIFEST. Compaction sets them from its inputs. The blob reference census + // uses them to subtract the references of those SSTs. If the edit deletes an + // exact SST whose summary is missing here, the census is rebuilt instead. + void SetBlobReferenceSummariesOfDeletedFiles( + std::vector> summaries) { + blob_reference_summaries_of_deleted_files_ = std::move(summaries); + } + const std::vector>& + GetBlobReferenceSummariesOfDeletedFiles() const { + return blob_reference_summaries_of_deleted_files_; + } + // Add a WAL (either just created or closed). // AddWal and DeleteWalsBefore cannot be called on the same VersionEdit. void AddWal(WalNumber number, WalMetadata metadata = WalMetadata()) { @@ -1053,7 +1260,14 @@ class VersionEdit { size_t NumEntries() const { return new_files_.size() + deleted_files_.size() + blob_file_additions_.size() + blob_file_garbages_.size() + - wal_additions_.size() + !wal_deletion_.IsEmpty(); + blob_relocation_file_additions_.size() + + blob_relocation_file_deletions_.size() + + blob_physical_file_losses_.size() + blob_file_retirements_.size() + + blob_relocation_routes_.size() + + blob_relocation_route_deletions_.size() + + has_blob_indirection_format_version_ + + has_db_blob_indirection_format_version_ + wal_additions_.size() + + !wal_deletion_.IsEmpty(); } void SetColumnFamily(uint32_t column_family_id) { @@ -1226,9 +1440,24 @@ class VersionEdit { // Compaction cursors for round-robin compaction policy CompactCursors compact_cursors_; NewFiles new_files_; + BlobFileAdditions blob_file_additions_; BlobFileGarbages blob_file_garbages_; std::vector dropped_blob_files_; + + bool has_blob_indirection_format_version_ = false; + uint32_t blob_indirection_format_version_ = 0; + bool has_db_blob_indirection_format_version_ = false; + uint32_t db_blob_indirection_format_version_ = 0; + std::vector blob_relocation_file_additions_; + std::vector blob_relocation_file_deletions_; + std::vector blob_physical_file_losses_; + std::vector blob_file_retirements_; + std::vector blob_relocation_routes_; + std::vector blob_relocation_route_deletions_; + std::vector> + blob_reference_summaries_of_deleted_files_; + WalAdditions wal_additions_; std::string db_id_; diff --git a/db/version_edit_handler.cc b/db/version_edit_handler.cc index b4b1c7b02d5c..d21e5c666288 100644 --- a/db/version_edit_handler.cc +++ b/db/version_edit_handler.cc @@ -9,6 +9,7 @@ #include "db/version_edit_handler.h" +#include #include #include @@ -33,7 +34,7 @@ void VersionEditHandlerBase::Iterate(log::Reader& reader, while (reader.LastRecordEnd() < max_manifest_read_size_ && s.ok() && reader.ReadRecord(&record, &scratch) && log_read_status->ok()) { VersionEdit edit; - s = edit.DecodeFrom(record); + s = DecodeVersionEdit(record, &edit); if (s.ok()) { s = read_buffer_.AddEdit(&edit); } @@ -45,6 +46,12 @@ void VersionEditHandlerBase::Iterate(log::Reader& reader, s = OnAtomicGroupReplayBegin(); for (size_t i = 0; s.ok() && i < read_buffer_.replay_buffer().size(); i++) { + TEST_SYNC_POINT_CALLBACK( + "VersionEditHandlerBase::Iterate:BeforeApplyAtomicGroupEdit", + &s); + if (!s.ok()) { + break; + } auto& e = read_buffer_.replay_buffer()[i]; s = ApplyVersionEdit(e, &cfd); if (s.ok()) { @@ -99,6 +106,14 @@ void VersionEditHandlerBase::Iterate(log::Reader& reader, &recovered_edits); } +Status VersionEditHandlerBase::DecodeVersionEdit(const Slice& record, + VersionEdit* edit) { + if (edit == nullptr) { + return Status::InvalidArgument("null VersionEdit output"); + } + return edit->DecodeFrom(record); +} + Status ListColumnFamiliesHandler::ApplyVersionEdit( VersionEdit& edit, ColumnFamilyData** /*unused*/) { Status s; @@ -169,6 +184,24 @@ Status FileChecksumRetriever::ApplyVersionEdit(VersionEdit& edit, new_blob_file.GetBlobFileNumber(), std::make_pair(checksum_value, checksum_method)); } + for (uint64_t file_number : edit.GetBlobRelocationFileDeletions()) { + const auto cf = cf_file_checksums_.find(column_family_id); + if (cf != cf_file_checksums_.end()) { + cf->second.erase(file_number); + } + } + for (const BlobRelocationFileAddition& addition : + edit.GetBlobRelocationFileAdditions()) { + std::string checksum_value = addition.GetChecksumValue(); + std::string checksum_method = addition.GetChecksumMethod(); + if (checksum_method.empty()) { + checksum_value = kUnknownFileChecksum; + checksum_method = kUnknownFileChecksumFuncName; + } + cf_file_checksums_[column_family_id].emplace( + addition.GetFileNumber(), + std::make_pair(std::move(checksum_value), std::move(checksum_method))); + } return Status::OK(); } @@ -237,6 +270,17 @@ Status VersionEditHandler::ApplyVersionEdit(VersionEdit& edit, assert(cfd != nullptr); s = ExtractInfoFromVersionEdit(*cfd, edit); } + if (s.ok()) { + replay_removed_blob_physical_files_.insert( + edit.GetBlobRelocationFileDeletions().begin(), + edit.GetBlobRelocationFileDeletions().end()); + replay_removed_blob_physical_files_.insert( + edit.GetBlobFileRetirements().begin(), + edit.GetBlobFileRetirements().end()); + replay_deleted_blob_relocation_routes_.insert( + edit.GetBlobRelocationRouteDeletions().begin(), + edit.GetBlobRelocationRouteDeletions().end()); + } return s; } @@ -518,9 +562,7 @@ ColumnFamilyData* VersionEditHandler::CreateCfAndInit( ColumnFamilyData* VersionEditHandler::DestroyCfAndCleanup( const VersionEdit& edit) { uint32_t cf_id = edit.GetColumnFamily(); - auto builder_iter = builders_.find(cf_id); - assert(builder_iter != builders_.end()); - builders_.erase(builder_iter); + DestroyVersionBuilder(cf_id); ColumnFamilyData* ret = version_set_->GetColumnFamilySet()->GetColumnFamily(cf_id); assert(ret != nullptr); @@ -530,6 +572,12 @@ ColumnFamilyData* VersionEditHandler::DestroyCfAndCleanup( return ret; } +void VersionEditHandler::DestroyVersionBuilder(uint32_t column_family_id) { + const auto builder = builders_.find(column_family_id); + assert(builder != builders_.end()); + builders_.erase(builder); +} + Status VersionEditHandler::MaybeCreateVersionBeforeApplyEdit( const VersionEdit& edit, ColumnFamilyData* cfd, bool force_create_version) { assert(cfd->initialized()); @@ -591,7 +639,28 @@ Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd, Status VersionEditHandler::ExtractInfoFromVersionEdit(ColumnFamilyData* cfd, const VersionEdit& edit) { + if (version_set_ == nullptr) { + return Status::Corruption("null VersionSet while replaying MANIFEST"); + } Status s; + if (edit.HasBlobIndirectionFormatVersion() && + edit.HasDbBlobIndirectionFormatVersion()) { + return Status::Corruption( + "Conflicting per-CF and DB-global blob indirection format fences"); + } + if (edit.HasBlobIndirectionFormatVersion() || + edit.HasDbBlobIndirectionFormatVersion()) { + const uint32_t format_version = + edit.HasBlobIndirectionFormatVersion() + ? edit.GetBlobIndirectionFormatVersion() + : edit.GetDbBlobIndirectionFormatVersion(); + if (version_set_->blob_indirection_format_version_ != 0 && + version_set_->blob_indirection_format_version_ != format_version) { + return Status::Corruption( + "Conflicting DB-global blob indirection format fence"); + } + version_set_->blob_indirection_format_version_ = format_version; + } if (edit.HasDbId()) { version_set_->db_id_ = edit.GetDbId(); version_edit_params_.SetDBId(edit.GetDbId()); @@ -742,6 +811,7 @@ VersionEditHandlerPointInTime::VersionEditHandlerPointInTime( read_options, /*skip_load_table_files=*/false, allow_incomplete_valid_version, epoch_number_requirement), + atomic_update_versions_missing_(0), trust_manifest_recovery_(trust_manifest_recovery), defer_sst_file_opening_(defer_sst_file_opening) { assert(!record_missing_blob_files_as_dropped || @@ -750,13 +820,21 @@ VersionEditHandlerPointInTime::VersionEditHandlerPointInTime( } VersionEditHandlerPointInTime::~VersionEditHandlerPointInTime() { - for (const auto& cfid_and_version : atomic_update_versions_) { - delete cfid_and_version.second.version; - } - for (const auto& elem : versions_) { + VersionEditHandlerPointInTime::DiscardVersions(&atomic_update_versions_); + VersionEditHandlerPointInTime::DiscardVersions(&versions_); +} + +void VersionEditHandlerPointInTime::DeleteVersion(Version* version) { + delete version; +} + +void VersionEditHandlerPointInTime::DiscardVersions( + PointInTimeVersions* versions) { + assert(versions != nullptr); + for (const auto& elem : *versions) { delete elem.second.version; } - versions_.clear(); + versions->clear(); } Status VersionEditHandlerPointInTime::OnAtomicGroupReplayBegin() { @@ -783,9 +861,7 @@ Status VersionEditHandlerPointInTime::OnAtomicGroupReplayBegin() { // An old AtomicGroup is incomplete. Throw away the versions that failed to // complete it. They must not be used for completing the upcoming // AtomicGroup since they are too old. - for (auto& cfid_and_version : atomic_update_versions_) { - delete cfid_and_version.second.version; - } + ResetAtomicUpdateState(); in_atomic_group_ = true; // We lazily assume the column families that exist at this point are all @@ -825,8 +901,33 @@ Status VersionEditHandlerPointInTime::OnAtomicGroupReplayEnd() { return Status::OK(); } +void VersionEditHandlerPointInTime::ResetAtomicUpdateState() { + DiscardVersions(&atomic_update_versions_); + atomic_update_versions_missing_ = 0; + in_atomic_group_ = false; +} + void VersionEditHandlerPointInTime::CheckIterationResult( const log::Reader& reader, Status* s) { + if (s->ok() && read_only_ && allow_incomplete_valid_version_) { + // Best-efforts recovery never marks a relocation file lost. When the + // current state needs a missing relocation file, a read-write open falls + // back to an older complete Version. It then writes a new MANIFEST, so the + // DB stays at that point and every later open sees the same state. A + // read-only open cannot write the MANIFEST. Its fallback would show old + // values of keys that later edits overwrote or deleted, while the MANIFEST + // still holds the later state. So the read-only open fails instead. + // Secondary and follower catch-up keep their usual rule of complete + // Versions only. Only the final builder state counts: a file that an old + // edit added and a later edit deleted is ignored. + for (const auto& entry : builders_) { + if (entry.second->version_builder()->HasMissingBlobRelocationFiles()) { + *s = Status::NotFound( + "Current MANIFEST requires a missing relocation file"); + break; + } + } + } VersionEditHandler::CheckIterationResult(reader, s); assert(s != nullptr); if (s->ok()) { @@ -852,17 +953,17 @@ void VersionEditHandlerPointInTime::CheckIterationResult( } } } else { - for (const auto& elem : versions_) { - delete elem.second.version; - } - versions_.clear(); + DiscardVersions(&versions_); } } ColumnFamilyData* VersionEditHandlerPointInTime::DestroyCfAndCleanup( const VersionEdit& edit) { - ColumnFamilyData* cfd = VersionEditHandler::DestroyCfAndCleanup(edit); uint32_t cfid = edit.GetColumnFamily(); + // Delete the candidate Versions of this column family first, while its + // builder still holds every added FileMetaData and the current Version still + // holds every base file. Then a candidate Version is never the last holder of + // a file, so deleting it (or the ManifestTailer) never marks a file obsolete. if (AtomicUpdateVersionsContains(cfid)) { AtomicUpdateVersionsDropCf(cfid); if (AtomicUpdateVersionsCompleted()) { @@ -871,9 +972,10 @@ ColumnFamilyData* VersionEditHandlerPointInTime::DestroyCfAndCleanup( } auto v_iter = versions_.find(cfid); if (v_iter != versions_.end()) { - delete v_iter->second.version; + DeleteVersion(v_iter->second.version); versions_.erase(v_iter); } + ColumnFamilyData* cfd = VersionEditHandler::DestroyCfAndCleanup(edit); installed_version_log_numbers_.erase(cfid); return cfd; } @@ -942,7 +1044,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersionBeforeApplyEdit( version_set_->db_options_->max_file_opening_threads, false, true, mopts, MaxFileSizeForL0MetaPin(mopts), read_options_); if (!s.ok()) { - delete version; + DeleteVersion(version); if (negative_edge) { builder->RedoLastApply(); } @@ -960,7 +1062,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersionBeforeApplyEdit( return s; } } - s = builder->SaveTo(version->storage_info()); + s = SaveVersionTo(builder, version->storage_info()); if (negative_edge) { builder->RedoLastApply(); } @@ -976,7 +1078,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersionBeforeApplyEdit( !version_set_->db_options_->skip_stats_update_on_db_open); auto v_iter = versions_.find(cfd->GetID()); if (v_iter != versions_.end()) { - delete v_iter->second.version; + DeleteVersion(v_iter->second.version); v_iter->second = PointInTimeVersion{version, log_number}; } else { versions_.emplace(cfd->GetID(), @@ -984,7 +1086,7 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersionBeforeApplyEdit( } } } else { - delete version; + DeleteVersion(version); } } @@ -1028,12 +1130,40 @@ Status VersionEditHandlerPointInTime::VerifyBlobFile( return s; } +Status VersionEditHandlerPointInTime::VerifyBlobRelocationFile( + ColumnFamilyData* cfd, const BlobRelocationFileAddition& addition) { + if (trust_manifest_recovery_) { + return Status::OK(); + } + assert(cfd != nullptr); + Status status = addition.Validate(); + if (!status.ok()) { + return status; + } + CacheHandleGuard reader; + const MutableCFOptions& mutable_options = cfd->GetLatestMutableCFOptions(); + BlobFileCache* const blob_file_cache = cfd->blob_file_cache(); + if (blob_file_cache == nullptr) { + return Status::Corruption("Missing blob relocation recovery dependency"); + } + return blob_file_cache->GetRelocationFileReader( + read_options_, addition.ToPhysicalFile(), mutable_options, &reader); +} + Status VersionEditHandlerPointInTime::LoadTables( ColumnFamilyData* /*cfd*/, bool /*prefetch_index_and_filter_in_cache*/, bool /*is_initial_load*/) { return Status::OK(); } +Status VersionEditHandlerPointInTime::SaveVersionTo( + VersionBuilder* builder, VersionStorageInfo* storage) { + if (builder == nullptr || storage == nullptr) { + return Status::InvalidArgument("null point-in-time Version output"); + } + return builder->SaveTo(storage); +} + bool VersionEditHandlerPointInTime::HasMissingFiles() const { for (const auto& builder : builders_) { if (builder.second->version_builder()->HasMissingFiles()) { @@ -1068,7 +1198,7 @@ void VersionEditHandlerPointInTime::AtomicUpdateVersionsDropCf(uint32_t cfid) { if (atomic_update_versions_iter->second.version == nullptr) { atomic_update_versions_missing_--; } else { - delete atomic_update_versions_iter->second.version; + DeleteVersion(atomic_update_versions_iter->second.version); } atomic_update_versions_.erase(atomic_update_versions_iter); } @@ -1083,7 +1213,7 @@ void VersionEditHandlerPointInTime::AtomicUpdateVersionsPut( if (atomic_update_versions_iter->second.version == nullptr) { atomic_update_versions_missing_--; } else { - delete atomic_update_versions_iter->second.version; + DeleteVersion(atomic_update_versions_iter->second.version); } atomic_update_versions_iter->second = pit_version; } @@ -1099,7 +1229,7 @@ void VersionEditHandlerPointInTime::AtomicUpdateVersionsApply() { !version_set_->db_options_->skip_stats_update_on_db_open); auto versions_iter = versions_.find(cfid); if (versions_iter != versions_.end()) { - delete versions_iter->second.version; + DeleteVersion(versions_iter->second.version); versions_iter->second = pit_version; } else { versions_.emplace(cfid, pit_version); @@ -1108,6 +1238,20 @@ void VersionEditHandlerPointInTime::AtomicUpdateVersionsApply() { atomic_update_versions_.clear(); } +void ManifestTailer::PrepareToReadNewManifest(InstrumentedMutex* db_mutex) { + assert(db_mutex_ == nullptr); + db_mutex_ = db_mutex; + ResetAtomicUpdateState(); + RunWithoutDbMutex("ManifestTailer::PrepareToReadNewManifest:Unlocked", [&] { + ResetReadState(); + ResetReplayBlobRelocationHistory(); + }); + db_mutex_ = nullptr; + initialized_ = false; + status_ = Status::OK(); + cfs_awaiting_from_previous_manifest_ = cfs_awaiting_file_list_; +} + Status ManifestTailer::Initialize() { if (Mode::kRecovery == mode_) { return VersionEditHandler::Initialize(); @@ -1119,8 +1263,7 @@ Status ManifestTailer::Initialize() { assert(cfd_set); ColumnFamilyData* default_cfd = cfd_set->GetDefault(); assert(default_cfd); - auto builder_iter = builders_.find(default_cfd->GetID()); - assert(builder_iter != builders_.end()); + assert(builders_.find(default_cfd->GetID()) != builders_.end()); Version* dummy_version = default_cfd->dummy_versions(); assert(dummy_version); @@ -1129,7 +1272,7 @@ Status ManifestTailer::Initialize() { VersionBuilderUPtr new_builder(new BaseReferencedVersionBuilder( default_cfd, base_version, this, track_found_and_missing_files_, allow_incomplete_valid_version_)); - builder_iter->second = std::move(new_builder); + ReplaceBuilder(default_cfd->GetID(), std::move(new_builder)); AwaitFileList(default_cfd->GetID()); initialized_ = true; @@ -1163,6 +1306,125 @@ Status ManifestTailer::ApplyVersionEdit(VersionEdit& edit, return s; } +Status ManifestTailer::DecodeVersionEdit(const Slice& record, + VersionEdit* edit) { + return RunWithoutDbMutex("ManifestTailer::DecodeVersionEdit:Unlocked", [&] { + return VersionEditHandlerBase::DecodeVersionEdit(record, edit); + }); +} + +Status ManifestTailer::VerifyBlobRelocationFile( + ColumnFamilyData* cfd, const BlobRelocationFileAddition& addition) { + // Called while the builder applies an edit under the DB mutex. The file check + // does I/O, so it runs without the mutex. This is safe because catch-up calls + // run one at a time and only this tailer uses the builder. + return RunWithoutDbMutex( + "ManifestTailer::VerifyBlobRelocationFile:Unlocked", [&] { + return VersionEditHandlerPointInTime::VerifyBlobRelocationFile( + cfd, addition); + }); +} + +Status ManifestTailer::SaveVersionTo(VersionBuilder* builder, + VersionStorageInfo* storage) { + if (builder == nullptr || storage == nullptr) { + return Status::InvalidArgument("null point-in-time Version output"); + } + // As in LogAndApply: save the SST files (and change their reference counts) + // under the DB mutex, and finalize the blob and relocation metadata without + // it. + Status status = builder->SaveTableFilesTo(storage); + if (status.ok()) { + status = RunWithoutDbMutex("ManifestTailer::SaveVersionTo:Unlocked", [&] { + return builder->SaveBlobFilesAndCheckTo(storage); + }); + } + return status; +} + +void ManifestTailer::DeleteVersion(Version* version) { + assert(version != nullptr); + if (db_mutex_ == nullptr) { + VersionEditHandlerPointInTime::DeleteVersion(version); + return; + } + // The Version is deleted under the DB mutex. Only its relocation state, which + // can be large, is freed without the mutex. + std::vector> holders; + holders.push_back(version->storage_info()->ReleaseBlobRelocationState()); + VersionEditHandlerPointInTime::DeleteVersion(version); + VersionSet::FreeBlobRelocationState(&holders, db_mutex_, + "ManifestTailer::DeleteVersion:Unlocked"); +} + +void ManifestTailer::DiscardVersions(PointInTimeVersions* versions) { + assert(versions != nullptr); + if (db_mutex_ == nullptr) { + VersionEditHandlerPointInTime::DiscardVersions(versions); + return; + } + std::vector> holders; + for (const auto& entry : *versions) { + if (entry.second.version != nullptr) { + holders.push_back( + entry.second.version->storage_info()->ReleaseBlobRelocationState()); + } + } + VersionEditHandlerPointInTime::DiscardVersions(versions); + VersionSet::FreeBlobRelocationState( + &holders, db_mutex_, "ManifestTailer::DiscardVersions:Unlocked"); +} + +ColumnFamilyData* ManifestTailer::DestroyCfAndCleanup(const VersionEdit& edit) { + assert(version_set_ != nullptr); + ColumnFamilySet* cfd_set = version_set_->GetColumnFamilySet(); + assert(cfd_set != nullptr); + ColumnFamilyData* cfd = cfd_set->GetColumnFamily(edit.GetColumnFamily()); + assert(cfd != nullptr); + cfds_changed_.erase(cfd); + + ColumnFamilyData* result = + VersionEditHandlerPointInTime::DestroyCfAndCleanup(edit); + assert(result == nullptr); + return result; +} + +void ManifestTailer::DestroyVersionBuilder(uint32_t column_family_id) { + const auto existing = builders_.find(column_family_id); + assert(existing != builders_.end()); + assert(existing->second != nullptr); + VersionBuilderUPtr guard = std::move(existing->second); + builders_.erase(existing); + ReleaseBuilderUnderMutex(std::move(guard), + "ManifestTailer::DestroyVersionBuilder:Unlocked"); +} + +void ManifestTailer::ReleaseBuilderUnderMutex(VersionBuilderUPtr guard, + const char* unlocked_sync_point) { + if (db_mutex_ == nullptr) { + return; + } + // The builder and its reference to the base Version are deleted under the DB + // mutex. Only its staged relocation changes, which can be large, are freed + // without the mutex. + std::vector> holders; + holders.push_back(guard->version_builder()->ReleaseBlobRelocationChanges()); + guard.reset(); + VersionSet::FreeBlobRelocationState(&holders, db_mutex_, unlocked_sync_point); +} + +void ManifestTailer::ReplaceBuilder(uint32_t column_family_id, + VersionBuilderUPtr replacement) { + assert(replacement != nullptr); + const auto existing = builders_.find(column_family_id); + assert(existing != builders_.end()); + assert(existing->second != nullptr); + VersionBuilderUPtr old = std::move(existing->second); + existing->second = std::move(replacement); + ReleaseBuilderUnderMutex(std::move(old), + "ManifestTailer::ReplaceBuilder:Unlocked"); +} + Status ManifestTailer::OnColumnFamilyAdd(VersionEdit& edit, ColumnFamilyData** cfd) { if (Mode::kRecovery == mode_) { @@ -1178,8 +1440,7 @@ Status ManifestTailer::OnColumnFamilyAdd(VersionEdit& edit, // For now, ignore new column families created after Recover() succeeds. return Status::OK(); } - auto builder_iter = builders_.find(edit.GetColumnFamily()); - assert(builder_iter != builders_.end()); + assert(builders_.find(edit.GetColumnFamily()) != builders_.end()); Version* dummy_version = tmp_cfd->dummy_versions(); assert(dummy_version); @@ -1187,7 +1448,7 @@ Status ManifestTailer::OnColumnFamilyAdd(VersionEdit& edit, assert(base_version); VersionBuilderUPtr new_builder(new BaseReferencedVersionBuilder( tmp_cfd, base_version, this, track_found_and_missing_files_)); - builder_iter->second = std::move(new_builder); + ReplaceBuilder(edit.GetColumnFamily(), std::move(new_builder)); AwaitFileList(tmp_cfd->GetID()); #ifndef NDEBUG @@ -1243,6 +1504,62 @@ Status ManifestTailer::VerifyFile(ColumnFamilyData* cfd, return s; } +namespace { +// Prints the blob indirection state of a column family for "ldb +// manifest_dump": the per-CF format fence, the relocation files and the routes +// (see "MANIFEST records" in db/blob/blob_relocation_manifest.h), then the +// kind of reference metadata of its SSTs. A column family without the fence, +// relocation files or routes prints nothing, so the dump of a DB that does not +// use blob indirection does not change. +void PrintBlobIndirectionSummary(const VersionStorageInfo& vstorage) { + const VersionStorageInfo::BlobRelocationFiles& files = + vstorage.GetBlobRelocationFiles(); + const size_t route_count = vstorage.GetBlobRelocationRouteCount(); + if (!vstorage.HasBlobIndirectionFormat() && files.empty() && + route_count == 0) { + return; + } + uint64_t file_bytes = 0; + uint64_t values = 0; + for (const std::shared_ptr& file : files) { + file_bytes += file->GetFileSize(); + values += file->GetTotalEntries(); + } + fprintf(stdout, + "--- blob indirection --- format version: %" PRIu32 + ", relocation files: %zu (%" PRIu64 " bytes, %" PRIu64 + " values), routes: %zu\n", + vstorage.GetBlobIndirectionFormatVersion(), files.size(), file_bytes, + values, route_count); + // The kind of blob reference metadata of each SST (see + // db/blob/blob_reference_metadata.h). Indirect blob GC does not collect a + // blob file numbered at or above the lowest frontier of the coarse SSTs. The + // coarse frontier of an SST is the oldest indirect-origin file number a + // coarse SST may reference (see "Coarse SST" in + // db/blob/blob_reference_metadata.h). + size_t exact_ssts = 0; + size_t coarse_ssts = 0; + size_t legacy_ssts = 0; + for (int level = 0; level < vstorage.num_levels(); ++level) { + for (const FileMetaData* f : vstorage.LevelFiles(level)) { + if (!f->blob_reference_descriptor.empty()) { + ++exact_ssts; + } else if (f->oldest_coarse_indirect_origin_file_number != + kInvalidBlobFileNumber) { + ++coarse_ssts; + } else if (f->blob_reference_legacy_metadata) { + ++legacy_ssts; + } + } + } + fprintf(stdout, + "--- blob reference metadata --- exact SSTs: %zu, coarse SSTs: %zu " + "(lowest frontier %" PRIu64 "), legacy SSTs: %zu\n", + exact_ssts, coarse_ssts, + vstorage.OldestCoarseIndirectOriginFileNumber(), legacy_ssts); +} +} // namespace + void DumpManifestHandler::CheckIterationResult(const log::Reader& reader, Status* s) { VersionEditHandler::CheckIterationResult(reader, s); @@ -1270,6 +1587,7 @@ void DumpManifestHandler::CheckIterationResult(const log::Reader& reader, // Print out DebugStrings. Can include non-terminating null characters. fwrite(cfd->current()->DebugString(hex_).data(), sizeof(char), cfd->current()->DebugString(hex_).size(), stdout); + PrintBlobIndirectionSummary(*cfd->current()->storage_info()); fprintf(stdout, "By default, manifest file dump prints LSM trees as if %d levels " @@ -1288,6 +1606,10 @@ void DumpManifestHandler::CheckIterationResult(const log::Reader& reader, version_set_->LastSequence(), version_set_->prev_log_number(), version_set_->column_family_set_->GetMaxColumnFamily(), version_set_->min_log_number_to_keep()); + if (version_set_->blob_indirection_format_version_ != 0) { + fprintf(stdout, "db blob indirection format version %" PRIu32 "\n", + version_set_->blob_indirection_format_version_); + } } } // namespace ROCKSDB_NAMESPACE diff --git a/db/version_edit_handler.h b/db/version_edit_handler.h index a6c902d7efb6..d8fa0162360f 100644 --- a/db/version_edit_handler.h +++ b/db/version_edit_handler.h @@ -44,6 +44,8 @@ class VersionEditHandlerBase { virtual Status ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) = 0; + virtual Status DecodeVersionEdit(const Slice& record, VersionEdit* edit); + virtual Status OnAtomicGroupReplayBegin() { return Status::OK(); } virtual Status OnAtomicGroupReplayEnd() { return Status::OK(); } @@ -190,6 +192,22 @@ class VersionEditHandler : public VersionEditHandlerBase { const BlobFileAddition& /*blob_addition*/) { return Status::OK(); } + virtual Status VerifyBlobRelocationFile( + ColumnFamilyData* /*cfd*/, + const BlobRelocationFileAddition& /*addition*/) { + return Status::OK(); + } + + // Moves out the file numbers of the relocation files deleted and the blob + // files removed by blob file retirement in the replayed MANIFEST, and the old + // file numbers of the routes deleted there. With reuse_manifest_on_open, + // LogAndApply then keeps rejecting an edit that adds them back to the same + // MANIFEST (see VersionSet::RejectReaddedBlobRelocationState()). + void TakeBlobRelocationDeletions(UnorderedSet* files, + UnorderedSet* routes) { + *files = std::move(replay_removed_blob_physical_files_); + *routes = std::move(replay_deleted_blob_relocation_routes_); + } protected: explicit VersionEditHandler( @@ -219,6 +237,11 @@ class VersionEditHandler : public VersionEditHandlerBase { void CheckColumnFamilyId(const VersionEdit& edit, bool* do_not_open_cf, bool* cf_in_builders) const; + void ResetReplayBlobRelocationHistory() { + replay_removed_blob_physical_files_.clear(); + replay_deleted_blob_relocation_routes_.clear(); + } + void CheckIterationResult(const log::Reader& reader, Status* s) override; ColumnFamilyData* CreateCfAndInit(const ColumnFamilyOptions& cf_options, @@ -226,6 +249,8 @@ class VersionEditHandler : public VersionEditHandlerBase { virtual ColumnFamilyData* DestroyCfAndCleanup(const VersionEdit& edit); + virtual void DestroyVersionBuilder(uint32_t column_family_id); + virtual Status MaybeCreateVersionBeforeApplyEdit(const VersionEdit& edit, ColumnFamilyData* cfd, bool force_create_version); @@ -249,6 +274,11 @@ class VersionEditHandler : public VersionEditHandlerBase { // families. In non read only mode, if those column families are not dropped // by subsequent manifest records, Recover() will return failure status. std::unordered_map do_not_open_column_families_; + // In the replayed MANIFEST: file numbers of the relocation files deleted and + // of the blob files removed by blob file retirement, and old file numbers of + // the routes deleted. + UnorderedSet replay_removed_blob_physical_files_; + UnorderedSet replay_deleted_blob_relocation_routes_; VersionEditParams version_edit_params_; bool no_error_if_files_missing_; std::shared_ptr io_tracer_; @@ -329,6 +359,9 @@ class VersionEditHandlerPointInTime : public VersionEditHandler { const FileMetaData& fmeta) override; Status VerifyBlobFile(ColumnFamilyData* cfd, uint64_t blob_file_num, const BlobFileAddition& blob_addition) override; + Status VerifyBlobRelocationFile( + ColumnFamilyData* cfd, + const BlobRelocationFileAddition& addition) override; protected: Status OnAtomicGroupReplayBegin() override; @@ -355,6 +388,15 @@ class VersionEditHandlerPointInTime : public VersionEditHandler { return true; } + virtual Status SaveVersionTo(VersionBuilder* builder, + VersionStorageInfo* storage); + + // Drops the Versions of an incomplete AtomicGroup and resets the AtomicGroup + // state. Called when a new AtomicGroup starts, and before the replay of a new + // MANIFEST. The Versions are freed through DiscardVersions(); ManifestTailer + // frees their relocation state without the DB mutex. + void ResetAtomicUpdateState(); + // A Version built from the MANIFEST records read up to some point in time, // together with the column family's log number that those records had put in // effect. Keeping the two together is what lets an installed Version report @@ -364,12 +406,17 @@ class VersionEditHandlerPointInTime : public VersionEditHandler { uint64_t log_number = 0; }; - std::unordered_map versions_; + using PointInTimeVersions = UnorderedMap; + + virtual void DeleteVersion(Version* version); + virtual void DiscardVersions(PointInTimeVersions* versions); + + PointInTimeVersions versions_; // `atomic_update_versions_` is for ensuring all-or-nothing AtomicGroup // recoveries. When `atomic_update_versions_` is nonempty, it serves as a // barrier to updating `versions_` until all its values are populated. - std::unordered_map atomic_update_versions_; + PointInTimeVersions atomic_update_versions_; // `atomic_update_versions_missing_` counts the null `version`s in // `atomic_update_versions_`. size_t atomic_update_versions_missing_; @@ -449,11 +496,7 @@ class ManifestTailer : public VersionEditHandlerPointInTime { Status VerifyFile(ColumnFamilyData* cfd, const std::string& fpath, int level, const FileMetaData& fmeta) override; - void PrepareToReadNewManifest() { - initialized_ = false; - ResetReadState(); - cfs_awaiting_from_previous_manifest_ = cfs_awaiting_file_list_; - } + void PrepareToReadNewManifest(InstrumentedMutex* db_mutex); std::unordered_set& GetUpdatedColumnFamilies() { return cfds_changed_; @@ -461,6 +504,32 @@ class ManifestTailer : public VersionEditHandlerPointInTime { std::vector GetAndClearIntermediateFiles(); + void SetCatchUpDBMutex(InstrumentedMutex* db_mutex) { + assert(db_mutex_ == nullptr); + assert(db_mutex != nullptr); + db_mutex_ = db_mutex; + } + + void ClearCatchUpDBMutex() { + assert(db_mutex_ != nullptr); + db_mutex_ = nullptr; + } + + private: + // Runs fn() without the DB mutex during catch-up (a null mutex means + // single-threaded recovery). `unlocked_sync_point` fires without the mutex. + template + auto RunWithoutDbMutex(const char* unlocked_sync_point, Fn&& fn) + -> decltype(fn()) { + if (db_mutex_ == nullptr) { + return fn(); + } + InstrumentedMutexUnlock unlock(db_mutex_); + TEST_SYNC_POINT_CALLBACK(unlocked_sync_point, db_mutex_); + (void)unlocked_sync_point; + return fn(); + } + protected: Status Initialize() override; @@ -468,6 +537,27 @@ class ManifestTailer : public VersionEditHandlerPointInTime { Status ApplyVersionEdit(VersionEdit& edit, ColumnFamilyData** cfd) override; + Status DecodeVersionEdit(const Slice& record, VersionEdit* edit) override; + + Status VerifyBlobRelocationFile( + ColumnFamilyData* cfd, + const BlobRelocationFileAddition& addition) override; + + Status SaveVersionTo(VersionBuilder* builder, + VersionStorageInfo* storage) override; + + void DeleteVersion(Version* version) override; + void DiscardVersions(PointInTimeVersions* versions) override; + ColumnFamilyData* DestroyCfAndCleanup(const VersionEdit& edit) override; + void DestroyVersionBuilder(uint32_t column_family_id) override; + + void ReplaceBuilder(uint32_t column_family_id, + VersionBuilderUPtr replacement); + // Destroys `guard` under the DB mutex and frees its staged relocation changes + // without it (VersionSet::FreeBlobRelocationState()). + void ReleaseBuilderUnderMutex(VersionBuilderUPtr guard, + const char* unlocked_sync_point); + Status OnColumnFamilyAdd(VersionEdit& edit, ColumnFamilyData** cfd) override; bool BuilderHoldsPointInTime(uint32_t cf_id) const override { @@ -489,9 +579,10 @@ class ManifestTailer : public VersionEditHandlerPointInTime { // replay a new MANIFEST, and that have not yet applied the record in it that // lists their files. std::unordered_set cfs_awaiting_file_list_; - // Column families that were awaiting their file list when the tailer - // switched to the current MANIFEST, and that it has not added again. + // Column families that were awaiting their file list when the tailer switched + // to the current MANIFEST, and that it has not added again. std::unordered_set cfs_awaiting_from_previous_manifest_; + InstrumentedMutex* db_mutex_ = nullptr; }; class DumpManifestHandler : public VersionEditHandler { diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index ee8bf3c829df..dc10f3688207 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -10,6 +10,7 @@ #include "db/version_edit.h" #include "db/blob/blob_index.h" +#include "db/blob/blob_reference_metadata.h" #include "rocksdb/advanced_options.h" #include "table/unique_id_impl.h" #include "test_util/sync_point.h" @@ -68,6 +69,34 @@ static void TestOptionsFileNumberField(const char* field_name, class VersionEditTest : public testing::Test {}; +static Status MakeBlobRelocationFileAddition( + uint64_t file_number, uint64_t origin_file_number, + BlobRelocationFileAddition* addition) { + if (addition == nullptr) { + return Status::InvalidArgument("Null relocation addition output"); + } + BlobRelocationDirectoryBuilder builder; + Status status = builder.Add(origin_file_number, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression); + if (status.ok()) { + status = builder.AddServedFile(origin_file_number); + } + if (!status.ok()) { + return status; + } + BlobRelocationDirectory directory; + status = builder.Finish(&directory); + if (!status.ok()) { + return status; + } + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + *addition = BlobRelocationFileAddition( + file_number, /*file_size=*/4096, "crc32c", "checksum", + std::move(encoded_directory), /*tail_size=*/1); + return addition->Validate(); +} + TEST_F(VersionEditTest, EncodeDecode) { static const uint64_t kBig = 1ull << 50; static const uint32_t kBig32Bit = 1ull << 30; @@ -444,6 +473,137 @@ TEST_F(VersionEditTest, DroppedBlobFiles) { std::string::npos); } +TEST_F(VersionEditTest, BlobRelocationStateEncodeDecode) { + constexpr uint32_t kColumnFamilyId = 7; + constexpr uint64_t kOriginFileNumber = 100; + constexpr uint64_t kRelocationFileNumber = 200; + + BlobRelocationFileAddition addition; + ASSERT_OK(MakeBlobRelocationFileAddition(kRelocationFileNumber, + kOriginFileNumber, &addition)); + + VersionEdit edit; + edit.SetColumnFamily(kColumnFamilyId); + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + edit.DeleteBlobRelocationFile(/*file_number=*/201); + edit.SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/kOriginFileNumber, kRelocationFileNumber)); + edit.DeleteBlobRelocationRoute(/*source_file_number=*/101); + edit.RetireBlobFile(/*file_number=*/102); + + TestEncodeDecode(edit); + + std::string encoded; + ASSERT_TRUE(edit.EncodeTo(&encoded, /*ts_sz=*/0)); + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded)); + ASSERT_TRUE(decoded.HasBlobIndirectionFormatVersion()); + ASSERT_EQ(decoded.GetBlobIndirectionFormatVersion(), + kBlobIndirectionManifestFormatVersion); + ASSERT_EQ(decoded.GetBlobRelocationFileAdditions().size(), 1U); + ASSERT_EQ(decoded.GetBlobRelocationFileAdditions().front(), addition); + ASSERT_EQ(decoded.GetBlobRelocationFileDeletions(), + std::vector{201}); + ASSERT_EQ(decoded.GetBlobRelocationRoutes().size(), 1U); + ASSERT_EQ(decoded.GetBlobRelocationRoutes().front(), + BlobRelocationRoute(kOriginFileNumber, kRelocationFileNumber)); + ASSERT_EQ(decoded.GetBlobRelocationRouteDeletions(), + std::vector{101}); + ASSERT_EQ(decoded.GetBlobFileRetirements(), std::vector{102}); + ASSERT_EQ(decoded.GetFilesToQuarantineIfCommitFail()->size(), 1U); + ASSERT_EQ(decoded.GetFilesToQuarantineIfCommitFail()->front(), + kRelocationFileNumber); +} + +TEST_F(VersionEditTest, DbBlobIndirectionFormatFenceEncodeDecode) { + VersionEdit edit; + edit.SetDbBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + TestEncodeDecode(edit); + + std::string encoded; + ASSERT_TRUE(edit.EncodeTo(&encoded)); + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded)); + ASSERT_TRUE(decoded.HasDbBlobIndirectionFormatVersion()); + ASSERT_EQ(decoded.GetDbBlobIndirectionFormatVersion(), + kBlobIndirectionManifestFormatVersion); + ASSERT_FALSE(decoded.HasBlobIndirectionFormatVersion()); +} + +TEST_F(VersionEditTest, RejectsConflictingBlobIndirectionFormatFences) { + std::string encoded; + PutVarint32(&encoded, kBlobIndirectionFormatVersion); + PutVarint32(&encoded, kBlobIndirectionManifestFormatVersion); + PutVarint32(&encoded, kDbBlobIndirectionFormatVersion); + PutVarint32(&encoded, kBlobIndirectionManifestFormatVersion); + VersionEdit decoded; + ASSERT_TRUE(decoded.DecodeFrom(encoded).IsCorruption()); + + VersionEdit conflicting; + conflicting.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + conflicting.SetDbBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + encoded.clear(); + ASSERT_FALSE(conflicting.EncodeTo(&encoded)); +} + +TEST_F(VersionEditTest, BlobRelocationRecordsRejectTrailingBytes) { + // The records carry no version of their own (the format fence covers it), + // so an unexpected extra field must fail instead of being skipped. + BlobRelocationFileAddition addition; + ASSERT_OK(MakeBlobRelocationFileAddition(/*file_number=*/200, + /*origin_file_number=*/100, + &addition)); + std::string encoded_addition; + addition.EncodeTo(&encoded_addition); + std::string encoded_route; + BlobRelocationRoute(/*source_file_number=*/100, + /*physical_file_number=*/200) + .EncodeTo(&encoded_route); + std::string encoded_route_deletion; + BlobRelocationRouteDeletion(/*source_file_number=*/100) + .EncodeTo(&encoded_route_deletion); + for (const auto& record : + {std::make_pair(kBlobRelocationFileAddition, encoded_addition), + std::make_pair(kBlobRelocationRoute, encoded_route), + std::make_pair(kBlobRelocationRouteDeletion, encoded_route_deletion)}) { + SCOPED_TRACE(record.first); + std::string encoded_edit; + PutVarint32(&encoded_edit, kBlobIndirectionFormatVersion); + PutVarint32(&encoded_edit, kBlobIndirectionManifestFormatVersion); + PutVarint32(&encoded_edit, record.first); + PutLengthPrefixedSlice(&encoded_edit, record.second); + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded_edit)); + + encoded_edit.clear(); + PutVarint32(&encoded_edit, record.first); + PutLengthPrefixedSlice(&encoded_edit, record.second + std::string(1, 1)); + ASSERT_TRUE(decoded.DecodeFrom(encoded_edit).IsCorruption()); + } +} + +TEST_F(VersionEditTest, BlobRelocationStateRejectsUnsupportedFormatFence) { + for (uint32_t tag : + {kBlobIndirectionFormatVersion, kDbBlobIndirectionFormatVersion}) { + for (uint32_t format_version : + {0U, kBlobIndirectionManifestFormatVersion + 1}) { + std::string encoded_edit; + PutVarint32(&encoded_edit, tag); + PutVarint32(&encoded_edit, format_version); + VersionEdit decoded; + const Status status = decoded.DecodeFrom(encoded_edit); + if (format_version == 0) { + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + } else { + ASSERT_TRUE(status.IsNotSupported()) << status.ToString(); + } + } + } +} + TEST_F(VersionEditTest, AddWalEncodeDecode) { VersionEdit edit; Random rnd(301); @@ -870,6 +1030,59 @@ TEST(FileMetaDataTest, UpdateBoundariesBlobIndex) { } } +// Compaction passes its own decode of each output BlobIndex to +// UpdateBoundaries(). The status and oldest_blob_file_number must be the same +// as when UpdateBoundaries() decodes the value itself. +TEST(FileMetaDataTest, UpdateBoundariesWithDecodedBlobIndex) { + std::string legacy; + BlobIndex::EncodeBlob(&legacy, /*file_number=*/25, /*offset=*/1000, + /*size=*/100, kNoCompression); + std::string ttl; + BlobIndex::EncodeBlobTTL(&ttl, /*expiration=*/9876543210, + /*file_number=*/15, /*offset=*/2000, /*size=*/500, + kNoCompression); + std::string inlined_ttl; + BlobIndex::EncodeInlinedTTL(&inlined_ttl, /*expiration=*/9876543210, "value"); + std::string invalid_file; + BlobIndex::EncodeBlob(&invalid_file, kInvalidBlobFileNumber, + /*offset=*/10000, /*size=*/1000, kNoCompression); + std::string unflattened; + BlobIndex::EncodeUnflattenedIndirectBlob(&unflattened, + {.origin_file_number = 20, + .original_value_offset = 200, + .stored_value_size = 60, + .compression = kNoCompression, + .stable_checksum = 1234, + .stored_checksum = 0}); + std::string flattened; + BlobIndex::EncodeIndirectBlob(&flattened, /*generation=*/40, + {.origin_file_number = 30, + .original_value_offset = 300, + .stored_value_size = 70, + .compression = kZSTD, + .stable_checksum = 5678, + .stored_checksum = 8765}); + + for (const std::string& value : + {legacy, ttl, inlined_ttl, invalid_file, unflattened, flattened}) { + BlobIndex decoded; + ASSERT_OK(decoded.DecodeFrom(value)); + SCOPED_TRACE(decoded.DebugString(/*output_hex=*/true)); + + FileMetaData self_decoded; + FileMetaData caller_decoded; + const Status status = self_decoded.UpdateBoundaries( + "foo", value, /*seqno=*/100, kTypeBlobIndex); + EXPECT_EQ(caller_decoded + .UpdateBoundaries("foo", value, /*seqno=*/100, kTypeBlobIndex, + &decoded) + .ToString(), + status.ToString()); + EXPECT_EQ(caller_decoded.oldest_blob_file_number, + self_decoded.oldest_blob_file_number); + } +} + class SubcompactionProgressTest : public VersionEditTest { protected: static constexpr uint64_t kTestFileSize = 1024; @@ -1503,6 +1716,64 @@ TEST_F(VersionEditTest, TryGetDbOwnershipStartTime) { EXPECT_EQ(f.TryGetDbOwnershipStartTime(), 2500u); } +TEST_F(VersionEditTest, BlobReferenceDescriptorEncodeDecode) { + VersionEdit edit; + edit.AddFile(3, 300, 0, 100, InternalKey("foo", 10, kTypeValue), + InternalKey("zoo", 20, kTypeValue), 10, 20, false, + Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncestorTime, kUnknownFileCreationTime, + 300 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + BlobReferenceMetadataDescriptor descriptor; + descriptor.index_handle = BlockHandle(10, 20); + descriptor.column_family_id = 0; + descriptor.sst_file_number = 300; + descriptor.EncodeTo( + &edit.GetMutableNewFiles()[0].second.blob_reference_descriptor); + + std::string encoded; + ASSERT_TRUE(edit.EncodeTo(&encoded, 0 /* ts_sz */)); + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded)); + ASSERT_EQ(decoded.GetNewFiles().size(), 1); + ASSERT_EQ(decoded.GetNewFiles()[0].second.blob_reference_descriptor, + edit.GetNewFiles()[0].second.blob_reference_descriptor); + ASSERT_NE(NewFileCustomTag::kBlobReferenceDescriptor & + NewFileCustomTag::kCustomTagNonSafeIgnoreMask, + 0U); + + std::string corrupt = encoded; + const std::string& descriptor_bytes = + edit.GetNewFiles()[0].second.blob_reference_descriptor; + const size_t descriptor_offset = corrupt.find(descriptor_bytes); + ASSERT_NE(descriptor_offset, std::string::npos); + corrupt[descriptor_offset] ^= 1; + ASSERT_TRUE(decoded.DecodeFrom(corrupt).IsCorruption()); +} + +TEST_F(VersionEditTest, BlobReferenceLegacyMetadataEncodeDecode) { + VersionEdit edit; + edit.AddFile(3, 300, 0, 100, InternalKey("foo", 10, kTypeValue), + InternalKey("zoo", 20, kTypeValue), 10, 20, false, + Temperature::kUnknown, kInvalidBlobFileNumber, + kUnknownOldestAncestorTime, kUnknownFileCreationTime, + 300 /* epoch_number */, kUnknownFileChecksum, + kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0, 0, true); + edit.GetMutableNewFiles()[0].second.blob_reference_legacy_metadata = true; + + std::string encoded; + ASSERT_TRUE(edit.EncodeTo(&encoded, 0 /* ts_sz */)); + VersionEdit decoded; + ASSERT_OK(decoded.DecodeFrom(encoded)); + ASSERT_EQ(decoded.GetNewFiles().size(), 1); + ASSERT_TRUE(decoded.GetNewFiles()[0].second.blob_reference_legacy_metadata); + ASSERT_TRUE( + decoded.GetNewFiles()[0].second.blob_reference_descriptor.empty()); + ASSERT_NE(NewFileCustomTag::kBlobReferenceLegacyMetadata & + NewFileCustomTag::kCustomTagNonSafeIgnoreMask, + 0U); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/version_set.cc b/db/version_set.cc index 9c4a49d24412..13d0b47e8147 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -27,6 +27,7 @@ #include "db/blob/blob_file_cache.h" #include "db/blob/blob_file_reader.h" #include "db/blob/blob_log_format.h" +#include "db/blob/blob_reference_census.h" #include "db/blob/blob_source.h" #include "db/compaction/compaction.h" #include "db/compaction/file_pri.h" @@ -60,6 +61,7 @@ #include "rocksdb/env.h" #include "rocksdb/merge_operator.h" #include "rocksdb/write_buffer_manager.h" +#include "table/adaptive/adaptive_table_factory.h" #include "table/compaction_merging_iterator.h" #include "table/format.h" #include "table/get_context.h" @@ -1048,6 +1050,16 @@ Version::~Version() { prev_->next_ = next_; next_->prev_ = prev_; + // The file table of the relocation state can be large. When this Version is + // its last owner, free the table later without the DB mutex (see + // VersionSet::AddObsoleteBlobRelocationState()). + std::shared_ptr relocation_state = + storage_info_.ReleaseBlobRelocationStateForDeferredFree(); + if (relocation_state != nullptr) { + TEST_SYNC_POINT("Version::~Version:DeferRelocationState"); + vset_->AddObsoleteBlobRelocationState(std::move(relocation_state)); + } + // Drop references to files for (int level = 0; level < storage_info_.num_levels_; level++) { for (size_t i = 0; i < storage_info_.files_[level].size(); i++) { @@ -2848,6 +2860,10 @@ VersionStorageInfo::VersionStorageInfo( offpeak_time_option_(std::move(offpeak_time_option)), periodic_compaction_phase_params_(periodic_compaction_phase_params) { if (ref_vstorage != nullptr) { + blob_relocation_state_fingerprint_high_ = + ref_vstorage->blob_relocation_state_fingerprint_high_; + blob_relocation_state_fingerprint_low_ = + ref_vstorage->blob_relocation_state_fingerprint_low_; accumulated_file_size_ = ref_vstorage->accumulated_file_size_; accumulated_raw_key_size_ = ref_vstorage->accumulated_raw_key_size_; accumulated_raw_value_size_ = ref_vstorage->accumulated_raw_value_size_; @@ -2869,7 +2885,9 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset, const MutableCFOptions& mutable_cf_options, const std::shared_ptr& io_tracer, uint64_t version_number, - EpochNumberRequirement epoch_number_requirement) + EpochNumberRequirement epoch_number_requirement, + std::optional + periodic_compaction_phase_params) : env_(vset->env_), clock_(vset->clock_), cfd_(column_family_data), @@ -2896,7 +2914,8 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset, vset->offpeak_time_option(), cfd_ == nullptr ? PeriodicCompactionPhaseParams{} - : vset->GetPeriodicCompactionPhaseParams(cfd_->GetID())), + : periodic_compaction_phase_params.value_or( + vset->GetPeriodicCompactionPhaseParams(cfd_->GetID()))), vset_(vset), next_(this), prev_(this), @@ -2908,6 +2927,100 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset, version_number_(version_number), io_tracer_(io_tracer) {} +void Version::AssertStableCacheHitHasFile(const BlobIndex& blob_index) const { +#ifdef NDEBUG + (void)blob_index; +#else + // The generation number is a file or a route in the BlobFileTable, or, when + // the BlobFileTable is empty and the generation number equals the origin, the + // origin is a normal blob file (see ResolveBlobPhysicalFile()). This does not + // call ResolveBlobPhysicalFile(), so tests can count the read path's lookups + // with its sync point. + const BlobFileTable& table = storage_info_.GetBlobFileTable(); + assert( + table.Find(blob_index.generation_file_number()) != 0 || + (table.size() == 0 && + blob_index.generation_file_number() == blob_index.origin_file_number() && + storage_info_.PeekBlobFileMetaData(blob_index.origin_file_number()) != + nullptr)); +#endif +} + +Status Version::ResolveBlobPhysicalFile(const BlobIndex& blob_index, + BlobPhysicalFile* physical_file) const { + assert(physical_file != nullptr); + TEST_SYNC_POINT("Version::ResolveBlobPhysicalFile"); + if (!blob_index.IsIndirect()) { + return Status::InvalidArgument("BlobIndex does not have a stable identity"); + } + + const uint64_t generation = blob_index.generation_file_number(); + const uint64_t origin = blob_index.origin_file_number(); + // A Version with the per-CF format fence keeps every live blob file, + // relocation file and route in its file table. So a direct read needs one + // table lookup, and a routed read needs two. + const BlobFileTable& table = storage_info_.GetBlobFileTable(); + uint64_t value = table.Find(generation); + // The file numbers of a reference, for Corruption messages. + const auto numbers = [&](uint64_t target) { + return "generation " + std::to_string(generation) + ", origin " + + std::to_string(origin) + + (target == 0 ? std::string() + : ", route target " + std::to_string(target)); + }; + if (BlobFileTable::IsRoute(value)) { + // A target is never a route key, so this is the last hop. + const uint64_t target = BlobFileTable::RouteTarget(value); + value = table.Find(target); + if (!BlobFileTable::IsRelocationFile(value)) { + return Status::Corruption("Blob relocation route target is missing", + numbers(target)); + } + RecordTick(db_statistics_, BLOB_DB_INDIRECT_READ_ROUTED); + } + if (value == 0 && table.size() == 0 && generation == origin && + storage_info_.PeekBlobFileMetaData(origin) != nullptr) { + // This branch only gives a better Corruption message. It handles an empty + // file table. A Version without the per-CF format fence has no file table, + // and none of its blob files is an indirect origin (VersionBuilder rejects + // an indirect-origin blob file without the fence). A fenced Version with an + // empty table has no blob files. So a kIndirectBlob reference to a blob + // file of this Version must not reach this branch. + return Status::Corruption("Indirect blob generation is not relocatable", + numbers(0)); + } + if (value == 0) { + return Status::Corruption("Invalid indirect blob physical file", + numbers(0)); + } + if (BlobFileTable::IsBlobFile(value)) { + // A live normal blob file: the value has not moved. So the generation + // number must be the origin, and the file must be an indirect origin. + const BlobFileMetaData* const normal = BlobFileTable::AsBlobFile(value); + if (generation != origin || !normal->IsIndirectOrigin()) { + return Status::Corruption("Indirect blob generation is not relocatable", + numbers(0)); + } + *physical_file = + BlobPhysicalFile{BlobFileOpenInfo{origin, normal->GetChecksumValue(), + normal->GetChecksumMethod()}, + normal->GetBlobFileSize()}; + return Status::OK(); + } + // A relocation file: the generation number itself (a flattened reference) or + // a route target. The reader seeks (origin, original offset), finds the group + // of values that can hold it, and checks that the group has the same origin + // and the exact original offset. Otherwise the read fails with Corruption. + if (BlobFileTable::IsLost(value)) { + // An accepted loss: the file and every old file routed to it are gone. + return Status::NotFound("Blob relocation file is marked lost"); + } + const BlobRelocationFileMetaData* const relocation = + BlobFileTable::AsRelocationFile(value); + *physical_file = relocation->GetAddition().ToPhysicalFile(); + return Status::OK(); +} + Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key, const BlobIndex& blob_index, FilePrefetchBuffer* prefetch_buffer, @@ -2918,9 +3031,34 @@ Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key, return Status::Corruption("Unexpected TTL/inlined blob index"); } - const uint64_t blob_file_number = blob_index.file_number(); + assert(blob_source_); + value->Reset(); + if (blob_index.IsIndirect()) { + // An indirect value is looked up in the stable blob cache first, before the + // route and file lookup (see BlobSource::GetIndirectBlobFromCache()). + Status status = blob_source_->GetIndirectBlobFromCache( + blob_index, /*range_offset=*/0, kWholeBlobLength, value); + if (!status.IsNotFound()) { + if (status.ok()) { + AssertStableCacheHitHasFile(blob_index); + } + return status; + } + BlobPhysicalFile physical_file; + status = ResolveBlobPhysicalFile(blob_index, &physical_file); + if (!status.ok()) { + return status; + } + return blob_source_->GetIndirectBlob(read_options, user_key, blob_index, + physical_file, mutable_cf_options_, + /*range_offset=*/0, kWholeBlobLength, + prefetch_buffer, value, bytes_read, + /*cache_checked=*/true); + } - auto blob_file_meta = storage_info_.GetBlobFileMetaData(blob_file_number); + const uint64_t blob_file_number = blob_index.file_number(); + const std::shared_ptr blob_file_meta = + storage_info_.GetBlobFileMetaData(blob_file_number); if (!blob_file_meta) { // INTEGRITY CHECK -- do not weaken. The blob index must reference a known // external blob file. No metadata (including file_number 0, the @@ -2931,8 +3069,6 @@ Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key, return UnknownBlobFileStatus(storage_info_, blob_file_number); } - assert(blob_source_); - value->Reset(); const BlobFileOpenInfo blob_file{blob_file_number, blob_file_meta->GetChecksumValue(), blob_file_meta->GetChecksumMethod()}; @@ -2954,15 +3090,40 @@ Status Version::GetBlobRange(const ReadOptions& read_options, return Status::Corruption("Unexpected TTL/inlined blob index"); } - // A strict sub-range of a compressed blob cannot be decompressed in - // isolation; the caller resolves such columns whole and slices instead. + // A value cannot be range-read when it is compressed, because a byte range + // cannot be decompressed alone; callers read the whole value and slice it. + // A range read of an uncompressed value checks no CRC. Callers send only + // strict parts of a value (see ReadPathBlobResolver::ResolveColumnRange). if (blob_index.compression() != kNoCompression) { return Status::Corruption("Cannot range-read a compressed blob"); } - const uint64_t blob_file_number = blob_index.file_number(); + assert(blob_source_); + value->Reset(); + if (blob_index.IsIndirect()) { + // As in GetBlob, the stable blob cache comes first for an indirect value. + Status status = blob_source_->GetIndirectBlobFromCache( + blob_index, range_offset, range_length, value); + if (!status.IsNotFound()) { + if (status.ok()) { + AssertStableCacheHitHasFile(blob_index); + } + return status; + } + BlobPhysicalFile physical_file; + status = ResolveBlobPhysicalFile(blob_index, &physical_file); + if (!status.ok()) { + return status; + } + return blob_source_->GetIndirectBlob( + read_options, user_key, blob_index, physical_file, mutable_cf_options_, + range_offset, range_length, /*prefetch_buffer=*/nullptr, value, + bytes_read, /*cache_checked=*/true); + } - auto blob_file_meta = storage_info_.GetBlobFileMetaData(blob_file_number); + const uint64_t blob_file_number = blob_index.file_number(); + const std::shared_ptr blob_file_meta = + storage_info_.GetBlobFileMetaData(blob_file_number); if (!blob_file_meta) { // INTEGRITY CHECK -- see Version::GetBlob. No metadata (including // file_number 0, the same-file "embedded" sentinel) means a corrupt index @@ -2970,8 +3131,6 @@ Status Version::GetBlobRange(const ReadOptions& read_options, return UnknownBlobFileStatus(storage_info_, blob_file_number); } - assert(blob_source_); - value->Reset(); const BlobFileOpenInfo blob_file{blob_file_number, blob_file_meta->GetChecksumValue(), blob_file_meta->GetChecksumMethod()}; @@ -2981,6 +3140,30 @@ Status Version::GetBlobRange(const ReadOptions& read_options, blob_index.compression(), range_offset, range_length, value, bytes_read); } +namespace { + +size_t GetOrCreateIndirectFileBatch( + const BlobPhysicalFile& file, + autovector* batches, + UnorderedMap* current_batch_by_file) { + assert(batches != nullptr); + assert(current_batch_by_file != nullptr); + const auto existing = current_batch_by_file->find(file.open_info.file_number); + if (existing != current_batch_by_file->end() && + (*batches)[existing->second].requests.size() < + MultiGetContext::MAX_BATCH_SIZE) { + return existing->second; + } + + const size_t index = batches->size(); + (*current_batch_by_file)[file.open_info.file_number] = index; + batches->push_back(BlobFileIndirectReadRequests{ + file, std::vector()}); + return index; +} + +} // namespace + void Version::MultiGetBlobLazy(const ReadOptions& read_options, autovector& reqs) const { assert(blob_source_); @@ -2997,8 +3180,10 @@ void Version::MultiGetBlobLazy(const ReadOptions& read_options, // of that file's current (most recently started, still fillable) batch. autovector whole_reqs; autovector range_reqs; + autovector indirect_reqs; std::unordered_map whole_idx; std::unordered_map range_idx; + UnorderedMap indirect_idx; for (auto& req : reqs) { assert(req.user_key); @@ -3011,9 +3196,33 @@ void Version::MultiGetBlobLazy(const ReadOptions& read_options, *req.status = Status::Corruption("Unexpected TTL/inlined blob index"); continue; } + if (blob_index.IsIndirect()) { + // A stable blob cache hit is filled at once. Only misses are resolved, + // grouped by file and read. + *req.status = blob_source_->GetIndirectBlobFromCache( + blob_index, req.range_offset, req.range_length, req.result); + if (!req.status->IsNotFound()) { + if (req.status->ok()) { + AssertStableCacheHitHasFile(blob_index); + } + continue; + } + BlobPhysicalFile physical_file; + *req.status = ResolveBlobPhysicalFile(blob_index, &physical_file); + if (!req.status->ok()) { + continue; + } + const size_t idx = GetOrCreateIndirectFileBatch( + physical_file, &indirect_reqs, &indirect_idx); + indirect_reqs[idx].requests.push_back(IndirectBlobReadRequest{ + *req.user_key, &blob_index, req.range_offset, req.range_length, + req.result, req.status, /*cache_checked=*/true}); + continue; + } const uint64_t file_number = blob_index.file_number(); - auto blob_file_meta = storage_info_.GetBlobFileMetaData(file_number); + const std::shared_ptr blob_file_meta = + storage_info_.GetBlobFileMetaData(file_number); if (!blob_file_meta) { // INTEGRITY CHECK -- see Version::GetBlob. A same-file/embedded reference // (file_number 0) must be resolved via SameFileBlobReader, not here. @@ -3083,6 +3292,12 @@ void Version::MultiGetBlobLazy(const ReadOptions& read_options, blob_source_->MultiGetBlobRange(read_options, range_reqs, /*bytes_read=*/nullptr); } + for (BlobFileIndirectReadRequests& file_requests : indirect_reqs) { + std::vector& batch = file_requests.requests; + blob_source_->MultiGetIndirectBlobFromOneFile( + read_options, file_requests.file, mutable_cf_options_, batch.size(), + batch.data(), /*bytes_read=*/nullptr); + } } void Version::MultiGetBlob( @@ -3091,10 +3306,15 @@ void Version::MultiGetBlob( assert(!blob_ctxs.empty()); autovector blob_reqs; + autovector indirect_file_reqs; + UnorderedMap indirect_file_indexes; for (auto& ctx : blob_ctxs) { const auto file_number = ctx.first; - const auto blob_file_meta = storage_info_.GetBlobFileMetaData(file_number); + // Looked up at the first legacy reference only: an indirect reference + // reads the stable blob cache first and resolves its file on a miss. + std::shared_ptr blob_file_meta; + bool blob_file_meta_looked_up = false; autovector blob_reqs_in_file; BlobReadContexts& blobs_in_file = ctx.second; @@ -3112,6 +3332,42 @@ void Version::MultiGetBlob( key_context->columns->Reset(); } + if (blob_index.HasTTL() || blob_index.IsInlined()) { + *key_context->s = + Status::Corruption("Unexpected TTL/inlined blob index"); + continue; + } + + if (blob_index.IsIndirect()) { + // A stable blob cache hit is filled at once. Only misses are resolved + // and read. + *key_context->s = blob_source_->GetIndirectBlobFromCache( + blob_index, /*range_offset=*/0, kWholeBlobLength, &blob.result); + if (!key_context->s->IsNotFound()) { + if (key_context->s->ok()) { + AssertStableCacheHitHasFile(blob_index); + } + continue; + } + BlobPhysicalFile physical_file; + *key_context->s = ResolveBlobPhysicalFile(blob_index, &physical_file); + if (!key_context->s->ok()) { + continue; + } + const size_t request_index = GetOrCreateIndirectFileBatch( + physical_file, &indirect_file_reqs, &indirect_file_indexes); + indirect_file_reqs[request_index].requests.push_back( + IndirectBlobReadRequest{ + key_context->get_context->ukey_to_get_blob_value(), &blob_index, + /*range_offset=*/0, kWholeBlobLength, &blob.result, + key_context->s, /*cache_checked=*/true}); + continue; + } + + if (!blob_file_meta_looked_up) { + blob_file_meta = storage_info_.GetBlobFileMetaData(file_number); + blob_file_meta_looked_up = true; + } if (!blob_file_meta) { // INTEGRITY CHECK -- do not weaken; see Version::GetBlob and // FileMetaData::UpdateBoundaries. A same-file/embedded reference must @@ -3120,12 +3376,6 @@ void Version::MultiGetBlob( continue; } - if (blob_index.HasTTL() || blob_index.IsInlined()) { - *key_context->s = - Status::Corruption("Unexpected TTL/inlined blob index"); - continue; - } - blob_reqs_in_file.emplace_back( key_context->get_context->ukey_to_get_blob_value(), blob_index.offset(), blob_index.size(), blob_index.compression(), @@ -3144,6 +3394,12 @@ void Version::MultiGetBlob( blob_source_->MultiGetBlob(read_options, blob_reqs, /*bytes_read=*/nullptr); } + for (BlobFileIndirectReadRequests& file_requests : indirect_file_reqs) { + std::vector& requests = file_requests.requests; + blob_source_->MultiGetIndirectBlobFromOneFile( + read_options, file_requests.file, mutable_cf_options_, requests.size(), + requests.data(), /*bytes_read=*/nullptr); + } for (auto& ctx : blob_ctxs) { BlobReadContexts& blobs_in_file = ctx.second; @@ -3428,9 +3684,110 @@ void VersionStorageInfo::GenerateLevelFilesBrief() { } } +bool VersionStorageInfo::ExpandExactL0Inputs( + std::vector* inputs) const { + if (inputs->empty() || !HasBlobIndirectionFormat()) { + return true; + } + // Inputs can be out of Version's epoch order (newest first): the L0 overlap + // search with range expansion adds files in discovery order, so a newer file + // found after the range grows comes after an older one. Expand a copy sorted + // by epoch, newest first, so no input file is lost. + const auto newer = [](const FileMetaData* lhs, const FileMetaData* rhs) { + return lhs->epoch_number > rhs->epoch_number; + }; + std::vector sorted; + const std::vector* ordered = inputs; + if (!std::is_sorted(inputs->begin(), inputs->end(), newer)) { + sorted = *inputs; + std::stable_sort(sorted.begin(), sorted.end(), newer); + ordered = &sorted; + } + const std::vector& l0 = files_[0]; + std::vector expanded; + size_t input = 0; + while (input < ordered->size()) { + const uint64_t epoch = (*ordered)[input]->epoch_number; + const size_t first_input = input++; + while (input < ordered->size() && + (*ordered)[input]->epoch_number == epoch) { + ++input; + } + const std::vector::const_iterator first = + std::lower_bound(l0.begin(), l0.end(), epoch, + [](const FileMetaData* file, uint64_t value) { + return file->epoch_number > value; + }); + std::vector::const_iterator last = first; + bool exact = false; + while (last != l0.end() && (*last)->epoch_number == epoch) { + exact |= !(*last)->blob_reference_descriptor.empty(); + ++last; + } + if (exact) { + if (epoch == kUnknownEpochNumber) { + return false; + } + for (std::vector::const_iterator file = first; + file != last; ++file) { + if ((*file)->being_compacted) { + return false; + } + expanded.push_back(*file); + } + } else { + expanded.insert(expanded.end(), ordered->begin() + first_input, + ordered->begin() + input); + } + } + inputs->swap(expanded); + return true; +} + +size_t VersionStorageInfo::L0SortedRunEnd(size_t first) const { + const std::vector& l0 = files_[0]; + assert(first < l0.size()); + const uint64_t epoch = l0[first]->epoch_number; + if (!HasBlobIndirectionFormat() || epoch == kUnknownEpochNumber) { + return first + 1; + } + bool exact = !l0[first]->blob_reference_descriptor.empty(); + size_t end = first + 1; + while (end < l0.size() && l0[end]->epoch_number == epoch) { + exact |= !l0[end]->blob_reference_descriptor.empty(); + ++end; + } + return exact ? end : first + 1; +} + +size_t VersionStorageInfo::CountL0SortedRuns( + const std::vector& files) const { + if (!HasBlobIndirectionFormat()) { + return files.size(); + } + size_t runs = 0; + size_t first = 0; + while (first < files.size()) { + const uint64_t epoch = files[first]->epoch_number; + bool exact = !files[first]->blob_reference_descriptor.empty(); + size_t end = first + 1; + if (epoch != kUnknownEpochNumber) { + while (end < files.size() && files[end]->epoch_number == epoch) { + exact |= !files[end]->blob_reference_descriptor.empty(); + ++end; + } + } + // Files of one epoch without an exact file are not a run: each one counts. + runs += exact ? 1 : end - first; + first = end; + } + return runs; +} + void VersionStorageInfo::PrepareForVersionAppend( const ImmutableOptions& immutable_options, const MutableCFOptions& mutable_cf_options) { + SealBlobRelocationState(); ComputeCompensatedSizes(); UpdateNumNonEmptyLevels(); CalculateBaseBytes(immutable_options, mutable_cf_options); @@ -3649,9 +4006,10 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded( for (auto* f : files_[0]) { level_size += f->fd.GetFileSize(); } - // Level 0 + // Level 0. An exact epoch run counts as one sorted run (see + // NumL0SortedRuns()), as in the L0 compaction score. bool level0_compact_triggered = false; - if (static_cast(files_[0].size()) >= + if (static_cast(CountL0SortedRuns(files_[0])) >= mutable_cf_options.level0_file_num_compaction_trigger || level_size >= mutable_cf_options.max_bytes_for_level_base) { level0_compact_triggered = true; @@ -3817,12 +4175,23 @@ void VersionStorageInfo::ComputeCompactionScore( // overwrites/deletions). int num_sorted_runs = 0; uint64_t total_size = 0; - for (auto* f : files_[level]) { - total_downcompact_bytes += static_cast(f->fd.GetFileSize()); - if (!f->being_compacted) { - total_size += f->compensated_file_size; + // An exact epoch run counts as one sorted run (see NumL0SortedRuns()). + // Its files are compacted together, so all or none of them are being + // compacted. + const std::vector& l0_files = files_[level]; + for (size_t first = 0; first < l0_files.size();) { + const size_t end = L0SortedRunEnd(first); + if (!l0_files[first]->being_compacted) { num_sorted_runs++; } + for (size_t i = first; i < end; ++i) { + FileMetaData* const f = l0_files[i]; + total_downcompact_bytes += static_cast(f->fd.GetFileSize()); + if (!f->being_compacted) { + total_size += f->compensated_file_size; + } + } + first = end; } if (compaction_style_ == kCompactionStyleUniversal) { // For universal compaction, we use level0 score to indicate @@ -4000,7 +4369,8 @@ void VersionStorageInfo::ComputeCompactionScore( ComputeFilesMarkedForForcedBlobGC( mutable_cf_options.blob_garbage_collection_age_cutoff, mutable_cf_options.blob_garbage_collection_force_threshold, - mutable_cf_options.enable_blob_garbage_collection); + mutable_cf_options.enable_blob_garbage_collection, + mutable_cf_options.enable_blob_indirection); ComputeFilesMarkedForReadTriggeredCompaction( mutable_cf_options.read_triggered_compaction_threshold, immutable_options.compaction_style); @@ -4205,100 +4575,10 @@ void VersionStorageInfo::ComputeFilesMarkedForForcedBlobGC( double blob_garbage_collection_age_cutoff, double blob_garbage_collection_force_threshold, bool enable_blob_garbage_collection) { - files_marked_for_forced_blob_gc_.clear(); - if (!(enable_blob_garbage_collection && - blob_garbage_collection_age_cutoff > 0.0 && - blob_garbage_collection_force_threshold < 1.0)) { - return; - } - - if (blob_files_.empty()) { - return; - } - - // Number of blob files eligible for GC based on age - const size_t cutoff_count = static_cast( - blob_garbage_collection_age_cutoff * blob_files_.size()); - if (!cutoff_count) { - return; - } - - // Compute the sum of total and garbage bytes over the batch of blob files - // currently eligible for garbage collection based on - // blob_garbage_collection_age_cutoff, and if the garbage ratio exceeds - // blob_garbage_collection_force_threshold, schedule compaction for the - // SST files that reference the oldest batch of blob files. Here is a toy - // example. Let's assume we have three SSTs 1, 2, and 3, and four blob files - // 10, 11, 12, and 13, which correspond to the range that is eligible for GC - // and satisfy the garbage ratio threshold. Also, let's say SSTs 1 and 2 both - // rely on blob file 10 and potentially some higher-numbered ones, while SST 3 - // relies on blob file 12 and potentially some higher-numbered ones. Then, the - // SST to oldest blob file mapping is as follows: - // - // SST file number Oldest blob file number - // 1 10 - // 2 10 - // 3 12 - // - // This is what the same thing looks like from the blob files' POV. (Note that - // the linked SSTs simply denote the inverse mapping of the above.) - // - // Blob file number Linked SST set - // 10 {1, 2} - // 11 {} - // 12 {3} - // 13 {} - // - // Then, the oldest batch of blob files consists of blob files 10 and 11, - // and we can get rid of them by forcing the compaction of SSTs 1 and 2. - const auto& oldest_meta = blob_files_.front(); - assert(oldest_meta); - - const auto& linked_ssts = oldest_meta->GetLinkedSsts(); - if (linked_ssts.empty()) { - // SST files that reference the oldest blob file are linked to an older, - // dropped blob file instead, so the SST files to compact are unknown. - assert(!dropped_blob_files_.empty()); - return; - } - - size_t count = 1; - uint64_t sum_total_blob_bytes = oldest_meta->GetTotalBlobBytes(); - uint64_t sum_garbage_blob_bytes = oldest_meta->GetGarbageBlobBytes(); - - assert(cutoff_count <= blob_files_.size()); - - for (; count < cutoff_count; ++count) { - const auto& meta = blob_files_[count]; - assert(meta); - - sum_total_blob_bytes += meta->GetTotalBlobBytes(); - sum_garbage_blob_bytes += meta->GetGarbageBlobBytes(); - } - - if (sum_garbage_blob_bytes < - blob_garbage_collection_force_threshold * sum_total_blob_bytes) { - return; - } - - for (uint64_t sst_file_number : linked_ssts) { - const FileLocation location = GetFileLocation(sst_file_number); - assert(location.IsValid()); - - const int level = location.GetLevel(); - assert(level >= 0); - - const size_t pos = location.GetPosition(); - - FileMetaData* const sst_meta = files_[level][pos]; - assert(sst_meta); - - if (sst_meta->being_compacted) { - continue; - } - - files_marked_for_forced_blob_gc_.emplace_back(level, sst_meta); - } + ComputeFilesMarkedForForcedBlobGC(blob_garbage_collection_age_cutoff, + blob_garbage_collection_force_threshold, + enable_blob_garbage_collection, + /*enable_blob_indirection=*/false); } void VersionStorageInfo::ComputeFilesMarkedForReadTriggeredCompaction( @@ -4394,6 +4674,149 @@ void VersionStorageInfo::AddBlobFile( blob_files_.emplace_back(std::move(blob_file_meta)); } +void VersionStorageInfo::AddBlobRelocationFile( + std::shared_ptr metadata) { + MutableBlobRelocationState().SetFile(std::move(metadata)); +} + +void VersionStorageInfo::AddBlobRelocationRoute( + const BlobRelocationRoute& route) { + MutableBlobRelocationState().SetRoute(route); +} + +std::shared_ptr +VersionStorageInfo::GetBlobRelocationFileMetaData(uint64_t file_number) const { + return BlobRelocationStateForRead().FindFile(file_number); +} + +void VersionStorageInfo::SyncBlobFileTable( + const std::vector& changed_blob_file_numbers, bool rebuild, + std::shared_ptr cache_res_mgr) { + if (blob_indirection_format_version_ == 0 || + (!rebuild && changed_blob_file_numbers.empty())) { + // No format fence, or no blob file changed: keep the shared state. + return; + } + BlobRelocationState& state = MutableBlobRelocationState(); + if (rebuild) { + // The first fenced Version: every live blob file enters the table. + state.SetCacheReservationManager(std::move(cache_res_mgr)); + for (const std::shared_ptr& file : blob_files_) { + state.SetBlobFile(file.get()); + } + return; + } + for (uint64_t file_number : changed_blob_file_numbers) { + const BlobFileMetaData* file = PeekBlobFileMetaData(file_number); + if (file != nullptr) { + state.SetBlobFile(file); + } else { + state.DeleteBlobFile(file_number); + } + } +} + +Status VersionStorageInfo::ValidateBlobRelocationState() const { + if (blob_indirection_format_version_ == 0) { + // VersionBuilder rejects each indirect-origin blob file addition without + // the fence, so a legacy Version build does not scan its blob files here. + if (!GetBlobRelocationFiles().empty() || + GetBlobRelocationRouteCount() != 0 || GetBlobFileTable().size() != 0) { + return Status::Corruption( + "Blob relocation state exists without the format fence"); + } + return Status::OK(); + } + if (blob_indirection_format_version_ != + kBlobIndirectionManifestFormatVersion) { + return Status::NotSupported("Unknown blob indirection format fence"); + } + const BlobRelocationState& state = BlobRelocationStateForRead(); + if (state.AggregateOverflow()) { + return Status::Corruption("Blob relocation physical-byte sum overflow"); + } +#ifdef NDEBUG + if (state.validated()) { + // The inherited immutable state has already been checked; the builder + // checked each change. Do not rescan every unchanged entry for each + // ordinary flush. + return Status::OK(); + } +#endif // NDEBUG + const BlobFileTable& table = state.table(); + // Check that the table agrees with the blob file and relocation file vectors: + // every live blob file and relocation file is in the table with the same + // metadata, and every other entry is a route. + size_t relocation_entries = 0; + size_t blob_entries = 0; + size_t route_entries = 0; + Status status; + table.ForEach([&](uint64_t file_number, uint64_t value, uint32_t count) { + if (!status.ok()) { + return; + } + if (count != 0 && !BlobFileTable::IsRelocationFile(value)) { + status = Status::Corruption("Blob file table has a stray route count"); + return; + } + if (BlobFileTable::IsBlobFile(value)) { + ++blob_entries; + if (PeekBlobFileMetaData(file_number) != + BlobFileTable::AsBlobFile(value)) { + status = + Status::Corruption("Blob file table disagrees with blob files"); + } + } else if (BlobFileTable::IsRelocationFile(value)) { + // The metadata and the per-target count are checked below. + ++relocation_entries; + } else { + ++route_entries; + const uint64_t target = BlobFileTable::RouteTarget(value); + const BlobRelocationRoute route(file_number, target); + status = route.Validate(); + if (!status.ok()) { + return; + } + // The target is a live relocation file, never a route key or a blob file + // (no chains, one hop). + if (!BlobFileTable::IsRelocationFile(table.Find(target))) { + status = Status::Corruption("Invalid blob relocation route target"); + } + } + }); + if (!status.ok()) { + return status; + } + if (blob_entries != blob_files_.size() || + relocation_entries != state.files().size() || + route_entries != state.route_count()) { + return Status::Corruption("Blob file table disagrees with its counts"); + } + UnorderedMap route_counts; + state.ForEachRoute([&](const BlobRelocationRoute& route) { + ++route_counts[route.GetTargetFileNumber()]; + }); + for (const std::shared_ptr& metadata : + state.files()) { + if (metadata == nullptr || metadata->GetFileNumber() == 0 || + state.PeekFile(metadata->GetFileNumber()) != metadata.get() || + GetFileLocation(metadata->GetFileNumber()).IsValid()) { + return Status::Corruption("Invalid blob relocation file index"); + } + status = metadata->GetAddition().Validate(); + if (!status.ok()) { + return status; + } + const auto count = route_counts.find(metadata->GetFileNumber()); + if (state.RouteCount(metadata->GetFileNumber()) != + (count != route_counts.end() ? count->second : 0)) { + return Status::Corruption("Invalid blob relocation route count"); + } + } + state.MarkValidated(); + return Status::OK(); +} + VersionStorageInfo::BlobFiles::const_iterator VersionStorageInfo::GetBlobFileMetaDataLB(uint64_t blob_file_number) const { return std::lower_bound( @@ -4699,6 +5122,14 @@ void VersionStorageInfo::GenerateBottommostFiles() { void VersionStorageInfo::GenerateFileLocationIndex() { size_t num_files = 0; + // Also compute the smallest coarse frontier (the oldest indirect-origin file + // number a coarse SST may reference) over all SSTs. A coarse SST can use any + // generation number at or above its frontier, so the census treats every + // indirect origin at or above the smallest frontier as in use (pinned). This + // SST walk already runs outside the DB mutex, so the census reads the result + // without scanning the SSTs under the mutex. + oldest_coarse_indirect_origin_file_number_ = kInvalidBlobFileNumber; + coarse_sst_count_ = 0; for (int level = 0; level < num_levels_; ++level) { num_files += files_[level].size(); @@ -4712,6 +5143,15 @@ void VersionStorageInfo::GenerateFileLocationIndex() { assert(meta); const uint64_t file_number = meta->fd.GetNumber(); + const uint64_t frontier = meta->oldest_coarse_indirect_origin_file_number; + if (frontier != kInvalidBlobFileNumber) { + ++coarse_sst_count_; + if (oldest_coarse_indirect_origin_file_number_ == + kInvalidBlobFileNumber || + frontier < oldest_coarse_indirect_origin_file_number_) { + oldest_coarse_indirect_origin_file_number_ = frontier; + } + } assert(file_locations_.find(file_number) == file_locations_.end()); file_locations_.emplace(file_number, FileLocation(level, pos)); @@ -4848,6 +5288,19 @@ bool Version::Unref() { return false; } +void Version::UnrefAndFreeBlobRelocationState(InstrumentedMutex* mu) { + assert(mu != nullptr); + mu->AssertHeld(); + assert(refs_ >= 1); + std::vector> holders; + if (refs_ == 1) { + holders.push_back(storage_info_.ReleaseBlobRelocationState()); + } + Unref(); + VersionSet::FreeBlobRelocationState( + &holders, mu, "Version::UnrefAndFreeBlobRelocationState:Unlocked"); +} + bool VersionStorageInfo::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { @@ -5262,7 +5715,9 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableOptions& ioptions, const MutableCFOptions& options) { // Special logic to set number of sorted runs. // It is to match the previous behavior when all files are in L0. - int num_l0_count = static_cast(files_[0].size()); + // An exact epoch run counts as one sorted run (see NumL0SortedRuns()). + num_l0_sorted_runs_ = static_cast(CountL0SortedRuns(files_[0])); + int num_l0_count = num_l0_sorted_runs_; if (compaction_style_ == kCompactionStyleUniversal) { // For universal compaction, we use level0 score to indicate // compaction score for the whole DB. Adding other levels as if @@ -5445,6 +5900,12 @@ uint64_t VersionStorageInfo::EstimateLiveDataSize() const { size += meta->GetTotalBlobBytes(); size -= meta->GetGarbageBlobBytes(); } + // Relocation files: add the full file size and subtract the garbage record + // bytes from the census. Only record bytes are subtracted, so the file + // metadata counts as live. The census garbage is a lower bound, and it is 0 + // until the census publishes it for this Version. + size += BlobRelocationStateForRead().PhysicalFileBytes(); + size -= blob_relocation_garbage_bytes_.load(std::memory_order_relaxed); return size; } @@ -5537,6 +5998,11 @@ void Version::AddLiveFiles(std::vector* live_table_files, live_blob_files->emplace_back(meta->GetBlobFileNumber()); } + for (const std::shared_ptr& metadata : + storage_info_.GetBlobRelocationFiles()) { + assert(metadata != nullptr); + live_blob_files->emplace_back(metadata->GetFileNumber()); + } } void Version::RemoveLiveFiles( @@ -5554,7 +6020,9 @@ void Version::RemoveLiveFiles( std::remove_if( blob_delete_candidates.begin(), blob_delete_candidates.end(), [this](ObsoleteBlobFileInfo& x) { - return storage_info()->GetBlobFileMetaData(x.GetBlobFileNumber()); + return storage_info()->GetBlobFileMetaData(x.GetBlobFileNumber()) || + storage_info()->GetBlobRelocationFileMetaData( + x.GetBlobFileNumber()); }), blob_delete_candidates.end()); } @@ -5632,18 +6100,47 @@ struct VersionSet::ManifestWriter { ColumnFamilyData* cfd; const autovector& edit_list; const std::function manifest_write_callback; + // Set only by DBImpl::SetOptions() (see MutableCFOptionsPublication). + // new_mutable_cf_options points to the entry of this column family. + // compaction_phase_anchor_time is the same for every writer of the call. + // ProcessManifestWrites() builds each new Version with these values and + // publishes them at install time. + const MutableCFOptions* new_mutable_cf_options; + std::optional compaction_phase_anchor_time; + // True for the first writer of a LogAndApply() call that passed a + // precondition callback (pre_cb). Such a writer is called a precondition + // writer. pre_cb checks that the caller's edit is still valid, and it runs + // only when this writer is at the queue head. Only the first writer of the + // call gets the flag, because the writers of one call are queued together + // and only the first one waits for the queue head. ProcessManifestWrites() + // uses the flag in three ways: + // - An earlier leader stops its batch before this writer, so this writer + // reaches the queue head and its pre_cb runs. + // - When this writer leads, its batch takes no writer of a later call. + // - When this writer leads a batch that writes a MANIFEST record, its edits + // that change a Version are applied without the DB mutex (see + // LogAndApplyHelper()). + // Intended user: indirect blob GC, which checks that its input is still + // installable. + bool requires_precondition; int max_file_opening_threads; explicit ManifestWriter( InstrumentedMutex* mu, ColumnFamilyData* _cfd, const autovector& e, const std::function& manifest_wcb, + bool _requires_precondition, + const MutableCFOptions* _new_mutable_cf_options, + std::optional _compaction_phase_anchor_time, int _max_file_opening_threads = 1) : done(false), cv(mu), cfd(_cfd), edit_list(e), manifest_write_callback(manifest_wcb), + new_mutable_cf_options(_new_mutable_cf_options), + compaction_phase_anchor_time(_compaction_phase_anchor_time), + requires_precondition(_requires_precondition), max_file_opening_threads(_max_file_opening_threads) {} // No copying allowed ManifestWriter(const ManifestWriter&) = delete; @@ -5901,6 +6398,9 @@ Status VersionSet::Close(FSDirectory* db_dir, InstrumentedMutex* mu) { VersionSet::~VersionSet() { // Must clean up column families to make all files "obsolete" column_family_set_.reset(); + // Free the deferred file tables. They hold only memory and a cache + // reservation, so freeing them needs no column family. + obsolete_blob_relocation_states_.clear(); for (auto& file : obsolete_files_) { // NOTE: DB is shutting down, so file is probably not obsolete, just @@ -5917,6 +6417,7 @@ VersionSet::~VersionSet() { file.DeleteMetadata(); } obsolete_files_.clear(); + obsolete_blob_files_.clear(); io_status_.PermitUncheckedError(); } @@ -5968,6 +6469,9 @@ void VersionSet::Reset() { options_file_size_ = 0; has_committed_options_file_number_ = false; pending_manifest_file_number_ = 0; + blob_indirection_format_version_ = 0; + manifest_removed_blob_physical_files_.clear(); + manifest_deleted_blob_relocation_routes_.clear(); last_sequence_.store(0); last_allocated_sequence_.store(0); last_published_sequence_.store(0); @@ -5983,6 +6487,10 @@ void VersionSet::Reset() { last_compacted_manifest_file_size_ = 0; TuneMaxManifestFileSize(); obsolete_files_.clear(); + { + MutexLock lock(&obsolete_blob_files_mutex_); + obsolete_blob_files_.clear(); + } obsolete_manifests_.clear(); wals_.Reset(); } @@ -6021,16 +6529,7 @@ void VersionSet::UpdatedMutableDbOptions( // the current Version) rather than only when the next Version is built. // Safe because periodic_compaction_phase_params_ is read only under the // DB mutex, which is held here. - for (auto* cfd : *column_family_set_) { - if (cfd->IsDropped()) { - continue; - } - Version* v = cfd->current(); - if (v != nullptr) { - v->storage_info_.periodic_compaction_phase_params_ = - GetPeriodicCompactionPhaseParams(cfd->GetID()); - } - } + RefreshCurrentVersionCompactionPhaseParams(); } } @@ -6042,17 +6541,26 @@ PeriodicCompactionPhaseParams VersionSet::GetPeriodicCompactionPhaseParams( return periodic_compaction_phaser_.ParamsForCf(cf_id); } -void VersionSet::ReanchorCompactionPhase() { - // Caller holds the DB mutex. Move the phasing anchor to now and refresh each - // CF's current Version's cached params, mirroring the refresh done for - // seed/recovery changes in UpdatedMutableDbOptions(). Used when a CF's - // periodic_compaction_seconds changes, so a turn-down's newly past-due cohort - // is spread rather than fired at once. Re-anchoring is DB-level and benign to - // CFs whose interval did not change (their not-past-due files keep phasing). +std::optional VersionSet::PrepareCompactionPhaseReanchor() const { int64_t now = 0; if (clock_ != nullptr && clock_->GetCurrentTime(&now).ok()) { - periodic_compaction_phaser_.Reanchor(static_cast(now)); + return static_cast(now); } + return std::nullopt; +} + +PeriodicCompactionPhaseParams +VersionSet::GetPeriodicCompactionPhaseParamsAtAnchor( + uint32_t cf_id, uint64_t anchor_time) const { + return periodic_compaction_phaser_.ParamsForCfAtAnchor(cf_id, anchor_time); +} + +void VersionSet::CommitCompactionPhaseReanchor(uint64_t anchor_time) { + periodic_compaction_phaser_.Reanchor(anchor_time); + RefreshCurrentVersionCompactionPhaseParams(); +} + +void VersionSet::RefreshCurrentVersionCompactionPhaseParams() { for (auto* cfd : *column_family_set_) { if (cfd->IsDropped()) { continue; @@ -6101,41 +6609,354 @@ void VersionSet::AppendVersion(ColumnFamilyData* column_family_data, v->next_->prev_ = v; } -Status VersionSet::ProcessManifestWrites( +struct VersionSet::BlobReferenceCensusInstallBatch { + // One entry per new Version: the reservation from + // BlobReferenceCensus::BeginInstallPreparation(), or 0 if the Version needs + // no install. Empty if no Version of the batch needs one. + std::vector reservations; + // The column family of each entry of `reservations`. + std::vector reservation_cfds; + // One prepared install per new Version (step 2). + std::vector installs; + // The old census states that the installs replaced (step 3). + std::vector obsolete_garbage; + // The census state of a dropped column family. + BlobReferenceCensusGarbage dropped_garbage; +}; + +void VersionSet::ReserveBlobReferenceCensusInstalls( + const autovector& versions, uint32_t pending_format_version, + BlobReferenceCensusInstallBatch* batch) const { + assert(batch != nullptr); + const auto needs_census_for_version = + [pending_format_version](Version* version) { + return version->cfd() + ->current() + ->storage_info() + ->HasBlobIndirectionFormat() || + (pending_format_version != 0 && + version->GetMutableCFOptions().enable_blob_indirection); + }; + bool needs_census_install = false; + if (pending_format_version != 0) { + for (Version* version : versions) { + if (needs_census_for_version(version)) { + needs_census_install = true; + break; + } + } + } + if (needs_census_install) { + // Reserve a census install for each new Version that needs one + // (BeginInstallPreparation() is O(1)). Other Versions get reservation 0. + batch->reservations.reserve(versions.size()); + batch->reservation_cfds.reserve(versions.size()); + for (Version* version : versions) { + ColumnFamilyData* cfd = version->cfd(); + batch->reservation_cfds.push_back(cfd); + if (needs_census_for_version(version)) { + batch->reservations.push_back( + cfd->GetOrCreateBlobReferenceCensus()->BeginInstallPreparation()); + } else { + batch->reservations.push_back(0); + } + } + } +} + +Status VersionSet::BuildBlobReferenceCensusDelta( + const autovector& batch_edits, Version* version, + std::shared_ptr* delta) const { + assert(version != nullptr); + assert(delta != nullptr); + ColumnFamilyData* const cfd = version->cfd(); + assert(cfd != nullptr); + *delta = std::make_shared(); + (*delta)->base_version_number = cfd->current()->GetVersionNumber(); + (*delta)->version_number = version->GetVersionNumber(); + (*delta)->forward_state = version->storage_info()->GetBlobRelocationState(); + (*delta)->coarse_indirect_origin_frontier = + version->storage_info()->OldestCoarseIndirectOriginFileNumber(); + + std::map added_files; + std::set removed_files; + std::map> + deleted_summaries; + std::set changed_routes; + std::set changed_physical_files; + changed_physical_files.insert( + version->storage_info()->RemovedBlobFiles().begin(), + version->storage_info()->RemovedBlobFiles().end()); + for (const VersionEdit* edit : batch_edits) { + if (edit->GetColumnFamily() != cfd->GetID()) { + continue; + } + for (const BlobRelocationRoute& route : edit->GetBlobRelocationRoutes()) { + changed_routes.insert(route.GetOldFileNumber()); + } + for (uint64_t source : edit->GetBlobRelocationRouteDeletions()) { + changed_routes.insert(source); + } + for (const BlobRelocationFileAddition& file : + edit->GetBlobRelocationFileAdditions()) { + changed_physical_files.insert(file.GetFileNumber()); + } + for (uint64_t file_number : edit->GetBlobRelocationFileDeletions()) { + changed_physical_files.insert(file_number); + } + for (uint64_t file_number : edit->GetBlobPhysicalFileLosses()) { + changed_physical_files.insert(file_number); + } + for (const BlobFileAddition& file : edit->GetBlobFileAdditions()) { + changed_physical_files.insert(file.GetBlobFileNumber()); + } + for (const BlobFileGarbage& file : edit->GetBlobFileGarbages()) { + changed_physical_files.insert(file.GetBlobFileNumber()); + } + for (uint64_t file_number : edit->GetBlobFileRetirements()) { + changed_physical_files.insert(file_number); + } + for (const std::shared_ptr& summary : + edit->GetBlobReferenceSummariesOfDeletedFiles()) { + if (summary != nullptr) { + deleted_summaries.emplace(summary->file_number, summary); + } + } + for (const auto& deletion : edit->GetDeletedFiles()) { + const uint64_t file_number = deletion.second; + if (added_files.erase(file_number) != 0) { + continue; + } + removed_files.insert(file_number); + } + for (const auto& addition : edit->GetNewFiles()) { + const FileMetaData& file = addition.second; + const uint64_t file_number = file.fd.GetNumber(); + if (removed_files.erase(file_number) != 0) { + // A trivial move deletes and re-adds the same physical SST at a + // different level. The SST keeps the same references, so the census + // does not change. + continue; + } + if (!added_files.emplace(file_number, &file).second) { + return Status::Corruption( + "Duplicate SST addition in blob reference census delta"); + } + } + } + + (*delta)->removed_files.assign(removed_files.begin(), removed_files.end()); + // To remove an exact SST, the delta needs the SST's reference summary + // from the edit (GetBlobReferenceSummariesOfDeletedFiles()). Every change, + // also a full compaction, is applied as a delta. If a removed exact SST has + // no summary there (for example, the compaction could not read it, or + // DeleteFilesInRange removed it), the census is rebuilt on the new + // Version instead. So a missing summary never fails the commit. + const VersionStorageInfo* const base_storage = cfd->current()->storage_info(); + (*delta)->removed_summaries.reserve(removed_files.size()); + for (uint64_t file_number : removed_files) { + if ((*delta)->rebuild) { + break; + } + const auto summary = deleted_summaries.find(file_number); + if (summary != deleted_summaries.end()) { + (*delta)->removed_summaries.push_back(summary->second); + continue; + } + const FileMetaData* const file = + base_storage->GetFileMetaDataByNumber(file_number); + if (file == nullptr || !file->blob_reference_descriptor.empty()) { + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:CensusRemovalWithoutGroups", + &file_number); + (*delta)->rebuild = true; + } + } + (*delta)->changed_routes.assign(changed_routes.begin(), changed_routes.end()); + (*delta)->changed_physical_files.assign(changed_physical_files.begin(), + changed_physical_files.end()); + const std::shared_ptr base_forward_state = + cfd->current()->storage_info()->GetBlobRelocationState(); + if ((*delta)->forward_state != nullptr && + (*delta)->forward_state != base_forward_state) { + const BlobRelocationState empty; + (*delta)->retained_forward_state_bytes = + (*delta)->forward_state->IncrementalRetentionBytes( + base_forward_state != nullptr ? *base_forward_state : empty, + (*delta)->changed_physical_files); + } + (*delta)->physical_files.reserve(changed_physical_files.size()); + for (uint64_t file_number : changed_physical_files) { + BlobGcPhysicalFileSummary summary; + summary.file_number = file_number; + const std::shared_ptr normal = + version->storage_info()->GetBlobFileMetaData(file_number); + if (normal != nullptr && normal->IsIndirectOrigin()) { + summary.total_record_bytes = normal->GetTotalBlobBytes(); + summary.is_indirect_origin = true; + } else { + const std::shared_ptr relocated = + version->storage_info()->GetBlobRelocationFileMetaData(file_number); + if (relocated != nullptr) { + summary.total_record_bytes = relocated->GetTotalRecordBytes(); + summary.is_lost = relocated->IsMarkedLost(); + } + } + (*delta)->physical_files.push_back(summary); + } + for (const auto& addition : added_files) { + const FileMetaData& file = *addition.second; + if (file.blob_reference_descriptor.empty()) { + if (version->storage_info()->HasBlobIndirectionFormat() && + !file.blob_reference_legacy_metadata && + file.oldest_coarse_indirect_origin_file_number == + kInvalidBlobFileNumber) { + return Status::Corruption( + "An SST added to a fenced column family has no exact blob " + "reference metadata, legacy marker or coarse frontier"); + } + continue; + } + if (file.blob_reference_summary == nullptr) { + // Flush and compaction prepare the summary of a new exact SST before they + // queue the edit. Recovered SSTs do not come here, ingestion into a + // fenced CF is rejected, and RepairDB rejects a DB that uses the format. + // So a missing summary is a bug. The SST has its reference metadata on + // disk, so the commit goes ahead and the census is rebuilt from disk. + // Indirect blob GC waits for the rebuild. +#ifndef NDEBUG + // Debug builds stop here; a test sets `expected` to check the + // release behavior below. + bool expected = false; + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:CensusAdditionWithoutSummary", + &expected); + assert(expected); +#endif // NDEBUG + ROCKS_LOG_ERROR(db_options_->info_log, + "New exact SST has no prepared blob reference " + "summary; the blob reference census is rebuilt. " + "CF %s, SST #%" PRIu64, + cfd->GetName().c_str(), addition.first); + cfd->blob_reference_census()->NoteMissingSummaryRebuild(); + (*delta)->rebuild = true; + continue; + } + (*delta)->added_files.emplace_back(file.blob_reference_summary); + } + if ((*delta)->rebuild) { + (*delta)->removed_summaries.clear(); + } + + return Status::OK(); +} + +Status VersionSet::PrepareBlobReferenceCensusInstalls( + const autovector& batch_edits, + const autovector& versions, + BlobReferenceCensusInstallBatch* batch) const { + assert(batch != nullptr); + if (batch->reservations.empty()) { + return Status::OK(); + } + batch->installs.clear(); + batch->installs.resize(versions.size()); + batch->obsolete_garbage.reserve(versions.size()); + for (size_t version_index = 0; version_index < versions.size(); + ++version_index) { + if (batch->reservations[version_index] == 0) { + continue; + } + Version* const version = versions[version_index]; + assert(version != nullptr); + ColumnFamilyData* const cfd = version->cfd(); + std::shared_ptr delta; + Status status = BuildBlobReferenceCensusDelta(batch_edits, version, &delta); + if (!status.ok()) { + return status; + } + std::shared_ptr log_block_cache; + const BlockBasedTableOptions* const table_options = + GetBlockBasedTableWriterOptions( + version->GetMutableCFOptions().table_factory.get()); + if (table_options != nullptr && table_options->block_cache != nullptr && + table_options->cache_usage_options.options_overrides + .at(CacheEntryRole::kBlobReferenceCensus) + .charged != CacheEntryRoleOptions::Decision::kDisabled) { + log_block_cache = table_options->block_cache; + } + // A failure to prepare the census install does not fail the commit: the + // census is invalidated and rebuilt instead. The sync point lets a test + // inject such a failure. + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ProcessManifestWrites:PrepareCensusInstall", &status); + if (status.ok()) { + status = cfd->blob_reference_census()->PrepareInstallWithCache( + std::move(delta), batch->reservations[version_index], + std::move(log_block_cache), &batch->installs[version_index]); + } + if (!status.ok()) { + status = cfd->blob_reference_census()->PrepareInvalidation( + batch->reservations[version_index], &batch->installs[version_index]); + if (!status.ok()) { + return status; + } + } + } + return Status::OK(); +} + +Status VersionSet::ProcessManifestWrites( std::deque& writers, InstrumentedMutex* mu, FSDirectory* dir_contains_current_file, bool new_descriptor_log, const ColumnFamilyOptions* new_cf_options, bool new_cf_options_are_sanitized, const ReadOptions& read_options, - const WriteOptions& write_options) { + const WriteOptions& write_options, const std::function& pre_cb) { mu->AssertHeld(); assert(!writers.empty()); ManifestWriter& first_writer = writers.front(); ManifestWriter* last_writer = &first_writer; + const auto publish_batch_callbacks = [&](const Status& status) { + mu->AssertHeld(); + // Runs the writer callbacks once for the whole batch. They finish + // in-memory changes that belong with this batch, for example removing the + // flushed memtables. Only finish_batch() calls this, at the end of this + // function, after the cleanup that can briefly release the DB mutex. So + // while that cleanup runs, other threads cannot see the callback effects + // yet. For example, a Flush() caller cannot see its memtables removed + // before the flush job installs its SuperVersion. See the callback + // contract in the comment of VersionSet::LogAndApply(). + for (ManifestWriter* writer : manifest_writers_) { + if (writer->manifest_write_callback) { + writer->manifest_write_callback(status); + } + if (writer == last_writer) { + break; + } + } + }; + // Takes this batch, from the queue head through `last_writer`, off - // manifest_writers_: records `s` on each writer, runs its callback, and wakes - // writers other threads wait on, then the next batch's leader. Every return - // path must do this, because the writers in `writers` live on the caller's - // stack and a queued pointer to one would outlive them. + // manifest_writers_. First it runs the batch callbacks (see + // publish_batch_callbacks). Then it records `s` on each writer, marks it + // done and signals it. Last, it signals the leader of the next batch. Every + // return path must do this, because the writers in `writers` live on the + // caller's stack and a queued pointer to one would outlive them. auto finish_batch = [&](const Status& s) { + // The callbacks run here on every path: with an OK status after the new + // Versions are installed, or with the error of a failed batch. + publish_batch_callbacks(s); while (true) { ManifestWriter* ready = manifest_writers_.front(); manifest_writers_.pop_front(); - bool need_signal = true; - for (const auto& w : writers) { - if (&w == ready) { - need_signal = false; - break; - } - } ready->status = s; ready->done = true; - if (ready->manifest_write_callback) { - (ready->manifest_write_callback)(s); - } - if (need_signal) { - ready->cv.Signal(); - } + // Only the first writer of a caller can have a waiting thread, so the + // signal does nothing for the other writers. Signalling every writer + // avoids a search of `writers` for each writer, which costs quadratic + // time in a large multi-column-family batch. + ready->cv.Signal(); if (ready == last_writer) { break; } @@ -6161,15 +6982,43 @@ Status VersionSet::ProcessManifestWrites( autovector> batch_edits_ts_sz; autovector versions; std::vector> builder_guards; + BlobReferenceCensusInstallBatch census_batch; autovector*> files_to_quarantine_if_commit_fail; autovector limbo_descriptor_log_file_number; + // Versions and builders are destroyed under the mutex; only their relocation + // state, which can be large, is freed without it. + const auto discard_candidate_versions = [&] { + std::vector> holders; + for (Version* version : versions) { + holders.push_back(version->storage_info()->ReleaseBlobRelocationState()); + delete version; + } + versions.clear(); + FreeBlobRelocationState(&holders, mu, + "VersionSet::ProcessManifestWrites:" + "CandidateRelocationStateFreedUnlocked"); + }; + const auto release_builder_guards = [&] { + std::vector> holders; + for (const auto& guard : builder_guards) { + holders.push_back( + guard->version_builder()->ReleaseBlobRelocationChanges()); + } + builder_guards.clear(); + FreeBlobRelocationState(&holders, mu, + "VersionSet::ProcessManifestWrites:" + "BuilderRelocationEditsFreedUnlocked"); + }; + // Tracking `max_last_sequence` is needed to ensure we write // `VersionEdit::last_sequence_`s in non-decreasing order according to the // recovery code's requirement. It also allows us to defer updating // `descriptor_last_sequence_` until the apply phase, after the log phase // succeeds. SequenceNumber max_last_sequence = descriptor_last_sequence_; + uint32_t pending_blob_indirection_format_version = + blob_indirection_format_version_; bool skip_manifest_write = first_writer.edit_list.front()->IsNoManifestWriteDummy(); @@ -6180,23 +7029,38 @@ Status VersionSet::ProcessManifestWrites( batch_edits.push_back(first_writer.edit_list.front()); batch_edits_ts_sz.push_back(std::nullopt); } else { - auto it = manifest_writers_.cbegin(); + // Walk the queue by index, not by iterator. LogAndApplyHelper() can + // release the DB mutex, and other callers can then add writers to + // manifest_writers_. std::deque::push_back invalidates iterators. Indexes + // stay valid, because no writer is removed from the queue during the + // batch. + size_t writer_index = 0; + // True once the batch holds every writer of the leader's caller. + bool past_leader_caller = false; size_t group_start = std::numeric_limits::max(); Status batch_apply_status; for (;;) { - assert(!(*it)->edit_list.front()->IsColumnFamilyManipulation()); - last_writer = *it; + assert(!manifest_writers_[writer_index] + ->edit_list.front() + ->IsColumnFamilyManipulation()); + last_writer = manifest_writers_[writer_index]; batch_max_file_opening_threads = std::max(batch_max_file_opening_threads, last_writer->max_file_opening_threads); assert(last_writer != nullptr); - assert(last_writer->cfd != nullptr); - if (!batch_apply_status.ok()) { + ColumnFamilyData* const writer_cfd = last_writer->cfd; + // Only a column family add has a null cfd, and a column family add or + // drop is always processed alone above. So a null cfd here is a caller + // bug. Fail the batch instead of crashing in release builds. + if (writer_cfd == nullptr) { + batch_apply_status = + Status::Corruption("null column family in MANIFEST writer"); + } else if (!batch_apply_status.ok()) { // An earlier edit failed, so apply no more edits, but keep grouping // until the loop conditions below end the batch as usual. LogAndApply() // returns once a caller's first writer is done, so none of its writers // may be left queued. - } else if (last_writer->cfd->IsDropped()) { + } else if (writer_cfd->IsDropped()) { // If we detect a dropped CF at this point, and the corresponding // version edits belong to an atomic group, then we need to find out // the preceding version edits in the same atomic group, and update @@ -6229,12 +7093,10 @@ Status VersionSet::ProcessManifestWrites( } } } else { - // We do a linear search on versions because versions is small. - // TODO(yanqin) maybe consider unordered_map Version* version = nullptr; VersionBuilder* builder = nullptr; for (int i = 0; i != static_cast(versions.size()); ++i) { - uint32_t cf_id = last_writer->cfd->GetID(); + uint32_t cf_id = writer_cfd->GetID(); if (versions[i]->cfd()->GetID() == cf_id) { version = versions[i]; assert(!builder_guards.empty() && @@ -6249,13 +7111,21 @@ Status VersionSet::ProcessManifestWrites( // WAL and OPTIONS protocol manipulations do not change LSM Versions. if (!last_writer->IsAllVersionIndependentEdits()) { version = new Version( - last_writer->cfd, this, file_options_, - last_writer->cfd ? last_writer->cfd->GetLatestMutableCFOptions() - : MutableCFOptions(*new_cf_options), - io_tracer_, current_version_number_++); + writer_cfd, this, file_options_, + last_writer->new_mutable_cf_options != nullptr + ? *last_writer->new_mutable_cf_options + : writer_cfd->GetLatestMutableCFOptions(), + io_tracer_, current_version_number_++, + EpochNumberRequirement::kMustPresent, + last_writer->compaction_phase_anchor_time.has_value() + ? std::make_optional( + GetPeriodicCompactionPhaseParamsAtAnchor( + writer_cfd->GetID(), + *last_writer->compaction_phase_anchor_time)) + : std::nullopt); versions.push_back(version); builder_guards.emplace_back(new BaseReferencedVersionBuilder( - last_writer->cfd, last_writer->cfd->current())); + writer_cfd, writer_cfd->current())); builder = builder_guards.back()->version_builder(); } assert(last_writer->IsAllVersionIndependentEdits() || builder); @@ -6263,7 +7133,7 @@ Status VersionSet::ProcessManifestWrites( TEST_SYNC_POINT_CALLBACK( "VersionSet::ProcessManifestWrites:NewVersion", version); } - const Comparator* ucmp = last_writer->cfd->user_comparator(); + const Comparator* ucmp = writer_cfd->user_comparator(); assert(ucmp); std::optional edit_ts_sz = ucmp->timestamp_size(); for (const auto& e : last_writer->edit_list) { @@ -6276,25 +7146,104 @@ Status VersionSet::ProcessManifestWrites( } else if (group_start != std::numeric_limits::max()) { group_start = std::numeric_limits::max(); } - batch_apply_status = LogAndApplyHelper(last_writer->cfd, builder, e, - &max_last_sequence, mu); + // The re-add check runs once the batch has the format fence: the DB + // already has it, an earlier edit of this batch set it, or this edit + // sets blob indirection state (see + // VersionEdit::SetsBlobIndirectionState()). Only an edit in a fenced + // batch can hold relocation records, so a legacy batch does no new + // work here. + if (pending_blob_indirection_format_version != 0 || + e->SetsBlobIndirectionState()) { + batch_apply_status = RejectReaddedBlobRelocationState(*e); + } + // Only the leader can be a precondition writer in this batch, + // because the loop stops before any other precondition writer (see + // the stop rules below). So pre_cb is this writer's own callback. A + // batch that writes no MANIFEST record (skip_manifest_write) applies + // its edits under the DB mutex, so its pre_cb runs only once. + if (batch_apply_status.ok()) { + batch_apply_status = LogAndApplyHelper( + writer_cfd, builder, e, &max_last_sequence, mu, + last_writer->requires_precondition && !skip_manifest_write, + pre_cb); + } if (!batch_apply_status.ok()) { break; // inner for-`e` loop } batch_edits.push_back(e); batch_edits_ts_sz.push_back(edit_ts_sz); + if (e->HasBlobIndirectionFormatVersion()) { + pending_blob_indirection_format_version = + e->GetBlobIndirectionFormatVersion(); + } else if (e->HasDbBlobIndirectionFormatVersion()) { + pending_blob_indirection_format_version = + e->GetDbBlobIndirectionFormatVersion(); + } } } - // Loop increment/conditions - ++it; - if (it == manifest_writers_.cend()) { + // Loop increment/conditions. The batch ends at the first of these stop + // rules, in this order: + // 1. The queue has no more writers. + // 2. A batch that writes no MANIFEST record reaches its caller's last + // writer. + // 3. The batch holds all writers of the leader's caller, and the leader + // or the next writer has prepared options. + // 4. The batch holds all writers of the leader's caller, and the leader + // is a precondition writer. + // 5. The next writer is a precondition writer. + // 6. The next writer adds or drops a column family, or is a dummy edit + // that writes no MANIFEST record. + // A batch that writes no MANIFEST record checks only rules 1 and 2. + ++writer_index; + if (writer_index == manifest_writers_.size()) { break; } if (skip_manifest_write) { - // no grouping when skipping manifest write + // A batch that writes nothing to the MANIFEST (from SetOptions) holds + // all the writers of its one caller, and ends at that caller's last + // writer. It never takes a writer of a later caller, whose edits can + // need a MANIFEST write or can name a column family that is already in + // this batch. + if (last_writer == &writers.back()) { + break; + } + continue; + } + // Grouping rule for prepared options: once the batch holds all writers + // of the leader's caller, stop if the leader has prepared options, or + // before a next writer that has prepared options. The writers of one + // caller are queued together, so every writer after writers.back() + // belongs to another caller. The rule matters only on the test-only + // MANIFEST path of SetOptions; see step 2 in the + // MutableCFOptionsPublication comment in version_set.h. + if (last_writer == &writers.back()) { + past_leader_caller = true; + } + if (past_leader_caller && + (first_writer.new_mutable_cf_options != nullptr || + manifest_writers_[writer_index]->new_mutable_cf_options != + nullptr)) { break; } - const auto* next = (*it)->edit_list.front(); + if (past_leader_caller && first_writer.requires_precondition) { + // A leader that is a precondition writer is a batch barrier: the + // batch ends after the writers of the leader's caller. Absorbing a + // later caller could invalidate the state that the precondition + // checked. Also, if the precondition fails after the unlocked apply + // (see LogAndApplyHelper()), its error must fail only its own caller, + // not the writers queued after it (also ones that joined while the DB + // mutex was released). We estimate that the lost grouping costs + // little, because precondition writers (for example indirect blob GC + // installs) are rare. + break; + } + if (manifest_writers_[writer_index]->requires_precondition) { + // Stop the group here. This writer becomes the queue head later, and + // its caller checks the precondition against all Versions committed + // before it. + break; + } + const auto* next = manifest_writers_[writer_index]->edit_list.front(); if (next->IsColumnFamilyManipulation() || next->IsNoManifestWriteDummy()) { // no group commits for column family add or drop @@ -6303,9 +7252,8 @@ Status VersionSet::ProcessManifestWrites( } } if (!batch_apply_status.ok()) { - for (auto v : versions) { - delete v; - } + discard_candidate_versions(); + release_builder_guards(); finish_batch(batch_apply_status); return batch_apply_status; } @@ -6313,12 +7261,10 @@ Status VersionSet::ProcessManifestWrites( assert(!builder_guards.empty() && builder_guards.size() == versions.size()); auto* builder = builder_guards[i]->version_builder(); - Status s = builder->SaveTo(versions[i]->storage_info()); + Status s = builder->SaveTableFilesTo(versions[i]->storage_info()); if (!s.ok()) { - // free up the allocated memory - for (auto v : versions) { - delete v; - } + discard_candidate_versions(); + release_builder_guards(); finish_batch(s); return s; } @@ -6360,8 +7306,7 @@ Status VersionSet::ProcessManifestWrites( k = i; } if (skip_manifest_write) { - // no grouping when skipping manifest write - assert(last_writer == &first_writer); + assert(last_writer == &writers.back()); } #endif // NDEBUG @@ -6430,27 +7375,77 @@ Status VersionSet::ProcessManifestWrites( uint64_t new_manifest_file_size = 0; Status s; + // True if blob finalization failed. That failure happens before any MANIFEST + // record is written, so the error path below keeps the current MANIFEST + // writer. + bool blob_finalization_failed = false; IOStatus io_s; IOStatus manifest_io_status; manifest_io_status.PermitUncheckedError(); std::unique_ptr new_desc_log_ptr; // Save before releasing mu uint64_t manifest_preallocation_size = manifest_preallocation_size_; + // Step 1 of the census install (see BlobReferenceCensusInstallBatch). + ReserveBlobReferenceCensusInstalls( + versions, pending_blob_indirection_format_version, &census_batch); + // Calls VersionBuilder::SaveBlobFilesAndCheckTo() for every new Version, with + // the DB mutex released. A column family add or drop builds no new Version, + // so for that batch this does nothing. This is safe in both the MANIFEST path + // and the no-MANIFEST path, because this batch stays at the head of + // manifest_writers_ while the mutex is released, and only the batch at the + // head installs Versions. Both paths check this in debug builds after they + // lock mu again below. + const auto finalize_blob_files = [&]() { + TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:BeforeBlobFinalization"); + assert(builder_guards.size() == versions.size()); + Status status; + for (size_t i = 0; status.ok() && i < versions.size(); ++i) { + status = builder_guards[i]->version_builder()->SaveBlobFilesAndCheckTo( + versions[i]->storage_info()); + } + TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:AfterBlobFinalization"); + blob_finalization_failed = !status.ok(); + return status; + }; if (skip_manifest_write) { + // As in the MANIFEST path below, PrepareAppend() runs without the DB + // mutex. The new Versions are not shared yet. PrepareAppend() can also + // fill in the stats of FileMetaData objects that older Versions share. + // This is the same as in the MANIFEST path, and only the batch at the head + // of manifest_writers_ does it. + mu->Unlock(); + s = finalize_blob_files(); if (s.ok()) { constexpr bool update_stats = true; for (int i = 0; i < static_cast(versions.size()); ++i) { - // NOTE: normally called with DB mutex released, but we don't - // want to release the DB mutex in this mode of LogAndApply versions[i]->PrepareAppend(read_options, update_stats); } } + // Step 2 of the census install. Run it after PrepareAppend(), which + // computes the coarse frontier that the census delta carries. + if (s.ok()) { + s = PrepareBlobReferenceCensusInstalls(batch_edits, versions, + &census_batch); + } + mu->Lock(); + // As in the MANIFEST path below: while the DB mutex was released, this + // batch stayed at the head of manifest_writers_, so the base Version of + // each builder is still the current Version of its column family. +#ifndef NDEBUG + assert(manifest_writers_.front() == &first_writer); + assert(builder_guards.size() == versions.size()); + for (size_t i = 0; i < versions.size(); ++i) { + assert(versions[i]->cfd_->current() == builder_guards[i]->base_version()); + } +#endif // NDEBUG } else { FileOptions opt_file_opts = GetFileOptionsForManifestWrite(); mu->Unlock(); TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifestStart"); TEST_SYNC_POINT_CALLBACK("VersionSet::LogAndApply:WriteManifest", nullptr); - if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) { + s = finalize_blob_files(); + if (s.ok() && + !first_writer.edit_list.front()->IsColumnFamilyManipulation()) { for (int i = 0; i < static_cast(versions.size()); ++i) { assert(!builder_guards.empty() && builder_guards.size() == versions.size()); @@ -6485,9 +7480,9 @@ Status VersionSet::ProcessManifestWrites( CreateManifestWriter(std::move(descriptor_file), descriptor_fname, opt_file_opts, manifest_preallocation_size); raw_desc_log_ptr = new_desc_log_ptr.get(); - s = WriteCurrentStateToManifest(write_options, curr_state, - wal_additions, options_manifest_state, - raw_desc_log_ptr, io_s); + s = WriteCurrentStateToManifest( + write_options, curr_state, wal_additions, options_manifest_state, + pending_blob_indirection_format_version, raw_desc_log_ptr, io_s); assert(s == io_s); } if (!io_s.ok()) { @@ -6503,8 +7498,13 @@ Status VersionSet::ProcessManifestWrites( for (int i = 0; i < static_cast(versions.size()); ++i) { versions[i]->PrepareAppend(read_options, update_stats); } + // Step 2 of the census install. Run it after PrepareAppend(), which + // computes the coarse frontier that the census delta carries. + s = PrepareBlobReferenceCensusInstalls(batch_edits, versions, + &census_batch); } - + } + if (s.ok()) { // Write new records to MANIFEST log #ifndef NDEBUG size_t idx = 0; @@ -6600,9 +7600,26 @@ Status VersionSet::ProcessManifestWrites( LogFlush(db_options_->info_log); TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifestDone"); mu->Lock(); + // While the DB mutex was released, this batch stayed at the head of + // manifest_writers_. Only the batch at the head installs Versions, so the + // current Version of each column family is still the base Version of its + // builder, and the new Versions built on top of it are still valid. The + // checks below verify this in debug builds. +#ifndef NDEBUG + assert(!manifest_writers_.empty()); + assert(manifest_writers_.front() == &first_writer); + if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) { + assert(builder_guards.size() == versions.size()); + for (size_t i = 0; i < versions.size(); ++i) { + assert(versions[i]->cfd_->current() == + builder_guards[i]->base_version()); + } + } +#endif } if (s.ok()) { + blob_indirection_format_version_ = pending_blob_indirection_format_version; // Apply WAL edits, DB mutex must be held. for (auto& e : batch_edits) { if (e->IsWalAddition()) { @@ -6616,6 +7633,8 @@ Status VersionSet::ProcessManifestWrites( } } + const bool recovering_manifest_authority = + s.ok() && new_descriptor_log && !skip_manifest_write && !io_status_.ok(); if (!io_s.ok()) { if (io_status_.ok()) { io_status_ = io_s; @@ -6653,6 +7672,8 @@ Status VersionSet::ProcessManifestWrites( } else if (first_writer.edit_list.front()->IsColumnFamilyDrop()) { assert(batch_edits.size() == 1); assert(max_last_sequence == descriptor_last_sequence_); + census_batch.dropped_garbage = + first_writer.cfd->CancelBlobReferenceCensus(); first_writer.cfd->SetDropped(); first_writer.cfd->UnrefAndTryDelete(); } else { @@ -6686,12 +7707,54 @@ Status VersionSet::ProcessManifestWrites( MarkMinLogNumberToKeep(last_min_log_number_to_keep); } +#ifndef NDEBUG + // Only the first caller's writers can carry prepared options (see the + // grouping rule for prepared options in the loop above). The first + // caller's writers are the first writers.size() writers of the queue. + { + size_t position = 0; + for (ManifestWriter* writer : manifest_writers_) { + assert(position < writers.size() || + writer->new_mutable_cf_options == nullptr); + if (writer == last_writer) { + break; + } + ++position; + } + } +#endif // NDEBUG + // Publish the options of a DBImpl::SetOptions() call: step 5 in the + // MutableCFOptionsPublication comment in version_set.h. A batch that + // holds a SetOptions() call holds only its writers, so first_writer + // decides for the whole batch. CommitCompactionPhaseReanchor() also + // refreshes the phasing parameters in the current Version of every + // column family that is not dropped. This matters for the column + // families that are not in this call. + if (first_writer.new_mutable_cf_options != nullptr) { + if (first_writer.compaction_phase_anchor_time.has_value()) { + CommitCompactionPhaseReanchor( + *first_writer.compaction_phase_anchor_time); + } + for (Version* version : versions) { + version->cfd_->PublishMutableCFOptions( + version->GetMutableCFOptions()); + } + } + for (int i = 0; i < static_cast(versions.size()); ++i) { ColumnFamilyData* cfd = versions[i]->cfd_; AppendVersion(cfd, versions[i]); + // Step 3 of the census install. + if (!census_batch.installs.empty() && + census_batch.reservations[i] != 0) { + census_batch.obsolete_garbage.emplace_back( + cfd->blob_reference_census()->Install(census_batch.installs[i])); + cfd->blob_reference_census()->NoteUnready(clock_->NowMicros()); + } } } if (!skip_manifest_write) { + RecordBlobRelocationDeletions(batch_edits, new_descriptor_log); for (const auto* edit : batch_edits) { assert(edit != nullptr); if (edit->HasCommittedOptionsFileNumber()) { @@ -6707,6 +7770,23 @@ Status VersionSet::ProcessManifestWrites( manifest_last_valid_record_end_file_number_ = manifest_file_number_; prev_log_number_ = first_writer.edit_list.front()->GetPrevLogNumber(); } + if (recovering_manifest_authority) { + // An earlier MANIFEST write failed, so it may or may not be on disk, and + // descriptor_log_ was reset. This write created a new MANIFEST with a + // full snapshot of all CFs and published it through CURRENT, so the + // on-disk state is known again. Only such a write calls + // ResolveMetadataCommitAmbiguity() on the census of every CF. A normal + // successful append does not. + for (ColumnFamilyData* cfd : *column_family_set_) { + if (cfd->blob_reference_census() != nullptr) { + cfd->blob_reference_census()->ResolveMetadataCommitAmbiguity(); + } + } + } + } else if (blob_finalization_failed) { + // Nothing was written to the MANIFEST. As for a SaveTableFilesTo() failure, + // drop the new Versions and keep the current MANIFEST writer. + discard_candidate_versions(); } else { std::string version_edits; for (auto& e : batch_edits) { @@ -6715,9 +7795,7 @@ Status VersionSet::ProcessManifestWrites( ROCKS_LOG_ERROR(db_options_->info_log, "Error in committing version edit to MANIFEST: %s", version_edits.c_str()); - for (auto v : versions) { - delete v; - } + discard_candidate_versions(); // If manifest append failed for whatever reason, the file could be // corrupted. So we need to force the next version update to start a // new manifest file. @@ -6765,6 +7843,56 @@ Status VersionSet::ProcessManifestWrites( } } + // The caller of a successful options publication installs the SuperVersions + // after this returns, so census payloads are freed after that, without the + // mutex. See ReleaseDeferredPublicationCleanup(). + const bool defer_publication_cleanup = + s.ok() && first_writer.new_mutable_cf_options != nullptr; + if (defer_publication_cleanup) { + for (size_t index = 0; index < census_batch.reservations.size(); ++index) { + if (census_batch.reservations[index] != 0) { + census_batch.reservation_cfds[index] + ->blob_reference_census() + ->CancelInstallPreparation(census_batch.reservations[index]); + } + } + census_batch.reservations.clear(); + for (auto& install : census_batch.installs) { + deferred_publication_census_installs_.push_back(std::move(install)); + } + census_batch.installs.clear(); + for (auto& garbage : census_batch.obsolete_garbage) { + deferred_publication_census_garbage_.push_back(std::move(garbage)); + } + census_batch.obsolete_garbage.clear(); + } + if (!census_batch.installs.empty() || !census_batch.reservations.empty() || + !census_batch.obsolete_garbage.empty()) { + // Step 4 of the census install: + // 1. Under the DB mutex, cancel the unused census reservations. A + // cancellation can wake a census build that waits between slices, and + // that changes the census queue of its CF. + // 2. Without the mutex, free the large objects: the census installs and the + // old census state. + for (size_t index = 0; index < census_batch.reservations.size(); ++index) { + if (census_batch.reservations[index] != 0) { + census_batch.reservation_cfds[index] + ->blob_reference_census() + ->CancelInstallPreparation(census_batch.reservations[index]); + } + } + mu->Unlock(); + TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:CleanupUnlocked"); + std::vector().swap( + census_batch.installs); + std::vector().swap(census_batch.reservations); + std::vector().swap(census_batch.reservation_cfds); + std::vector().swap( + census_batch.obsolete_garbage); + census_batch.dropped_garbage = {}; + mu->Lock(); + } + pending_manifest_file_number_ = 0; #ifndef NDEBUG @@ -6784,10 +7912,49 @@ Status VersionSet::ProcessManifestWrites( } #endif // NDEBUG + release_builder_guards(); finish_batch(s); return s; } +void VersionSet::ReleaseDeferredPublicationCleanup(InstrumentedMutex* mu) { + mu->AssertHeld(); + if (deferred_publication_census_installs_.empty() && + deferred_publication_census_garbage_.empty()) { + return; + } + std::vector census_installs; + census_installs.swap(deferred_publication_census_installs_); + std::vector census_garbage; + census_garbage.swap(deferred_publication_census_garbage_); + mu->Unlock(); + TEST_SYNC_POINT_CALLBACK( + "VersionSet::ReleaseDeferredPublicationCleanup:Unlocked", mu); + census_installs.clear(); + census_garbage.clear(); + mu->Lock(); +} + +void VersionSet::FreeBlobRelocationState( + std::vector>* holders, InstrumentedMutex* mu, + [[maybe_unused]] const char* unlocked_sync_point) { + assert(holders != nullptr); + mu->AssertHeld(); + // A holder that is not the last owner frees nothing; drop it here. + holders->erase(std::remove_if(holders->begin(), holders->end(), + [](const std::shared_ptr& holder) { + return holder.use_count() != 1; + }), + holders->end()); + if (holders->empty()) { + return; + } + mu->Unlock(); + TEST_SYNC_POINT_CALLBACK(unlocked_sync_point, mu); + holders->clear(); + mu->Lock(); +} + void VersionSet::WakeUpWaitingManifestWriters() { // wake up all the waiting writers // Notify new head of manifest write queue. @@ -6805,12 +7972,13 @@ Status VersionSet::LogAndApply( InstrumentedMutex* mu, FSDirectory* dir_contains_current_file, bool new_descriptor_log, const ColumnFamilyOptions* new_cf_options, const std::vector>& manifest_wcbs, - const std::function& pre_cb, int max_file_opening_threads) { + const std::function& pre_cb, int max_file_opening_threads, + const MutableCFOptionsPublication* options_publication) { return LogAndApplyImpl(column_family_datas, read_options, write_options, edit_lists, mu, dir_contains_current_file, new_descriptor_log, new_cf_options, /*new_cf_options_are_sanitized=*/false, manifest_wcbs, - pre_cb, max_file_opening_threads); + pre_cb, max_file_opening_threads, options_publication); } Status VersionSet::LogAndApplyColumnFamilyAddWithSanitizedOptions( @@ -6828,7 +7996,8 @@ Status VersionSet::LogAndApplyColumnFamilyAddWithSanitizedOptions( /*new_descriptor_log=*/false, &sanitized_options, /*new_cf_options_are_sanitized=*/true, /*manifest_wcbs=*/{}, /*pre_cb=*/{}, - /*max_file_opening_threads=*/1); + /*max_file_opening_threads=*/1, + /*options_publication=*/nullptr); } Status VersionSet::LogAndApplyImpl( @@ -6839,7 +8008,8 @@ Status VersionSet::LogAndApplyImpl( bool new_descriptor_log, const ColumnFamilyOptions* new_cf_options, bool new_cf_options_are_sanitized, const std::vector>& manifest_wcbs, - const std::function& pre_cb, int max_file_opening_threads) { + const std::function& pre_cb, int max_file_opening_threads, + const MutableCFOptionsPublication* options_publication) { mu->AssertHeld(); int num_edits = 0; for (const auto& elist : edit_lists) { @@ -6849,16 +8019,31 @@ Status VersionSet::LogAndApplyImpl( return Status::OK(); } else if (num_edits > 1) { #ifndef NDEBUG + const bool no_manifest_batch = + edit_lists.front().front()->IsNoManifestWriteDummy(); for (const auto& edit_list : edit_lists) { for (const auto& edit : edit_list) { assert(!edit->IsColumnFamilyManipulation()); - assert(!edit->IsNoManifestWriteDummy()); + assert(edit->IsNoManifestWriteDummy() == no_manifest_batch); } } #endif /* ! NDEBUG */ } int num_cfds = static_cast(column_family_datas.size()); +#ifndef NDEBUG + if (options_publication != nullptr) { + // ProcessManifestWrites() gives all writers of one column family one shared + // Version, built with the options of the first writer. The options of the + // other writers would be lost, so when `options_publication` is set, + // each column family may appear only once in `column_family_datas`. + UnorderedSet unique_cfds; + for (ColumnFamilyData* cfd : column_family_datas) { + assert(cfd != nullptr); + assert(unique_cfds.insert(cfd).second); + } + } +#endif if (num_cfds == 1 && column_family_datas[0] == nullptr) { assert(edit_lists.size() == 1 && edit_lists[0].size() == 1); assert(edit_lists[0][0]->IsColumnFamilyAdd()); @@ -6867,12 +8052,23 @@ Status VersionSet::LogAndApplyImpl( std::deque writers; if (num_cfds > 0) { assert(static_cast(num_cfds) == edit_lists.size()); + assert(options_publication == nullptr || + static_cast(num_cfds) == + options_publication->new_mutable_cf_options.size()); } for (int i = 0; i < num_cfds; ++i) { const auto wcb = manifest_wcbs.empty() ? [](const Status&) {} : manifest_wcbs[i]; - writers.emplace_back(mu, column_family_datas[i], edit_lists[i], wcb, - max_file_opening_threads); + writers.emplace_back( + mu, column_family_datas[i], edit_lists[i], wcb, + /*requires_precondition=*/i == 0 && !!pre_cb, + options_publication == nullptr + ? nullptr + : &options_publication->new_mutable_cf_options[i], + options_publication == nullptr + ? std::nullopt + : options_publication->compaction_phase_anchor_time, + max_file_opening_threads); manifest_writers_.push_back(&writers[i]); } assert(!writers.empty()); @@ -6928,7 +8124,7 @@ Status VersionSet::LogAndApplyImpl( return ProcessManifestWrites(writers, mu, dir_contains_current_file, new_descriptor_log, new_cf_options, new_cf_options_are_sanitized, read_options, - write_options); + write_options, pre_cb); } } @@ -6946,10 +8142,64 @@ void VersionSet::LogAndApplyCFHelper(VersionEdit* edit, } } +Status VersionSet::RejectReaddedBlobRelocationState( + const VersionEdit& edit) const { + for (const BlobRelocationFileAddition& addition : + edit.GetBlobRelocationFileAdditions()) { + if (manifest_removed_blob_physical_files_.count(addition.GetFileNumber()) != + 0) { + return Status::Corruption( + "Relocation file was deleted earlier in this MANIFEST"); + } + } + for (const BlobFileAddition& addition : edit.GetBlobFileAdditions()) { + if (manifest_removed_blob_physical_files_.count( + addition.GetBlobFileNumber()) != 0) { + return Status::Corruption( + "Blob file was retired earlier in this MANIFEST"); + } + } + for (const BlobRelocationRoute& route : edit.GetBlobRelocationRoutes()) { + if (manifest_deleted_blob_relocation_routes_.count( + route.GetOldFileNumber()) != 0) { + return Status::Corruption( + "Relocation route was deleted earlier in this MANIFEST"); + } + } + return Status::OK(); +} + +void VersionSet::RecordBlobRelocationDeletions( + const autovector& edits, bool new_manifest) { + if (new_manifest) { + // The snapshot that starts a MANIFEST has no deletion history. + manifest_removed_blob_physical_files_.clear(); + manifest_deleted_blob_relocation_routes_.clear(); + } + for (const VersionEdit* edit : edits) { + manifest_removed_blob_physical_files_.insert( + edit->GetBlobRelocationFileDeletions().begin(), + edit->GetBlobRelocationFileDeletions().end()); + manifest_removed_blob_physical_files_.insert( + edit->GetBlobFileRetirements().begin(), + edit->GetBlobFileRetirements().end()); + manifest_deleted_blob_relocation_routes_.insert( + edit->GetBlobRelocationRouteDeletions().begin(), + edit->GetBlobRelocationRouteDeletions().end()); + } +} + Status VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd, VersionBuilder* builder, VersionEdit* edit, SequenceNumber* max_last_sequence, - InstrumentedMutex* mu) { + InstrumentedMutex* mu, bool apply_unlocked, + const std::function& pre_cb) { + // mu is used on every path (AssertHeld() below), and the unlocked apply must + // release it and take it again. A null mu is a caller bug, so return an + // error instead of crashing. + if (mu == nullptr) { + return Status::InvalidArgument("Null DB mutex in LogAndApply"); + } #ifdef NDEBUG (void)cfd; #endif @@ -6976,7 +8226,36 @@ Status VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd, // LSM Version, so they do not need a VersionBuilder. assert(builder || edit->IsWalManipulation() || edit->IsOptionsFileManipulation()); - return builder ? builder->Apply(edit) : Status::OK(); + if (builder == nullptr) { + return Status::OK(); + } + if (!apply_unlocked) { + return builder->Apply(edit); + } + + // Unlocked apply: a precondition writer (see + // ManifestWriter::requires_precondition) applies the edit without the DB + // mutex, because the edit can be large (one change per route). This is safe + // for these reasons: + // - The edit and the builder belong to this writer. The builder holds a + // reference to its base Version, so that Version stays alive. + // - The writer stays at the queue head, so no other Version is installed + // and no other MANIFEST edit is applied meanwhile. + // - The caller, ProcessManifestWrites(), already ran the re-add check + // (RejectReaddedBlobRelocationState()) of a batch with the format fence + // under the DB mutex, against that same state. + // State outside the Versions that pre_cb depends on can change, so pre_cb + // runs again below. + mu->Unlock(); + TEST_SYNC_POINT("VersionSet::LogAndApplyHelper:ApplyUnlocked"); + Status status = builder->Apply(edit); + mu->Lock(); + // The caller's state may change while Apply() runs without the DB mutex. + // Revalidate before this candidate Version can be written or installed. + if (status.ok() && pre_cb) { + status = pre_cb(); + } + return status; } FileOptions VersionSet::GetFileOptionsForManifestWrite() const { @@ -7327,6 +8606,8 @@ Status VersionSet::Recover( uint64_t current_manifest_file_size = 0; uint64_t log_number = 0; + UnorderedSet removed_blob_physical_files; + UnorderedSet deleted_blob_relocation_routes; { force_new_manifest_on_open_ = false; VersionSet::LogReporter reporter; @@ -7349,6 +8630,8 @@ Status VersionSet::Recover( handler.GetDbId(db_id); } if (s.ok()) { + handler.TakeBlobRelocationDeletions(&removed_blob_physical_files, + &deleted_blob_relocation_routes); RecoverEpochNumbers(); } if (log_status) { @@ -7383,6 +8666,14 @@ Status VersionSet::Recover( if (s.ok() && !read_only && db_options_->reuse_manifest_on_open && manifest_recovery_last_valid_record_end_ > 0) { s = ReopenManifestForAppend(manifest_path); + if (s.ok() && descriptor_log_ != nullptr) { + // Appends go to the replayed MANIFEST, which already holds these + // deletions. + manifest_removed_blob_physical_files_ = + std::move(removed_blob_physical_files); + manifest_deleted_blob_relocation_routes_ = + std::move(deleted_blob_relocation_routes); + } } return s; @@ -8052,6 +9343,22 @@ Status VersionSet::GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) { return s; } } + + for (const std::shared_ptr& metadata : + vstorage->GetBlobRelocationFiles()) { + assert(metadata != nullptr); + std::string checksum_value = metadata->GetChecksumValue(); + std::string checksum_method = metadata->GetChecksumMethod(); + if (checksum_method.empty()) { + checksum_value = kUnknownFileChecksum; + checksum_method = kUnknownFileChecksumFuncName; + } + s = checksum_list->InsertOneFileChecksum(metadata->GetFileNumber(), + checksum_value, checksum_method); + if (!s.ok()) { + return s; + } + } } return s; @@ -8134,7 +9441,8 @@ Status VersionSet::WriteCurrentStateToManifest( const std::unordered_map& curr_state, const VersionEdit& wal_additions, const OptionsFileManifestState& options_file_manifest_state, - log::Writer* log, IOStatus& io_s) { + uint32_t blob_indirection_format_version, log::Writer* log, + IOStatus& io_s) { // TODO: Break up into multiple records to reduce memory usage on recovery? // WARNING: This method doesn't hold a mutex!! @@ -8144,6 +9452,24 @@ Status VersionSet::WriteCurrentStateToManifest( // LogAndApply (the same single thread), so we're safe to iterate. assert(io_s.ok()); + if (log == nullptr) { + return Status::InvalidArgument("null MANIFEST writer"); + } + if (blob_indirection_format_version != 0) { + VersionEdit format_fence; + format_fence.SetColumnFamily(0); + format_fence.SetDbBlobIndirectionFormatVersion( + blob_indirection_format_version); + std::string record; + if (!format_fence.EncodeTo(&record)) { + return Status::Corruption( + "Unable to encode DB-global blob indirection format fence"); + } + io_s = log->AddRecord(write_options, record); + if (!io_s.ok()) { + return io_s; + } + } if (db_options_->write_dbid_to_manifest) { VersionEdit edit_for_db_id; assert(!db_id_.empty()); @@ -8241,6 +9567,12 @@ Status VersionSet::WriteCurrentStateToManifest( // so live SSTs still benefit from fast_sst_open after a MANIFEST // rotation. edit.AddExistingFile(level, *f, /*provenance_flags_to_set=*/0); + FileMetaData& snapshot_file = edit.GetMutableNewFiles().back().second; + snapshot_file.blob_reference_legacy_metadata = + vstorage->HasBlobIndirectionFormat() && + f->blob_reference_descriptor.empty() && + f->oldest_coarse_indirect_origin_file_number == + kInvalidBlobFileNumber; } } @@ -8254,7 +9586,7 @@ Status VersionSet::WriteCurrentStateToManifest( edit.AddBlobFile(blob_file_number, meta->GetTotalBlobCount(), meta->GetTotalBlobBytes(), meta->GetChecksumMethod(), - meta->GetChecksumValue()); + meta->GetChecksumValue(), meta->IsIndirectOrigin()); if (meta->GetGarbageBlobCount() > 0) { edit.AddBlobFileGarbage(blob_file_number, meta->GetGarbageBlobCount(), meta->GetGarbageBlobBytes()); @@ -8264,6 +9596,22 @@ Status VersionSet::WriteCurrentStateToManifest( for (const uint64_t blob_file_number : vstorage->GetDroppedBlobFiles()) { edit.AddDroppedBlobFile(blob_file_number); } + if (vstorage->HasBlobIndirectionFormat()) { + edit.SetBlobIndirectionFormatVersion( + vstorage->GetBlobIndirectionFormatVersion()); + } + for (const std::shared_ptr& metadata : + vstorage->GetBlobRelocationFiles()) { + assert(metadata != nullptr); + edit.AddBlobRelocationFile(metadata->GetAddition()); + if (metadata->IsMarkedLost()) { + edit.MarkBlobPhysicalFileLost(metadata->GetFileNumber()); + } + } + vstorage->ForEachBlobRelocationRoute( + [&](const BlobRelocationRoute& route) { + edit.SetBlobRelocationRoute(route); + }); const auto iter = curr_state.find(cfd->GetID()); assert(iter != curr_state.end()); @@ -8696,7 +10044,8 @@ void VersionSet::AddLiveFiles(std::vector* live_table_files, total_table_files += vstorage->LevelFiles(level).size(); } - total_blob_files += vstorage->GetBlobFiles().size(); + total_blob_files += vstorage->GetBlobFiles().size() + + vstorage->GetBlobRelocationFiles().size(); } } @@ -8946,15 +10295,18 @@ void VersionSet::GetObsoleteFiles(std::vector* files, // min_pending_output still holds back blob files: VersionBuilder can report // one obsolete without it having been part of an installed Version, so the // reasoning above does not apply to them. - std::vector pending_blob_files; - for (auto& blob_file : obsolete_blob_files_) { - if (blob_file.GetBlobFileNumber() < min_pending_output) { - blob_files->emplace_back(std::move(blob_file)); - } else { - pending_blob_files.emplace_back(std::move(blob_file)); + { + MutexLock lock(&obsolete_blob_files_mutex_); + std::vector pending_blob_files; + for (auto& blob_file : obsolete_blob_files_) { + if (blob_file.GetBlobFileNumber() < min_pending_output) { + blob_files->emplace_back(std::move(blob_file)); + } else { + pending_blob_files.emplace_back(std::move(blob_file)); + } } + obsolete_blob_files_.swap(pending_blob_files); } - obsolete_blob_files_.swap(pending_blob_files); obsolete_manifests_.swap(*manifest_filenames); } @@ -9051,6 +10403,13 @@ uint64_t VersionSet::GetTotalBlobFileSize(Version* dummy_versions) { all_versions_blob_file_size += meta->GetBlobFileSize(); } } + for (const std::shared_ptr& metadata : + vstorage->GetBlobRelocationFiles()) { + assert(metadata != nullptr); + if (unique_blob_files.insert(metadata->GetFileNumber()).second) { + all_versions_blob_file_size += metadata->GetFileSize(); + } + } } return all_versions_blob_file_size; @@ -9128,7 +10487,8 @@ Status ReactiveVersionSet::Recover( manifest_reporter->reset(new LogReporter()); static_cast_with_check(manifest_reporter->get())->status = manifest_reader_status->get(); - Status s = MaybeSwitchManifest(manifest_reporter->get(), manifest_reader); + Status s = MaybeSwitchManifest(manifest_reporter->get(), manifest_reader, + /*db_mutex=*/nullptr); if (!s.ok()) { return s; } @@ -9158,15 +10518,28 @@ Status ReactiveVersionSet::ReadAndApply( assert(manifest_reader != nullptr); assert(cfds_changed != nullptr); mu->AssertHeld(); + // A catch-up releases the DB mutex to decode records, check files and build + // Versions. The caller holds its catch-up mutex + // (DBImplSecondary::catch_up_mutex_), so catch-ups never overlap. + assert(!manifest_tail_in_progress_); + manifest_tail_in_progress_ = true; Status s; log::Reader* reader = manifest_reader->get(); assert(reader); - s = MaybeSwitchManifest(reader->GetReporter(), manifest_reader); + s = MaybeSwitchManifest(reader->GetReporter(), manifest_reader, mu); if (!s.ok()) { + manifest_tail_in_progress_ = false; return s; } + if (!manifest_tailer_->status().ok()) { + manifest_tail_in_progress_ = false; + return manifest_tailer_->status(); + } + manifest_tailer_->SetCatchUpDBMutex(mu); manifest_tailer_->Iterate(*(manifest_reader->get()), manifest_read_status); + manifest_tailer_->ClearCatchUpDBMutex(); + manifest_tail_in_progress_ = false; s = manifest_tailer_->status(); if (s.ok()) { *cfds_changed = std::move(manifest_tailer_->GetUpdatedColumnFamilies()); @@ -9186,7 +10559,8 @@ uint64_t ReactiveVersionSet::GetInstalledVersionLogNumber( Status ReactiveVersionSet::MaybeSwitchManifest( log::Reader::Reporter* reporter, - std::unique_ptr* manifest_reader) { + std::unique_ptr* manifest_reader, + InstrumentedMutex* db_mutex) { assert(manifest_reader != nullptr); Status s; std::string manifest_path; @@ -9237,7 +10611,7 @@ Status ReactiveVersionSet::MaybeSwitchManifest( ROCKS_LOG_INFO(db_options_->info_log, "Switched to new manifest: %s\n", manifest_path.c_str()); if (manifest_tailer_) { - manifest_tailer_->PrepareToReadNewManifest(); + manifest_tailer_->PrepareToReadNewManifest(db_mutex); } } else if (s.IsPathNotFound()) { // This can happen if the primary switches to a new MANIFEST after the @@ -9262,6 +10636,266 @@ std::vector& ReactiveVersionSet::replay_buffer() { return manifest_tailer_->GetReadBuffer().replay_buffer(); } +void VersionStorageInfo::ComputeBlobRelocationStateFingerprint() { + BlobRelocationStateForRead().ComputeFingerprint( + blob_indirection_format_version_, + &blob_relocation_state_fingerprint_high_, + &blob_relocation_state_fingerprint_low_); +} + +bool VersionStorageInfo::IsBlobReferenceLost(const BlobIndex& index) const { + if (!index.IsIndirect() || !HasLostBlobRelocationFiles()) { + return false; + } + // A reference is lost when its generation is a lost relocation file, or an + // old file routed to a lost relocation file. + const BlobFileTable& table = GetBlobFileTable(); + uint64_t value = table.Find(index.generation_file_number()); + if (BlobFileTable::IsRoute(value)) { + value = table.Find(BlobFileTable::RouteTarget(value)); + } + return BlobFileTable::IsLost(value); +} + +void VersionStorageInfo::ComputeFilesMarkedForForcedBlobGC( + double blob_garbage_collection_age_cutoff, + double blob_garbage_collection_force_threshold, + bool enable_blob_garbage_collection, bool enable_blob_indirection) { + files_marked_for_forced_blob_gc_.clear(); + if (!(enable_blob_garbage_collection && + blob_garbage_collection_age_cutoff > 0.0 && + blob_garbage_collection_force_threshold < 1.0)) { + return; + } + + if (blob_files_.empty()) { + return; + } + // With enable_blob_indirection on, compaction keeps the references to an + // indirect origin as they are. So compacting the SSTs linked to the oldest + // file never frees it, and the same SSTs would be marked again and again. + // Instead, the census ranks such files by garbage ratio as indirect blob GC + // candidates (see SetBlobGcCandidates()). + if (enable_blob_indirection && blob_files_.front()->IsIndirectOrigin()) { + return; + } + + // Number of blob files eligible for GC based on age + const size_t cutoff_count = static_cast( + blob_garbage_collection_age_cutoff * blob_files_.size()); + if (!cutoff_count) { + return; + } + + // Compute the sum of total and garbage bytes over the batch of blob files + // currently eligible for garbage collection based on + // blob_garbage_collection_age_cutoff, and if the garbage ratio exceeds + // blob_garbage_collection_force_threshold, schedule compaction for the + // SST files that reference the oldest batch of blob files. Here is a toy + // example. Let's assume we have three SSTs 1, 2, and 3, and four blob files + // 10, 11, 12, and 13, which correspond to the range that is eligible for GC + // and satisfy the garbage ratio threshold. Also, let's say SSTs 1 and 2 both + // rely on blob file 10 and potentially some higher-numbered ones, while SST 3 + // relies on blob file 12 and potentially some higher-numbered ones. Then, the + // SST to oldest blob file mapping is as follows: + // + // SST file number Oldest blob file number + // 1 10 + // 2 10 + // 3 12 + // + // This is what the same thing looks like from the blob files' POV. (Note that + // the linked SSTs simply denote the inverse mapping of the above.) + // + // Blob file number Linked SST set + // 10 {1, 2} + // 11 {} + // 12 {3} + // 13 {} + // + // Then, the oldest batch of blob files consists of blob files 10 and 11, + // and we can get rid of them by forcing the compaction of SSTs 1 and 2. + const auto& oldest_meta = blob_files_.front(); + assert(oldest_meta); + + const auto& linked_ssts = oldest_meta->GetLinkedSsts(); + // In a fenced CF an SST links only the blob file named by its + // oldest_blob_file_number. Two kinds of files stay without linked SSTs: a + // relocated indirect origin that waits for its retirement record, and a + // normal blob file newer than a retired indirect origin. Either kind can be + // the oldest. The first kind reaches this check only with + // enable_blob_indirection off (with it on, the check above already + // returned). The second kind can reach it with either setting. No SST can be + // compacted to free such a file, so mark nothing. In any CF, the SSTs that + // reference the oldest blob file can also be linked to an older, dropped blob + // file instead (see GetDroppedBlobFiles()), so the SSTs to compact are + // unknown. + if (linked_ssts.empty()) { + assert(HasBlobIndirectionFormat() || !dropped_blob_files_.empty()); + return; + } + + size_t count = 1; + uint64_t sum_total_blob_bytes = oldest_meta->GetTotalBlobBytes(); + uint64_t sum_garbage_blob_bytes = oldest_meta->GetGarbageBlobBytes(); + + assert(cutoff_count <= blob_files_.size()); + + for (; count < cutoff_count; ++count) { + const auto& meta = blob_files_[count]; + assert(meta); + + sum_total_blob_bytes += meta->GetTotalBlobBytes(); + sum_garbage_blob_bytes += meta->GetGarbageBlobBytes(); + } + + if (sum_garbage_blob_bytes < + blob_garbage_collection_force_threshold * sum_total_blob_bytes) { + return; + } + + for (uint64_t sst_file_number : linked_ssts) { + const FileLocation location = GetFileLocation(sst_file_number); + assert(location.IsValid()); + + const int level = location.GetLevel(); + assert(level >= 0); + + const size_t pos = location.GetPosition(); + + FileMetaData* const sst_meta = files_[level][pos]; + assert(sst_meta); + + if (sst_meta->being_compacted) { + continue; + } + + files_marked_for_forced_blob_gc_.emplace_back(level, sst_meta); + } +} + +BlobRelocationState& VersionStorageInfo::MutableBlobRelocationState() { + if (blob_relocation_state_ == nullptr) { + blob_relocation_state_ = std::make_shared(); + } else if (blob_relocation_state_.use_count() != 1) { + blob_relocation_state_ = + std::make_shared(*blob_relocation_state_); + } + return *blob_relocation_state_; +} + +std::shared_ptr +VersionStorageInfo::ReleaseBlobRelocationStateForDeferredFree() { + std::shared_ptr state = + std::move(blob_relocation_state_); + if (state == nullptr || state.use_count() != 1) { + return nullptr; + } + state->ReleaseFiles(); + return state; +} + +void VersionStorageInfo::SealBlobRelocationState() { + if (blob_relocation_state_ != nullptr && !blob_relocation_state_->sealed()) { + blob_relocation_state_->Seal(); + } +} + +void VersionStorageInfo::DeleteBlobRelocationFile(uint64_t file_number) { + MutableBlobRelocationState().DeleteFile(file_number); +} + +void VersionStorageInfo::DeleteBlobRelocationRoute(uint64_t old_file_number) { + MutableBlobRelocationState().DeleteRoute(old_file_number); +} + +void VersionStorageInfo::SetBlobGcCandidates( + const std::array& files) { + blob_files_for_indirect_blob_gc_ = files; + blob_files_for_indirect_blob_gc_count_ = 0; + for (uint64_t file_number : files) { + if (file_number == kInvalidBlobFileNumber) { + break; + } + ++blob_files_for_indirect_blob_gc_count_; + } +} + +void VersionStorageInfo::SuppressBlobFileForIndirectBlobGC( + uint64_t physical_file_number) { + for (size_t i = 0; i < empty_blob_files_for_indirect_blob_gc_.size(); ++i) { + if (empty_blob_files_for_indirect_blob_gc_[i] == physical_file_number) { + std::move(empty_blob_files_for_indirect_blob_gc_.begin() + i + 1, + empty_blob_files_for_indirect_blob_gc_.end(), + empty_blob_files_for_indirect_blob_gc_.begin() + i); + empty_blob_files_for_indirect_blob_gc_.back() = 0; + break; + } + } + for (size_t i = 0; i < blob_files_for_indirect_blob_gc_count_; ++i) { + if (blob_files_for_indirect_blob_gc_[i] == physical_file_number) { + for (size_t next = i + 1; next < blob_files_for_indirect_blob_gc_count_; + ++next) { + blob_files_for_indirect_blob_gc_[next - 1] = + blob_files_for_indirect_blob_gc_[next]; + } + blob_files_for_indirect_blob_gc_ + [--blob_files_for_indirect_blob_gc_count_] = 0; + return; + } + } +} + +void VersionStorageInfo::DeferBlobFilesForIndirectBlobGC( + const std::vector& physical_file_numbers) { + for (uint64_t file_number : physical_file_numbers) { + for (size_t i = 0; i < blob_files_for_indirect_blob_gc_count_; ++i) { + if (blob_files_for_indirect_blob_gc_[i] != file_number) { + continue; + } + for (size_t next = i + 1; next < blob_files_for_indirect_blob_gc_count_; + ++next) { + blob_files_for_indirect_blob_gc_[next - 1] = + blob_files_for_indirect_blob_gc_[next]; + } + blob_files_for_indirect_blob_gc_[blob_files_for_indirect_blob_gc_count_ - + 1] = file_number; + break; + } + } +} + +uint64_t VersionStorageInfo::BlobPhysicalFileForIndirectBlobGC() const { + return blob_files_for_indirect_blob_gc_count_ != 0 + ? blob_files_for_indirect_blob_gc_[0] + : kInvalidBlobFileNumber; +} + +void VersionStorageInfo::GetBlobPhysicalFilesForIndirectBlobGC( + size_t max_files, std::vector* physical_file_numbers) const { + assert(physical_file_numbers != nullptr); + physical_file_numbers->clear(); + const size_t count = + std::min(max_files, blob_files_for_indirect_blob_gc_count_); + for (size_t i = 0; i < count; ++i) { + physical_file_numbers->push_back(blob_files_for_indirect_blob_gc_[i]); + } +} + +void VersionStorageInfo::GetEmptyBlobGcCandidates( + size_t max_files, std::vector* files) const { + assert(files != nullptr); + files->clear(); + for (size_t i = 0; + i < std::min(max_files, empty_blob_files_for_indirect_blob_gc_.size()); + ++i) { + if (empty_blob_files_for_indirect_blob_gc_[i] == kInvalidBlobFileNumber) { + break; + } + files->push_back(empty_blob_files_for_indirect_blob_gc_[i]); + } +} + } // namespace ROCKSDB_NAMESPACE // Generate the regular and coroutine versions of the functions defined in diff --git a/db/version_set.h b/db/version_set.h index 0511bafbedd8..aaabae109390 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -35,6 +35,7 @@ #include "cache/cache_helpers.h" #include "db/blob/blob_file_meta.h" #include "db/blob/blob_index.h" +#include "db/blob/blob_relocation_manifest.h" #include "db/column_family.h" #include "db/compaction/compaction.h" #include "db/compaction/compaction_picker.h" @@ -53,6 +54,7 @@ #include "monitoring/instrumented_mutex.h" #include "options/db_options.h" #include "options/offpeak_time_info.h" +#include "port/lang.h" #include "port/port.h" #include "rocksdb/env.h" #include "rocksdb/file_checksum.h" @@ -62,6 +64,7 @@ #include "util/autovector.h" #include "util/coro_utils.h" #include "util/hash_containers.h" +#include "util/mutexlock.h" namespace ROCKSDB_NAMESPACE { @@ -70,6 +73,9 @@ class Writer; } class BlobIndex; +struct BlobPhysicalFile; +struct BlobReferenceCensusDelta; +struct BlobReferenceCensusPreparedInstall; class Compaction; class LogBuffer; class LookupKey; @@ -271,6 +277,10 @@ class VersionStorageInfo { double blob_garbage_collection_age_cutoff, double blob_garbage_collection_force_threshold, bool enable_blob_garbage_collection); + void ComputeFilesMarkedForForcedBlobGC( + double blob_garbage_collection_age_cutoff, + double blob_garbage_collection_force_threshold, + bool enable_blob_garbage_collection, bool enable_blob_indirection); // This computes read_triggered_compaction_files_ and is called by // ComputeCompactionScore() @@ -399,6 +409,67 @@ class VersionStorageInfo { return files_[level]; } + // An exact compaction to L0 can write several files with one epoch number, + // because the reference cap (blob_indirection_max_references_per_sst) can + // cut its output. These files are an epoch run. Later picks must take all + // files of such a run, or none of them. + // + // Why: a compaction output gets the smallest epoch number of its inputs. In + // this example, the file numbers, epochs and keys are only examples: + // + // L0, newest first: + // file 25 epoch 12 keys [j..n] + // file 21 epoch 9 keys [m..z] <- one epoch run, cut by the + // file 20 epoch 9 keys [a..k] <- reference cap + // + // Suppose a compaction to L0 takes files 25 and 21, but not file 20. Its + // output gets epoch 9 and keys [j..z]. The output holds the newer versions + // of the keys in [j..k] (from file 25), and file 20 holds older versions of + // the same keys, with the same epoch. L0 orders overlapping files by epoch, + // so the two files have no order. The Version consistency check (on by + // default; see force_consistency_checks) rejects them ("L0 files of same + // epoch number but overlapping range"), so the compaction cannot install. + // Taking the whole run avoids this. + // + // Only runs that hold an exact file are expanded, because only an exact + // output can be cut by the reference cap. Files of the other epochs are + // kept as given. Only the epochs of the inputs are inspected, not the whole + // Version. `inputs` may be in any order; the result is in Version's epoch + // order (newest first). When the DB has no blob indirection format, + // `inputs` is returned unchanged. Returns false, leaving `inputs` + // unchanged, when an exact run has a file being compacted or has + // kUnknownEpochNumber (0). A file with epoch 0 has no known epoch, so its + // run cannot be found. + bool ExpandExactL0Inputs(std::vector* inputs) const; + + // L0 sorted runs. In a column family with the blob indirection format, the + // files of one exact epoch run (adjacent L0 files with one known epoch + // number, at least one of them exact; see ExpandExactL0Inputs()) count as + // one sorted run. This is correct because the files share one epoch and + // their key ranges do not overlap, so each key is in at most one of them, + // the same as in a single file. Every other L0 file is its own sorted run, as + // in base RocksDB. See "Reference cap and epoch runs" in + // db/blob/blob_reference_metadata.h for why runs exist. Counting each file of + // a run would keep the L0 count at about references / cap: + // compactions would rewrite the same run again and again, and writes would + // slow down or stop. + // + // L0SortedRunEnd() returns the L0 position one past the last file of the + // sorted run that starts at L0 position `first`. + size_t L0SortedRunEnd(size_t first) const; + + // The number of L0 sorted runs among `files`: L0 files of this Version in + // its order (newest first), where each run is whole. Without the blob + // indirection format, it is files.size(). O(files.size()). + size_t CountL0SortedRuns(const std::vector& files) const; + + // The number of L0 sorted runs in this Version, set by CalculateBaseBytes() + // (called from PrepareForVersionAppend()). Without the blob indirection + // format, it is the L0 file count. The L0 compaction score, the write stall + // count (l0_delay_trigger_count()) and the leveled intra-L0 triggers use + // it. rocksdb.num-files-at-level0 and NumLevelFiles(0) stay file counts. + int NumL0SortedRuns() const { return num_l0_sorted_runs_; } + bool HasMissingEpochNumber() const; uint64_t GetMaxEpochNumberOfFiles() const; EpochNumberRequirement GetEpochNumberRequirement() const { @@ -473,6 +544,58 @@ class VersionStorageInfo { // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) using BlobFiles = std::vector>; const BlobFiles& GetBlobFiles() const { return blob_files_; } + uint64_t OldestCoarseIndirectOriginFileNumber() const { + return oldest_coarse_indirect_origin_file_number_; + } + // The number of coarse SSTs (SSTs with a nonzero coarse frontier, the oldest + // indirect-origin file number a coarse SST may reference). + size_t CoarseSstCount() const { return coarse_sst_count_; } + bool IsCoarseIndirectOriginBlobPinned(uint64_t file_number) const { + return oldest_coarse_indirect_origin_file_number_ != + kInvalidBlobFileNumber && + file_number >= oldest_coarse_indirect_origin_file_number_; + } + + // Indirect blob GC candidate hints of this Version. The census worker keeps + // one heap per column family that ranks the physical blob files by garbage + // ratio. Outside the DB mutex it selects at most kBlobGcMaxSelectedFiles (8) + // files; under the mutex it stores them here. The fixed-size array keeps + // publishing and reading the hints cheap, and building a Version never scans + // or sorts all blob files. A hint is not proof of garbage: GC checks the + // exact references before it moves any value. + void SetBlobGcCandidates( + const std::array& files); + void SetBlobRelocationGarbageBytes(uint64_t bytes) { + blob_relocation_garbage_bytes_.store( + std::min(bytes, BlobRelocationStateForRead().RecordBytes()), + std::memory_order_relaxed); + } + void SetEmptyBlobGcCandidates( + const std::array& files) { + empty_blob_files_for_indirect_blob_gc_ = files; + } + bool HasEmptyBlobGcCandidate() const { + return empty_blob_files_for_indirect_blob_gc_[0] != kInvalidBlobFileNumber; + } + void GetEmptyBlobGcCandidates(size_t max_files, + std::vector* files) const; + void RecordRemovedBlobFile(uint64_t file_number) { + removed_blob_files_.push_back(file_number); + } + const std::vector& RemovedBlobFiles() const { + return removed_blob_files_; + } + + // Intended user: indirect blob GC. It picks candidate physical files of + // this Version (BlobPhysicalFileForIndirectBlobGC(), + // GetBlobPhysicalFilesForIndirectBlobGC()), and suppresses or defers files + // that it cannot use now. + void SuppressBlobFileForIndirectBlobGC(uint64_t physical_file_number); + void DeferBlobFilesForIndirectBlobGC( + const std::vector& physical_file_numbers); + uint64_t BlobPhysicalFileForIndirectBlobGC() const; + void GetBlobPhysicalFilesForIndirectBlobGC( + size_t max_files, std::vector* physical_file_numbers) const; // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) BlobFiles::const_iterator GetBlobFileMetaDataLB( @@ -571,6 +694,159 @@ class VersionStorageInfo { dropped_blob_files_ = std::move(dropped_blob_files); } + using BlobRelocationFiles = BlobRelocationState::Files; + + void ShareBlobRelocationState(const VersionStorageInfo& base) { + blob_relocation_state_ = base.blob_relocation_state_; + blob_relocation_state_fingerprint_high_ = + base.blob_relocation_state_fingerprint_high_; + blob_relocation_state_fingerprint_low_ = + base.blob_relocation_state_fingerprint_low_; + } + + // Moves out the relocation state before this Version is destroyed under the + // DB mutex (see VersionSet::FreeBlobRelocationState()). + std::shared_ptr ReleaseBlobRelocationState() { + return std::move(blob_relocation_state_); + } + + // Called by ~Version() under the DB mutex. When this Version is the last + // owner of its relocation state, frees the relocation file metadata now and + // returns the rest: the file table, which holds only memory and its cache + // reservation. The metadata must be freed now, while the column family is + // alive, because its deleter uses the column family's options (cf_paths) and + // blob file cache. The caller frees the table later without the mutex. + // Returns null when other owners remain. The use_count() check can race with + // another owner that drops its reference at the same time. Then whichever + // side drops the last reference frees the whole state. + std::shared_ptr ReleaseBlobRelocationStateForDeferredFree(); + + // The state's file table does not own its blob file entries: this Version's + // blob file vector does. A holder that may outlive the Version can use the + // relocation files and routes, but must not read a blob file entry of the + // table. + std::shared_ptr GetBlobRelocationState() const { + return blob_relocation_state_; + } + void AddBlobRelocationFile( + std::shared_ptr metadata); + void AddBlobRelocationRoute(const BlobRelocationRoute& route); + void DeleteBlobRelocationFile(uint64_t file_number); + void DeleteBlobRelocationRoute(uint64_t old_file_number); + + // Applies the buffered file table edits of the calls above. The table must be + // sealed before anyone reads it. VersionBuilder::SaveBlobFilesAndCheckTo() + // calls it once per saved Version; PrepareForVersionAppend() calls it again, + // which does nothing when the state is already sealed. + void SealBlobRelocationState(); + + // Sorted by file number. + const BlobRelocationFiles& GetBlobRelocationFiles() const { + return BlobRelocationStateForRead().files(); + } + size_t GetBlobRelocationRouteCount() const { + return BlobRelocationStateForRead().route_count(); + } + // The total size of the live relocation files. + uint64_t GetBlobRelocationFileBytes() const { + return BlobRelocationStateForRead().PhysicalFileBytes(); + } + // Visits every route, in no particular order. + template + void ForEachBlobRelocationRoute(Fn&& fn) const { + BlobRelocationStateForRead().ForEachRoute(std::forward(fn)); + } + + // The file table: file number -> live blob file, live relocation file or + // route (see BlobFileTable). It is filled only when this Version has the + // per-CF format fence (HasBlobIndirectionFormat()); otherwise it is empty. + const BlobFileTable& GetBlobFileTable() const { + return BlobRelocationStateForRead().table(); + } + bool HasLostBlobRelocationFiles() const { + return BlobRelocationStateForRead().HasLostFiles(); + } + bool IsBlobReferenceLost(const BlobIndex& index) const; + + std::shared_ptr GetBlobRelocationFileMetaData( + uint64_t file_number) const; + // Raw-pointer lookups for the read path; valid while this Version lives. + const BlobFileMetaData* PeekBlobFileMetaData(uint64_t file_number) const { + const auto it = GetBlobFileMetaDataLB(file_number); + return it != blob_files_.end() && (*it)->GetBlobFileNumber() == file_number + ? it->get() + : nullptr; + } + const BlobRelocationFileMetaData* PeekBlobRelocationFileMetaData( + uint64_t file_number) const { + return BlobRelocationStateForRead().PeekFile(file_number); + } + + // The target of the route of old file `old_file_number`, or + // kInvalidBlobFileNumber. + uint64_t GetBlobRelocationRouteTarget(uint64_t old_file_number) const { + return BlobRelocationStateForRead().FindRouteTarget(old_file_number); + } + + // The number of routes to relocation file `target_file_number`. + size_t GetBlobRelocationRouteTargetCount(uint64_t target_file_number) const { + return BlobRelocationStateForRead().RouteCount(target_file_number); + } + + void SetBlobIndirectionFormatVersion(uint32_t format_version) { + blob_indirection_format_version_ = format_version; + } + uint32_t GetBlobIndirectionFormatVersion() const { + return blob_indirection_format_version_; + } + bool HasBlobIndirectionFormat() const { + return blob_indirection_format_version_ != 0; + } + + // Identity of this Version's relocation state. The fingerprint has 128 bits + // and depends only on the relocation files, the routes and the format + // version, not on the order of the changes. Its intended user is remote + // compaction: the job sends the fingerprint of the Version it was planned on, + // so the worker can check that it builds SSTs with the same routes. 128 bits + // make an accidental match of two different states very unlikely (an + // estimate, not measured). + uint64_t GetBlobRelocationStateFingerprintHigh() const { + return blob_relocation_state_fingerprint_high_; + } + uint64_t GetBlobRelocationStateFingerprintLow() const { + return blob_relocation_state_fingerprint_low_; + } + + // Stores the fingerprint of this Version's relocation files and routes (see + // BlobRelocationState::ComputeFingerprint()). It is O(1). VersionBuilder + // calls it outside the DB mutex when the relocation state or the format + // version changes. + void ComputeBlobRelocationStateFingerprint(); + // Brings the file table of a fenced Version up to date with its blob files: + // `rebuild` adds every live blob file (the first fenced Version); otherwise + // only the blob files in `changed_blob_file_numbers` are set or erased. A + // rebuilt table charges its memory to `cache_res_mgr` (may be null). + void SyncBlobFileTable( + const std::vector& changed_blob_file_numbers, bool rebuild, + std::shared_ptr cache_res_mgr); + // Checks the relocation state and the file table. A release build skips a + // state that is already marked validated: an inherited state, or a new state + // after MarkBlobRelocationChangesValidated(). A debug build always checks + // that the table agrees with the blob file and relocation file vectors. + Status ValidateBlobRelocationState() const; + + // VersionBuilder::Apply() checks every changed relocation file and route, + // also in release builds: a route target is a live relocation file, the old + // file of a route is not a live file, file numbers do not collide, and no + // route forms a chain. The inherited state is immutable and was already + // checked. So the new state is valid without a scan of the untouched routes, + // and this call marks it validated. + void MarkBlobRelocationChangesValidated() const { + if (blob_relocation_state_ != nullptr) { + blob_relocation_state_->MarkValidated(); + } + } + // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) struct BlobStats { uint64_t total_file_size = 0; @@ -588,6 +864,14 @@ class VersionStorageInfo { total_file_size += meta->GetBlobFileSize(); total_garbage_size += meta->GetGarbageBlobBytes(); } + // Relocation files take disk space too. Their total size comes from the + // relocation state, so it is counted even before the census is ready. The + // blob size, garbage and space-amp DB properties (internal_stats.cc) read + // these totals. The garbage of relocation files comes from the census, and + // it is 0 until the census publishes it for this Version. + total_file_size += BlobRelocationStateForRead().PhysicalFileBytes(); + total_garbage_size += + blob_relocation_garbage_bytes_.load(std::memory_order_relaxed); double space_amp = 0.0; if (total_file_size > total_garbage_size) { @@ -643,6 +927,11 @@ class VersionStorageInfo { files_marked_for_periodic_compaction_.emplace_back(level, f); } + const PeriodicCompactionPhaseParams& TEST_GetPeriodicCompactionPhaseParams() + const { + return periodic_compaction_phase_params_; + } + // REQUIRES: PrepareForVersionAppend has been called const autovector>& BottommostFiles() const { assert(finalized_); @@ -842,10 +1131,48 @@ class VersionStorageInfo { // Vector of blob files in version sorted by blob file number. BlobFiles blob_files_; + uint64_t oldest_coarse_indirect_origin_file_number_ = kInvalidBlobFileNumber; + size_t coarse_sst_count_ = 0; + // Blob files of the base Version that are gone from this Version. + // VersionBuilder::SaveBlobFilesAndCheckTo() records them outside the DB mutex + // for the census delta. This is not the set of retired files, and it is never + // written to the MANIFEST. + std::vector removed_blob_files_; + std::array + blob_files_for_indirect_blob_gc_{}; + size_t blob_files_for_indirect_blob_gc_count_ = 0; + // Garbage record bytes in the relocation files, from the census. It is 0 + // until the census publishes a value for this Version. Atomic because + // approximate live-data properties can read a pinned Version without the DB + // mutex. + std::atomic blob_relocation_garbage_bytes_{0}; + std::array + empty_blob_files_for_indirect_blob_gc_{}; // See GetDroppedBlobFiles(). std::vector dropped_blob_files_; + // The live relocation files, routes and file table of this Version (see + // BlobRelocationState). It can be null when there are none; readers then see + // an empty state. Copy-on-write: a Version that changes none of them shares + // its base Version's state. Only an unpublished builder makes a private copy + // (MutableBlobRelocationState()); the copy shares every file table shard it + // does not change. LogAndApply and the secondary catch-up make this copy + // without the DB mutex; recovery makes it with the mutex held. An old Version + // keeps its own state alive. The state has no deletion history: a deleted + // file or route simply leaves it. + std::shared_ptr blob_relocation_state_; + const BlobRelocationState& BlobRelocationStateForRead() const { + STATIC_AVOID_DESTRUCTION(BlobRelocationState, empty); + return blob_relocation_state_ != nullptr ? *blob_relocation_state_ : empty; + } + BlobRelocationState& MutableBlobRelocationState(); + uint32_t blob_indirection_format_version_ = 0; + // A hash of the relocation files and routes of this Version (see + // ComputeBlobRelocationStateFingerprint()). + uint64_t blob_relocation_state_fingerprint_high_ = 0; + uint64_t blob_relocation_state_fingerprint_low_ = 0; + // Level that L0 data should be compacted to. All levels < base_level_ should // be empty. -1 if it is not level-compaction so it's not applicable. int base_level_; @@ -932,6 +1259,7 @@ class VersionStorageInfo { std::vector compaction_score_; std::vector compaction_level_; int l0_delay_trigger_count_ = 0; // Count used to trigger slow down and stop + int num_l0_sorted_runs_ = 0; // See NumL0SortedRuns() // for number of L0 files. // Compact cursors for round-robin compactions in each level @@ -979,6 +1307,19 @@ class VersionStorageInfo { friend class VersionSet; }; +// True if the column family uses blob indirection now (the option +// enable_blob_indirection) or has used it (the format fence of `storage`, see +// VersionStorageInfo::HasBlobIndirectionFormat()). In such a column family, +// flush, WAL recovery flush and compaction record blob reference metadata: +// exact metadata, or the coarse classification (see +// TableBuilderOptions::track_blob_reference_classification). `storage` may be +// nullptr; then only the option counts. +inline bool UsesBlobIndirection(const MutableCFOptions& mutable_cf_options, + const VersionStorageInfo* storage) { + return mutable_cf_options.enable_blob_indirection || + (storage != nullptr && storage->HasBlobIndirectionFormat()); +} + struct ObsoleteFileInfo { FileMetaData* metadata; std::string path; @@ -1138,7 +1479,10 @@ class Version { // TTL/inlined/same-file or compressed blob index (the lazy caller only routes // uncompressed separate-file references here; other cases take the // whole-value GetBlob path). The caller must ensure range_offset + - // range_length <= blob_index.size(). + // range_length <= blob_index.size(). For an indirect index, a range of an + // uncompressed value never checks a CRC. Callers send only strict parts of a + // value (see ReadPathBlobResolver::ResolveColumnRange), and a range is not + // turned into a whole read. Status GetBlobRange(const ReadOptions& read_options, const Slice& user_key, const BlobIndex& blob_index, uint64_t range_offset, size_t range_length, PinnableSlice* value, @@ -1191,6 +1535,13 @@ class Version { // and return true. Otherwise, return false. bool Unref(); + // Like Unref(), for a Version pinned by background work. The Version is + // destroyed under `mu` like every Version; when this drops the last + // reference, only its relocation state, which can be large, is freed + // without `mu` (see VersionSet::FreeBlobRelocationState()). + // REQUIRES: `mu` (the DB mutex) held. + void UnrefAndFreeBlobRelocationState(InstrumentedMutex* mu); + // Add all files listed in the current version to *live_table_files and // *live_blob_files. void AddLiveFiles(std::vector* live_table_files, @@ -1299,6 +1650,17 @@ class Version { int level, bool allow_unprepared_value); private: + // Resolves indirect `blob_index` to the physical file that holds its bytes. + // GetBlob, GetBlobRange and both MultiGet paths call it after a stable blob + // cache miss. It looks up the generation number in the file table: a live + // file takes one lookup, and a route takes two (the target is always a live + // relocation file). It returns Corruption when the generation number is + // unknown, when a route target is missing, or when a normal blob file is not + // the indirect origin of the reference. It returns NotFound when the + // physical file is a relocation file that an accepted loss marked lost. + Status ResolveBlobPhysicalFile(const BlobIndex& blob_index, + BlobPhysicalFile* physical_file) const; + Env* env_; SystemClock* clock_; @@ -1317,6 +1679,12 @@ class Version { return storage_info_.user_comparator_; } + // Debug builds only. A stable blob cache hit skips the blob file and route + // lookup, so a hit for a BlobIndex whose file this Version does not have + // would go unnoticed. This asserts that the file or route of `blob_index` is + // in this Version. + void AssertStableCacheHitHasFile(const BlobIndex& blob_index) const; + // Returns true if the filter blocks in the specified level will not be // checked during read operations. In certain cases (trivial move or preload), // the filter block may already be cached, but we still do not access it such @@ -1390,12 +1758,19 @@ class Version { uint64_t version_number_; std::shared_ptr io_tracer_; + // `periodic_compaction_phase_params` are the phasing parameters of this + // Version. std::nullopt (the default) means the current parameters of the + // VersionSet. For a SetOptions() call that moves the phasing anchor, + // LogAndApply() passes the parameters for the new anchor, which is not + // published yet (see VersionSet::PrepareCompactionPhaseReanchor()). Version(ColumnFamilyData* cfd, VersionSet* vset, const FileOptions& file_opt, const MutableCFOptions& mutable_cf_options, const std::shared_ptr& io_tracer, uint64_t version_number = 0, EpochNumberRequirement epoch_number_requirement = - EpochNumberRequirement::kMustPresent); + EpochNumberRequirement::kMustPresent, + std::optional + periodic_compaction_phase_params = std::nullopt); ~Version(); @@ -1426,6 +1801,42 @@ class AtomicGroupReadBuffer { std::vector replay_buffer_; }; +// The new options that SetOptions publishes together with the new Versions +// (see VersionSet::LogAndApply()). SetOptions owns it. +// +// How the options of one SetOptions() call are published: +// 1. DBImpl::SetOptions() fills one entry per column family with a copy of +// the latest options. +// 2. LogAndApply() gives each ManifestWriter a pointer to its entry. A batch +// that holds a SetOptions() call holds only the writers of that call. +// SetOptions() edits are dummy edits, and a batch of dummy edits ends at +// the caller's last writer. On the test-only MANIFEST path (a test clears +// the dummy flag through the sync point DBImpl::SetOptions:dummy_edit), +// the grouping rule for prepared options in +// VersionSet::ProcessManifestWrites() keeps the call alone in its batch. +// 3. When the call reaches the head of the MANIFEST writer queue, pre_cb +// applies the input maps to the entries under the DB mutex +// (ColumnFamilyData::PrepareMutableCFOptions()). +// 4. VersionSet::ProcessManifestWrites() builds each new Version with its +// entry, and with the phasing parameters of the new anchor if one is set. +// 5. In the same mutex hold as AppendVersion(), it commits the anchor and +// calls ColumnFamilyData::PublishMutableCFOptions() with the options of +// each new Version. +struct MutableCFOptionsPublication { + // The new options of each column family, one entry per element of the + // `cfds` argument of LogAndApply(), stored by value. Each column family + // appears once. The pre_cb of SetOptions() writes the entries through + // SetOptions' own non-const reference, under the DB mutex, before any + // Version is built. LogAndApply() only reads them. The vector is not resized + // after LogAndApply() starts, because each ManifestWriter keeps the address + // of its entry. + autovector new_mutable_cf_options; + // If set, the new periodic compaction phasing anchor (see + // VersionSet::PrepareCompactionPhaseReanchor()). The new Versions are built + // with it, and it is published with them. + std::optional compaction_phase_anchor_time; +}; + // VersionSet is the collection of versions of all the column families of the // database. Each database owns one VersionSet. A VersionSet has access to all // column families via ColumnFamilySet, i.e. set of the column families. @@ -1526,9 +1937,41 @@ class VersionSet { column_family_options, {manifest_wcb}, pre_cb); } - // The across-multi-cf batch version. If edit_lists contain more than - // 1 version edits, caller must ensure that no edit in the []list is column - // family manipulation. + // The batch version for several column families. If edit_lists hold more than + // one edit, no edit may add or drop a column family. Also, either every edit + // or no edit is a dummy edit that writes nothing to the MANIFEST (only + // SetOptions makes these). `options_publication` is set only by + // SetOptions(). It holds the new options (one entry per element of `cfds`) + // and an optional new phasing anchor. LogAndApply() publishes both with the + // new Versions. When it is set, each column family may appear only once in + // `cfds`, because all writers of one column family share one Version. + // + // LogAndApply() can release the DB mutex while it waits in the MANIFEST + // writer queue, and also during the batch, both when it writes a MANIFEST + // record and when it gets dummy edits and writes nothing (see + // VersionSet::ProcessManifestWrites()). The callbacks in `manifest_wcbs` run + // once per batch, under the DB mutex, at the end of the batch: on success + // after the new Versions are installed, and on a failure inside the batch + // with the error. The DB mutex is not released between the callbacks and the + // point where the writers of the batch are marked done. They do not run if + // LogAndApply() returns before the batch starts: when there are no edits, + // when every column family is dropped, or when pre_cb fails. They must not + // call user code or do I/O. + // + // pre_cb: when pre_cb is set, the call gets no group commit with other + // calls. An earlier leader does not take the writers of this call into its + // batch, and the batch of this call does not take the writers of a later + // call. When the call reaches the queue head, pre_cb runs under the DB mutex. + // If the batch writes a MANIFEST record, each edit of the call's first + // column family that changes a Version is then applied without the DB mutex + // (VersionSet::LogAndApplyHelper()), and pre_cb runs again after each such + // edit, because the caller's state can change meanwhile. So a call with one + // such edit runs pre_cb twice, and pre_cb must be safe to run more than + // once. The edits of the other column families of the call are applied + // under the DB mutex. Dummy edits (SetOptions()) write no MANIFEST record + // and are applied under the DB mutex, so their pre_cb runs once. + // Intended user: indirect blob GC, which checks that its input is still + // installable. virtual Status LogAndApply( const autovector& cfds, const ReadOptions& read_options, const WriteOptions& write_options, @@ -1538,10 +1981,74 @@ class VersionSet { const ColumnFamilyOptions* new_cf_options = nullptr, const std::vector>& manifest_wcbs = {}, const std::function& pre_cb = {}, - int max_file_opening_threads = 1); + int max_file_opening_threads = 1, + const MutableCFOptionsPublication* options_publication = nullptr); void WakeUpWaitingManifestWriters(); + // An options publication (new_mutable_cf_options) must not release the DB + // mutex between publishing its Versions and the caller installing their + // SuperVersions. So on success LogAndApply keeps its census payloads here + // instead of freeing them unlocked. The caller calls this after it has + // installed the SuperVersions. REQUIRES: *mu held; releases and reacquires + // it while freeing the payloads. + void ReleaseDeferredPublicationCleanup(InstrumentedMutex* mu); + + // Freeing relocation state + // + // Why: the relocation state of a fenced Version (mainly its file table) can + // be large, and freeing it under the DB mutex can stall writers. Versions, + // builders and file reference counts still change only under the DB mutex. + // Holders are std::shared_ptr because they hold different types (a + // Version's BlobRelocationState, a state whose files were released, or a + // builder's change maps), and the free step only needs to drop the last + // reference. + // + // There are two paths: + // 1. Explicit release (the candidate Versions of a failed LogAndApply + // batch, the builders of every LogAndApply batch, and the Versions and + // builders that ManifestTailer deletes): the caller moves the state + // into `holders` with VersionStorageInfo::ReleaseBlobRelocationState() + // and VersionBuilder::ReleaseBlobRelocationChanges(), destroys the + // Version or builder under the mutex, and then calls this function, + // which frees the state without the mutex. + // 2. ~Version(), when the last reference to a Version is dropped (under the + // DB mutex), and only if this Version is the last owner of its + // relocation state (Versions can share one state): the Version frees its + // relocation file metadata under the mutex, because the metadata deleter + // uses the column family. It hands only the file table to + // AddObsoleteBlobRelocationState() + // (VersionStorageInfo::ReleaseBlobRelocationStateForDeferredFree()). The + // next DBImpl::FindObsoleteFiles() moves the table to its JobContext + // (TakeObsoleteBlobRelocationStates()), and JobContext::Clean() frees it + // without the mutex. This also works while file deletion is disabled. + // + // Lifecycle of a relocation file (its metadata, cached reader and disk + // file): + // 1. The last owner of the file's metadata drops it: usually the last + // Version that lists the file, destroyed under the DB mutex, or, for a + // failed LogAndApply batch or the ManifestTailer, relocation state that + // is freed without the mutex. + // 2. The metadata deleter (VersionBuilder) adds the file to the obsolete + // blob files and drops a remembered failed open. It does not evict the + // cached reader, because a reader open can hold the stripe mutex of the + // file (see BlobFileCache::Evict()). + // 3. The file table is freed without the DB mutex (path 1 or path 2 + // above). + // 4. DBImpl::PurgeObsoleteFiles() evicts the cached reader + // (TableCache::Evict()), checks that the file starts with the + // relocation file prefix, and deletes the file. + // + // This function (path 1) first drops every holder that is null or shared with + // another owner, because freeing it frees nothing. If a holder is left, it + // releases `mu`, frees the holders, and locks `mu` again; + // `unlocked_sync_point` fires while `mu` is released. Otherwise (for example, + // legacy Versions and builders give up nothing) `mu` stays held. File + // reference counts are not touched here: they change only under `mu`. + static void FreeBlobRelocationState( + std::vector>* holders, InstrumentedMutex* mu, + const char* unlocked_sync_point); + // Recover the last saved descriptor (MANIFEST) from persistent storage. // Unlike `unchanging` on the VersionSet, `read_only` here and in other // functions below refers to the CF receiving no writes or modifications @@ -1921,9 +2428,30 @@ class VersionSet { void GetLiveFilesMetaData(std::vector* metadata); void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) { + MutexLock lock(&obsolete_blob_files_mutex_); obsolete_blob_files_.emplace_back(blob_file_number, std::move(path)); } + // Keeps the file table of a destroyed Version's relocation state, which can + // be large, so that it is freed later without the DB mutex (path 2 of + // "Freeing relocation state" at FreeBlobRelocationState()). + void AddObsoleteBlobRelocationState(std::shared_ptr state) { + MutexLock lock(&obsolete_blob_files_mutex_); + obsolete_blob_relocation_states_.push_back(std::move(state)); + } + + // Moves out the states kept by AddObsoleteBlobRelocationState(). The caller + // frees them without the DB mutex (JobContext::Clean()). + void TakeObsoleteBlobRelocationStates( + std::vector>* states) { + MutexLock lock(&obsolete_blob_files_mutex_); + states->insert( + states->end(), + std::make_move_iterator(obsolete_blob_relocation_states_.begin()), + std::make_move_iterator(obsolete_blob_relocation_states_.end())); + obsolete_blob_relocation_states_.clear(); + } + // Obsolete blob files numbered at or above `min_pending_output`, and // obsolete SST files numbered at or above `min_listener_pending_output`, are // left for a later call. @@ -1969,12 +2497,45 @@ class VersionSet { PeriodicCompactionPhaseParams GetPeriodicCompactionPhaseParams( uint32_t cf_id) const; - // (Re)anchor periodic-compaction phasing to now and refresh the cached phase - // params on every column family's current Version. Called when a CF's - // periodic_compaction_seconds changes via SetOptions, so a turn-down's newly - // past-due cohort is spread (over the phase grid within ~N/4 of now) instead - // of firing all at once. Caller must hold the DB mutex. - void ReanchorCompactionPhase(); + // Periodic compaction phasing anchor (used only when + // DBOptions::periodic_compaction_phase_recovery_percent > 0). The anchor is + // the time from which PeriodicCompactionPhaser::TriggerTime() spreads + // catch-up compactions. A catch-up compaction is a periodic compaction of a + // file that is already due. A lower periodic_compaction_seconds can make many + // files due at the same time. So when an options map of any column family + // sets periodic_compaction_seconds and the call succeeds, + // DBImpl::SetOptions() moves the anchor to the current time. This also + // happens when the value does not change, unless the call returns early + // because no option changes. It does not move when there is no clock or the + // clock fails. TriggerTime() then spreads these compactions out after the + // anchor instead of running them all at once. The anchor is DB-wide, so it + // also applies to the column families that keep their period. The move has + // two steps, so that the new Versions can be built with the new anchor before + // it is published: + // 1. PrepareCompactionPhaseReanchor() reads the clock. + // 2. CommitCompactionPhaseReanchor() publishes the anchor, inside + // LogAndApply(), together with the new options and Versions. + // Until step 2, the live phaser and the current Versions keep the old + // anchor. + + // Step 1: returns a new phasing anchor (the current time) for SetOptions(), + // without changing the live phasing state. If there is no clock or the clock + // fails, it returns no anchor, and SetOptions() keeps the current one. + std::optional PrepareCompactionPhaseReanchor() const; + + // Returns the phasing parameters that column family `cf_id` would get with + // the anchor `anchor_time`, without changing the live phasing state. + // LogAndApply() uses it to build a new Version for SetOptions() before the + // anchor is published. + PeriodicCompactionPhaseParams GetPeriodicCompactionPhaseParamsAtAnchor( + uint32_t cf_id, uint64_t anchor_time) const; + + // Step 2: publishes a phasing anchor, and refreshes the cached phasing + // parameters in the current Version of every column family that is not + // dropped. The new Versions that SetOptions() prepared already have these + // parameters. + // REQUIRES: DB mutex held. + void CommitCompactionPhaseReanchor(uint64_t anchor_time); const ImmutableDBOptions* db_options() const { return db_options_; } @@ -2017,6 +2578,9 @@ class VersionSet { size_t TEST_GetManifestPreallocationSize() { return manifest_preallocation_size_; } + uint32_t TEST_GetDbBlobIndirectionFormatVersion() const { + return blob_indirection_format_version_; + } // Appends a kColumnFamilyDrop record for each id in cf_ids to the MANIFEST // file at manifest_path, whose valid content length is manifest_size bytes. @@ -2036,6 +2600,7 @@ class VersionSet { friend class Version; friend class VersionEditHandler; friend class VersionEditHandlerPointInTime; + friend class ManifestTailer; friend class DumpManifestHandler; friend class DBImpl; friend class DBImplReadOnly; @@ -2079,7 +2644,8 @@ class VersionSet { const std::unordered_map& curr_state, const VersionEdit& wal_additions, const OptionsFileManifestState& options_file_manifest_state, - log::Writer* log, IOStatus& io_s); + uint32_t blob_indirection_format_version, log::Writer* log, + IOStatus& io_s); // Reopen the existing MANIFEST file for append at the end of Recover() // when reuse_manifest_on_open is set, so the next LogAndApply appends @@ -2119,6 +2685,8 @@ class VersionSet { void AppendVersion(ColumnFamilyData* column_family_data, Version* v); + void RefreshCurrentVersionCompactionPhaseParams(); + ColumnFamilyData* CreateColumnFamily(const ColumnFamilyOptions& cf_options, const ReadOptions& read_options, const VersionEdit* edit, bool read_only, @@ -2156,6 +2724,19 @@ class VersionSet { uint64_t options_file_size_; bool has_committed_options_file_number_; uint64_t pending_manifest_file_number_; + // The DB-global format fence (0 = not set). A format fence is the MANIFEST + // record that marks blob indirection as in use (see "MANIFEST records" in + // db/blob/blob_relocation_manifest.h). The per-CF fence is in + // VersionStorageInfo. This one stays set when the last fenced column family + // is dropped, and every new MANIFEST writes it as its first record + // (WriteCurrentStateToManifest()). + uint32_t blob_indirection_format_version_ = 0; + // The relocation files deleted, the blob files removed by blob file + // retirement, and the routes deleted in the current MANIFEST. See the "no + // re-add" rule at RejectReaddedBlobRelocationState(). Protected by the DB + // mutex. + UnorderedSet manifest_removed_blob_physical_files_; + UnorderedSet manifest_deleted_blob_relocation_routes_; // The last seq visible to reads. It normally indicates the last sequence in // the memtable but when using two write queues it could also indicate the // last sequence in the WAL visible to reads. @@ -2188,6 +2769,11 @@ class VersionSet { // Queue of writers to the manifest file std::deque manifest_writers_; + // See ReleaseDeferredPublicationCleanup(). Protected by the DB mutex. + std::vector + deferred_publication_census_installs_; + std::vector deferred_publication_census_garbage_; + // Current size of manifest file uint64_t manifest_file_size_; @@ -2239,7 +2825,17 @@ class VersionSet { // Obsolete files, or during DB shutdown any files not referenced by what's // left of the in-memory LSM state. std::vector obsolete_files_; + // Protects obsolete_blob_files_ and obsolete_blob_relocation_states_. + // Relocation state can be freed without the DB mutex (see + // FreeBlobRelocationState()); the deleter of a relocation file's metadata + // then adds the file to obsolete_blob_files_. + port::Mutex obsolete_blob_files_mutex_; std::vector obsolete_blob_files_; + // File tables of destroyed Versions' relocation state, freed without the DB + // mutex. Their relocation file metadata was already freed under it + // (VersionStorageInfo::ReleaseBlobRelocationStateForDeferredFree()), so they + // do not depend on any column family. + std::vector> obsolete_blob_relocation_states_; std::vector obsolete_manifests_; // env options for all reads and writes except compactions @@ -2261,15 +2857,55 @@ class VersionSet { ErrorHandler* const error_handler_; private: + // The blob reference census installs of one ProcessManifestWrites() batch. + // The census of a column family (BlobReferenceCensus, see + // db/blob/blob_reference_census.h) follows each new Version of the column + // family in four steps: + // 1. Reserve, under the DB mutex: ReserveBlobReferenceCensusInstalls() + // reserves an install for each new Version that needs one. + // 2. Prepare, without the DB mutex: PrepareBlobReferenceCensusInstalls() + // builds the delta of each such Version (BuildBlobReferenceCensusDelta()) + // and prepares its install. + // 3. Install, under the DB mutex: ProcessManifestWrites() installs each + // prepared install when it appends the new Version. The old census state + // becomes garbage. + // 4. Free, without the DB mutex: ProcessManifestWrites() cancels the unused + // reservations under the mutex, then frees the installs and the garbage + // without it, or defers that (see ReleaseDeferredPublicationCleanup()). + // The struct is defined in version_set.cc, which includes the census types. + struct BlobReferenceCensusInstallBatch; + + // Step 1 (see BlobReferenceCensusInstallBatch). REQUIRES: DB mutex held. + // `pending_format_version` is the DB-global blob indirection format version + // after this batch commits: the current one, or the one this batch + // activates. It is 0 only when the DB has no format fence, and then no + // column family needs a census. + void ReserveBlobReferenceCensusInstalls( + const autovector& versions, uint32_t pending_format_version, + BlobReferenceCensusInstallBatch* batch) const; + + // Step 2. Runs without the DB mutex. Returns Corruption for an edit that the + // census cannot apply. A failure to prepare one install does not fail the + // batch: that census is invalidated and rebuilt instead. + Status PrepareBlobReferenceCensusInstalls( + const autovector& batch_edits, + const autovector& versions, + BlobReferenceCensusInstallBatch* batch) const; + + // Builds the census delta from the current Version of the column family of + // `version` to `version`, from the edits of that column family in + // `batch_edits`. Part of step 2. + Status BuildBlobReferenceCensusDelta( + const autovector& batch_edits, Version* version, + std::shared_ptr* delta) const; + // REQUIRES db mutex at beginning. may release and re-acquire db mutex - Status ProcessManifestWrites(std::deque& writers, - InstrumentedMutex* mu, - FSDirectory* dir_contains_current_file, - bool new_descriptor_log, - const ColumnFamilyOptions* new_cf_options, - bool new_cf_options_are_sanitized, - const ReadOptions& read_options, - const WriteOptions& write_options); + Status ProcessManifestWrites( + std::deque& writers, InstrumentedMutex* mu, + FSDirectory* dir_contains_current_file, bool new_descriptor_log, + const ColumnFamilyOptions* new_cf_options, + bool new_cf_options_are_sanitized, const ReadOptions& read_options, + const WriteOptions& write_options, const std::function& pre_cb); Status LogAndApplyImpl( const autovector& column_family_datas, @@ -2279,13 +2915,40 @@ class VersionSet { bool new_descriptor_log, const ColumnFamilyOptions* new_cf_options, bool new_cf_options_are_sanitized, const std::vector>& manifest_wcbs, - const std::function& pre_cb, int max_file_opening_threads); + const std::function& pre_cb, int max_file_opening_threads, + const MutableCFOptionsPublication* options_publication); void LogAndApplyCFHelper(VersionEdit* edit, SequenceNumber* max_last_sequence); Status LogAndApplyHelper(ColumnFamilyData* cfd, VersionBuilder* b, VersionEdit* edit, SequenceNumber* max_last_sequence, - InstrumentedMutex* mu); + InstrumentedMutex* mu, bool apply_unlocked, + const std::function& pre_cb); + // The "no re-add" rule. File numbers are never reused. So an edit that adds + // back a relocation file, a route, or a blob file that an earlier edit of the + // same MANIFEST deleted (or removed by blob file retirement) is a writer bug. + // LogAndApply applies the rule when it writes, so the bug cannot first show + // up at the next open. The rule is checked in three places: + // 1. VersionBuilder rejects a re-added relocation file or route within the + // edits it applies: one LogAndApply batch, or the whole MANIFEST at + // replay (IsBlobRelocationFileDeletedInBuilder() and + // IsBlobRelocationRouteDeletedInBuilder()). + // 2. This function rejects a re-add of a number that an earlier batch of + // the current MANIFEST deleted. RecordBlobRelocationDeletions() keeps + // that history in manifest_removed_blob_physical_files_ and + // manifest_deleted_blob_relocation_routes_. + // 3. With reuse_manifest_on_open, the open keeps writing to the replayed + // MANIFEST. VersionEditHandler::TakeBlobRelocationDeletions() then seeds + // the history of (2) with the deletions found in the replay. + Status RejectReaddedBlobRelocationState(const VersionEdit& edit) const; + + // Adds the relocation files and routes that `edits` delete, and the blob + // files that their blob file retirement records remove, to the deletion + // history of the current MANIFEST. RejectReaddedBlobRelocationState() uses + // this history to reject an edit that adds them back. When `new_manifest` is + // true, the history is cleared first. + void RecordBlobRelocationDeletions(const autovector& edits, + bool new_manifest); const bool unchanging_; bool closed_; @@ -2363,7 +3026,8 @@ class ReactiveVersionSet : public VersionSet { Status MaybeSwitchManifest( log::Reader::Reporter* reporter, - std::unique_ptr* manifest_reader); + std::unique_ptr* manifest_reader, + InstrumentedMutex* db_mutex); private: std::unique_ptr manifest_tailer_; @@ -2375,6 +3039,8 @@ class ReactiveVersionSet : public VersionSet { // Defers initial SecondaryDB SST validation and opening until after // OpenAsSecondary returns. Catch-up remains synchronous. const bool defer_sst_file_opening_; + // Debug check that catch-ups do not overlap (see ReadAndApply()). + bool manifest_tail_in_progress_ = false; // TODO: plumb Env::IOActivity, Env::IOPriority const ReadOptions read_options_; using VersionSet::LogAndApply; @@ -2389,7 +3055,8 @@ class ReactiveVersionSet : public VersionSet { bool /*new_descriptor_log*/, const ColumnFamilyOptions* /*new_cf_option*/, const std::vector>& /*manifest_wcbs*/, const std::function& /*pre_cb*/, - int /*max_file_opening_threads*/) override { + int /*max_file_opening_threads*/, + const MutableCFOptionsPublication* /*options_publication*/) override { return Status::NotSupported("not supported in reactive mode"); } diff --git a/db/version_set_test.cc b/db/version_set_test.cc index f1706d8d69e7..e0980c6fc38f 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -190,11 +190,12 @@ class VersionStorageInfoTestBase : public testing::Test { void AddBlob(uint64_t blob_file_number, uint64_t total_blob_count, uint64_t total_blob_bytes, BlobFileMetaData::LinkedSsts linked_ssts, - uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) { + uint64_t garbage_blob_count, uint64_t garbage_blob_bytes, + bool is_indirect_origin = false) { auto shared_meta = SharedBlobFileMetaData::Create( blob_file_number, total_blob_count, total_blob_bytes, /* checksum_method */ std::string(), - /* checksum_value */ std::string()); + /* checksum_value */ std::string(), is_indirect_origin); auto meta = BlobFileMetaData::Create(std::move(shared_meta), std::move(linked_ssts), garbage_blob_count, garbage_blob_bytes); @@ -659,6 +660,73 @@ TEST_F(VersionStorageInfoTest, ForcedBlobGCEmpty) { ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); } +TEST_F(VersionStorageInfoTest, ForcedBlobGCUnlinkedIndirectOriginIsOldest) { + // Indirect origin #10 was relocated, so no SST links it, but it stays in the + // Version until its blob file retirement record. Normal blob file #11 is + // linked by SST #1. Nothing is marked: with enable_blob_indirection on, the + // oldest file is an indirect origin, and with it off, #10 has no SST to + // compact. + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + constexpr int level = 0; + constexpr uint64_t sst = 1; + constexpr uint64_t indirect_origin = 10; + constexpr uint64_t legacy_blob = 11; + + Add(level, sst, "bar", "foo", /*file_size=*/1000, legacy_blob); + AddBlob(indirect_origin, /*total_blob_count=*/10, /*total_blob_bytes=*/1000, + BlobFileMetaData::LinkedSsts{}, /*garbage_blob_count=*/9, + /*garbage_blob_bytes=*/900, /*is_indirect_origin=*/true); + AddBlob(legacy_blob, /*total_blob_count=*/10, /*total_blob_bytes=*/1000, + BlobFileMetaData::LinkedSsts{sst}, /*garbage_blob_count=*/9, + /*garbage_blob_bytes=*/900); + UpdateVersionStorageInfo(); + + constexpr double age_cutoff = 1.0; + constexpr double force_threshold = 0.5; + for (const bool enable_blob_indirection : {false, true}) { + SCOPED_TRACE(enable_blob_indirection); + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true, + enable_blob_indirection); + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); + } +} + +TEST_F(VersionStorageInfoTest, ForcedBlobGCLinkedIndirectOriginIsOldest) { + // Indirect origin #10 still holds its values, so SST #1 links it. + vstorage_.SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + constexpr int level = 0; + constexpr uint64_t sst = 1; + constexpr uint64_t indirect_origin = 10; + + Add(level, sst, "bar", "foo", /*file_size=*/1000, indirect_origin); + AddBlob(indirect_origin, /*total_blob_count=*/10, /*total_blob_bytes=*/1000, + BlobFileMetaData::LinkedSsts{sst}, /*garbage_blob_count=*/9, + /*garbage_blob_bytes=*/900, /*is_indirect_origin=*/true); + UpdateVersionStorageInfo(); + + constexpr double age_cutoff = 1.0; + constexpr double force_threshold = 0.5; + + // With enable_blob_indirection off, compaction rewrites the values of #10 + // into new normal blob files, so compacting SST #1 frees #10. + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true, + /*enable_blob_indirection=*/false); + const auto& marked = vstorage_.FilesMarkedForForcedBlobGC(); + ASSERT_EQ(marked.size(), 1); + ASSERT_EQ(marked[0], std::make_pair(level, vstorage_.LevelFiles(level)[0])); + + // With enable_blob_indirection on, compaction keeps the references to #10 as + // they are, so compacting SST #1 cannot free #10. + vstorage_.ComputeFilesMarkedForForcedBlobGC( + age_cutoff, force_threshold, /*enable_blob_garbage_collection=*/true, + /*enable_blob_indirection=*/true); + ASSERT_TRUE(vstorage_.FilesMarkedForForcedBlobGC().empty()); +} + TEST_F(VersionStorageInfoTest, ForcedBlobGCSingleBatch) { // Test the edge case when all blob files are part of the oldest batch. // We have one L0 SST file #1, and four blob files #10, #11, #12, and #13. @@ -1556,6 +1624,160 @@ TEST_F(VersionSetTest, SameColumnFamilyGroupCommit) { EXPECT_EQ(kGroupSize - 1, count); } +// While the edit is applied without the DB mutex, the test checks that the +// DB mutex is free and then makes the precondition invalid. The second pre_cb +// run must fail the call. pre_cb must have run twice, and the current Version +// must not change. +TEST_F(VersionSetTest, UnlockedApplyRevalidatesPrecondition) { + NewDB(); + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + const uint64_t original_version_number = cfd->current()->GetVersionNumber(); + + std::atomic precondition_valid{true}; + size_t precondition_checks = 0; + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + sync_point->SetCallBack("VersionSet::LogAndApplyHelper:ApplyUnlocked", + [&](void*) { + ASSERT_TRUE(mutex_.TryLock()); + mutex_.Unlock(); + precondition_valid.store(false); + }); + sync_point->EnableProcessing(); + + VersionEdit edit; + edit.SetDBId("candidate"); + mutex_.Lock(); + const Status status = versions_->LogAndApply( + cfd, read_options_, write_options_, &edit, &mutex_, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, + /*manifest_wcb=*/{}, [&]() { + ++precondition_checks; + if (!precondition_valid.load()) { + return Status::Incomplete("precondition changed during apply"); + } + return Status::OK(); + }); + mutex_.Unlock(); + + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + ASSERT_TRUE(status.IsIncomplete()) << status.ToString(); + ASSERT_EQ(precondition_checks, 2U); + ASSERT_EQ(cfd->current()->GetVersionNumber(), original_version_number); +} + +TEST_F(VersionSetTest, BlobFinalizationDoesNotHoldDBMutex) { + NewDB(); + + std::promise finalization_started; + std::future finalization_started_future = + finalization_started.get_future(); + std::promise release_finalization; + std::future release_finalization_future = + release_finalization.get_future(); + std::promise follower_queued; + std::future follower_queued_future = follower_queued.get_future(); + std::atomic block_first_finalization{true}; + std::atomic expect_follower_queue{false}; + std::atomic follower_queue_signaled{false}; + const auto wait_or_abort = [](std::future* future, + const char* description) { + if (future->wait_for(std::chrono::seconds(30)) != + std::future_status::ready) { + fprintf(stderr, "Timed out waiting for %s\n", description); + abort(); + } + }; + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeBlobFinalization", [&](void*) { + if (block_first_finalization.exchange(false, + std::memory_order_relaxed)) { + finalization_started.set_value(); + release_finalization_future.wait(); + } + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) { + if (expect_follower_queue.load(std::memory_order_relaxed) && + !follower_queue_signaled.exchange(true, + std::memory_order_relaxed)) { + follower_queued.set_value(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + Status leader_status; + Status follower_status; + VersionEdit leader_edit; + leader_edit.SetDBId("leader"); + port::Thread leader( + [&] { leader_status = LogAndApplyToDefaultCF(leader_edit); }); + wait_or_abort(&finalization_started_future, "blob finalization"); + expect_follower_queue.store(true, std::memory_order_relaxed); + + VersionEdit follower_edit; + follower_edit.SetDBId("follower"); + port::Thread follower( + [&] { follower_status = LogAndApplyToDefaultCF(follower_edit); }); + wait_or_abort(&follower_queued_future, "the following MANIFEST writer"); + + release_finalization.set_value(); + leader.join(); + follower.join(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_OK(leader_status); + ASSERT_OK(follower_status); +} + +TEST_F(VersionSetTest, BlobFinalizationFailureKeepsManifestWriter) { + NewDB(); + // The first edit opens the MANIFEST writer that the failure must keep. + VersionEdit first_edit; + first_edit.SetDBId("first"); + ASSERT_OK(LogAndApplyToDefaultCF(first_edit)); + const uint64_t manifest_file_number = versions_->manifest_file_number(); + + // Fail only the consistency check run by blob finalization, which happens + // before any MANIFEST record is written. + std::atomic in_finalization{false}; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeBlobFinalization", + [&](void*) { in_finalization.store(true); }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:AfterBlobFinalization", + [&](void*) { in_finalization.store(false); }); + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) { + if (in_finalization.load()) { + *static_cast(arg) = Status::Corruption("finalization"); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + VersionEdit failed_edit; + failed_edit.SetDBId("failed"); + ASSERT_TRUE(LogAndApplyToDefaultCF(failed_edit).IsCorruption()); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + // No MANIFEST write was attempted, so the next edit reuses the writer. + VersionEdit next_edit; + next_edit.SetDBId("next"); + ASSERT_OK(LogAndApplyToDefaultCF(next_edit)); + ASSERT_EQ(versions_->manifest_file_number(), manifest_file_number); +} + namespace { // Runs `fn` on its own thread and aborts if it does not return in time. A // LogAndApply queued behind a stranded writer is never woken, and the @@ -1612,7 +1834,8 @@ TEST_F(VersionSetTest, FailedEditReleasesEveryWriterInTheBatch) { mutex_.Lock(); const Status s = versions_->LogAndApply( cfds, read_options_, write_options_, edit_lists, &mutex_, - /*dir_contains_current_file=*/nullptr, /*new_descriptor_log=*/false, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/false, /*new_cf_options=*/nullptr, manifest_wcbs); mutex_.Unlock(); ASSERT_TRUE(s.IsCorruption()) << s.ToString(); @@ -1725,6 +1948,252 @@ TEST_F(VersionSetTest, FailedBatchSignalsWriterFromAnotherCaller) { })); } +TEST_F(VersionSetTest, PreconditionedWriterIsNotGroupedBehindEarlierWriter) { + NewDB(); + + std::promise first_manifest_write_started; + std::future first_manifest_write_started_future = + first_manifest_write_started.get_future(); + std::promise release_first_manifest_write; + std::future release_first_manifest_write_future = + release_first_manifest_write.get_future(); + std::promise plain_writer_queued; + std::future plain_writer_queued_future = + plain_writer_queued.get_future(); + std::promise preconditioned_writer_queued; + std::future preconditioned_writer_queued_future = + preconditioned_writer_queued.get_future(); + std::atomic queued_writers{0}; + std::atomic block_first_manifest_write{true}; + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) { + const int num_queued = + queued_writers.fetch_add(1, std::memory_order_relaxed) + 1; + if (num_queued == 2) { + plain_writer_queued.set_value(); + } else if (num_queued == 3) { + preconditioned_writer_queued.set_value(); + } + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::LogAndApply:WriteManifestStart", [&](void*) { + if (block_first_manifest_write.exchange(false, + std::memory_order_relaxed)) { + first_manifest_write_started.set_value(); + release_first_manifest_write_future.wait(); + } + }); + SyncPoint::GetInstance()->EnableProcessing(); + + // Three writers: the leader blocks in its MANIFEST write, then the plain + // writer and the preconditioned writer queue behind it. The plain writer + // leads the next batch. That batch must not take the preconditioned writer, + // because its precondition runs only when it reaches the head of the queue. + Status leader_status; + Status plain_status; + Status preconditioned_status; + int precondition_calls = 0; + ASSERT_OK(RunOrAbortOnHang([&] { + VersionEdit leader_edit; + leader_edit.SetDBId("leader_db_id"); + port::Thread leader( + [&] { leader_status = LogAndApplyToDefaultCF(leader_edit); }); + first_manifest_write_started_future.wait(); + + VersionEdit plain_edit; + plain_edit.SetDBId("plain_db_id"); + port::Thread plain( + [&] { plain_status = LogAndApplyToDefaultCF(plain_edit); }); + plain_writer_queued_future.wait(); + + VersionEdit preconditioned_edit; + preconditioned_edit.SetDBId("preconditioned_db_id"); + port::Thread preconditioned([&] { + mutex_.Lock(); + preconditioned_status = versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), read_options_, + write_options_, &preconditioned_edit, &mutex_, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, /*manifest_wcb=*/{}, [&] { + ++precondition_calls; + return Status::OK(); + }); + mutex_.Unlock(); + }); + preconditioned_writer_queued_future.wait(); + + release_first_manifest_write.set_value(); + leader.join(); + plain.join(); + preconditioned.join(); + return Status::OK(); + })); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + + ASSERT_OK(leader_status); + ASSERT_OK(plain_status); + ASSERT_OK(preconditioned_status); + // If the plain writer's batch took the preconditioned writer, the + // preconditioned writer would finish without running its precondition. + ASSERT_GT(precondition_calls, 0); +} + +// A LogAndApply() call that passes a precondition callback (pre_cb) is called +// a preconditioned leader here, once it reaches the head of the MANIFEST +// writer queue. A batch is the group of queued writers that one leader writes +// to the MANIFEST together. The leader applies its edit without the DB mutex +// (see VersionSet::LogAndApplyHelper()), so other callers can queue behind it +// meanwhile. These tests check that such a leader is a batch barrier: its +// batch holds only its own caller's writers, and a caller queued behind it +// runs in a later batch with its own result. +class VersionSetPreconditionBarrierTest : public VersionSetTest { + protected: + struct BarrierResult { + Status leader_status; + Status plain_status; + int precondition_checks = 0; + // The number of LogAndApply callers that led a batch. It is 2 when the + // plain writer ran in its own batch, and 1 when the leader's batch took + // it. + int batch_leaders = 0; + }; + + // Runs a preconditioned leader on the default column family. While the + // leader applies its edit without the DB mutex, a second thread queues a + // plain LogAndApply (no precondition) on column family kColumnFamilyName1. + // The plain edit sets the log number of that column family to + // `plain_log_number`. The leader's precondition passes on its first check. + // On the second check, after the unlocked apply, it fails when + // `fail_second_check` is true. + BarrierResult RunPreconditionedLeaderWithQueuedPlainWriter( + bool fail_second_check, uint64_t plain_log_number) { + BarrierResult result; + std::promise plain_writer_queued; + std::future plain_writer_queued_future = + plain_writer_queued.get_future(); + std::atomic expect_plain_queue{false}; + std::atomic plain_started{false}; + std::unique_ptr plain; + ColumnFamilyData* const plain_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(kColumnFamilyName1); + EXPECT_NE(plain_cfd, nullptr); + VersionEdit plain_edit; + plain_edit.SetColumnFamily(plain_cfd->GetID()); + plain_edit.SetLogNumber(plain_log_number); + + SyncPoint* const sync_point = SyncPoint::GetInstance(); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + sync_point->SetCallBack( + "VersionSet::LogAndApplyHelper:ApplyUnlocked", [&](void*) { + if (plain_started.exchange(true, std::memory_order_relaxed)) { + return; + } + // The leader released the DB mutex, so the plain writer can take + // it and queue behind the leader before the leader continues. + expect_plain_queue.store(true, std::memory_order_relaxed); + plain.reset(new port::Thread([&] { + mutex_.Lock(); + result.plain_status = + versions_->LogAndApply(plain_cfd, read_options_, write_options_, + &plain_edit, &mutex_, nullptr); + mutex_.Unlock(); + })); + plain_writer_queued_future.wait(); + }); + sync_point->SetCallBack( + "VersionSet::LogAndApply:BeforeWriterWaiting", [&](void*) { + if (expect_plain_queue.exchange(false, std::memory_order_relaxed)) { + plain_writer_queued.set_value(); + } + }); + sync_point->SetCallBack("VersionSet::LogAndApply:WakeUpAndNotDone", + [&](void*) { ++result.batch_leaders; }); + sync_point->EnableProcessing(); + + EXPECT_OK(RunOrAbortOnHang([&] { + VersionEdit leader_edit; + leader_edit.SetDBId("leader_db_id"); + mutex_.Lock(); + result.leader_status = versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), read_options_, + write_options_, &leader_edit, &mutex_, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, + /*manifest_wcb=*/{}, [&]() { + ++result.precondition_checks; + if (fail_second_check && result.precondition_checks >= 2) { + return Status::Incomplete("precondition changed during apply"); + } + return Status::OK(); + }); + mutex_.Unlock(); + if (plain != nullptr) { + plain->join(); + } + return Status::OK(); + })); + sync_point->DisableProcessing(); + sync_point->ClearAllCallBacks(); + EXPECT_TRUE(plain_started.load()); + return result; + } + + // Checks that the log number of column family kColumnFamilyName1 is + // `log_number` now and after the VersionSet is recovered from the MANIFEST. + void VerifyPlainLogNumber(uint64_t log_number) { + ColumnFamilyData* cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(kColumnFamilyName1); + ASSERT_NE(cfd, nullptr); + ASSERT_EQ(cfd->GetLogNumber(), log_number); + CloseDB(); + ReopenDB(); + cfd = versions_->GetColumnFamilySet()->GetColumnFamily(kColumnFamilyName1); + ASSERT_NE(cfd, nullptr); + ASSERT_EQ(cfd->GetLogNumber(), log_number); + } +}; + +TEST_F(VersionSetPreconditionBarrierTest, + FailedPreconditionDoesNotFailWriterQueuedDuringUnlockedApply) { + // The leader's precondition fails after the unlocked apply. The error + // belongs to the leader's caller only. The plain writer that queued while + // the DB mutex was released must run in its own batch, succeed, and have + // its edit in the MANIFEST. + NewDB(); + const uint64_t plain_log_number = versions_->FetchAddFileNumber(1); + const BarrierResult result = RunPreconditionedLeaderWithQueuedPlainWriter( + /*fail_second_check=*/true, plain_log_number); + ASSERT_TRUE(result.leader_status.IsIncomplete()) + << result.leader_status.ToString(); + ASSERT_EQ(result.precondition_checks, 2); + ASSERT_OK(result.plain_status); + ASSERT_EQ(result.batch_leaders, 2); + VerifyPlainLogNumber(plain_log_number); +} + +TEST_F(VersionSetPreconditionBarrierTest, + PassingPreconditionLeaderDoesNotTakeQueuedWriter) { + // The leader's precondition passes on both checks. The leader's batch still + // ends with its own caller's writers: a later caller could change the state + // that the precondition checked. So the plain writer leads its own batch. + NewDB(); + const uint64_t plain_log_number = versions_->FetchAddFileNumber(1); + const BarrierResult result = RunPreconditionedLeaderWithQueuedPlainWriter( + /*fail_second_check=*/false, plain_log_number); + ASSERT_OK(result.leader_status); + ASSERT_EQ(result.precondition_checks, 2); + ASSERT_OK(result.plain_status); + ASSERT_EQ(result.batch_leaders, 2); + VerifyPlainLogNumber(plain_log_number); +} + TEST_F(VersionSetTest, FailedVersionBuildReleasesTheBatch) { NewDB(); @@ -1749,6 +2218,8 @@ TEST_F(VersionSetTest, FailedVersionBuildReleasesTheBatch) { // failure lands in SaveTo() after the batch is assembled. bool armed = false; bool injected = false; + size_t candidate_cleanup_unlocked = 0; + size_t builder_cleanup_unlocked = 0; SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->SetCallBack("VersionBuilder::CheckConsistency1", @@ -1761,6 +2232,19 @@ TEST_F(VersionSetTest, FailedVersionBuildReleasesTheBatch) { injected = true; } }); + const auto assert_unlocked = [&](void* arg, size_t* count) { + auto* db_mutex = static_cast(arg); + ASSERT_EQ(db_mutex, &mutex_); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + ++*count; + }; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CandidateRelocationStateFreedUnlocked", + [&](void* arg) { assert_unlocked(arg, &candidate_cleanup_unlocked); }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BuilderRelocationEditsFreedUnlocked", + [&](void* arg) { assert_unlocked(arg, &builder_cleanup_unlocked); }); SyncPoint::GetInstance()->EnableProcessing(); int callback_calls = 0; Status::Code callback_code = Status::kOk; @@ -1777,6 +2261,10 @@ TEST_F(VersionSetTest, FailedVersionBuildReleasesTheBatch) { SyncPoint::GetInstance()->DisableProcessing(); SyncPoint::GetInstance()->ClearAllCallBacks(); ASSERT_TRUE(injected); + // A legacy candidate and builder hold no relocation state: both are destroyed + // under the mutex, without an unlock. + ASSERT_EQ(candidate_cleanup_unlocked, 0U); + ASSERT_EQ(builder_cleanup_unlocked, 0U); ASSERT_TRUE(s.IsCorruption()) << s.ToString(); EXPECT_EQ(1, callback_calls); EXPECT_EQ(Status::kCorruption, callback_code); @@ -1788,65 +2276,299 @@ TEST_F(VersionSetTest, FailedVersionBuildReleasesTheBatch) { })); } -TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) { - // Initialize the database and add a couple of blob files, one with some - // garbage in it, and one without any garbage. +TEST_F(VersionSetTest, FailedFencedVersionBuildFreesRelocationStateUnlocked) { + // As above, but the edit also activates the format and adds a relocation + // file. The candidate Version and the builder are destroyed under the mutex; + // their relocation state, which only they own, is freed without it. NewDB(); - assert(versions_); - assert(versions_->GetColumnFamilySet()); - - ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); - assert(cfd); + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + // Add two legacy SSTs in an edit before the fence. The fenced edit adds only + // the relocation file: a new SST in a fenced edit would need exact reference + // metadata. + std::vector file_metas; + CreateDummyTableFiles({SstInfo(100, kDefaultColumnFamilyName, "a", + /*level=*/1, /*epoch_number=*/100), + SstInfo(101, kDefaultColumnFamilyName, "c", + /*level=*/1, /*epoch_number=*/101)}, + &file_metas); + ASSERT_EQ(file_metas.size(), 2U); + VersionEdit files; + for (const FileMetaData& meta : file_metas) { + files.AddFile(/*level=*/1, meta); + } + ASSERT_OK(LogAndApplyToDefaultCF(files)); + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(BlobRelocationFileAddition( + /*file_number=*/102, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1)); - Version* const version = cfd->current(); - assert(version); + bool armed = false; + bool injected = false; + size_t candidate_cleanup_unlocked = 0; + size_t builder_cleanup_unlocked = 0; + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + // Arm the failure when blob finalization starts. It then fails the + // consistency check at the end of finalization, when the candidate Version + // already holds its relocation state. + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BeforeBlobFinalization", + [&](void*) { armed = !injected; }); + SyncPoint::GetInstance()->SetCallBack( + "VersionBuilder::CheckConsistencyBeforeReturn", [&](void* arg) { + if (armed) { + *static_cast(arg) = Status::Corruption("Inject corruption"); + armed = false; + injected = true; + } + }); + const auto assert_unlocked = [&](void* arg, size_t* count) { + auto* db_mutex = static_cast(arg); + ASSERT_EQ(db_mutex, &mutex_); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + ++*count; + }; + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:CandidateRelocationStateFreedUnlocked", + [&](void* arg) { assert_unlocked(arg, &candidate_cleanup_unlocked); }); + SyncPoint::GetInstance()->SetCallBack( + "VersionSet::ProcessManifestWrites:BuilderRelocationEditsFreedUnlocked", + [&](void* arg) { assert_unlocked(arg, &builder_cleanup_unlocked); }); + SyncPoint::GetInstance()->EnableProcessing(); + int callback_calls = 0; + Status::Code callback_code = Status::kOk; + mutex_.Lock(); + const Status s = versions_->LogAndApply( + versions_->GetColumnFamilySet()->GetDefault(), read_options_, + write_options_, &edit, &mutex_, /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/false, /*column_family_options=*/nullptr, + [&](const Status& cb_status) { + ++callback_calls; + callback_code = cb_status.code(); + }); + mutex_.Unlock(); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + ASSERT_TRUE(injected) << s.ToString(); + ASSERT_EQ(candidate_cleanup_unlocked, 1U); + ASSERT_EQ(builder_cleanup_unlocked, 1U); + ASSERT_TRUE(s.IsCorruption()) << s.ToString(); + EXPECT_EQ(1, callback_calls); + EXPECT_EQ(Status::kCorruption, callback_code); - VersionStorageInfo* const storage_info = version->storage_info(); - assert(storage_info); + ASSERT_OK(RunOrAbortOnHang([&] { + VersionEdit next; + next.SetDBId("next_db_id"); + return LogAndApplyToDefaultCF(next); + })); +} +TEST_F(VersionSetTest, LogAndApplyRejectsReaddedRelocationState) { + // File numbers are never reused, so replay rejects adding back a relocation + // route or file that an earlier edit of the same MANIFEST deleted. + // LogAndApply rejects it when it writes, so the bug cannot first show up at + // the next open. A new MANIFEST starts without deletion history, for replay + // and for LogAndApply. + NewDB(); + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + const BlobRelocationFileAddition addition( + /*file_number=*/102, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1); + const BlobRelocationRoute route(/*source_file_number=*/77, + /*physical_file_number=*/102); { - constexpr uint64_t blob_file_number = 123; - constexpr uint64_t total_blob_count = 456; - constexpr uint64_t total_blob_bytes = 77777777; - constexpr char checksum_method[] = "SHA1"; - constexpr char checksum_value[] = - "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c" - "\x52\x5c\xbd"; - - auto shared_meta = SharedBlobFileMetaData::Create( - blob_file_number, total_blob_count, total_blob_bytes, checksum_method, - checksum_value); - - constexpr uint64_t garbage_blob_count = 89; - constexpr uint64_t garbage_blob_bytes = 1000000; - - auto meta = BlobFileMetaData::Create( - std::move(shared_meta), BlobFileMetaData::LinkedSsts(), - garbage_blob_count, garbage_blob_bytes); - - storage_info->AddBlobFile(std::move(meta)); + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + edit.SetBlobRelocationRoute(route); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); } - { - constexpr uint64_t blob_file_number = 234; - constexpr uint64_t total_blob_count = 555; - constexpr uint64_t total_blob_bytes = 66666; - constexpr char checksum_method[] = "CRC32"; - constexpr char checksum_value[] = "\x3d\x87\xff\x57"; - - auto shared_meta = SharedBlobFileMetaData::Create( - blob_file_number, total_blob_count, total_blob_bytes, checksum_method, - checksum_value); - - constexpr uint64_t garbage_blob_count = 0; - constexpr uint64_t garbage_blob_bytes = 0; - - auto meta = BlobFileMetaData::Create( - std::move(shared_meta), BlobFileMetaData::LinkedSsts(), - garbage_blob_count, garbage_blob_bytes); - - storage_info->AddBlobFile(std::move(meta)); + VersionEdit edit; + edit.DeleteBlobRelocationRoute(/*source_file_number=*/77); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + { + VersionEdit edit; + edit.SetBlobRelocationRoute(route); + ASSERT_TRUE(LogAndApplyToDefaultCF(edit).IsCorruption()); + } + { + VersionEdit edit; + edit.DeleteBlobRelocationFile(/*file_number=*/102); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + { + VersionEdit edit; + edit.AddBlobRelocationFile(addition); + ASSERT_TRUE(LogAndApplyToDefaultCF(edit).IsCorruption()); + } + // The rejected edits were not written: the MANIFEST still replays. + CloseDB(); + ReopenDB(); + ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobRelocationFileMetaData(102), + nullptr); + // After reopen, the next write starts a new MANIFEST with a snapshot, which + // has no deletion history. + { + VersionEdit edit; + edit.AddBlobRelocationFile(addition); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + CloseDB(); + ReopenDB(); + cfd = versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd->current()->storage_info()->GetBlobRelocationFileMetaData(102), + nullptr); +} + +TEST_F(VersionSetTest, ReusedManifestRejectsReaddedRelocationState) { + // With reuse_manifest_on_open, writes after reopen go to the replayed + // MANIFEST, which already holds the deletions. LogAndApply must still reject + // adding the deleted state back; otherwise the next open fails. + NewDB(); + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + const BlobRelocationFileAddition addition( + /*file_number=*/102, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1); + const BlobRelocationRoute route(/*source_file_number=*/77, + /*physical_file_number=*/102); + { + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(addition); + edit.SetBlobRelocationRoute(route); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + { + VersionEdit edit; + edit.DeleteBlobRelocationRoute(/*source_file_number=*/77); + edit.DeleteBlobRelocationFile(/*file_number=*/102); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + std::string manifest_before; + GetManifestPath(&manifest_before); + CloseDB(); + imm_db_options_.reuse_manifest_on_open = true; + ReopenDB(); + { + VersionEdit edit; + edit.SetBlobRelocationRoute(route); + edit.AddBlobRelocationFile(addition); + const Status status = LogAndApplyToDefaultCF(edit); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + } + { + VersionEdit edit; + edit.AddBlobRelocationFile(addition); + ASSERT_TRUE(LogAndApplyToDefaultCF(edit).IsCorruption()); + } + // An unrelated write goes to the same MANIFEST, and it still replays. + { + VersionEdit edit; + edit.SetLastSequence(1); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + std::string manifest_after; + GetManifestPath(&manifest_after); + ASSERT_EQ(manifest_after, manifest_before); + CloseDB(); + ReopenDB(); + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobRelocationFileMetaData(102), + nullptr); +} + +TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) { + // Initialize the database and add a couple of blob files, one with some + // garbage in it, and one without any garbage. + NewDB(); + + assert(versions_); + assert(versions_->GetColumnFamilySet()); + + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + assert(cfd); + + Version* const version = cfd->current(); + assert(version); + + VersionStorageInfo* const storage_info = version->storage_info(); + assert(storage_info); + + { + constexpr uint64_t blob_file_number = 123; + constexpr uint64_t total_blob_count = 456; + constexpr uint64_t total_blob_bytes = 77777777; + constexpr char checksum_method[] = "SHA1"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c" + "\x52\x5c\xbd"; + + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value); + + constexpr uint64_t garbage_blob_count = 89; + constexpr uint64_t garbage_blob_bytes = 1000000; + + auto meta = BlobFileMetaData::Create( + std::move(shared_meta), BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + storage_info->AddBlobFile(std::move(meta)); + } + + { + constexpr uint64_t blob_file_number = 234; + constexpr uint64_t total_blob_count = 555; + constexpr uint64_t total_blob_bytes = 66666; + constexpr char checksum_method[] = "CRC32"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; + + auto shared_meta = SharedBlobFileMetaData::Create( + blob_file_number, total_blob_count, total_blob_bytes, checksum_method, + checksum_value); + + constexpr uint64_t garbage_blob_count = 0; + constexpr uint64_t garbage_blob_bytes = 0; + + auto meta = BlobFileMetaData::Create( + std::move(shared_meta), BlobFileMetaData::LinkedSsts(), + garbage_blob_count, garbage_blob_bytes); + + storage_info->AddBlobFile(std::move(meta)); } // Force the creation of a new manifest file and make sure metadata for @@ -2028,6 +2750,101 @@ TEST_F(VersionSetTest, AddLiveBlobFiles) { first_version->Unref(); } +TEST_F(VersionSetTest, LastUnrefFreesOnlyRelocationStateUnlocked) { + // A Version is destroyed under the DB mutex. When the last reference drops + // and the Version holds relocation state, only that state is freed without + // the mutex. + NewDB(); + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + int unlocked = 0; + SyncPoint::GetInstance()->SetCallBack( + "Version::UnrefAndFreeBlobRelocationState:Unlocked", [&](void* arg) { + auto* db_mutex = static_cast(arg); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + ++unlocked; + }); + SyncPoint::GetInstance()->EnableProcessing(); + for (const bool with_relocation_state : {false, true}) { + SCOPED_TRACE(with_relocation_state); + Version* const old_version = cfd->current(); + ASSERT_NE(old_version, nullptr); + mutex_.Lock(); + old_version->Ref(); + if (with_relocation_state) { + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + old_version->storage_info()->SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + old_version->storage_info()->AddBlobRelocationFile( + BlobRelocationFileMetaData::Create(BlobRelocationFileAddition( + /*file_number=*/102, /*file_size=*/4096, std::string(), + std::string(), encoded_directory, /*tail_size=*/1))); + old_version->storage_info()->SealBlobRelocationState(); + } + versions_->TEST_CreateAndAppendVersion(cfd); + old_version->UnrefAndFreeBlobRelocationState(&mutex_); + mutex_.Unlock(); + ASSERT_EQ(unlocked, with_relocation_state ? 1 : 0); + } + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); +} + +TEST_F(VersionSetTest, IndirectReferenceToUnfencedBlobFileIsNotRelocatable) { + // A read of a kIndirectBlob reference finds the physical file in the file + // table of the Version (Version::ResolveBlobPhysicalFile()). A Version + // without the per-CF format fence has an empty file table. Valid edits never + // give such a Version a blob file that a kIndirectBlob reference names, + // because VersionBuilder rejects an indirect-origin blob file without the + // fence. So this test adds blob file 234 to the storage info by hand, as + // AddLiveBlobFiles does, and reads a reference whose origin and generation + // are file 234. The read returns Corruption before it opens any file. + NewDB(); + ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + Version* const version = cfd->current(); + VersionStorageInfo* const storage_info = version->storage_info(); + ASSERT_FALSE(storage_info->HasBlobIndirectionFormat()); + constexpr uint64_t kBlobFileNumber = 234; + storage_info->AddBlobFile(BlobFileMetaData::Create( + SharedBlobFileMetaData::Create(kBlobFileNumber, /*total_blob_count=*/1, + /*total_blob_bytes=*/100, + /*checksum_method=*/std::string(), + /*checksum_value=*/std::string()), + BlobFileMetaData::LinkedSsts(), /*garbage_blob_count=*/0, + /*garbage_blob_bytes=*/0)); + ASSERT_EQ(storage_info->GetBlobFileTable().size(), 0U); + + std::string encoded; + BlobIndex::EncodeUnflattenedIndirectBlob( + &encoded, {.origin_file_number = kBlobFileNumber, + .original_value_offset = 100, + .stored_value_size = 5, + .compression = kNoCompression, + .stable_checksum = 0, + .stored_checksum = 0}); + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(encoded)); + PinnableSlice value; + const Status status = version->GetBlob(ReadOptions(), "key", blob_index, + /*prefetch_buffer=*/nullptr, &value, + /*bytes_read=*/nullptr); + ASSERT_EQ(status.code(), Status::kCorruption) << status.ToString(); + ASSERT_NE( + status.ToString().find("Indirect blob generation is not relocatable"), + std::string::npos) + << status.ToString(); +} + TEST_F(VersionSetTest, ObsoleteBlobFile) { // Initialize the database and add a blob file that is entirely garbage // and thus can immediately be marked obsolete. @@ -3150,168 +3967,932 @@ class VersionSetAtomicGroupTest : public VersionSetTestBase, CreateCurrentFile(); } - void SetupIncompleteTrailingAtomicGroup(int atomic_group_size) { - edits_.resize(atomic_group_size); - int remaining = atomic_group_size; - for (size_t i = 0; i != edits_.size(); ++i) { - edits_[i].SetLogNumber(0); - edits_[i].SetNextFile(2); - edits_[i].MarkAtomicGroup(--remaining); - edits_[i].SetLastSequence(last_seqno_++); - } - CreateCurrentFile(); - } + void SetupIncompleteTrailingAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + edits_[i].MarkAtomicGroup(--remaining); + edits_[i].SetLastSequence(last_seqno_++); + } + CreateCurrentFile(); + } + + void SetupCorruptedAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + if (i != ((size_t)atomic_group_size / 2)) { + edits_[i].MarkAtomicGroup(--remaining); + } + edits_[i].SetLastSequence(last_seqno_++); + } + CreateCurrentFile(); + } + + void SetupIncorrectAtomicGroup(int atomic_group_size) { + edits_.resize(atomic_group_size); + int remaining = atomic_group_size; + for (size_t i = 0; i != edits_.size(); ++i) { + edits_[i].SetLogNumber(0); + edits_[i].SetNextFile(2); + if (i != 1) { + edits_[i].MarkAtomicGroup(--remaining); + } else { + edits_[i].MarkAtomicGroup(remaining--); + } + edits_[i].SetLastSequence(last_seqno_++); + } + CreateCurrentFile(); + } + + void SetupTestSyncPoints() { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:FirstInAtomicGroup", [&](void* arg) { + VersionEdit* e = static_cast(arg); + EXPECT_EQ(edits_.front().DebugString(), + e->DebugString()); // compare based on value + first_in_atomic_group_ = true; + }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:LastInAtomicGroup", [&](void* arg) { + VersionEdit* e = static_cast(arg); + EXPECT_EQ(edits_.back().DebugString(), + e->DebugString()); // compare based on value + EXPECT_TRUE(first_in_atomic_group_); + last_in_atomic_group_ = true; + }); + SyncPoint::GetInstance()->SetCallBack( + "VersionEditHandlerBase::Iterate:Finish", + [&](void* arg) { num_recovered_edits_ = *static_cast(arg); }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:AtomicGroup", + [&](void* /* arg */) { ++num_edits_in_atomic_group_; }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:AtomicGroupMixedWithNormalEdits", + [&](void* arg) { corrupted_edit_ = *static_cast(arg); }); + SyncPoint::GetInstance()->SetCallBack( + "AtomicGroupReadBuffer::AddEdit:IncorrectAtomicGroupSize", + [&](void* arg) { + edit_with_incorrect_group_size_ = *static_cast(arg); + }); + SyncPoint::GetInstance()->EnableProcessing(); + } + + void AddNewEditsToLog(int num_edits) { + for (int i = 0; i < num_edits; i++) { + std::string record; + edits_[i].EncodeTo(&record, 0 /* ts_sz */); + ASSERT_OK(log_writer_->AddRecord(WriteOptions(), record)); + } + } + + void TearDown() override { + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + log_writer_.reset(); + } + + protected: + std::vector column_families_; + SequenceNumber last_seqno_; + std::vector edits_; + bool first_in_atomic_group_ = false; + bool last_in_atomic_group_ = false; + int num_edits_in_atomic_group_ = 0; + size_t num_recovered_edits_ = 0; + VersionEdit corrupted_edit_; + VersionEdit edit_with_incorrect_group_size_; + std::unique_ptr log_writer_; +}; + +TEST_F(VersionSetAtomicGroupTest, HandleValidAtomicGroupWithVersionSetRecover) { + const int kAtomicGroupSize = 3; + SetupValidAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + EXPECT_OK(versions_->Recover(column_families_, false)); + EXPECT_EQ(column_families_.size(), + versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_TRUE(last_in_atomic_group_); + EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleValidAtomicGroupWithReactiveVersionSetRecover) { + const int kAtomicGroupSize = 3; + SetupValidAtomicGroup(kAtomicGroupSize); + AddNewEditsToLog(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(column_families_.size(), + reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_TRUE(last_in_atomic_group_); + // The recover should clean up the replay buffer. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); + EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + HandleValidAtomicGroupWithReactiveVersionSetReadAndApply) { + const int kAtomicGroupSize = 3; + SetupValidAtomicGroup(kAtomicGroupSize); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + EXPECT_EQ(num_initial_edits_, num_recovered_edits_); + AddNewEditsToLog(kAtomicGroupSize); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + EXPECT_OK(reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr)); + mu.Unlock(); + EXPECT_TRUE(first_in_atomic_group_); + EXPECT_TRUE(last_in_atomic_group_); + // The recover should clean up the replay buffer. + EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); + EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); + EXPECT_EQ(kAtomicGroupSize, num_recovered_edits_); +} + +TEST_F(VersionSetAtomicGroupTest, + ReactiveVersionSetRejectsDeletionOfAnotherColumnFamilysFile) { + // Column family 2 deletes relocation file 100 during catch-up. Column family + // 2 has no live relocation file 100 (it is an SST of column family 1), so + // the catch-up fails and column family 2 keeps its Version. + constexpr uint64_t kTableFileNumber = 100; + std::vector file_metas; + CreateDummyTableFiles({SstInfo(kTableFileNumber, kColumnFamilyName1, "key", + /*epoch_number=*/kTableFileNumber)}, + &file_metas); + ASSERT_EQ(file_metas.size(), 1U); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddFile(/*level=*/0, file_metas.front()); + edits_.back().SetNextFile(kTableFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/2); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + + ColumnFamilyData* second_cfd = + reactive_versions_->GetColumnFamilySet()->GetColumnFamily(2); + ASSERT_NE(second_cfd, nullptr); + Version* const version_before_collision = second_cfd->current(); + ASSERT_NE(version_before_collision, nullptr); + ASSERT_EQ( + version_before_collision->storage_info()->GetBlobRelocationFileMetaData( + kTableFileNumber), + nullptr); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/2); + edits_.back().DeleteBlobRelocationFile(kTableFileNumber); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + Status status = reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr); + mu.Unlock(); + + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << status.ToString(); + ASSERT_EQ(second_cfd->current(), version_before_collision); + ASSERT_EQ( + second_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + kTableFileNumber), + nullptr); +} + +TEST_F(VersionSetAtomicGroupTest, + ReactivePointInTimeFallbackKeepsVersionAndReleasesMutex) { + table_factory_.reset(NewBlockBasedTableFactory()); + cf_options_.table_factory = table_factory_; + mutable_cf_options_.table_factory = table_factory_; + for (auto& column_family : column_families_) { + column_family.options.table_factory = table_factory_; + } + reactive_versions_ = std::make_shared( + dbname_, &imm_db_options_, mutable_db_options_, env_options_, + table_cache_.get(), &write_buffer_manager_, &write_controller_, + /*io_tracer=*/nullptr, /*db_id=*/"test-db", + /*db_session_id=*/"test-session", /*defer_sst_file_opening=*/false); + + constexpr uint64_t kTableFileNumber = 100; + constexpr uint64_t kMissingRelocationFileNumber = 101; + std::vector file_metas; + CreateDummyTableFiles({SstInfo(kTableFileNumber, kColumnFamilyName1, "key", + /*epoch_number=*/kTableFileNumber)}, + &file_metas); + ASSERT_EQ(file_metas.size(), 1U); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddFile(/*level=*/0, file_metas.front()); + edits_.back().SetNextFile(kMissingRelocationFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + for (uint32_t column_family_id : {1U, 2U}) { + edits_.emplace_back(); + edits_.back().SetColumnFamily(column_family_id); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().SetLastSequence(last_seqno_++); + } + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().DeleteFile(/*level=*/0, kTableFileNumber); + edits_.back().AddBlobRelocationFile(BlobRelocationFileAddition( + kMissingRelocationFileNumber, + /*file_size=*/4096, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), encoded_directory, /*tail_size=*/1)); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + + InstrumentedMutex mu; + size_t decode_unlocked = 0; + size_t verify_unlocked = 0; + size_t save_unlocked = 0; + const auto assert_unlocked = [](void* arg, size_t* count) { + auto* db_mutex = static_cast(arg); + ASSERT_NE(db_mutex, nullptr); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + ++*count; + }; + SyncPoint::GetInstance()->SetCallBack( + "ManifestTailer::DecodeVersionEdit:Unlocked", + [&](void* arg) { assert_unlocked(arg, &decode_unlocked); }); + SyncPoint::GetInstance()->SetCallBack( + "ManifestTailer::VerifyBlobRelocationFile:Unlocked", + [&](void* arg) { assert_unlocked(arg, &verify_unlocked); }); + SyncPoint::GetInstance()->SetCallBack( + "ManifestTailer::SaveVersionTo:Unlocked", + [&](void* arg) { assert_unlocked(arg, &save_unlocked); }); + + std::unordered_set cfds_changed; + mu.Lock(); + Status status = reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr); + mu.Unlock(); + ASSERT_OK(status); + ASSERT_GT(decode_unlocked, 0U); + ASSERT_GT(verify_unlocked, 0U); + ASSERT_GT(save_unlocked, 0U); + // There is no unlocked hook for builder Apply() and the Version install: they + // change file reference counts, so they run under the DB mutex. Only the + // relocation file check inside Apply() runs without it. + + ColumnFamilyData* first_cfd = + reactive_versions_->GetColumnFamilySet()->GetColumnFamily(1); + ASSERT_NE(first_cfd, nullptr); + ASSERT_TRUE(first_cfd->current() + ->storage_info() + ->GetFileLocation(kTableFileNumber) + .IsValid()); + ASSERT_EQ(first_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + kMissingRelocationFileNumber), + nullptr); +} + +TEST_F(VersionSetAtomicGroupTest, + RecoveryAcceptsRelocationFileDeletionOfOwnColumnFamily) { + // Column family 1 adds relocation file 100 and then deletes it, and the + // MANIFEST repeats the deletion. Replay accepts both deletions: the first + // one names a live relocation file of column family 1, and the repeat names + // a file that the builder already deleted. The DB opens without the file. + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + constexpr uint64_t kRelocationFileNumber = 100; + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().AddBlobRelocationFile(BlobRelocationFileAddition( + kRelocationFileNumber, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1)); + edits_.back().SetNextFile(kRelocationFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + for (int i = 0; i < 2; ++i) { + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().DeleteBlobRelocationFile(kRelocationFileNumber); + edits_.back().SetLastSequence(last_seqno_++); + } + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + ASSERT_OK(versions_->Recover(column_families_, /*read_only=*/false)); + ColumnFamilyData* const cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(1); + ASSERT_NE(cfd, nullptr); + ASSERT_TRUE(cfd->current()->storage_info()->HasBlobIndirectionFormat()); + ASSERT_EQ(cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + kRelocationFileNumber), + nullptr); +} + +TEST_F(VersionSetAtomicGroupTest, + RecoveryRejectsRelocationRecordOfAnotherColumnFamily) { + // VersionBuilder applies a relocation record to the state of the edit's own + // column family. Column family 2 deletes relocation file 100, which only + // column family 1 lists, so replay rejects the record. + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + constexpr uint64_t kRelocationFileNumber = 100; + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().AddBlobRelocationFile(BlobRelocationFileAddition( + kRelocationFileNumber, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1)); + edits_.back().SetNextFile(kRelocationFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/2); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().DeleteBlobRelocationFile(kRelocationFileNumber); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + const Status status = + versions_->Recover(column_families_, /*read_only=*/false); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find( + "Blob relocation file deletion names no live file"), + std::string::npos) + << status.ToString(); +} + +TEST_F(VersionSetAtomicGroupTest, + RecoveryRejectsLossRecordOfAnotherColumnFamily) { + // VersionBuilder applies a loss record to the state of the edit's own column + // family. Column family 2 marks relocation file 100 lost, but only column + // family 1 lists that file, so replay rejects the record. + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + constexpr uint64_t kRelocationFileNumber = 100; + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().AddBlobRelocationFile(BlobRelocationFileAddition( + kRelocationFileNumber, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1)); + edits_.back().SetNextFile(kRelocationFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/2); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().MarkBlobPhysicalFileLost(kRelocationFileNumber); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + const Status status = + versions_->Recover(column_families_, /*read_only=*/false); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("Blob relocation loss names no live target"), + std::string::npos) + << status.ToString(); +} + +TEST_F(VersionSetAtomicGroupTest, + ReactiveFailedApplyIsTerminalUntilManifestSwitch) { + constexpr uint64_t kRelocationFileNumber = 100; + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + reactive_versions_->SetTrustManifestRecovery(true); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddBlobRelocationFile(BlobRelocationFileAddition( + kRelocationFileNumber, + /*file_size=*/4096, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), encoded_directory, /*tail_size=*/1)); + edits_.back().DeleteFile(/*level=*/0, /*file=*/999); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + const Status first_status = reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr); + mu.Unlock(); + ASSERT_TRUE(first_status.IsCorruption()) << first_status.ToString(); + + ColumnFamilyData* first_cfd = + reactive_versions_->GetColumnFamilySet()->GetColumnFamily(1); + ASSERT_NE(first_cfd, nullptr); + ASSERT_EQ(first_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + kRelocationFileNumber), + nullptr); + + mu.Lock(); + const Status retry_status = reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr); + mu.Unlock(); + ASSERT_EQ(retry_status.ToString(), first_status.ToString()); + ASSERT_EQ(first_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + kRelocationFileNumber), + nullptr); +} + +TEST_F(VersionSetAtomicGroupTest, + ReactiveManifestSwitchAbortsFailedAtomicReplayState) { + CreateCurrentFile(); + log_writer_.reset(); + ASSERT_OK(versions_->Recover(column_families_, /*read_only=*/false)); + + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); - void SetupCorruptedAtomicGroup(int atomic_group_size) { - edits_.resize(atomic_group_size); - int remaining = atomic_group_size; - for (size_t i = 0; i != edits_.size(); ++i) { - edits_[i].SetLogNumber(0); - edits_[i].SetNextFile(2); - if (i != ((size_t)atomic_group_size / 2)) { - edits_[i].MarkAtomicGroup(--remaining); - } - edits_[i].SetLastSequence(last_seqno_++); - } - CreateCurrentFile(); - } + ColumnFamilyData* primary_first_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(1); + ColumnFamilyData* primary_second_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(2); + ASSERT_NE(primary_first_cfd, nullptr); + ASSERT_NE(primary_second_cfd, nullptr); + edits_.clear(); + edits_.resize(2); + edits_[0].SetColumnFamily(/*column_family_id=*/1); + edits_[0].MarkAtomicGroup(/*remaining_entries=*/1); + edits_[1].SetColumnFamily(/*column_family_id=*/2); + edits_[1].MarkAtomicGroup(/*remaining_entries=*/0); + autovector cfds{primary_first_cfd, primary_second_cfd}; + autovector> edit_lists; + edit_lists.emplace_back(autovector{&edits_[0]}); + edit_lists.emplace_back(autovector{&edits_[1]}); + InstrumentedMutex primary_mu; + primary_mu.Lock(); + Status status = versions_->LogAndApply( + cfds, ReadOptions(), WriteOptions(), edit_lists, &primary_mu, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/false, /*new_cf_options=*/nullptr, + /*manifest_wcbs=*/{}, /*pre_cb=*/{}, + /*max_file_opening_threads=*/1); + primary_mu.Unlock(); + ASSERT_OK(status); - void SetupIncorrectAtomicGroup(int atomic_group_size) { - edits_.resize(atomic_group_size); - int remaining = atomic_group_size; - for (size_t i = 0; i != edits_.size(); ++i) { - edits_[i].SetLogNumber(0); - edits_[i].SetNextFile(2); - if (i != 1) { - edits_[i].MarkAtomicGroup(--remaining); - } else { - edits_[i].MarkAtomicGroup(remaining--); - } - edits_[i].SetLastSequence(last_seqno_++); - } - CreateCurrentFile(); - } + SyncPoint::GetInstance()->SetCallBack( + "VersionEditHandlerBase::Iterate:BeforeApplyAtomicGroupEdit", + [](void* arg) { + *static_cast(arg) = + Status::Corruption("injected atomic replay failure"); + }); + InstrumentedMutex reactive_mu; + std::unordered_set cfds_changed; + reactive_mu.Lock(); + status = reactive_versions_->ReadAndApply( + &reactive_mu, &manifest_reader, manifest_reader_status.get(), + &cfds_changed, /*files_to_delete=*/nullptr); + reactive_mu.Unlock(); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + + SetupTestSyncPoints(); + // Replaced builders are destroyed under the DB mutex; only staged relocation + // changes they alone own are freed without it. + SyncPoint::GetInstance()->SetCallBack( + "ManifestTailer::ReplaceBuilder:Unlocked", [&](void* arg) { + auto* db_mutex = static_cast(arg); + ASSERT_EQ(db_mutex, &reactive_mu); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + }); + VersionEdit rotate; + ColumnFamilyData* primary_default_cfd = + versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(primary_default_cfd, nullptr); + primary_mu.Lock(); + status = versions_->LogAndApply(primary_default_cfd, ReadOptions(), + WriteOptions(), &rotate, &primary_mu, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/true); + primary_mu.Unlock(); + ASSERT_OK(status); + + reactive_mu.Lock(); + status = reactive_versions_->ReadAndApply( + &reactive_mu, &manifest_reader, manifest_reader_status.get(), + &cfds_changed, /*files_to_delete=*/nullptr); + reactive_mu.Unlock(); + ASSERT_OK(status); + ASSERT_EQ(reactive_versions_->TEST_read_edits_in_atomic_group(), 0U); + ASSERT_TRUE(reactive_versions_->replay_buffer().empty()); +} - void SetupTestSyncPoints() { - SyncPoint::GetInstance()->DisableProcessing(); - SyncPoint::GetInstance()->ClearAllCallBacks(); - SyncPoint::GetInstance()->SetCallBack( - "AtomicGroupReadBuffer::AddEdit:FirstInAtomicGroup", [&](void* arg) { - VersionEdit* e = static_cast(arg); - EXPECT_EQ(edits_.front().DebugString(), - e->DebugString()); // compare based on value - first_in_atomic_group_ = true; - }); - SyncPoint::GetInstance()->SetCallBack( - "AtomicGroupReadBuffer::AddEdit:LastInAtomicGroup", [&](void* arg) { - VersionEdit* e = static_cast(arg); - EXPECT_EQ(edits_.back().DebugString(), - e->DebugString()); // compare based on value - EXPECT_TRUE(first_in_atomic_group_); - last_in_atomic_group_ = true; - }); - SyncPoint::GetInstance()->SetCallBack( - "VersionEditHandlerBase::Iterate:Finish", - [&](void* arg) { num_recovered_edits_ = *static_cast(arg); }); - SyncPoint::GetInstance()->SetCallBack( - "AtomicGroupReadBuffer::AddEdit:AtomicGroup", - [&](void* /* arg */) { ++num_edits_in_atomic_group_; }); - SyncPoint::GetInstance()->SetCallBack( - "AtomicGroupReadBuffer::AddEdit:AtomicGroupMixedWithNormalEdits", - [&](void* arg) { corrupted_edit_ = *static_cast(arg); }); - SyncPoint::GetInstance()->SetCallBack( - "AtomicGroupReadBuffer::AddEdit:IncorrectAtomicGroupSize", - [&](void* arg) { - edit_with_incorrect_group_size_ = *static_cast(arg); - }); - SyncPoint::GetInstance()->EnableProcessing(); - } +TEST_F(VersionSetAtomicGroupTest, ReactiveIdenticalRelocationRepeatIsAccepted) { + // Catch-up replays a relocation file addition that the builder already has. + // An identical repeat is accepted. + constexpr uint64_t kRelocationFileNumber = 100; + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + BlobRelocationFileAddition addition( + kRelocationFileNumber, + /*file_size=*/4096, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), encoded_directory, /*tail_size=*/1); - void AddNewEditsToLog(int num_edits) { - for (int i = 0; i < num_edits; i++) { - std::string record; - edits_[i].EncodeTo(&record, 0 /* ts_sz */); - ASSERT_OK(log_writer_->AddRecord(WriteOptions(), record)); - } - } + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().AddBlobRelocationFile(addition); + edits_.back().SetNextFile(kRelocationFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); - void TearDown() override { - SyncPoint::GetInstance()->DisableProcessing(); - SyncPoint::GetInstance()->ClearAllCallBacks(); - log_writer_.reset(); - } + reactive_versions_->SetTrustManifestRecovery(true); + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); - protected: - std::vector column_families_; - SequenceNumber last_seqno_; - std::vector edits_; - bool first_in_atomic_group_ = false; - bool last_in_atomic_group_ = false; - int num_edits_in_atomic_group_ = 0; - size_t num_recovered_edits_ = 0; - VersionEdit corrupted_edit_; - VersionEdit edit_with_incorrect_group_size_; - std::unique_ptr log_writer_; -}; + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddBlobRelocationFile(addition); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); -TEST_F(VersionSetAtomicGroupTest, HandleValidAtomicGroupWithVersionSetRecover) { - const int kAtomicGroupSize = 3; - SetupValidAtomicGroup(kAtomicGroupSize); - AddNewEditsToLog(kAtomicGroupSize); - EXPECT_OK(versions_->Recover(column_families_, false)); - EXPECT_EQ(column_families_.size(), - versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); - EXPECT_TRUE(first_in_atomic_group_); - EXPECT_TRUE(last_in_atomic_group_); - EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_); + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + Status status = reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr); + mu.Unlock(); + ASSERT_OK(status); } TEST_F(VersionSetAtomicGroupTest, - HandleValidAtomicGroupWithReactiveVersionSetRecover) { - const int kAtomicGroupSize = 3; - SetupValidAtomicGroup(kAtomicGroupSize); - AddNewEditsToLog(kAtomicGroupSize); + ReactiveConflictingRelocationRepeatFailsInApply) { + // Catch-up replays a relocation file addition that differs from the file + // the builder has, then drops the column family. Apply() compares the repeat + // and fails with Corruption, so the drop is not applied and the column + // family keeps the original file. + constexpr uint64_t kRelocationFileNumber = 100; + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + const BlobRelocationFileAddition original( + kRelocationFileNumber, + /*file_size=*/4096, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), encoded_directory, /*tail_size=*/1); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().AddBlobRelocationFile(original); + edits_.back().SetNextFile(kRelocationFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + reactive_versions_->SetTrustManifestRecovery(true); std::unique_ptr manifest_reader; std::unique_ptr manifest_reporter; std::unique_ptr manifest_reader_status; - EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, &manifest_reporter, &manifest_reader_status)); - EXPECT_EQ(column_families_.size(), - reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies()); - EXPECT_TRUE(first_in_atomic_group_); - EXPECT_TRUE(last_in_atomic_group_); - // The recover should clean up the replay buffer. - EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); - EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); - EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddBlobRelocationFile(BlobRelocationFileAddition( + kRelocationFileNumber, + /*file_size=*/8192, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), encoded_directory, /*tail_size=*/1)); + edits_.back().SetLastSequence(last_seqno_++); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().DropColumnFamily(); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + + InstrumentedMutex mu; + std::unordered_set cfds_changed; + mu.Lock(); + const Status status = reactive_versions_->ReadAndApply( + &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, + /*files_to_delete=*/nullptr); + mu.Unlock(); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ColumnFamilyData* first_cfd = + reactive_versions_->GetColumnFamilySet()->GetColumnFamily(1); + ASSERT_NE(first_cfd, nullptr); + const std::shared_ptr metadata = + first_cfd->current()->storage_info()->GetBlobRelocationFileMetaData( + kRelocationFileNumber); + ASSERT_NE(metadata, nullptr); + ASSERT_EQ(metadata->GetFileSize(), 4096U); } TEST_F(VersionSetAtomicGroupTest, - HandleValidAtomicGroupWithReactiveVersionSetReadAndApply) { - const int kAtomicGroupSize = 3; - SetupValidAtomicGroup(kAtomicGroupSize); + ReactiveColumnFamilyDropRemovesChangedPointer) { + constexpr uint64_t kTableFileNumber = 100; + std::vector file_metas; + CreateDummyTableFiles({SstInfo(kTableFileNumber, kColumnFamilyName1, "key", + /*epoch_number=*/kTableFileNumber)}, + &file_metas); + ASSERT_EQ(file_metas.size(), 1U); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddFile(/*level=*/0, file_metas.front()); + edits_.back().SetNextFile(kTableFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/2); + edits_.back().SetBlobIndirectionFormatVersion( + kBlobIndirectionManifestFormatVersion); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + std::unique_ptr manifest_reader; std::unique_ptr manifest_reporter; std::unique_ptr manifest_reader_status; - EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, &manifest_reporter, &manifest_reader_status)); - EXPECT_EQ(num_initial_edits_, num_recovered_edits_); - AddNewEditsToLog(kAtomicGroupSize); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().SetLogNumber(7); + edits_.back().SetLastSequence(last_seqno_++); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().DropColumnFamily(); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + InstrumentedMutex mu; + size_t builder_destroyed_unlocked = 0; + SyncPoint::GetInstance()->SetCallBack( + "ManifestTailer::DestroyVersionBuilder:Unlocked", [&](void* arg) { + auto* db_mutex = static_cast(arg); + ASSERT_EQ(db_mutex, &mu); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + ++builder_destroyed_unlocked; + }); + std::unordered_set cfds_changed; mu.Lock(); - EXPECT_OK(reactive_versions_->ReadAndApply( + Status status = reactive_versions_->ReadAndApply( &mu, &manifest_reader, manifest_reader_status.get(), &cfds_changed, - /*files_to_delete=*/nullptr)); + /*files_to_delete=*/nullptr); mu.Unlock(); - EXPECT_TRUE(first_in_atomic_group_); - EXPECT_TRUE(last_in_atomic_group_); - // The recover should clean up the replay buffer. - EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0); - EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0); - EXPECT_EQ(kAtomicGroupSize, num_recovered_edits_); + ASSERT_OK(status); + // The dropped CF's builder staged no relocation change, so it is destroyed + // under the DB mutex without an unlock. + ASSERT_EQ(builder_destroyed_unlocked, 0U); + ASSERT_EQ(reactive_versions_->GetColumnFamilySet()->GetColumnFamily(1), + nullptr); + for (ColumnFamilyData* cfd : cfds_changed) { + ASSERT_NE(cfd->GetID(), 1U); + } +} + +TEST_F(VersionSetAtomicGroupTest, + ReactiveManifestSwitchKeepsSecondaryOnlyColumnFamily) { + constexpr uint64_t kTableFileNumber = 100; + std::vector file_metas; + CreateDummyTableFiles({SstInfo(kTableFileNumber, kColumnFamilyName1, "key", + /*epoch_number=*/kTableFileNumber)}, + &file_metas); + ASSERT_EQ(file_metas.size(), 1U); + + edits_.clear(); + edits_.emplace_back(); + edits_.back().SetColumnFamily(/*column_family_id=*/1); + edits_.back().AddFile(/*level=*/0, file_metas.front()); + edits_.back().SetNextFile(kTableFileNumber + 1); + edits_.back().SetLastSequence(last_seqno_++); + AddNewEditsToLog(static_cast(edits_.size())); + CreateCurrentFile(); + + std::unique_ptr manifest_reader; + std::unique_ptr manifest_reporter; + std::unique_ptr manifest_reader_status; + ASSERT_OK(reactive_versions_->Recover(column_families_, &manifest_reader, + &manifest_reporter, + &manifest_reader_status)); + + log_writer_.reset(); + ASSERT_OK(versions_->Recover(column_families_, /*read_only=*/false)); + ColumnFamilyData* primary_first_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(1); + ColumnFamilyData* primary_second_cfd = + versions_->GetColumnFamilySet()->GetColumnFamily(2); + ASSERT_NE(primary_first_cfd, nullptr); + ASSERT_NE(primary_second_cfd, nullptr); + + InstrumentedMutex primary_mu; + primary_first_cfd->Ref(); + VersionEdit drop; + drop.SetColumnFamily(/*column_family_id=*/1); + drop.DropColumnFamily(); + primary_mu.Lock(); + Status status = versions_->LogAndApply(primary_first_cfd, ReadOptions(), + WriteOptions(), &drop, &primary_mu, + /*dir_contains_current_file=*/nullptr); + primary_mu.Unlock(); + ASSERT_OK(status); + + VersionEdit rotate; + ColumnFamilyData* primary_default_cfd = + versions_->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(primary_default_cfd, nullptr); + primary_mu.Lock(); + status = versions_->LogAndApply(primary_default_cfd, ReadOptions(), + WriteOptions(), &rotate, &primary_mu, + /*dir_contains_current_file=*/nullptr, + /*new_descriptor_log=*/true); + primary_mu.Unlock(); + ASSERT_OK(status); + + InstrumentedMutex reactive_mu; + std::unordered_set cfds_changed; + size_t unlocked_candidate_deletions = 0; + SyncPoint::GetInstance()->SetCallBack( + "ManifestTailer::DiscardVersions:Unlocked", [&](void* arg) { + auto* db_mutex = static_cast(arg); + ASSERT_NE(db_mutex, nullptr); + ASSERT_TRUE(db_mutex->TryLock()); + db_mutex->Unlock(); + ++unlocked_candidate_deletions; + }); + reactive_mu.Lock(); + status = reactive_versions_->ReadAndApply( + &reactive_mu, &manifest_reader, manifest_reader_status.get(), + &cfds_changed, /*files_to_delete=*/nullptr); + reactive_mu.Unlock(); + // The new MANIFEST starts with a complete snapshot, which has no CF 1 + // (dropped on the primary). The secondary still has CF 1, which lists table + // 100. + ASSERT_OK(status); + // Discarded candidates are deleted under the DB mutex. A candidate that is + // the last owner of its relocation state frees it without the mutex. + ASSERT_LE(unlocked_candidate_deletions, 1U); + + ColumnFamilyData* reactive_first_cfd = + reactive_versions_->GetColumnFamilySet()->GetColumnFamily(1); + ColumnFamilyData* reactive_second_cfd = + reactive_versions_->GetColumnFamilySet()->GetColumnFamily(2); + ASSERT_NE(reactive_first_cfd, nullptr); + ASSERT_NE(reactive_second_cfd, nullptr); + ASSERT_TRUE(reactive_first_cfd->current() + ->storage_info() + ->GetFileLocation(kTableFileNumber) + .IsValid()); + + primary_mu.Lock(); + primary_first_cfd->UnrefAndTryDelete(); + primary_mu.Unlock(); } TEST_F(VersionSetAtomicGroupTest, @@ -5165,6 +6746,222 @@ TEST_P(ChargeFileMetadataTestWithParam, Basic) { EXPECT_TRUE(s.ok()); } } + +TEST_F(VersionSetTest, IndirectBlobReadRouteSignals) { + // Checks the signals of Version::ResolveBlobPhysicalFile(): + // - BLOB_DB_INDIRECT_READ_ROUTED goes up by one for each read that finds its + // file through a route, and does not change for other reads. + // - Its Corruption messages name the generation, origin and route target + // file numbers, so a LOG line shows which reference failed. + // For example, relocation file 102 holds the moved values of blob file 77, + // and route 77 -> 102 sends reads of file 77 to file 102. + imm_db_options_.statistics = CreateDBStatistics(); + Statistics* const stats = imm_db_options_.statistics.get(); + // The column family reads the raw pointer. + imm_db_options_.stats = stats; + NewDB(); + BlobRelocationDirectoryBuilder directory_builder; + ASSERT_OK(directory_builder.Add( + /*origin_file_number=*/77, /*original_value_offset=*/11, + /*stored_value_size=*/17, kNoCompression)); + ASSERT_OK(directory_builder.AddServedFile(/*generation_file_number=*/77)); + BlobRelocationDirectory directory; + ASSERT_OK(directory_builder.Finish(&directory)); + std::string encoded_directory; + directory.EncodeTo(&encoded_directory); + { + VersionEdit edit; + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(BlobRelocationFileAddition( + /*file_number=*/102, /*file_size=*/4096, + /*checksum_method=*/std::string(), /*checksum_value=*/std::string(), + encoded_directory, /*tail_size=*/1)); + edit.SetBlobRelocationRoute(BlobRelocationRoute( + /*source_file_number=*/77, /*physical_file_number=*/102)); + ASSERT_OK(LogAndApplyToDefaultCF(edit)); + } + Version* const version = + versions_->GetColumnFamilySet()->GetDefault()->current(); + // Reads the reference with generation number `generation` and origin + // `origin`. This test DB has no relocation file reader (its table factory is + // not block-based) and file 102 does not exist on disk, so a read that gets + // past the route and file lookup fails when it opens the file. + const auto read = [&](uint64_t generation, uint64_t origin) { + std::string encoded; + BlobIndex::EncodeIndirectBlob(&encoded, generation, + {.origin_file_number = origin, + .original_value_offset = 11, + .stored_value_size = 17, + .compression = kNoCompression, + .stable_checksum = 0, + .stored_checksum = 0}); + BlobIndex blob_index; + EXPECT_OK(blob_index.DecodeFrom(encoded)); + PinnableSlice value; + return version->GetBlob(ReadOptions(), "key", blob_index, + /*prefetch_buffer=*/nullptr, &value, + /*bytes_read=*/nullptr); + }; + const auto routed = [&]() { + return stats->getTickerCount(BLOB_DB_INDIRECT_READ_ROUTED); + }; + ASSERT_EQ(routed(), 0); + + // A reference to file 77 goes through the route. + ASSERT_NOK(read(/*generation=*/77, /*origin=*/77)); + ASSERT_EQ(routed(), 1); + // A flattened reference names file 102 itself: no route. + ASSERT_NOK(read(/*generation=*/102, /*origin=*/77)); + ASSERT_EQ(routed(), 1); + + // An unknown generation number: the message names the generation and the + // origin, and there is no route. + { + const Status status = read(/*generation=*/999, /*origin=*/999); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE(status.ToString().find("generation 999, origin 999"), + std::string::npos) + << status.ToString(); + ASSERT_EQ(status.ToString().find("route target"), std::string::npos) + << status.ToString(); + ASSERT_EQ(routed(), 1); + } + + // A route whose target is not a live relocation file. Valid edits cannot + // make such a route (VersionBuilder checks every route target), so the test + // changes the file table of this Version directly, then puts it back. The + // message names the target, for example file 555. The read finds no file, + // so the ticker does not change. + { + BlobFileTable& table = + const_cast(version->storage_info()->GetBlobFileTable()); + uint32_t count = 0; + const uint64_t old_value = table.Find(77, &count); + ASSERT_TRUE(BlobFileTable::IsRoute(old_value)); + table.Apply({{77, BlobFileTable::RouteValue(555), count}}); + const Status status = read(/*generation=*/77, /*origin=*/77); + table.Apply({{77, old_value, count}}); + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + ASSERT_NE( + status.ToString().find("generation 77, origin 77, route target 555"), + std::string::npos) + << status.ToString(); + ASSERT_EQ(routed(), 1); + } +} + +class BlobIndirectionPropertyTest : public DBTestBase { + public: + BlobIndirectionPropertyTest() + : DBTestBase("blob_indirection_property_test", /*env_do_fsync=*/false) {} +}; + +TEST_F(BlobIndirectionPropertyTest, FormatVersionAndRoutedReads) { + // rocksdb.blob-indirection-format-version is the per-CF format fence: 0 for + // a column family without a fence, and the format version after the first + // flush with enable_blob_indirection. The fence stays when the option is + // turned off again. Reads without routes never tick + // BLOB_DB_INDIRECT_READ_ROUTED. A stable checksum mismatch names the file + // numbers and the offset of the reference. + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.enable_blob_files = true; + options.min_blob_size = 0; + options.disable_auto_compactions = true; + // A compressed value has a stored checksum of its own, so a wrong stable + // checksum is found by the stable checksum check, not by the record check. + const bool compressed = Snappy_Supported(); + if (compressed) { + options.blob_compression_type = kSnappyCompression; + } + DestroyAndReopen(options); + const auto format_version = [&]() { + uint64_t value = 0; + EXPECT_TRUE(db_->GetIntProperty( + DB::Properties::kBlobIndirectionFormatVersion, &value)); + return value; + }; + const auto routed = [&]() { + return options.statistics->getTickerCount(BLOB_DB_INDIRECT_READ_ROUTED); + }; + + // Blob files without blob indirection. + ASSERT_OK(Put("key1", "value1")); + ASSERT_OK(Flush()); + ASSERT_EQ(Get("key1"), "value1"); + ASSERT_EQ(format_version(), 0); + ASSERT_EQ(routed(), 0); + + options.enable_blob_indirection = true; + Reopen(options); + ASSERT_OK(Put("key2", "value2")); + ASSERT_OK(Flush()); + ASSERT_EQ(format_version(), kBlobIndirectionManifestFormatVersion); + ASSERT_EQ(Get("key1"), "value1"); + ASSERT_EQ(Get("key2"), "value2"); + ASSERT_EQ(routed(), 0); + + if (compressed) { + // Reads the reference of a compressible value with its stable checksum + // changed. The record itself is valid. + const std::string value(1000, 'c'); + ASSERT_OK(Put("key3", value)); + ASSERT_OK(Flush()); + PinnableSlice raw; + bool is_blob_index = false; + DBImpl::GetImplOptions get_options; + get_options.column_family = db_->DefaultColumnFamily(); + get_options.value = &raw; + get_options.is_blob_index = &is_blob_index; + ASSERT_OK(dbfull()->GetImpl(ReadOptions(), "key3", get_options)); + ASSERT_TRUE(is_blob_index); + BlobIndex blob_index; + ASSERT_OK(blob_index.DecodeFrom(raw)); + ASSERT_TRUE(blob_index.IsIndirect()); + ASSERT_NE(blob_index.compression(), kNoCompression); + std::string encoded; + BlobIndex::EncodeIndirectBlob( + &encoded, blob_index.generation_file_number(), + {.origin_file_number = blob_index.origin_file_number(), + .original_value_offset = blob_index.original_value_offset(), + .stored_value_size = blob_index.size(), + .compression = blob_index.compression(), + .stable_checksum = blob_index.stable_checksum() + 1, + .stored_checksum = blob_index.stored_checksum()}); + BlobIndex wrong_checksum; + ASSERT_OK(wrong_checksum.DecodeFrom(encoded)); + ColumnFamilyData* const cfd = + static_cast(db_->DefaultColumnFamily())->cfd(); + Version* version = nullptr; + { + InstrumentedMutexLock lock(dbfull()->mutex()); + version = cfd->current(); + version->Ref(); + } + PinnableSlice result; + const Status status = version->GetBlob( + ReadOptions(), "key3", wrong_checksum, /*prefetch_buffer=*/nullptr, + &result, /*bytes_read=*/nullptr); + { + InstrumentedMutexLock lock(dbfull()->mutex()); + version->Unref(); + } + ASSERT_TRUE(status.IsCorruption()) << status.ToString(); + const std::string expected = + "Stable blob checksum mismatch: generation " + + std::to_string(blob_index.generation_file_number()) + ", origin " + + std::to_string(blob_index.origin_file_number()) + ", original offset " + + std::to_string(blob_index.original_value_offset()); + ASSERT_NE(status.ToString().find(expected), std::string::npos) + << status.ToString(); + ASSERT_EQ(routed(), 0); + } + + options.enable_blob_indirection = false; + Reopen(options); + ASSERT_EQ(format_version(), kBlobIndirectionManifestFormatVersion); + ASSERT_EQ(Get("key2"), "value2"); +} } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/db/wide/db_lazy_entity_test.cc b/db/wide/db_lazy_entity_test.cc index 0bb21ded93cf..1c75be738074 100644 --- a/db/wide/db_lazy_entity_test.cc +++ b/db/wide/db_lazy_entity_test.cc @@ -1621,6 +1621,50 @@ TEST_F(DBLazyEntityTest, BatchForceVerifyNotSkippedAcrossDuplicateKeys) { SyncPoint::GetInstance()->ClearAllCallBacks(); } +// Two reads of the same embedded record in one batch make one storage read, +// and the read statistics count it once, for whole values and for ranges. +TEST_F(DBLazyEntityTest, BatchEmbeddedDuplicateReadCountsOnce) { + Options options = GetLazyTestOptions(); + DestroyAndReopen(options); + const std::string value(2000, 'd'); + IngestEmbeddedEntity( + options, "ek", + WideColumns{{kDefaultWideColumnName, "inline"}, {"data", value}}); + const std::array key_slices{Slice("ek"), Slice("ek")}; + for (const bool range : {false, true}) { + SCOPED_TRACE(range); + LazyWideColumnsBatch batch; + std::array get_statuses; + db_->MultiGetEntityLazy(ReadOptions(), db_->DefaultColumnFamily(), + key_slices.size(), key_slices.data(), &batch, + get_statuses.data()); + ASSERT_OK(get_statuses[0]); + ASSERT_OK(get_statuses[1]); + const uint64_t reads_before = LazyReadCount(options); + const uint64_t bytes_before = BlobBytesRead(options); + std::array results; + std::array statuses; + std::array reads; + for (size_t i = 0; i < reads.size(); ++i) { + reads[i].column = &batch[i][1]; + if (range) { + reads[i].offset = 10; + reads[i].length = 100; + } + reads[i].result = &results[i]; + reads[i].status = &statuses[i]; + } + ASSERT_OK(batch.MultiResolve(reads.size(), reads.data())); + for (size_t i = 0; i < reads.size(); ++i) { + ASSERT_OK(statuses[i]); + EXPECT_EQ(results[i], range ? value.substr(10, 100) : value); + } + EXPECT_EQ(LazyReadCount(options), reads_before + 1); + EXPECT_EQ(BlobBytesRead(options) - bytes_before, + range ? 100U : value.size() + 5U); + } +} + // A read that omits its status out-param is a caller error (there is nowhere to // report that read's outcome), so MultiResolve fails the whole call with // InvalidArgument rather than silently resolving. diff --git a/db/wide/wide_column_serialization.cc b/db/wide/wide_column_serialization.cc index a8b4373420fc..4398582275e5 100644 --- a/db/wide/wide_column_serialization.cc +++ b/db/wide/wide_column_serialization.cc @@ -345,9 +345,11 @@ Status WideColumnSerialization::DeserializeV1( return Status::OK(); } -Status WideColumnSerialization::DeserializeV2Impl( - Slice& input, uint32_t num_columns, std::vector& columns, - std::vector& column_types) { +template +Status WideColumnSerialization::WalkV2Columns(Slice& input, + uint32_t num_columns, + bool stop_without_blobs, + Visit&& visit) { // Section 2: SKIP INFO (3 varints) uint32_t name_sizes_bytes = 0; uint32_t value_sizes_bytes = 0; @@ -366,18 +368,20 @@ Status WideColumnSerialization::DeserializeV2Impl( if (input.size() < num_columns) { return Status::Corruption("Error decoding wide column types"); } - column_types.resize(num_columns); + const char* types = input.data(); for (uint32_t i = 0; i < num_columns; ++i) { - column_types[i] = static_cast(input[i]); - if (!IsValidColumnValueType(column_types[i])) { + if (!IsValidColumnValueType(static_cast(types[i]))) { return Status::Corruption("Unsupported wide column ValueType"); } } + if (stop_without_blobs && !ContainsBlobType(types, num_columns)) { + return Status::OK(); + } input.remove_prefix(num_columns); // Validate that sections 4-6 fit in the remaining input const size_t metadata_size = - name_sizes_bytes + value_sizes_bytes + names_bytes; + static_cast(name_sizes_bytes) + value_sizes_bytes + names_bytes; if (input.size() < metadata_size) { return Status::Corruption("Error decoding wide column sections"); } @@ -390,12 +394,11 @@ Status WideColumnSerialization::DeserializeV2Impl( const char* s5_limit = s5 + value_sizes_bytes; const char* s6 = s5_limit; // section 6: names const char* s7 = s6 + names_bytes; // section 7: values - const char* input_end = input.data() + input.size(); + const size_t total_values_bytes = input.size() - metadata_size; - columns.reserve(num_columns); size_t name_pos = 0; size_t value_pos = 0; - + Slice prev_name; for (uint32_t i = 0; i < num_columns; ++i) { // Decode name size from section 4 uint32_t ns = 0; @@ -414,24 +417,26 @@ Status WideColumnSerialization::DeserializeV2Impl( s5 = s5_next; // Read name from section 6 - if (name_pos + ns > names_bytes) { + if (ns > names_bytes - name_pos) { return Status::Corruption("Error decoding wide column name"); } - Slice name(s6 + name_pos, ns); - - if (!columns.empty()) { - if (Status so = ValidateColumnOrder(columns.back().name(), name); - !so.ok()) { + const Slice name(s6 + name_pos, ns); + if (i > 0) { + if (Status so = ValidateColumnOrder(prev_name, name); !so.ok()) { return so; } } // Read value from section 7 - if (s7 + value_pos + vs > input_end) { + if (vs > total_values_bytes - value_pos) { return Status::Corruption("Error decoding wide column value payload"); } - - columns.emplace_back(name, Slice(s7 + value_pos, vs)); + if (Status sv = visit(i, static_cast(types[i]), name, + Slice(s7 + value_pos, vs)); + !sv.ok()) { + return sv; + } + prev_name = name; name_pos += ns; value_pos += vs; } @@ -439,7 +444,6 @@ Status WideColumnSerialization::DeserializeV2Impl( // Validate that the consumed bytes match the declared section sizes. Because // a V2 entity's values section is defined to extend to the end of `input`, // this also rejects any trailing bytes after the entity. - const size_t total_values_bytes = static_cast(input_end - s7); if (s4 != s4_limit || s5 != s5_limit || name_pos != names_bytes || value_pos != total_values_bytes) { return Status::Corruption("Wide column section size mismatch"); @@ -448,6 +452,20 @@ Status WideColumnSerialization::DeserializeV2Impl( return Status::OK(); } +Status WideColumnSerialization::DeserializeV2Impl( + Slice& input, uint32_t num_columns, std::vector& columns, + std::vector& column_types) { + column_types.resize(num_columns); + columns.reserve(num_columns); + return WalkV2Columns( + input, num_columns, /*stop_without_blobs=*/false, + [&](uint32_t i, ValueType type, const Slice& name, const Slice& value) { + column_types[i] = type; + columns.emplace_back(name, value); + return Status::OK(); + }); +} + Status WideColumnSerialization::Deserialize( const Slice& entity, WideColumns& columns, std::vector>* blob_columns) { @@ -565,7 +583,7 @@ Status WideColumnSerialization::HasBlobColumns(const Slice& input, return Status::OK(); } -Status WideColumnSerialization::ForEachBlobFileNumber( +Status WideColumnSerialization::ForEachBlobIndex( const Slice& input, const std::function& callback) { Slice input_ref = input; @@ -575,9 +593,6 @@ Status WideColumnSerialization::ForEachBlobFileNumber( return Status::Corruption("Error decoding wide column version"); } - if (version < kVersion2) { - return Status::OK(); - } if (version > kVersion2) { return Status::Corruption("Corrupt or unsupported wide column version"); } @@ -587,99 +602,46 @@ Status WideColumnSerialization::ForEachBlobFileNumber( return Status::Corruption("Error decoding number of wide columns"); } - if (!num_columns) { - return Status::OK(); - } - - // Read SKIP INFO - uint32_t name_sizes_bytes = 0; - uint32_t value_sizes_bytes = 0; - uint32_t names_bytes = 0; - if (!GetVarint32(&input_ref, &name_sizes_bytes) || - !GetVarint32(&input_ref, &value_sizes_bytes) || - !GetVarint32(&input_ref, &names_bytes)) { - return Status::Corruption("Error decoding wide column skip info"); - } - - // Read COLUMN TYPES - if (input_ref.size() < num_columns) { - return Status::Corruption("Error decoding wide column types"); - } - - // Validate every column type, and note whether any blob columns are present. - // An unrecognized/unsupported type byte is Corruption (consistent with - // Deserialize() and HasBlobColumns()) rather than silently treated as a - // non-blob column. - bool has_any_blob = false; - for (uint32_t i = 0; i < num_columns; ++i) { - const auto type = lossless_cast(input_ref[i]); - if (!IsValidColumnValueType(type)) { - return Status::Corruption("Unsupported wide column ValueType"); - } - if (type == kTypeBlobIndex) { - has_any_blob = true; - } - } - - if (!has_any_blob) { + // V1 entities cannot hold blob columns. Callers need only the blob + // references, so the framing checks are skipped for V1. + if (version < kVersion2) { return Status::OK(); } - // Need to skip to value sizes to find blob column value offsets, - // then skip to the values section to decode blob indices. - const char* type_bytes = input_ref.data(); - input_ref.remove_prefix(num_columns); - - // Skip NAME SIZES section - if (input_ref.size() < name_sizes_bytes) { - return Status::Corruption("Error decoding wide column name sizes"); - } - input_ref.remove_prefix(name_sizes_bytes); - - // Read VALUE SIZES section to find blob column value offsets - if (input_ref.size() < value_sizes_bytes) { - return Status::Corruption("Error decoding wide column value sizes"); - } - const char* vs_ptr = input_ref.data(); - const char* vs_limit = vs_ptr + value_sizes_bytes; - input_ref.remove_prefix(value_sizes_bytes); - - // Skip NAMES section - if (input_ref.size() < names_bytes) { - return Status::Corruption("Error decoding wide column names"); + // First check the whole entity, and decode and validate every blob index. + // Call `callback` only after all checks pass, so a malformed later column + // cannot give callers only part of the blob indexes. + autovector blob_indexes; + Status s = WalkV2Columns( + input_ref, num_columns, /*stop_without_blobs=*/true, + [&](uint32_t /*index*/, ValueType type, const Slice& /*name*/, + const Slice& value) { + if (type != kTypeBlobIndex) { + return Status::OK(); + } + if (value.empty()) { + return Status::Corruption("Empty blob index in wide column"); + } + BlobIndex blob_index; + Status bs = blob_index.DecodeFrom(value); + if (bs.ok()) { + bs = blob_index.Validate(); + } + if (!bs.ok()) { + return Status::Corruption("Error decoding blob index in wide column"); + } + blob_indexes.push_back(blob_index); + return Status::OK(); + }); + if (!s.ok()) { + return s; } - input_ref.remove_prefix(names_bytes); - // Now input_ref points to VALUES section. Walk through value sizes - // and decode only blob column values. - size_t value_pos = 0; - for (uint32_t i = 0; i < num_columns; ++i) { - uint32_t vs = 0; - const char* vs_next = GetVarint32Ptr(vs_ptr, vs_limit, &vs); - if (vs_next == nullptr) { - return Status::Corruption("Error decoding wide column value size"); + for (const BlobIndex& blob_index : blob_indexes) { + Status callback_status = callback(blob_index); + if (!callback_status.ok()) { + return callback_status; } - vs_ptr = vs_next; - - if (static_cast(type_bytes[i]) == kTypeBlobIndex) { - if (value_pos + vs > input_ref.size()) { - return Status::Corruption("Error decoding wide column blob index"); - } - Slice blob_slice(input_ref.data() + value_pos, vs); - if (blob_slice.empty()) { - return Status::Corruption("Empty blob index in wide column"); - } - BlobIndex blob_idx; - Status s = blob_idx.DecodeFrom(blob_slice); - if (!s.ok()) { - return Status::Corruption("Error decoding blob index in wide column"); - } - Status cb_s = callback(blob_idx); - if (!cb_s.ok()) { - return cb_s; - } - } - value_pos += vs; } return Status::OK(); @@ -693,7 +655,7 @@ Status WideColumnSerialization::RemoveBlobColumns( // Most entities lose no column. Find out from the blob indexes alone before // paying for a full deserialization. bool any_removed = false; - Status s = ForEachBlobFileNumber( + Status s = ForEachBlobIndex( input, [&is_removed, &any_removed](const BlobIndex& blob_index) { any_removed |= is_removed(blob_index); return Status::OK(); diff --git a/db/wide/wide_column_serialization.h b/db/wide/wide_column_serialization.h index 33f974444cce..0010df2f5bf0 100644 --- a/db/wide/wide_column_serialization.h +++ b/db/wide/wide_column_serialization.h @@ -184,11 +184,19 @@ class WideColumnSerialization { // Returns Status::Corruption on decode errors. static Status HasBlobColumns(const Slice& input, bool& has_blob_columns); - // Lightweight extraction of blob file numbers from a V2 entity. - // Calls the callback for each blob column's BlobIndex. Avoids full - // deserialization of column names and inline values. - // Returns OK if no blob columns or not V2 (callback not invoked). - static Status ForEachBlobFileNumber( + // Calls `callback` once for each blob column's BlobIndex, in column order. + // Code that needs only the blob references of an entity (for example the + // table builder and BlobGarbageMeter) uses it instead of Deserialize(). + // - V1 entity: it cannot have blob columns. The function returns OK after + // the header and does not check the rest of the entity. + // - V2 entity without blob columns: the function returns OK after the + // column types and does not check the rest of the entity. + // - V2 entity with blob columns: the function checks the whole entity + // (section sizes, name order, no trailing bytes), and decodes and + // validates (BlobIndex::Validate()) every blob index. Only then does it + // call `callback`, so a malformed entity fails before any callback runs. + // Column names and inline values are not copied. + static Status ForEachBlobIndex( const Slice& input, const std::function& callback); @@ -338,6 +346,18 @@ class WideColumnSerialization { std::vector& columns, std::vector& column_types); + // Walks the columns of a V2 entity. DeserializeV2Impl() and + // ForEachBlobIndex() both use it, so the V2 format checks are in one place. + // It parses sections 2-7 in one pass and checks the whole framing: the skip + // info, every column type, strict name order, and that the sections fill + // `input` exactly. It calls `visit(index, type, name, value)`, which returns + // a Status, for each column in order. With `stop_without_blobs`, it returns + // OK right after the type check when no column is a blob index, without + // visiting any column. + template + static Status WalkV2Columns(Slice& input, uint32_t num_columns, + bool stop_without_blobs, Visit&& visit); + // Returns true if t is a supported per-column ValueType. Currently only // kTypeValue (inline) and kTypeBlobIndex are supported. Notably, // kTypeWideColumnEntity is rejected to prevent recursive nesting. diff --git a/db/wide/wide_column_serialization_test.cc b/db/wide/wide_column_serialization_test.cc index 31b272f554ae..8acfbef27135 100644 --- a/db/wide/wide_column_serialization_test.cc +++ b/db/wide/wide_column_serialization_test.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include "db/blob/blob_index.h" #include "db/wide/wide_columns_helper.h" @@ -212,9 +213,10 @@ TEST_F(WideColumnSerializationTest, FutureVersionRejectedConsistently) { ASSERT_TRUE(WideColumnSerialization::HasBlobColumns(input, has_blob_columns) .IsCorruption()); - ASSERT_TRUE(WideColumnSerialization::ForEachBlobFileNumber( - input, [](const BlobIndex&) { return Status::OK(); }) - .IsCorruption()); + ASSERT_TRUE( + WideColumnSerialization::ForEachBlobIndex(input, [](const BlobIndex&) { + return Status::OK(); + }).IsCorruption()); } TEST_F(WideColumnSerializationTest, DeserializeNumberOfColumnsError) { @@ -404,7 +406,7 @@ TEST_F(WideColumnSerializationTest, DeserializeV2RejectsRecursiveType) { } TEST_F(WideColumnSerializationTest, FastPathsRejectUnsupportedColumnType) { - // GetValueOfDefaultColumn() and ForEachBlobFileNumber() read the COLUMN TYPES + // GetValueOfDefaultColumn() and ForEachBlobIndex() read the COLUMN TYPES // section on a fast path. An unrecognized/unsupported type byte (here // kTypeWideColumnEntity) must be rejected as Corruption -- consistent with // Deserialize()/HasBlobColumns() -- rather than silently treated as an inline @@ -439,7 +441,7 @@ TEST_F(WideColumnSerializationTest, FastPathsRejectUnsupportedColumnType) { } { - const Status s = WideColumnSerialization::ForEachBlobFileNumber( + const Status s = WideColumnSerialization::ForEachBlobIndex( Slice(buf), [](const BlobIndex&) { return Status::OK(); }); ASSERT_TRUE(s.IsCorruption()); ASSERT_TRUE(std::strstr(s.getState(), "Unsupported wide column ValueType")); @@ -930,6 +932,74 @@ TEST_F(WideColumnSerializationTest, ResolveEntityForMergeNullBlobFetcher) { ASSERT_TRUE(s.IsCorruption()); } +TEST_F(WideColumnSerializationTest, RemoveBlobColumns) { + const std::vector> columns{ + {"", "inline"}, {"a", ""}, {"b", ""}, {"c", "x"}}; + const std::vector> blob_columns{ + {1, MakeBlobIndex(10, 100, 5)}, {2, MakeBlobIndex(20, 200, 6)}}; + std::string entity; + ASSERT_OK( + WideColumnSerialization::SerializeV2(columns, blob_columns, entity)); + const auto remove_files = [](std::set files) { + return [files](const BlobIndex& index) { + return files.count(index.file_number()) != 0; + }; + }; + + // Nothing to remove: the output is not changed. + std::string output = "unchanged"; + using Removal = WideColumnSerialization::BlobColumnRemoval; + Removal removal = Removal::kAll; + ASSERT_OK(WideColumnSerialization::RemoveBlobColumns( + entity, remove_files({30}), &output, removal)); + ASSERT_EQ(removal, Removal::kNone); + ASSERT_EQ(output, "unchanged"); + + // One blob column: the other blob column moves to position 1. + ASSERT_OK(WideColumnSerialization::RemoveBlobColumns( + entity, remove_files({10}), &output, removal)); + ASSERT_EQ(removal, Removal::kSome); + WideColumns result; + std::vector> result_blobs; + ASSERT_OK( + WideColumnSerialization::Deserialize(output, result, &result_blobs)); + ASSERT_EQ(result.size(), 3U); + ASSERT_EQ(result[0].name(), kDefaultWideColumnName); + ASSERT_EQ(result[0].value(), "inline"); + ASSERT_EQ(result[1].name(), "b"); + ASSERT_EQ(result[2].name(), "c"); + ASSERT_EQ(result[2].value(), "x"); + ASSERT_EQ(result_blobs.size(), 1U); + ASSERT_EQ(result_blobs[0].first, 1U); + ASSERT_EQ(result_blobs[0].second.file_number(), 20U); + + // Every blob column: a plain entity remains. + ASSERT_OK(WideColumnSerialization::RemoveBlobColumns( + entity, remove_files({10, 20}), &output, removal)); + ASSERT_EQ(removal, Removal::kSome); + result.clear(); + result_blobs.clear(); + ASSERT_OK( + WideColumnSerialization::Deserialize(output, result, &result_blobs)); + ASSERT_EQ(result, + WideColumns({{kDefaultWideColumnName, "inline"}, {"c", "x"}})); + ASSERT_TRUE(result_blobs.empty()); + + // Every column: the caller turns the entity into a deletion. + std::string blob_only; + ASSERT_OK(WideColumnSerialization::SerializeV2( + std::vector>{{"a", ""}}, + {{0, MakeBlobIndex(10, 100, 5)}}, blob_only)); + ASSERT_OK(WideColumnSerialization::RemoveBlobColumns( + blob_only, remove_files({10}), &output, removal)); + ASSERT_EQ(removal, Removal::kAll); + + // Malformed input fails. + ASSERT_TRUE(WideColumnSerialization::RemoveBlobColumns( + "\xff", remove_files({10}), &output, removal) + .IsCorruption()); +} + TEST_F(WideColumnSerializationTest, DeserializeRejectsTrailingData) { const WideColumns columns{{kDefaultWideColumnName, "d"}, {"attr", "val"}}; @@ -1169,6 +1239,129 @@ TEST(PinnableWideColumnsTest, MultiBufferMovePreservesPointers) { EXPECT_EQ(move_assigned.columns()[1].value().data(), resolved_data); } +TEST_F(WideColumnSerializationTest, ForEachBlobIndexValidatesBeforeCallbacks) { + const std::vector> columns{ + {"", "inline"}, {"a", "value-a"}, {"b", "value-b"}}; + const BlobIndex first = MakeBlobIndex(/*file_number=*/11, /*offset=*/101, + /*size=*/1001); + const BlobIndex second = MakeBlobIndex(/*file_number=*/12, /*offset=*/102, + /*size=*/1002); + std::string serialized; + ASSERT_OK(WideColumnSerialization::SerializeV2( + columns, {{1, first}, {2, second}}, serialized)); + + std::vector file_numbers; + ASSERT_OK(WideColumnSerialization::ForEachBlobIndex( + serialized, [&](const BlobIndex& blob_index) { + file_numbers.push_back(blob_index.file_number()); + return Status::OK(); + })); + EXPECT_EQ(file_numbers, std::vector({11, 12})); + + // With blob and inline columns mixed, the callback sees only the blob + // columns, in column order. + std::string mixed; + ASSERT_OK(WideColumnSerialization::SerializeV2( + columns, {{0, second}, {2, first}}, mixed)); + file_numbers.clear(); + ASSERT_OK(WideColumnSerialization::ForEachBlobIndex( + mixed, [&](const BlobIndex& blob_index) { + file_numbers.push_back(blob_index.file_number()); + return Status::OK(); + })); + EXPECT_EQ(file_numbers, std::vector({12, 11})); + + serialized.push_back('x'); + size_t callback_count = 0; + Status status = WideColumnSerialization::ForEachBlobIndex( + serialized, [&](const BlobIndex&) { + ++callback_count; + return Status::OK(); + }); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_EQ(callback_count, 0); + + std::string invalid_encoded; + BlobIndex::EncodeBlob(&invalid_encoded, kCurrentFileBlobIndexFileNumber, + /*offset=*/103, + /*size=*/1003, static_cast(0xff)); + BlobIndex invalid_same_file; + ASSERT_OK(invalid_same_file.DecodeFrom(invalid_encoded)); + serialized.clear(); + ASSERT_OK(WideColumnSerialization::SerializeV2( + columns, {{1, first}, {2, invalid_same_file}}, serialized)); + status = WideColumnSerialization::ForEachBlobIndex(serialized, + [&](const BlobIndex&) { + ++callback_count; + return Status::OK(); + }); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_EQ(callback_count, 0); +} + +TEST_F(WideColumnSerializationTest, + ForEachBlobIndexSkipsFramingWithoutBlobColumns) { + // Without blob columns, only the header, the skip info and the column types + // are read, so damage in later sections (here a trailing byte) is not + // reported. + const std::vector> columns{ + {"", "inline"}, {"a", "value-a"}}; + std::string serialized; + ASSERT_OK(WideColumnSerialization::SerializeV2(columns, {}, serialized)); + serialized.push_back('x'); + + size_t callback_count = 0; + ASSERT_OK(WideColumnSerialization::ForEachBlobIndex(serialized, + [&](const BlobIndex&) { + ++callback_count; + return Status::OK(); + })); + EXPECT_EQ(callback_count, 0); + + // Entities with a version below 2 (here 1, then 0) cannot hold blob columns, + // so a trailing byte is not reported for them either. + const WideColumns v1_columns{{"", "inline"}, {"a", "value-a"}}; + serialized.clear(); + ASSERT_OK(WideColumnSerialization::Serialize(v1_columns, serialized)); + serialized.push_back('x'); + ASSERT_OK(WideColumnSerialization::ForEachBlobIndex(serialized, + [&](const BlobIndex&) { + ++callback_count; + return Status::OK(); + })); + serialized[0] = 0; + ASSERT_OK(WideColumnSerialization::ForEachBlobIndex(serialized, + [&](const BlobIndex&) { + ++callback_count; + return Status::OK(); + })); + EXPECT_EQ(callback_count, 0); +} + +TEST_F(WideColumnSerializationTest, + ForEachBlobIndexRejectsUnorderedNamesWithBlobColumns) { + const std::vector> columns{ + {"x1", "value-1"}, {"x2", "value-2"}}; + const BlobIndex blob_index = MakeBlobIndex(/*file_number=*/11, + /*offset=*/101, /*size=*/1001); + std::string serialized; + ASSERT_OK(WideColumnSerialization::SerializeV2(columns, {{1, blob_index}}, + serialized)); + const size_t names_pos = serialized.find("x1x2"); + ASSERT_NE(names_pos, std::string::npos); + serialized.replace(names_pos, 4, "x2x1"); + + size_t callback_count = 0; + const Status status = WideColumnSerialization::ForEachBlobIndex( + serialized, [&](const BlobIndex&) { + ++callback_count; + return Status::OK(); + }); + EXPECT_TRUE(status.IsCorruption()) << status.ToString(); + EXPECT_TRUE(std::strstr(status.getState(), "Wide columns out of order")); + EXPECT_EQ(callback_count, 0); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/file/sst_file_manager_impl.cc b/file/sst_file_manager_impl.cc index 95f19016afa2..96f73bc3391a 100644 --- a/file/sst_file_manager_impl.cc +++ b/file/sst_file_manager_impl.cc @@ -91,7 +91,6 @@ Status SstFileManagerImpl::OnDeleteFile(const std::string& file_path) { } void SstFileManagerImpl::OnCompactionCompletion(Compaction* c) { - MutexLock l(&mu_); uint64_t size_added_by_compaction = 0; for (size_t i = 0; i < c->num_input_levels(); i++) { for (size_t j = 0; j < c->num_input_files(i); j++) { @@ -99,8 +98,13 @@ void SstFileManagerImpl::OnCompactionCompletion(Compaction* c) { size_added_by_compaction += filemeta->fd.GetFileSize(); } } - assert(cur_compactions_reserved_size_ >= size_added_by_compaction); - cur_compactions_reserved_size_ -= size_added_by_compaction; + OnCompactionCompletion(size_added_by_compaction); +} + +void SstFileManagerImpl::OnCompactionCompletion(uint64_t reserved_size) { + MutexLock l(&mu_); + assert(cur_compactions_reserved_size_ >= reserved_size); + cur_compactions_reserved_size_ -= reserved_size; } Status SstFileManagerImpl::OnMoveFile(const std::string& old_path, @@ -168,11 +172,33 @@ bool SstFileManagerImpl::EnoughRoomForCompaction( size_added_by_compaction += filemeta->fd.GetFileSize(); } } + return EnoughRoomForCompactionLocked( + size_added_by_compaction, + [&]() { + return TableFileName(cfd->ioptions().cf_paths, + inputs[0][0]->fd.GetNumber(), + inputs[0][0]->fd.GetPathId()); + }, + bg_error); +} +bool SstFileManagerImpl::EnoughRoomForCompaction( + uint64_t size_added_by_compaction, const std::string& output_path, + const Status& bg_error) { + MutexLock l(&mu_); + return EnoughRoomForCompactionLocked( + size_added_by_compaction, [&]() { return output_path; }, bg_error); +} + +bool SstFileManagerImpl::EnoughRoomForCompactionLocked( + uint64_t size_added_by_compaction, + const std::function& free_space_path, + const Status& bg_error) { + mu_.AssertHeld(); // Update cur_compactions_reserved_size_ so concurrent compaction // don't max out space - size_t needed_headroom = cur_compactions_reserved_size_ + - size_added_by_compaction + compaction_buffer_size_; + uint64_t needed_headroom = cur_compactions_reserved_size_ + + size_added_by_compaction + compaction_buffer_size_; if (max_allowed_space_ != 0 && (needed_headroom + total_files_size_ > max_allowed_space_)) { return false; @@ -183,11 +209,9 @@ bool SstFileManagerImpl::EnoughRoomForCompaction( // misbehaving DB instance and prevent it from slowing down compactions of // other DB instances if (bg_error.IsNoSpace() && CheckFreeSpace()) { - auto fn = - TableFileName(cfd->ioptions().cf_paths, inputs[0][0]->fd.GetNumber(), - inputs[0][0]->fd.GetPathId()); uint64_t free_space = 0; - Status s = fs_->GetFreeSpace(fn, IOOptions(), &free_space, nullptr); + Status s = + fs_->GetFreeSpace(free_space_path(), IOOptions(), &free_space, nullptr); s.PermitUncheckedError(); // TODO: Check the status // needed_headroom is based on current size reserved by compactions, // minus any files created by running compactions as they would count @@ -202,7 +226,7 @@ bool SstFileManagerImpl::EnoughRoomForCompaction( ROCKS_LOG_ERROR(logger_, "free space [%" PRIu64 " bytes] is less than " - "needed headroom [%" ROCKSDB_PRIszt " bytes]\n", + "needed headroom [%" PRIu64 " bytes]\n", free_space, needed_headroom); return false; } diff --git a/file/sst_file_manager_impl.h b/file/sst_file_manager_impl.h index cef717e0dc7e..dd054159f142 100644 --- a/file/sst_file_manager_impl.h +++ b/file/sst_file_manager_impl.h @@ -5,6 +5,7 @@ #pragma once +#include #include #include @@ -84,9 +85,18 @@ class SstFileManagerImpl : public SstFileManager { const std::vector& inputs, const Status& bg_error); + // Same check as above, for background work that has no CompactionInputFiles, + // such as indirect blob GC. `estimated_output_size` is the number of bytes + // the job expects to write. On success, it reserves that many bytes. The + // caller releases them with OnCompactionCompletion(uint64_t). + bool EnoughRoomForCompaction(uint64_t estimated_output_size, + const std::string& output_path, + const Status& bg_error); + // Bookkeeping so total_file_sizes_ goes back to normal after compaction // finishes void OnCompactionCompletion(Compaction* c); + void OnCompactionCompletion(uint64_t reserved_size); uint64_t GetCompactionsReservedSize(); @@ -185,6 +195,14 @@ class SstFileManagerImpl : public SstFileManager { void OnDeleteFileImpl(const std::string& file_path); void ClearError(); + + // The body of both EnoughRoomForCompaction() overloads. REQUIRES: mu_ held. + // `free_space_path` returns the path whose file system free space is + // checked; it is called only after a NoSpace error. + bool EnoughRoomForCompactionLocked( + uint64_t size_added_by_compaction, + const std::function& free_space_path, + const Status& bg_error); bool CheckFreeSpace() { return bg_err_.severity() == Status::Severity::kSoftError; } diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h index 80e95503df30..826450af9a97 100644 --- a/include/rocksdb/advanced_options.h +++ b/include/rocksdb/advanced_options.h @@ -1087,6 +1087,108 @@ struct AdvancedColumnFamilyOptions { // Dynamically changeable through the SetOptions() API bool enable_blob_files = false; + // Experimental, and not complete yet. Option validation returns NotSupported + // for this option until the rest of the feature is in place. + // + // Purpose: let indirect blob GC move the live values out of a blob file that + // is mostly garbage without rewriting the SSTs that point to them. When this + // option is on, each new blob value gets a kIndirectBlob BlobIndex, which + // keeps the value's identity (origin file and offset) when GC moves it. + // Compaction outputs at L1 or deeper, outputs of a bottommost compaction, + // and L0 compaction outputs with an exact input save the exact list of blob + // references they keep. Flush outputs save only a coarse classification, + // and a later compaction rewrites them with the exact list. See + // db/blob/blob_reference_metadata.h for how it works. + // + // Requirements and limits (option validation checks these after the + // NotSupported check above): + // - enable_blob_files must be true (else InvalidArgument). + // - The table format must be block-based (else NotSupported). + // - enable_blob_direct_write must be false (else NotSupported). + // - The FIFO compaction style is not supported (NotSupported). FIFO + // column families that do not set this option keep their existing + // BlobDB behavior. + // Turning the option off stops new values from using kIndirectBlob. The + // SSTs and blob files that were already written keep their format. + // + // Default: false + // + // Dynamically changeable through the SetOptions() API + bool enable_blob_indirection = false; + + // Experimental. Used only with enable_blob_indirection. + // + // Purpose: make reads of values that indirect blob GC moved into relocation + // files faster. A relocation file keeps the location of each value in its + // data blocks (the "location blocks"), which take a few bytes per relocated + // value. With this option on, a relocation file reader reads all of its + // location blocks when it opens and keeps them in memory while it is open. + // A lookup then needs no index or block cache lookup, so warm Gets and + // MultiGets of relocated values get faster. + // + // Cost: + // - The kept blocks are charged to the block cache, and they stay there + // while the reader is open. Without a block cache, they stay in the + // reader's own memory and are not charged. With allow_mmap_reads and no + // compression, the blocks point into the mapped file and are not + // charged. + // - Opening a relocation file reads all of its location blocks, so the + // open takes longer and reads more bytes. + // - With strict_capacity_limit, the kept blocks hold block cache capacity + // that eviction cannot free, so other block cache inserts can fail + // instead of evicting older blocks. If the block cache cannot charge the + // blocks, the reader logs a warning and does normal lookups for that + // file. + // + // When to use: when all kept location blocks are small compared with the + // block cache. Otherwise they push SST blocks out of the cache, and cold + // reads can get slower. + // + // Default: false + // + // Dynamically changeable through the SetOptions() API. A reader reads the + // option when it opens, so a change applies to the relocation files that are + // opened after it. + bool pin_blob_relocation_location_blocks = false; + + // Experimental. Applies to compaction outputs with exact blob reference + // metadata (the per-SST list of blob references that indirect blob GC reads). + // With enable_blob_indirection on, or after the DB has used it, these are + // outputs at L1 or deeper, outputs of a bottommost compaction, and L0 + // outputs that have an exact L0 input. When such an output SST holds this + // many blob references, compaction starts a new output at the next user key. + // + // Build memory: while an SST is built, its blob references are kept in memory + // until the SST is finished, charged to the block cache. Each reference uses + // about the size of its encoded entry plus a small chunk overhead: about 8 + // to 14 bytes (an estimate). A compressed value is at the high end, because + // its entry also stores the CRC of its stored bytes (see + // BlobReferenceMetadataCollector in db/blob/blob_reference_metadata.h). Many + // small groups in one SST cost more. This cap bounds that memory for each + // output being built. The default (1M references) uses about 8 to 14 MB. A + // lower value makes more, smaller output files. + // + // Output size: when every value is in a blob file, each SST row holds only + // the key and the BlobIndex, and the reference metadata adds a few bytes per + // reference. With small keys, an output at the default cap is well below the + // default target_file_size_base (64 MB). Larger keys or no compression make + // each row larger. With a larger target_file_size_base, raise this cap so + // that it does not cut outputs first. The build memory above then grows by + // about 8 to 14 bytes per extra reference. + // + // Soft limit: compaction starts a new output only before a new user key, once + // the output already holds the cap. The history of one user key (all of its + // versions that compaction keeps) is never split, so an output may pass the + // cap by one key's history. + // + // Scope: flush output carries no exact blob reference metadata, so this + // option applies to compaction outputs only. + // + // Valid range: [4096, 268435456] Default: 1048576 + // + // Dynamically changeable through the SetOptions() API + uint64_t blob_indirection_max_references_per_sst = 1 << 20; + // The size of the smallest value to be stored separately in a blob file. // Values which have an uncompressed size smaller than this threshold are // stored alongside the keys in SST files in the usual fashion. A value of @@ -1171,6 +1273,18 @@ struct AdvancedColumnFamilyOptions { // Dynamically changeable through the SetOptions() API double blob_garbage_collection_force_threshold = 1.0; + // Experimental. Used by indirect blob GC with enable_blob_indirection. GC + // reads the live values of an input file in offset order. Two neighbor values + // are read with one I/O when the bytes between them (dead values, and in a + // normal blob file also the next record's header and key) are at most this + // many; a larger gap starts a new read, so dead regions are skipped. One read + // is at most 1 MB. 0 reads each value on its own. + // + // Default: 4096 (one page; buffered I/O reads the whole page anyway) + // + // Dynamically changeable through the SetOptions() API + uint64_t indirect_blob_gc_read_coalesce_bytes = 4096; + // Compaction readahead for blob files. // // Default: 0 diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index 60abb1b7c6de..2cc23d2d6996 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -5547,6 +5547,27 @@ rocksdb_options_set_preserve_internal_time_seconds(rocksdb_options_t* opt, extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_get_preserve_internal_time_seconds(rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_indirection( + rocksdb_options_t* opt, unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_enable_blob_indirection(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_pin_blob_relocation_location_blocks(rocksdb_options_t* opt, + unsigned char v); + +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_pin_blob_relocation_location_blocks(rocksdb_options_t* opt); + +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt, uint64_t v); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_blob_indirection_max_references_per_sst( + rocksdb_options_t* opt); + extern ROCKSDB_LIBRARY_API void rocksdb_options_set_blob_file_writable_file_max_buffer_size( rocksdb_options_t* opt, uint64_t v); @@ -5555,6 +5576,14 @@ extern ROCKSDB_LIBRARY_API uint64_t rocksdb_options_get_blob_file_writable_file_max_buffer_size( rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_indirect_blob_gc_read_coalesce_bytes(rocksdb_options_t* opt, + uint64_t v); + +extern ROCKSDB_LIBRARY_API uint64_t +rocksdb_options_get_indirect_blob_gc_read_coalesce_bytes( + rocksdb_options_t* opt); + extern ROCKSDB_LIBRARY_API void rocksdb_options_set_enable_blob_direct_write( rocksdb_options_t* opt, unsigned char v); diff --git a/include/rocksdb/cache.h b/include/rocksdb/cache.h index 888b95b715c4..ab1c6bb977a7 100644 --- a/include/rocksdb/cache.h +++ b/include/rocksdb/cache.h @@ -84,6 +84,20 @@ enum class CacheEntryRole { kBlobCache, // Readahead (prefetch) buffers' charge to account for their memory usage kReadaheadBuffer, + // Buffers that record the blob references of an SST while the SST is built + // (only with enable_blob_indirection) + kBlobReferenceMetadataBuildingBuffer, + // Blob reference census: the reverse index from blob files to the SSTs that + // reference them, its build state, its queued changes, and the SST summaries + // that a compaction reads for the census + kBlobReferenceCensus, + // Working memory of indirect blob GC. Indirect blob GC is a background + // job that copies the live values out of a few mostly-garbage blob files. + // A job charges the live references of its input files, its census snapshot + // (a copy of the census entries of the inputs: the SSTs that reference them + // and their reference totals), its list of planned SST reads, its SST + // readers, and its read buffers + kIndirectBlobGcWorkingMemory, // Default bucket, for miscellaneous cache entries. Do not use for // entries that could potentially add up to large usage. kMisc, diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 41615a110474..030ccb390786 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -1951,6 +1951,45 @@ class DB { // "rocksdb.blob-cache-pinned-usage" - returns the memory size for the // entries being pinned in blob cache. static const std::string kBlobCachePinnedUsage; + + // "rocksdb.indirect-blob-gc-idle-micros-with-candidates" - returns how long + // the column family has had indirect blob GC candidates while no GC job + // began, in microseconds (a duration). A large value means that GC is + // starved, for example by LSM compactions. 0 means no candidates, or a GC + // job began since the candidates appeared. The LOG warning comes from the + // periodic stats dump; with stats_dump_period_sec = 0, use this property. + static const std::string kIndirectBlobGcIdleMicrosWithCandidates; + + // "rocksdb.indirect-blob-gc-stats" - returns a map of the indirect blob GC + // state of the column family. Keys: census-state (ready, building, failed, + // or none without blob indirection), census-unready-micros (0 when ready), + // census-consecutive-failures, census-builds, census-bytes (an estimate of + // the census memory that stays: the worker state and the change log; + // rocksdb.block-cache-entry-stats has the exact total of the census cache + // role, including short-lived copies), route-count, relocation-file-count, + // relocation-file-bytes, relocation-garbage-bytes, tracked-files (the files + // the census tracks for GC, including files below the garbage threshold), + // garbage-bytes-collectable, garbage-bytes-suppressed, coarse-pinned-files, + // coarse-pinned-bytes, coarse-pinned-garbage-bytes, coarse-sst-count, + // lowest-coarse-frontier, idle-micros-with-candidates. Every *-micros key + // is a duration, never a timestamp. + static const std::string kIndirectBlobGcStats; + + // "rocksdb.blob-reference-census-unready-micros" - returns how long the + // blob reference census of the column family has been unready, in + // microseconds (a duration, not a timestamp). 0 means that it is ready, or + // that the column family does not use blob indirection. While it is + // unready, the work that needs the census waits. The LOG warning comes + // from the periodic stats dump; with stats_dump_period_sec = 0, use this + // property. + static const std::string kBlobReferenceCensusUnreadyMicros; + + // "rocksdb.blob-indirection-format-version" - returns the blob indirection + // format version of the column family (the per-CF format fence). 0 means + // that the column family has no format fence yet (no SST built with + // enable_blob_indirection on was installed). A release that does not know + // the format cannot open a DB where it is not 0. + static const std::string kBlobIndirectionFormatVersion; }; // DB implementations export properties about their state via this method. diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 49df7e04cf9a..a4caf7d9559e 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -385,17 +385,21 @@ enum Tickers : uint32_t { // size of blob index evicted from base DB by BlobDB compaction filter // because of corresponding file deleted. Only applicable to legacy BlobDB. BLOB_DB_BLOB_INDEX_EVICTED_SIZE, - // # of blob files that were obsoleted by garbage collection. Only applicable - // to legacy BlobDB. + // # of blob files that were obsoleted by garbage collection: legacy BlobDB, + // and the input files that indirect blob GC removed. BLOB_DB_GC_NUM_FILES, - // # of blob files generated by garbage collection. Only applicable to legacy - // BlobDB. + // # of blob files generated by garbage collection: legacy BlobDB, and the + // relocation files that indirect blob GC wrote. BLOB_DB_GC_NUM_NEW_FILES, - // # of BlobDB garbage collection failures. Only applicable to legacy BlobDB. + // # of garbage collection failures: legacy BlobDB, and failed indirect blob + // GC jobs (see INDIRECT_BLOB_GC_JOBS_FAILED). BLOB_DB_GC_FAILURES, - // # of keys relocated to new blob file by garbage collection. + // # of keys relocated to new blob file by garbage collection. The total of + // all blob GCs: compaction-time blob GC and indirect blob GC (see + // INDIRECT_BLOB_GC_KEYS_RELOCATED). BLOB_DB_GC_NUM_KEYS_RELOCATED, - // # of bytes relocated to new blob file by garbage collection. + // # of bytes relocated to new blob file by garbage collection. The total of + // all blob GCs (see INDIRECT_BLOB_GC_BYTES_RELOCATED). BLOB_DB_GC_BYTES_RELOCATED, // # of blob files evicted because of BlobDB is full. Only applicable to // legacy BlobDB. @@ -691,6 +695,92 @@ enum Tickers : uint32_t { // hide. An iterator can count an entry again when it moves back to it. BLOB_DB_DROPPED_BLOB_READ_COUNT, + // Number of times a blob reference census build overflowed its change log + // again after an overflow restart, before it published. The census then + // retries after a backoff (DBs that use enable_blob_indirection). + BLOB_DB_CENSUS_RESTART_OVERFLOW_COUNT, + + // Number of compaction output SSTs that were closed because they already held + // blob_indirection_max_references_per_sst blob references when the next user + // key arrived (DBs that use or have used enable_blob_indirection). + COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // times a relocation file reader failed to open (the file is missing, + // corrupt, or could not be read). Opens of normal blob files count in + // NO_FILE_ERRORS only. + BLOB_RELOCATION_FILE_OPEN_FAILURES, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // reads that got the remembered result of an earlier failed relocation file + // open, without opening the file again (see BlobFileCache). + BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // blob cache misses that found the value through a route (the reference names + // an old file whose live values moved to a relocation file). + BLOB_DB_INDIRECT_READ_ROUTED, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // times an L0 trivial move (leveled compaction) stopped before a coarse L0 + // file, because a coarse file must be rewritten with exact blob reference + // metadata when it leaves L0. + COMPACTION_COARSE_L0_TRIVIAL_MOVE_BLOCKED, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // indirect blob GC jobs that began (registered with the census). + INDIRECT_BLOB_GC_JOBS_STARTED, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // indirect blob GC jobs that installed their result. + INDIRECT_BLOB_GC_JOBS_INSTALLED, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // indirect blob GC jobs that ended as stale (their input changed); a later + // job retries. + INDIRECT_BLOB_GC_JOBS_STALE, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // indirect blob GC jobs that ended without an error and installed nothing + // (for example, every input was skipped, or the job was deferred or + // interrupted). + INDIRECT_BLOB_GC_JOBS_NOTHING_INSTALLED, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // indirect blob GC jobs that failed with an error. Each one also counts in + // BLOB_DB_GC_FAILURES. + INDIRECT_BLOB_GC_JOBS_FAILED, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // inputs that indirect blob GC skipped because it found corruption (a blob + // record, a relocation file, or exact SST metadata). Each one also has an + // ERROR line in the LOG. + INDIRECT_BLOB_GC_CORRUPT_INPUTS, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // times indirect blob GC found SSTs that do not match the blob reference + // census. + INDIRECT_BLOB_GC_CENSUS_MISMATCHES, + + // Used only by DBs that use or have used enable_blob_indirection. Number of + // values that indirect blob GC moved into relocation files. + // BLOB_DB_GC_NUM_KEYS_RELOCATED counts them too. + INDIRECT_BLOB_GC_KEYS_RELOCATED, + + // Used only by DBs that use or have used enable_blob_indirection. Stored + // bytes of the values that indirect blob GC moved into relocation files. + // BLOB_DB_GC_BYTES_RELOCATED counts them too. + INDIRECT_BLOB_GC_BYTES_RELOCATED, + + // Used only by DBs that use or have used enable_blob_indirection. Bytes that + // indirect blob GC read from its input files. + INDIRECT_BLOB_GC_BYTES_READ, + + // Used only by DBs that use or have used enable_blob_indirection. File bytes + // that installed indirect blob GC jobs freed: the size of the removed input + // files minus the size of the new relocation files. + INDIRECT_BLOB_GC_BYTES_RECLAIMED, + TICKER_ENUM_MAX }; @@ -879,6 +969,15 @@ enum Histograms : uint32_t { // Time spent opening the secondary DB inside DB::OpenAndCompact(). OPEN_AND_COMPACT_DB_OPEN_MICROS, + // Number of blob references in each compaction output SST that has exact blob + // reference metadata (DBs that use or have used enable_blob_indirection). + COMPACTION_OUTPUT_BLOB_REFERENCES, + + // Used only by DBs that use or have used enable_blob_indirection. Time of + // each indirect blob GC job that began, from its start to its end, in + // microseconds. + INDIRECT_BLOB_GC_JOB_MICROS, + HISTOGRAM_ENUM_MAX }; diff --git a/include/rocksdb/table_reader_caller.h b/include/rocksdb/table_reader_caller.h index 10ec08130fbb..18aee122ec55 100644 --- a/include/rocksdb/table_reader_caller.h +++ b/include/rocksdb/table_reader_caller.h @@ -35,6 +35,9 @@ enum TableReaderCaller : char { // A list of callers that are either not interesting for analysis or are // calling from a test environment, e.g., unit test, benchmark, etc. kUncategorized = 14, + // Reads of a blob relocation file (blob indirection): its location blocks + // and raw records. The relocation reader checks the raw values itself. + kBlobRelocationFileRead = 15, // All callers should be added before kMaxBlockCacheLookupCaller. kMaxBlockCacheLookupCaller }; diff --git a/include/rocksdb/types.h b/include/rocksdb/types.h index 6d51cc451230..f017a77218e2 100644 --- a/include/rocksdb/types.h +++ b/include/rocksdb/types.h @@ -39,6 +39,8 @@ enum class BlobFileCreationReason { kFlush, kCompaction, kRecovery, + // A relocation file written by indirect blob GC (enable_blob_indirection). + kIndirectBlobGC, }; // The types of files RocksDB uses in a DB directory. (Available for diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 93615b451000..d33bf78ad391 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -6098,6 +6098,10 @@ class HistogramTypeJni { case ROCKSDB_NAMESPACE::Histograms:: IO_DISPATCHER_ASYNC_READ_PREFETCH_LEAD_MICROS: return 0x4E; + case ROCKSDB_NAMESPACE::Histograms::COMPACTION_OUTPUT_BLOB_REFERENCES: + return 0x4F; + case ROCKSDB_NAMESPACE::Histograms::INDIRECT_BLOB_GC_JOB_MICROS: + return 0x50; case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX: // 0x3E is reserved for backwards compatibility on current minor // version. @@ -6280,6 +6284,10 @@ class HistogramTypeJni { case 0x4E: return ROCKSDB_NAMESPACE::Histograms:: IO_DISPATCHER_ASYNC_READ_PREFETCH_LEAD_MICROS; + case 0x4F: + return ROCKSDB_NAMESPACE::Histograms::COMPACTION_OUTPUT_BLOB_REFERENCES; + case 0x50: + return ROCKSDB_NAMESPACE::Histograms::INDIRECT_BLOB_GC_JOB_MICROS; case 0x3E: // 0x3E is reserved for backwards compatibility on current minor // version. diff --git a/java/src/main/java/org/rocksdb/HistogramType.java b/java/src/main/java/org/rocksdb/HistogramType.java index 6cec59b4c566..67eafafcf727 100644 --- a/java/src/main/java/org/rocksdb/HistogramType.java +++ b/java/src/main/java/org/rocksdb/HistogramType.java @@ -291,6 +291,20 @@ public enum HistogramType { */ IO_DISPATCHER_ASYNC_READ_PREFETCH_LEAD_MICROS((byte) 0x4E), + /** + * Number of blob references in each compaction output SST that has exact + * blob reference metadata (DBs that use or have used + * enable_blob_indirection). + */ + COMPACTION_OUTPUT_BLOB_REFERENCES((byte) 0x4F), + + /** + * Used only by DBs that use or have used enable_blob_indirection. Time of + * each indirect blob GC job that began, from its start to its end, in + * microseconds. + */ + INDIRECT_BLOB_GC_JOB_MICROS((byte) 0x50), + // 0x3E is reserved for backwards compatibility on current minor version. HISTOGRAM_ENUM_MAX((byte) 0x3E); diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index 5c9e442d914f..8a2d2b0b56be 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -349,6 +349,35 @@ const std::vector> TickersNameMap = { {REMOTE_COMPACT_BLOB_RESULT_DECLINED_COUNT, "rocksdb.remote.compact.blob.result.declined.count"}, {BLOB_DB_DROPPED_BLOB_READ_COUNT, "rocksdb.blobdb.dropped.blob.read.count"}, + {BLOB_DB_CENSUS_RESTART_OVERFLOW_COUNT, + "rocksdb.blobdb.census.restart.overflow.count"}, + {COMPACTION_OUTPUT_BLOB_REFERENCE_LIMIT_CUTS, + "rocksdb.compaction.output.blob.reference.limit.cuts"}, + {BLOB_RELOCATION_FILE_OPEN_FAILURES, + "rocksdb.blob.relocation.file.open.failures"}, + {BLOB_RELOCATION_FAILED_OPEN_CACHE_HITS, + "rocksdb.blob.relocation.failed.open.cache.hits"}, + {BLOB_DB_INDIRECT_READ_ROUTED, "rocksdb.blobdb.indirect.read.routed"}, + {COMPACTION_COARSE_L0_TRIVIAL_MOVE_BLOCKED, + "rocksdb.compaction.coarse.l0.trivial.move.blocked"}, + {INDIRECT_BLOB_GC_JOBS_STARTED, "rocksdb.indirect.blob.gc.jobs.started"}, + {INDIRECT_BLOB_GC_JOBS_INSTALLED, + "rocksdb.indirect.blob.gc.jobs.installed"}, + {INDIRECT_BLOB_GC_JOBS_STALE, "rocksdb.indirect.blob.gc.jobs.stale"}, + {INDIRECT_BLOB_GC_JOBS_NOTHING_INSTALLED, + "rocksdb.indirect.blob.gc.jobs.nothing.installed"}, + {INDIRECT_BLOB_GC_JOBS_FAILED, "rocksdb.indirect.blob.gc.jobs.failed"}, + {INDIRECT_BLOB_GC_CORRUPT_INPUTS, + "rocksdb.indirect.blob.gc.corrupt.inputs"}, + {INDIRECT_BLOB_GC_CENSUS_MISMATCHES, + "rocksdb.indirect.blob.gc.census.mismatches"}, + {INDIRECT_BLOB_GC_KEYS_RELOCATED, + "rocksdb.indirect.blob.gc.keys.relocated"}, + {INDIRECT_BLOB_GC_BYTES_RELOCATED, + "rocksdb.indirect.blob.gc.bytes.relocated"}, + {INDIRECT_BLOB_GC_BYTES_READ, "rocksdb.indirect.blob.gc.bytes.read"}, + {INDIRECT_BLOB_GC_BYTES_RECLAIMED, + "rocksdb.indirect.blob.gc.bytes.reclaimed"}, }; const std::vector> HistogramsNameMap = { @@ -446,6 +475,9 @@ const std::vector> HistogramsNameMap = { "rocksdb.flush.write_buffer_manager.memtable.memory.bytes"}, {OPEN_AND_COMPACT_DB_OPEN_MICROS, "rocksdb.open.and.compact.db.open.micros"}, + {COMPACTION_OUTPUT_BLOB_REFERENCES, + "rocksdb.compaction.output.blob.references"}, + {INDIRECT_BLOB_GC_JOB_MICROS, "rocksdb.indirect.blob.gc.job.micros"}, }; std::shared_ptr CreateDBStatistics() { diff --git a/options/cf_options.cc b/options/cf_options.cc index 363fc5af3949..37736ae1a970 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -644,6 +644,25 @@ static std::unordered_map {offsetof(struct MutableCFOptions, enable_blob_files), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kMutable}}, + {"enable_blob_indirection", + {offsetof(struct MutableCFOptions, enable_blob_indirection), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"pin_blob_relocation_location_blocks", + {offsetof(struct MutableCFOptions, + pin_blob_relocation_location_blocks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"blob_indirection_max_references_per_sst", + {offsetof(struct MutableCFOptions, + blob_indirection_max_references_per_sst), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, + {"indirect_blob_gc_read_coalesce_bytes", + {offsetof(struct MutableCFOptions, + indirect_blob_gc_read_coalesce_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable}}, {"min_blob_size", {offsetof(struct MutableCFOptions, min_blob_size), OptionType::kUInt64T, OptionVerificationType::kNormal, @@ -1377,6 +1396,14 @@ void MutableCFOptions::Dump(Logger* log) const { // Blob file related options ROCKS_LOG_INFO(log, " enable_blob_files: %s", enable_blob_files ? "true" : "false"); + ROCKS_LOG_INFO(log, " enable_blob_indirection: %s", + enable_blob_indirection ? "true" : "false"); + ROCKS_LOG_INFO(log, " pin_blob_relocation_location_blocks: %s", + pin_blob_relocation_location_blocks ? "true" : "false"); + ROCKS_LOG_INFO(log, " blob_indirection_max_references_per_sst: %" PRIu64, + blob_indirection_max_references_per_sst); + ROCKS_LOG_INFO(log, " indirect_blob_gc_read_coalesce_bytes: %" PRIu64, + indirect_blob_gc_read_coalesce_bytes); ROCKS_LOG_INFO(log, " min_blob_size: %" PRIu64, min_blob_size); ROCKS_LOG_INFO(log, " blob_file_size: %" PRIu64, diff --git a/options/cf_options.h b/options/cf_options.h index c95523668075..93397e5a2e8e 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -164,6 +164,13 @@ struct MutableCFOptions { preserve_internal_time_seconds(options.preserve_internal_time_seconds), verify_output_flags(options.verify_output_flags), enable_blob_files(options.enable_blob_files), + enable_blob_indirection(options.enable_blob_indirection), + pin_blob_relocation_location_blocks( + options.pin_blob_relocation_location_blocks), + blob_indirection_max_references_per_sst( + options.blob_indirection_max_references_per_sst), + indirect_blob_gc_read_coalesce_bytes( + options.indirect_blob_gc_read_coalesce_bytes), min_blob_size(options.min_blob_size), blob_file_size(options.blob_file_size), blob_file_writable_file_max_buffer_size( @@ -242,6 +249,10 @@ struct MutableCFOptions { preserve_internal_time_seconds(0), verify_output_flags(VerifyOutputFlags::kVerifyNone), enable_blob_files(false), + enable_blob_indirection(false), + pin_blob_relocation_location_blocks(false), + blob_indirection_max_references_per_sst(1 << 20), + indirect_blob_gc_read_coalesce_bytes(4096), min_blob_size(0), blob_file_size(0), blob_file_writable_file_max_buffer_size(0), @@ -350,6 +361,10 @@ struct MutableCFOptions { // Blob file related options bool enable_blob_files; + bool enable_blob_indirection; + bool pin_blob_relocation_location_blocks; + uint64_t blob_indirection_max_references_per_sst; + uint64_t indirect_blob_gc_read_coalesce_bytes; uint64_t min_blob_size; uint64_t blob_file_size; uint64_t blob_file_writable_file_max_buffer_size; diff --git a/options/options.cc b/options/options.cc index c60fefe34ada..473e953783d0 100644 --- a/options/options.cc +++ b/options/options.cc @@ -101,6 +101,11 @@ AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options) options.preclude_last_level_data_seconds), preserve_internal_time_seconds(options.preserve_internal_time_seconds), enable_blob_files(options.enable_blob_files), + enable_blob_indirection(options.enable_blob_indirection), + pin_blob_relocation_location_blocks( + options.pin_blob_relocation_location_blocks), + blob_indirection_max_references_per_sst( + options.blob_indirection_max_references_per_sst), min_blob_size(options.min_blob_size), blob_file_size(options.blob_file_size), blob_file_writable_file_max_buffer_size( @@ -112,6 +117,8 @@ AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options) options.blob_garbage_collection_age_cutoff), blob_garbage_collection_force_threshold( options.blob_garbage_collection_force_threshold), + indirect_blob_gc_read_coalesce_bytes( + options.indirect_blob_gc_read_coalesce_bytes), blob_compaction_readahead_size(options.blob_compaction_readahead_size), blob_file_starting_level(options.blob_file_starting_level), blob_cache(options.blob_cache), @@ -443,6 +450,16 @@ void ColumnFamilyOptions::Dump(Logger* log) const { preserve_internal_time_seconds); ROCKS_LOG_HEADER(log, " Options.enable_blob_files: %s", enable_blob_files ? "true" : "false"); + ROCKS_LOG_HEADER(log, " Options.enable_blob_indirection: %s", + enable_blob_indirection ? "true" : "false"); + ROCKS_LOG_HEADER(log, " Options.pin_blob_relocation_location_blocks: %s", + pin_blob_relocation_location_blocks ? "true" : "false"); + ROCKS_LOG_HEADER(log, + "Options.blob_indirection_max_references_per_sst: %" PRIu64, + blob_indirection_max_references_per_sst); + ROCKS_LOG_HEADER(log, + "Options.indirect_blob_gc_read_coalesce_bytes: %" PRIu64, + indirect_blob_gc_read_coalesce_bytes); ROCKS_LOG_HEADER(log, " Options.min_blob_size: %" PRIu64, min_blob_size); diff --git a/options/options_helper.cc b/options/options_helper.cc index 313e506ff661..1d66a6959c1f 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -328,6 +328,10 @@ ColumnFamilyOptions BuildColumnFamilyOptions( void UpdateColumnFamilyOptions(const MutableCFOptions& moptions, ColumnFamilyOptions* cf_opts) { + assert(cf_opts != nullptr); + if (cf_opts == nullptr) { + return; + } // Memtable related options cf_opts->write_buffer_size = moptions.write_buffer_size; cf_opts->max_write_buffer_number = moptions.max_write_buffer_number; @@ -393,6 +397,13 @@ void UpdateColumnFamilyOptions(const MutableCFOptions& moptions, // Blob file related options cf_opts->enable_blob_files = moptions.enable_blob_files; + cf_opts->enable_blob_indirection = moptions.enable_blob_indirection; + cf_opts->pin_blob_relocation_location_blocks = + moptions.pin_blob_relocation_location_blocks; + cf_opts->blob_indirection_max_references_per_sst = + moptions.blob_indirection_max_references_per_sst; + cf_opts->indirect_blob_gc_read_coalesce_bytes = + moptions.indirect_blob_gc_read_coalesce_bytes; cf_opts->min_blob_size = moptions.min_blob_size; cf_opts->blob_file_size = moptions.blob_file_size; cf_opts->blob_file_writable_file_max_buffer_size = diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index de358f7548cf..79460fedfdab 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -762,6 +762,10 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { "read_triggered_compaction_threshold=0.5;" "sample_for_compression=0;" "enable_blob_files=true;" + "enable_blob_indirection=true;" + "pin_blob_relocation_location_blocks=true;" + "blob_indirection_max_references_per_sst=4096;" + "indirect_blob_gc_read_coalesce_bytes=8192;" "enable_blob_direct_write=true;" "min_blob_size=256;" "blob_file_size=1000000;" diff --git a/options/options_test.cc b/options/options_test.cc index 05b6e6379504..f83aa23b7dba 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -131,6 +131,10 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"experimental_mempurge_threshold", "0.003"}, {"optimize_filters_for_hits", "true"}, {"enable_blob_files", "true"}, + {"enable_blob_indirection", "true"}, + {"pin_blob_relocation_location_blocks", "true"}, + {"blob_indirection_max_references_per_sst", "8192"}, + {"indirect_blob_gc_read_coalesce_bytes", "16384"}, {"min_blob_size", "1K"}, {"blob_file_size", "1G"}, {"blob_file_writable_file_max_buffer_size", "128K"}, @@ -293,6 +297,10 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31"); ASSERT_EQ(new_cf_opt.experimental_mempurge_threshold, 0.003); ASSERT_EQ(new_cf_opt.enable_blob_files, true); + ASSERT_EQ(new_cf_opt.enable_blob_indirection, true); + ASSERT_EQ(new_cf_opt.pin_blob_relocation_location_blocks, true); + ASSERT_EQ(new_cf_opt.blob_indirection_max_references_per_sst, 8192U); + ASSERT_EQ(new_cf_opt.indirect_blob_gc_read_coalesce_bytes, 16384U); ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10); ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30); ASSERT_EQ(new_cf_opt.blob_file_writable_file_max_buffer_size, 128ULL << 10); @@ -2735,6 +2743,10 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { {"experimental_mempurge_threshold", "0.003"}, {"optimize_filters_for_hits", "true"}, {"enable_blob_files", "true"}, + {"enable_blob_indirection", "true"}, + {"pin_blob_relocation_location_blocks", "true"}, + {"blob_indirection_max_references_per_sst", "8192"}, + {"indirect_blob_gc_read_coalesce_bytes", "16384"}, {"min_blob_size", "1K"}, {"blob_file_size", "1G"}, {"blob_file_writable_file_max_buffer_size", "128K"}, @@ -2894,6 +2906,10 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { ASSERT_EQ(new_cf_opt.experimental_mempurge_threshold, 0.003); ASSERT_EQ(new_cf_opt.verify_output_flags, VerifyOutputFlags::kVerifyNone); ASSERT_EQ(new_cf_opt.enable_blob_files, true); + ASSERT_EQ(new_cf_opt.enable_blob_indirection, true); + ASSERT_EQ(new_cf_opt.pin_blob_relocation_location_blocks, true); + ASSERT_EQ(new_cf_opt.blob_indirection_max_references_per_sst, 8192U); + ASSERT_EQ(new_cf_opt.indirect_blob_gc_read_coalesce_bytes, 16384U); ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10); ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30); ASSERT_EQ(new_cf_opt.blob_file_writable_file_max_buffer_size, 128ULL << 10); diff --git a/src.mk b/src.mk index 8455f0213273..6d1d1fd10133 100644 --- a/src.mk +++ b/src.mk @@ -29,6 +29,11 @@ LIB_SOURCES = \ db/blob/blob_log_format.cc \ db/blob/blob_log_sequential_reader.cc \ db/blob/blob_log_writer.cc \ + db/blob/blob_relocation_file.cc \ + db/blob/blob_relocation_gc.cc \ + db/blob/blob_relocation_manifest.cc \ + db/blob/blob_reference_census.cc \ + db/blob/blob_reference_metadata.cc \ db/blob/blob_source.cc \ db/blob/blob_write_batch_transformer.cc \ db/blob/prefetch_buffer_collection.cc \ diff --git a/table/adaptive/adaptive_table_factory.cc b/table/adaptive/adaptive_table_factory.cc index db3f7625a710..095172e7e13c 100644 --- a/table/adaptive/adaptive_table_factory.cc +++ b/table/adaptive/adaptive_table_factory.cc @@ -6,11 +6,41 @@ #include "table/adaptive/adaptive_table_factory.h" #include "port/port.h" +#include "table/block_based/block_based_table_factory.h" #include "table/format.h" #include "table/table_builder.h" namespace ROCKSDB_NAMESPACE { +const BlockBasedTableOptions* GetBlockBasedTableWriterOptions( + const TableFactory* table_factory) { + if (table_factory == nullptr) { + return nullptr; + } + const AdaptiveTableFactory* adaptive = + table_factory->CheckedCast(); + return adaptive != nullptr + ? adaptive->GetBlockBasedTableWriterOptions() + : table_factory->GetOptions(); +} + +std::shared_ptr +GetBlobReferenceCacheReservationManager(const TableFactory* table_factory) { + if (table_factory == nullptr) { + return nullptr; + } + const AdaptiveTableFactory* adaptive = + table_factory->CheckedCast(); + if (adaptive != nullptr) { + return adaptive->GetBlobReferenceCacheReservationManager(); + } + const BlockBasedTableFactory* block_based = + table_factory->CheckedCast(); + return block_based != nullptr + ? block_based->GetBlobReferenceCacheReservationManager() + : nullptr; +} + AdaptiveTableFactory::AdaptiveTableFactory( std::shared_ptr table_factory_to_write, std::shared_ptr block_based_table_factory, @@ -105,6 +135,20 @@ std::string AdaptiveTableFactory::GetPrintableOptions() const { return ret; } +const BlockBasedTableOptions* +AdaptiveTableFactory::GetBlockBasedTableWriterOptions() const { + return table_factory_to_write_->GetOptions(); +} + +std::shared_ptr +AdaptiveTableFactory::GetBlobReferenceCacheReservationManager() const { + const BlockBasedTableFactory* block_based = + table_factory_to_write_->CheckedCast(); + return block_based != nullptr + ? block_based->GetBlobReferenceCacheReservationManager() + : nullptr; +} + TableFactory* NewAdaptiveTableFactory( std::shared_ptr table_factory_to_write, std::shared_ptr block_based_table_factory, diff --git a/table/adaptive/adaptive_table_factory.h b/table/adaptive/adaptive_table_factory.h index 0519f967f372..4992f4c317b6 100644 --- a/table/adaptive/adaptive_table_factory.h +++ b/table/adaptive/adaptive_table_factory.h @@ -5,6 +5,7 @@ #pragma once +#include #include #include "rocksdb/options.h" @@ -17,18 +18,40 @@ struct EnvOptions; class Status; class RandomAccessFile; class WritableFile; +class CacheReservationManager; class Table; class TableBuilder; +// Returns the BlockBasedTableOptions that `table_factory` uses to write new +// tables: the factory's own options for a BlockBasedTableFactory, or the +// options of its write factory for an AdaptiveTableFactory. Returns nullptr +// when `table_factory` is null or writes another table format. The column +// family passes the result to BlobFileCache as the relocation table options; +// with nullptr, relocation file reads return NotSupported. +const BlockBasedTableOptions* GetBlockBasedTableWriterOptions( + const TableFactory* table_factory); +// Returns the cache reservation manager that charges blob reference metadata to +// the block cache (CacheEntryRole::kBlobReferenceMetadataBuildingBuffer). It +// belongs to the block-based writer: the factory itself, or the write factory +// of an AdaptiveTableFactory. The table builders of that writer use it. +// TableCache also uses it to read the reference summary of an SST when the +// caller passes no reservation manager. Returns nullptr when there is no +// block-based writer, no block cache, or charging is disabled. +std::shared_ptr +GetBlobReferenceCacheReservationManager(const TableFactory* table_factory); + class AdaptiveTableFactory : public TableFactory { public: + // Method to allow CheckedCast to work for this class. + static const char* kClassName() { return "AdaptiveTableFactory"; } + explicit AdaptiveTableFactory( std::shared_ptr table_factory_to_write, std::shared_ptr block_based_table_factory, std::shared_ptr plain_table_factory, std::shared_ptr cuckoo_table_factory); - const char* Name() const override { return "AdaptiveTableFactory"; } + const char* Name() const override { return kClassName(); } using TableFactory::NewTableReader; Status NewTableReader( @@ -43,6 +66,14 @@ class AdaptiveTableFactory : public TableFactory { std::string GetPrintableOptions() const override; + // Returns the block-based options used for newly written tables, or nullptr + // when the configured write factory uses another table format. + const BlockBasedTableOptions* GetBlockBasedTableWriterOptions() const; + // The blob reference metadata cache reservation manager of the write + // factory, or nullptr (see the free function of the same name). + std::shared_ptr + GetBlobReferenceCacheReservationManager() const; + std::unique_ptr Clone() const override { return std::make_unique(*this); } diff --git a/table/block_based/block_based_table_builder.cc b/table/block_based/block_based_table_builder.cc index 531217ad5700..c6544658a9ef 100644 --- a/table/block_based/block_based_table_builder.cc +++ b/table/block_based/block_based_table_builder.cc @@ -33,6 +33,8 @@ #include "db/blob/blob_gen2_format.h" #include "db/blob/blob_index.h" #include "db/blob/blob_log_format.h" +#include "db/blob/blob_reference_metadata.h" +#include "db/blob/blob_relocation_file.h" #include "db/dbformat.h" #include "db/wide/wide_column_serialization.h" #include "logging/logging.h" @@ -1308,6 +1310,28 @@ struct BlockBasedTableBuilder::Rep { uint64_t buffer_limit = 0; std::shared_ptr compression_dict_buffer_cache_res_mgr; + std::unique_ptr blob_reference_collector; + // The reference cap of this output (blob_indirection_max_references_per_sst). + // Set only when blob_reference_collector is set. + uint64_t max_blob_references = 0; + bool track_blob_reference_classification = false; + uint64_t oldest_coarse_indirect_origin_file_number = kInvalidBlobFileNumber; + std::shared_ptr blob_reference_summary; + std::unique_ptr + blob_relocation_directory_builder; + std::optional blob_relocation_file_metadata; + // The open group of a relocation file: values whose location entry is not + // written yet (see PrepareBlobRelocationGroup()). + struct BlobRelocationGroupState { + uint64_t origin_file_number = 0; + uint64_t first_record_offset = 0; + // Where the next record of the group must start (debug check only). + uint64_t next_record_offset = 0; + size_t count = 0; + std::array values; + // Encoding buffer of the location entry value. + std::string encoded; + } blob_relocation_group; const bool use_delta_encoding_for_index_values; std::unique_ptr filter_builder; OffsetableCacheKey base_cache_key; @@ -1548,7 +1572,8 @@ struct BlockBasedTableBuilder::Rep { } Rep(const BlockBasedTableOptions& table_opt, const TableBuilderOptions& tbo, - WritableFileWriter* f) + WritableFileWriter* f, + std::shared_ptr blob_reference_cache_res_mgr) : ioptions(tbo.ioptions), prefix_extractor(tbo.moptions.prefix_extractor), write_options(tbo.write_options), @@ -1836,6 +1861,26 @@ struct BlockBasedTableBuilder::Rep { compression_dict_buffer_cache_res_mgr = nullptr; } + // The caller decides whether this SST gets exact blob reference metadata + // (TableBuilderOptions::collect_blob_reference_metadata) or only the coarse + // classification (track_blob_reference_classification). + track_blob_reference_classification = + tbo.track_blob_reference_classification; + if (tbo.collect_blob_reference_metadata) { + max_blob_references = + tbo.moptions.blob_indirection_max_references_per_sst; + blob_reference_collector = + std::make_unique( + std::move(blob_reference_cache_res_mgr), + BlobReferenceSstIdentity{ + .column_family_id = tbo.column_family_id, + .sst_file_number = tbo.cur_file_num, + .route_snapshot_fingerprint_high = + tbo.blob_route_snapshot_fingerprint_high, + .route_snapshot_fingerprint_low = + tbo.blob_route_snapshot_fingerprint_low}); + } + // Common user-key prefix feature (format_version >= 8) for index-like // blocks: gated uniformly for all index tiers (leaf/partition and the // partitioned index / partitioned filter top-level). See @@ -2182,12 +2227,14 @@ struct BlockBasedTableBuilder::Rep { BlockBasedTableBuilder::BlockBasedTableBuilder( const BlockBasedTableOptions& table_options, const TableBuilderOptions& tbo, - WritableFileWriter* file) { + WritableFileWriter* file, + std::shared_ptr blob_reference_cache_res_mgr) { BlockBasedTableOptions sanitized_table_options(table_options); auto ucmp = tbo.internal_comparator.user_comparator(); assert(ucmp); (void)ucmp; // avoids unused variable error. - rep_ = std::make_unique(sanitized_table_options, tbo, file); + rep_ = std::make_unique(sanitized_table_options, tbo, file, + std::move(blob_reference_cache_res_mgr)); TEST_SYNC_POINT_CALLBACK( "BlockBasedTableBuilder::BlockBasedTableBuilder:PreSetupBaseCacheKey", @@ -2209,6 +2256,11 @@ BlockBasedTableBuilder::~BlockBasedTableBuilder() { } void BlockBasedTableBuilder::Add(const Slice& ikey, const Slice& value) { + AddImpl(ikey, value, /*decoded_blob_index=*/nullptr); +} + +void BlockBasedTableBuilder::AddImpl(const Slice& ikey, const Slice& value, + const BlobIndex* decoded_blob_index) { Rep* r = rep_.get(); assert(rep_->state != Rep::State::kClosed); if (UNLIKELY(!ok())) { @@ -2240,7 +2292,10 @@ void BlockBasedTableBuilder::Add(const Slice& ikey, const Slice& value) { Slice entry_value = value; if (IsValueType(value_type)) { - if (r->embedded_blob_options) { + // A relocation file writes its records itself; its kTypeValue entries are + // location entries, which stay in the data blocks. + if (r->embedded_blob_options && + r->blob_relocation_directory_builder == nullptr) { if (!MaybeExtractEmbeddedBlobs(seq, value_type, &entry_ikey, &entry_value)) { return; // failed status already recorded @@ -2269,16 +2324,31 @@ void BlockBasedTableBuilder::Add(const Slice& ikey, const Slice& value) { blob_index.size() + blob_record_overhead; } }; + // A kTypeBlobIndex value is decoded once here, for the referenced bytes + // property and for the reference collector below. The caller's decode is + // used when it decoded this same value (MaybeExtractEmbeddedBlobs() does + // not change a kTypeBlobIndex value). + BlobIndex own_blob_index; + const BlobIndex* value_blob_index = nullptr; + Status blob_index_status; if (value_type == kTypeBlobIndex) { - BlobIndex blob_index; - Status s = blob_index.DecodeFrom(entry_value); - if (!s.ok()) { + if (decoded_blob_index != nullptr && entry_value.data() == value.data()) { + value_blob_index = decoded_blob_index; + } else { + blob_index_status = own_blob_index.DecodeFrom(entry_value); + if (blob_index_status.ok()) { + value_blob_index = &own_blob_index; + } + } + } + if (value_type == kTypeBlobIndex) { + if (value_blob_index == nullptr) { r->props.referenced_blob_bytes.reset(); } else { - account_blob_index(blob_index); + account_blob_index(*value_blob_index); } } else if (value_type == kTypeWideColumnEntity) { - Status s = WideColumnSerialization::ForEachBlobFileNumber( + Status s = WideColumnSerialization::ForEachBlobIndex( entry_value, [&](const BlobIndex& blob_index) { account_blob_index(blob_index); return Status::OK(); @@ -2308,6 +2378,54 @@ void BlockBasedTableBuilder::Add(const Slice& ikey, const Slice& value) { } } + auto* const blob_reference_collector = r->blob_reference_collector.get(); + if (blob_reference_collector != nullptr || + r->track_blob_reference_classification) { + const uint64_t user_key_size = ExtractUserKey(entry_ikey).size(); + // MaybeExtractEmbeddedBlobs() can turn a kTypeValue entry into a + // kTypeBlobIndex entry, but only with a same-file blob index. The + // collector skips same-file indexes, so it is enough to check the input + // `value_type`. + assert(ExtractValueType(entry_ikey) == value_type || + (value_type == kTypeValue && + ExtractValueType(entry_ikey) == kTypeBlobIndex)); + const auto add_reference = [&](const BlobIndex& blob_index) -> Status { + if (blob_reference_collector != nullptr) { + return blob_reference_collector->Add(blob_index, user_key_size); + } + if (blob_index.IsIndirect()) { + if (blob_index.generation_file_number() != + blob_index.origin_file_number()) { + return Status::Corruption( + "Coarse SST cannot contain a flattened blob reference " + "(generation number != origin)"); + } + const uint64_t origin = blob_index.origin_file_number(); + if (r->oldest_coarse_indirect_origin_file_number == + kInvalidBlobFileNumber || + origin < r->oldest_coarse_indirect_origin_file_number) { + r->oldest_coarse_indirect_origin_file_number = origin; + } + } + return Status::OK(); + }; + Status reference_status; + if (value_type == kTypeBlobIndex) { + reference_status = value_blob_index == nullptr + ? blob_index_status + : add_reference(*value_blob_index); + } else if (value_type == kTypeWideColumnEntity) { + reference_status = WideColumnSerialization::ForEachBlobIndex( + entry_value, [&](const BlobIndex& blob_index) { + return add_reference(blob_index); + }); + } + if (!reference_status.ok()) { + r->SetStatus(reference_status); + return; + } + } + // NOTE: WriteBatch guarantees keys < 4GB; value size checked above r->data_block.AddWithLastKey(entry_ikey, entry_value, r->last_ikey); r->last_ikey.assign(entry_ikey.data(), entry_ikey.size()); @@ -2359,10 +2477,22 @@ TableBuilderAddResult BlockBasedTableBuilder::TryAdd( const Slice& ikey, const Slice& value, TableBuilderAddContext* context) { assert(context != nullptr); context->new_table_reason = TableBuilderNewTableReason::kNone; - Add(ikey, value); + AddImpl(ikey, value, context->decoded_blob_index); return TableBuilderAddResult::kAdded; } +bool BlockBasedTableBuilder::BlobReferenceLimitReached() const { + const BlobReferenceMetadataCollector* const collector = + rep_->blob_reference_collector.get(); + if (collector == nullptr) { + return false; + } + uint64_t limit = rep_->max_blob_references; + TEST_SYNC_POINT_CALLBACK("BlockBasedTableBuilder::BlobReferenceLimit", + &limit); + return collector->reference_count() >= limit; +} + void BlockBasedTableBuilder::Flush(const Slice* first_key_in_next_block) { Rep* r = rep_.get(); assert(rep_->state != Rep::State::kClosed); @@ -2961,12 +3091,23 @@ void BlockBasedTableBuilder::WriteMaybeCompressedBlock( BlockType block_type, const Slice* uncompressed_block_data, bool* skip_delta_encoding, const uint32_t* precomputed_block_contents_crc32c) { + WriteMaybeCompressedBlock(block_contents, comp_type, handle, block_type, + uncompressed_block_data, skip_delta_encoding, + precomputed_block_contents_crc32c, + /*warm_cache=*/true); +} + +void BlockBasedTableBuilder::WriteMaybeCompressedBlock( + const Slice& block_contents, CompressionType comp_type, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data, + bool* skip_delta_encoding, + const uint32_t* precomputed_block_contents_crc32c, bool warm_cache) { // Must have pre-checked status in single-threaded context assert(status().ok()); assert(io_status().ok()); rep_->SetIOStatus(WriteMaybeCompressedBlockImpl( block_contents, comp_type, handle, block_type, uncompressed_block_data, - skip_delta_encoding, precomputed_block_contents_crc32c)); + skip_delta_encoding, precomputed_block_contents_crc32c, warm_cache)); } IOStatus BlockBasedTableBuilder::WriteMaybeCompressedBlockImpl( @@ -2974,6 +3115,17 @@ IOStatus BlockBasedTableBuilder::WriteMaybeCompressedBlockImpl( BlockType block_type, const Slice* uncompressed_block_data, bool* skip_delta_encoding, const uint32_t* precomputed_block_contents_crc32c) { + return WriteMaybeCompressedBlockImpl( + block_contents, comp_type, handle, block_type, uncompressed_block_data, + skip_delta_encoding, precomputed_block_contents_crc32c, + /*warm_cache=*/true); +} + +IOStatus BlockBasedTableBuilder::WriteMaybeCompressedBlockImpl( + const Slice& block_contents, CompressionType comp_type, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data, + bool* skip_delta_encoding, + const uint32_t* precomputed_block_contents_crc32c, bool warm_cache) { // File format contains a sequence of blocks where each block has: // block_data: uint8[n] // compression_type: uint8 @@ -3117,7 +3269,7 @@ IOStatus BlockBasedTableBuilder::WriteMaybeCompressedBlockImpl( } } - if (r->warm_cache_config.enabled) { + if (r->warm_cache_config.enabled && warm_cache) { io_s = status_to_io_status( InsertBlockInCacheHelper(*uncompressed_block_data, handle, block_type)); if (UNLIKELY(!io_s.ok())) { @@ -3258,7 +3410,9 @@ IOStatus BlockBasedTableBuilder::io_status() const { bool BlockBasedTableBuilder::ok() const { return rep_->StatusOk(); } -Status BlockBasedTableBuilder::WriteEmbeddedBlobRecord(const Slice& payload) { +Status BlockBasedTableBuilder::WriteEmbeddedBlobRecord( + const Slice& payload, CompressionType compression, + uint64_t* record_offset) { Rep* r = rep_.get(); assert(r->embedded_blob_options); // Inline blob writing requires single-threaded writes so the running table @@ -3267,8 +3421,8 @@ Status BlockBasedTableBuilder::WriteEmbeddedBlobRecord(const Slice& payload) { // Embedded blob payloads share the 32-bit value-size limit with the rest of // the table format; enforcing it here keeps record sizes in range and gives - // an early, cheap corruption check. - if (payload.size() > std::numeric_limits::max()) { + // an early, cheap size check. + if (payload.size() > kMaxSimpleGen2BlobPayloadSize) { return Status::InvalidArgument("Embedded blob value is too large"); } @@ -3279,8 +3433,6 @@ Status BlockBasedTableBuilder::WriteEmbeddedBlobRecord(const Slice& payload) { // The builder's running table offset equals the current file size, so the // record begins exactly here. const uint64_t offset = r->get_offset(); - // Placeholder for future embedded blob compression support. - const CompressionType compression = kNoCompression; IOStatus io_s = WriteSimpleGen2BlobRecord( r->file, r->write_options, r->table_options.checksum, r->base_context_checksum, offset, payload, compression); @@ -3297,6 +3449,246 @@ Status BlockBasedTableBuilder::WriteEmbeddedBlobRecord(const Slice& payload) { BlobIndex::EncodeBlob(&r->embedded_blob_state->blob_index_buf, kCurrentFileBlobIndexFileNumber, offset, payload.size(), compression); + if (record_offset != nullptr) { + *record_offset = offset; + } + return Status::OK(); +} + +Status BlockBasedTableBuilder::StartBlobRelocationFile() { + Rep* r = rep_.get(); + if (!ok()) { + return status(); + } + if (r->blob_relocation_directory_builder != nullptr || + r->props.num_entries != 0 || r->get_offset() != 0 || + r->embedded_blob_options == nullptr) { + return Status::InvalidArgument( + "Blob relocation file must start on a fresh embedded table builder"); + } + if (r->props.orig_file_number == 0 || r->props.db_session_id.empty()) { + return Status::InvalidArgument( + "Blob relocation file requires persisted file identity"); + } + if (r->internal_comparator.user_comparator() != BytewiseComparator()) { + return Status::InvalidArgument( + "Blob relocation file requires the bytewise comparator without " + "timestamps"); + } + assert(!r->IsParallelCompressionActive()); + + IOOptions io_options; + IOStatus io_status = + WritableFileWriter::PrepareIOOptions(r->write_options, io_options); + if (io_status.ok()) { + io_status = r->file->Append( + Slice(kBlobRelocationFilePrefix, kBlobRelocationFilePrefixSize), + io_options); + } + if (!io_status.ok()) { + r->SetIOStatus(io_status); + return io_status; + } + r->set_offset(kBlobRelocationFilePrefixSize); + // Compression sampling reports stats for user data blocks; relocation data + // blocks hold only location entries. + r->sample_for_compression = 0; + r->blob_relocation_directory_builder = + std::make_unique(); + return Status::OK(); +} + +Status BlockBasedTableBuilder::AddBlobRelocationServedFile( + uint64_t file_number) { + Rep* r = rep_.get(); + if (r->blob_relocation_directory_builder == nullptr) { + return Status::InvalidArgument("Blob relocation file has not been started"); + } + if (!ok()) { + return status(); + } + // A relocation file serves only file numbers smaller than its own. Every file + // number it serves (an input file of indirect blob GC, or an old file number + // routed to one) was allocated before this file was created, and file numbers + // only grow. Serving its own number would make a route to itself. + if (file_number >= r->props.orig_file_number) { + Status invalid = Status::InvalidArgument( + "Blob relocation file can serve only older files"); + r->SetStatus(invalid); + return invalid; + } + Status add_status = + r->blob_relocation_directory_builder->AddServedFile(file_number); + if (!add_status.ok()) { + r->SetStatus(add_status); + } + return add_status; +} + +Status BlockBasedTableBuilder::AddBlobRelocation(uint64_t origin_file_number, + uint64_t original_value_offset, + const Slice& stored_payload, + CompressionType compression) { + Rep* r = rep_.get(); + if (r->blob_relocation_directory_builder == nullptr) { + return Status::InvalidArgument("Blob relocation file has not been started"); + } + if (!ok()) { + return status(); + } + if (stored_payload.size() > kMaxSimpleGen2BlobPayloadSize) { + return Status::InvalidArgument("Blob relocation value is too large"); + } + if (!CompressionTypeSupported(compression)) { + return Status::InvalidArgument( + "Blob relocation compression type is not supported"); + } + + if (origin_file_number >= r->props.orig_file_number) { + Status invalid = Status::InvalidArgument( + "Blob relocation origin must be older than the file"); + r->SetStatus(invalid); + return invalid; + } + if (stored_payload.empty()) { + return Status::InvalidArgument("Blob relocation value is empty"); + } + Status add_status = r->blob_relocation_directory_builder->Add( + origin_file_number, original_value_offset, stored_payload.size(), + compression); + if (!add_status.ok()) { + r->SetStatus(add_status); + return add_status; + } + PrepareBlobRelocationGroup(origin_file_number); + if (!ok()) { + return status(); + } + uint64_t record_offset = 0; + add_status = + WriteEmbeddedBlobRecord(stored_payload, compression, &record_offset); + if (!add_status.ok()) { + return add_status; + } + AddToBlobRelocationGroup(origin_file_number, original_value_offset, + stored_payload.size(), record_offset); + return status(); +} + +void BlockBasedTableBuilder::PrepareBlobRelocationGroup( + uint64_t origin_file_number) { + const Rep::BlobRelocationGroupState& group = rep_->blob_relocation_group; + // An open group is never full: AddToBlobRelocationGroup() writes a full + // group at once. + if (group.count != 0 && group.origin_file_number != origin_file_number) { + FinishBlobRelocationGroup(); + } +} + +void BlockBasedTableBuilder::AddToBlobRelocationGroup( + uint64_t origin_file_number, uint64_t original_value_offset, + uint64_t stored_value_size, uint64_t record_offset) { + Rep::BlobRelocationGroupState& group = rep_->blob_relocation_group; + if (group.count == 0) { + group.origin_file_number = origin_file_number; + group.first_record_offset = record_offset; + } + // The records of one group are back to back (see + // PrepareBlobRelocationGroup()). + assert(group.count == 0 || record_offset == group.next_record_offset); + group.next_record_offset = + record_offset + stored_value_size + kSimpleGen2BlobTrailerSize; + group.values[group.count].original_value_offset = original_value_offset; + group.values[group.count].stored_value_size = stored_value_size; + ++group.count; + if (group.count == kBlobRelocationGroupSize) { + FinishBlobRelocationGroup(); + } +} + +void BlockBasedTableBuilder::FinishBlobRelocationGroup() { + Rep::BlobRelocationGroupState& group = rep_->blob_relocation_group; + if (group.count == 0) { + return; + } + EncodeBlobRelocationGroup(group.first_record_offset, group.values.data(), + group.count, &group.encoded); + // Build the 24-byte internal key on the stack: the 16-byte relocation key + // of the group's last value, then 8 bytes that pack sequence 0 and + // kTypeValue. The type is kTypeValue because the value is a location entry, + // not a BlobIndex. Add() keeps it in the data block because this builder is + // in relocation mode (see the check of blob_relocation_directory_builder in + // Add()). + char internal_key[kBlobRelocationKeySize + kNumInternalBytes]; + EncodeBlobRelocationKeyTo(group.origin_file_number, + group.values[group.count - 1].original_value_offset, + internal_key); + EncodeFixed64(internal_key + kBlobRelocationKeySize, + PackSequenceAndType(0, kTypeValue)); + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTableBuilder::FinishBlobRelocationGroup:InternalKey", + internal_key); + group.count = 0; + Add(Slice(internal_key, sizeof(internal_key)), group.encoded); +} + +Status BlockBasedTableBuilder::AddBlobRelocation( + uint64_t origin_file_number, uint64_t original_value_offset, + uint64_t stored_value_size, CompressionType compression, + uint32_t stored_checksum, const VerifiedBlobStoredSource& source) { + Rep* r = rep_.get(); + if (!ok()) { + return status(); + } + if (r->blob_relocation_directory_builder == nullptr || + r->embedded_blob_options == nullptr || stored_value_size == 0 || + stored_value_size > kMaxSimpleGen2BlobPayloadSize || + !CompressionTypeSupported(compression)) { + return Status::InvalidArgument("Invalid streaming blob relocation"); + } + assert(!r->IsParallelCompressionActive()); + Status status = r->blob_relocation_directory_builder->Add( + origin_file_number, original_value_offset, stored_value_size, + compression); + if (status.ok()) { + PrepareBlobRelocationGroup(origin_file_number); + status = this->status(); + } + const uint64_t offset = r->get_offset(); + if (status.ok()) { + status = WriteSimpleGen2BlobRecordStream( + r->file, r->write_options, r->table_options.checksum, + r->base_context_checksum, offset, stored_value_size, compression, + stored_checksum, source); + } + if (!status.ok()) { + r->SetStatus(status); + return status; + } + if (!r->embedded_blob_state) { + r->embedded_blob_state = std::make_unique(); + } + r->set_offset(offset + stored_value_size + kSimpleGen2BlobTrailerSize); + ++r->embedded_blob_state->stats.blob_count; + r->embedded_blob_state->stats.payload_bytes += stored_value_size; + AddToBlobRelocationGroup(origin_file_number, original_value_offset, + stored_value_size, offset); + return this->status(); +} + +Status BlockBasedTableBuilder::GetBlobRelocationFileMetadata( + BlobRelocationFileMetadata* metadata) const { + if (metadata == nullptr) { + return Status::InvalidArgument("Null blob relocation metadata output"); + } + if (!ok()) { + return status(); + } + if (!rep_->blob_relocation_file_metadata.has_value()) { + return Status::InvalidArgument("Blob relocation file is not finished"); + } + *metadata = *rep_->blob_relocation_file_metadata; + metadata->tail_size = GetTailSize(); return Status::OK(); } @@ -3322,7 +3714,8 @@ Status BlockBasedTableBuilder::BuildEmbeddedWideColumnEntity(const Slice& value, if (column_value.size() < min_blob_size) { continue; } - s = WriteEmbeddedBlobRecord(column_value); + s = WriteEmbeddedBlobRecord(column_value, kNoCompression, + /*record_offset=*/nullptr); if (!s.ok()) { return s; } @@ -3362,7 +3755,8 @@ bool BlockBasedTableBuilder::MaybeExtractEmbeddedBlobs(SequenceNumber seq, if (value->size() < r->embedded_blob_options->min_blob_size) { return true; // leave small values inline } - Status s = WriteEmbeddedBlobRecord(*value); + Status s = WriteEmbeddedBlobRecord(*value, kNoCompression, + /*record_offset=*/nullptr); if (!s.ok()) { r->SetStatus(std::move(s)); return false; @@ -3400,6 +3794,8 @@ Status BlockBasedTableBuilder::InsertBlockInCacheHelper( BlockType block_type) { Cache* block_cache = rep_->table_options.block_cache.get(); Status s; + TEST_SYNC_POINT_CALLBACK("BlockBasedTableBuilder::InsertBlockInCacheHelper", + &block_type); auto helper = GetCacheItemHelper(block_type, rep_->ioptions.lowest_used_cache_tier); if (block_cache && helper && helper->create_cb) { @@ -3647,9 +4043,13 @@ void BlockBasedTableBuilder::WritePropertiesBlock( // rep_->props.index_size is set in WriteIndexBlock(), which has direct // access to the index block handle (needed for accurate kCustomOnly stub // reporting). - rep_->props.comparator_name = rep_->ioptions.user_comparator != nullptr - ? rep_->ioptions.user_comparator->Name() - : "nullptr"; + const bool is_blob_relocation_file = + rep_->blob_relocation_directory_builder != nullptr; + rep_->props.comparator_name = + is_blob_relocation_file ? BytewiseComparator()->Name() + : (rep_->ioptions.user_comparator != nullptr + ? rep_->ioptions.user_comparator->Name() + : "nullptr"); rep_->props.merge_operator_name = rep_->ioptions.merge_operator != nullptr ? rep_->ioptions.merge_operator->Name() @@ -3724,7 +4124,7 @@ void BlockBasedTableBuilder::WritePropertiesBlock( rep_->uncompressible_input_data_bytes.LoadRelaxed(); } rep_->props.user_defined_timestamps_persisted = - rep_->persist_user_defined_timestamps; + is_blob_relocation_file ? false : rep_->persist_user_defined_timestamps; if (rep_->embedded_blob_state) { std::string encoded_stats; @@ -3776,6 +4176,96 @@ void BlockBasedTableBuilder::WritePropertiesBlock( } } +void BlockBasedTableBuilder::WriteBlobReferenceMetadata( + MetaIndexBuilder* meta_index_builder) { + // The collector's memory and charge are released when it is destroyed. + std::unique_ptr collector = + std::move(rep_->blob_reference_collector); + if (collector == nullptr) { + if (rep_->track_blob_reference_classification && ok()) { + std::string encoded; + EncodeBlobReferenceClassification( + rep_->oldest_coarse_indirect_origin_file_number, &encoded); + const bool inserted = + rep_->props.user_collected_properties + .emplace(kBlobReferenceClassificationPropertyName, + std::move(encoded)) + .second; + if (!inserted) { + rep_->SetStatus( + Status::Corruption("Duplicate blob reference classification")); + } + } + return; + } + if (!ok()) { + return; + } + + // The reference data blocks use the compressor of the SST's own index + // blocks, which never has a dictionary, so readers need only the table's + // decompressor. As for other blocks, a reference data block stays raw when + // compression does not pay. The reference index block is never compressed + // (see db/blob/blob_reference_metadata.h). + BlobReferenceMetadataDescriptor descriptor; + Status status = collector->Finish( + [&](const Slice& block, bool is_index_block, + BlockHandle* handle) -> Status { + CompressionType type = kNoCompression; + uint32_t compressed_crc32c = 0; + if (!is_index_block) { + Status compress_status = CompressAndVerifyBlock( + block, /*is_data_block=*/false, rep_->index_block_working_area, + &rep_->single_threaded_compressed_output, &type, + &compressed_crc32c); + if (!compress_status.ok()) { + return compress_status; + } + } + // The reference blocks use BlockType::kUserDefinedIndex, so RocksDB + // keeps their contents as plain bytes and never parses them as a + // Block. The reference index block also needs a type in + // BlockBasedTable::GetBlockTypeForMetaBlockByName(); see the comment + // there. + WriteMaybeCompressedBlock( + type == kNoCompression + ? block + : Slice(rep_->single_threaded_compressed_output), + type, handle, BlockType::kUserDefinedIndex, + is_index_block ? nullptr : &block, + /*skip_delta_encoding=*/nullptr, + type == kNoCompression || rep_->table_options.checksum != kCRC32c + ? nullptr + : &compressed_crc32c, + // No reader looks up reference blocks in the block cache, so + // warming them would only take cache space. + /*warm_cache=*/false); + rep_->single_threaded_compressed_output.Reset(); + return rep_->GetStatus(); + }, + &descriptor, &rep_->blob_reference_summary); + if (!status.ok()) { + rep_->SetStatus(status); + return; + } + meta_index_builder->Add(kBlobReferenceIndexBlockName, + descriptor.index_handle); + std::string encoded_descriptor; + descriptor.EncodeTo(&encoded_descriptor); + const auto [unused, inserted] = rep_->props.user_collected_properties.emplace( + kBlobReferenceDescriptorPropertyName, std::move(encoded_descriptor)); + (void)unused; + if (!inserted) { + rep_->SetStatus(Status::Corruption( + "Duplicate blob reference descriptor table property")); + } +} + +std::shared_ptr +BlockBasedTableBuilder::GetBlobReferenceSstSummary() const { + return ok() ? rep_->blob_reference_summary : nullptr; +} + void BlockBasedTableBuilder::WriteCompressionDictBlock( MetaIndexBuilder* meta_index_builder) { Slice compression_dict; @@ -4064,6 +4554,10 @@ Status BlockBasedTableBuilder::Finish() { } } #endif // !NDEBUG + // The last location entry of a relocation file. + if (r->blob_relocation_directory_builder != nullptr) { + FinishBlobRelocationGroup(); + } // To make sure properties block is able to keep the accurate size of index // block, we will finish writing all index entries first, in Flush(). Flush(/*first_key_in_next_block=*/nullptr); @@ -4079,6 +4573,25 @@ Status BlockBasedTableBuilder::Finish() { // if any, have joined so the estimate is now quiescent). r->AutoSkipWriteBack(); + if (ok() && r->blob_relocation_directory_builder != nullptr) { + BlobRelocationDirectory directory; + BlobRelocationFileMetadata metadata; + Status directory_status = r->blob_relocation_directory_builder->Finish( + &directory, &metadata.encoded_directory); + if (directory_status.ok()) { + r->props.user_collected_properties[kBlobRelocationDirectoryPropertyName] = + metadata.encoded_directory; + r->blob_relocation_file_metadata = std::move(metadata); + } else { + r->SetStatus(std::move(directory_status)); + } + } + + // The census, indirect blob GC and RepairDB read the exact blob reference + // metadata; a normal table open does not. So these blocks are written before + // the tail (tail_start_offset), and a table open does not prefetch them. + MetaIndexBuilder meta_index_builder; + WriteBlobReferenceMetadata(&meta_index_builder); r->props.tail_start_offset = r->offset.LoadRelaxed(); // Read the clock when finishing rather than when starting, so that this @@ -4101,7 +4614,6 @@ Status BlockBasedTableBuilder::Finish() { // 7. Footer BlockHandle metaindex_block_handle; BlockHandle index_block_handle; - MetaIndexBuilder meta_index_builder; WriteFilterBlock(&meta_index_builder); WriteIndexBlock(&meta_index_builder, &index_block_handle); WriteCompressionDictBlock(&meta_index_builder); @@ -4144,6 +4656,7 @@ void BlockBasedTableBuilder::Abandon() { if (rep_->IsParallelCompressionActive()) { StopParallelCompression(/*abort=*/true); } + rep_->blob_reference_collector.reset(); rep_->state = Rep::State::kClosed; rep_->GetIOStatus().PermitUncheckedError(); } @@ -4209,13 +4722,22 @@ uint64_t BlockBasedTableBuilder::EstimatedTailSize() const { estimated_tail_size += rep_->range_del_block.CurrentSizeEstimate(); } - // 5. Estimate properties block size conservatively (~1-2KB) + // 5. Add an upper bound of the exact blob reference metadata size. These + // blocks are written before tail_start_offset, so they are not part of the + // tail. They are counted here because output file size planning uses this + // estimate. + if (rep_->blob_reference_collector) { + estimated_tail_size += + rep_->blob_reference_collector->EstimatedEncodedSize(); + } + + // 6. Estimate properties block size conservatively (~1-2KB) estimated_tail_size += 2048; - // 6. Estimate meta-index block size conservatively (~1KB) + // 7. Estimate meta-index block size conservatively (~1KB) estimated_tail_size += 1024; - // 7. Add footer size + // 8. Add footer size estimated_tail_size += Footer::kMaxEncodedLength; return estimated_tail_size; @@ -4263,4 +4785,55 @@ const std::string BlockBasedTable::kFullFilterBlockPrefix = "fullfilter."; const std::string BlockBasedTable::kPartitionedFilterBlockPrefix = "partitionedfilter."; +Status NewBlobRelocationTableBuilder( + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const BlockBasedTableOptions& table_options, + const ReadOptions& read_options, const WriteOptions& write_options, + const InternalKeyComparator& comparator, + const BlobRelocationTableFileInfo& file_info, WritableFileWriter* file, + std::unique_ptr* builder) { + assert(builder != nullptr); + MutableCFOptions relocation_options = mutable_cf_options; + relocation_options.enable_blob_indirection = false; + relocation_options.prefix_extractor.reset(); + relocation_options.table_factory = std::make_shared( + MakeBlobRelocationTableOptions(table_options)); + InternalTblPropCollFactories collectors; + // Compression choice: see the NewBlobRelocationTableBuilder() comment in the + // header. + const bool use_bottommost = + mutable_cf_options.bottommost_compression != kDisableCompressionOption; + const CompressionType compression = + use_bottommost ? mutable_cf_options.bottommost_compression + : mutable_cf_options.compression; + CompressionOptions compression_options = + use_bottommost && mutable_cf_options.bottommost_compression_opts.enabled + ? mutable_cf_options.bottommost_compression_opts + : mutable_cf_options.compression_opts; + compression_options.max_dict_bytes = 0; + compression_options.zstd_max_train_bytes = 0; + compression_options.parallel_threads = 1; + EmbeddedBlobSstBuilderOptions embedded_options; + embedded_options.min_blob_size = 0; + const TableBuilderOptions builder_options( + ioptions, relocation_options, read_options, write_options, comparator, + &collectors, compression, compression_options, file_info.column_family_id, + file_info.column_family_name, /*level=*/-1, kUnknownNewestKeyTime, + /*is_bottommost=*/false, file_info.reason, + /*oldest_key_time=*/0, file_info.db_id, file_info.db_session_id, + file_info.target_file_size, file_info.file_number, kMaxSequenceNumber, + Slice(file_info.db_name), /*is_remote_compaction=*/false, + &embedded_options); + builder->reset(static_cast_with_check( + relocation_options.table_factory->NewTableBuilder(builder_options, + file))); + const Status status = (*builder)->StartBlobRelocationFile(); + if (!status.ok()) { + (*builder)->Abandon(); + builder->reset(); + } + return status; +} + } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_builder.h b/table/block_based/block_based_table_builder.h index 71f1049a070d..e147584c55b5 100644 --- a/table/block_based/block_based_table_builder.h +++ b/table/block_based/block_based_table_builder.h @@ -17,6 +17,7 @@ #include #include +#include "db/blob/blob_gen2_format.h" #include "db/version_edit.h" #include "rocksdb/flush_block_policy.h" #include "rocksdb/listener.h" @@ -34,6 +35,7 @@ class BlockBuilder; class BlockHandle; class WritableFile; struct BlockBasedTableOptions; +struct BlobRelocationFileMetadata; extern const uint64_t kBlockBasedTableMagicNumber; @@ -42,9 +44,14 @@ class BlockBasedTableBuilder : public TableBuilder { // Create a builder that will store the contents of the table it is // building in *file. Does not close the file. It is up to the // caller to close the file after calling Finish(). - BlockBasedTableBuilder(const BlockBasedTableOptions& table_options, - const TableBuilderOptions& table_builder_options, - WritableFileWriter* file); + // `blob_reference_cache_res_mgr` charges the block cache for the memory of + // the exact blob reference metadata collector. The table factory shares one + // manager between all its builders. Null means the memory is not charged. + BlockBasedTableBuilder( + const BlockBasedTableOptions& table_options, + const TableBuilderOptions& table_builder_options, + WritableFileWriter* file, + std::shared_ptr blob_reference_cache_res_mgr); // No copying allowed BlockBasedTableBuilder(const BlockBasedTableBuilder&) = delete; @@ -75,6 +82,56 @@ class BlockBasedTableBuilder : public TableBuilder { TableBuilderAddResult TryAdd(const Slice& key, const Slice& value, TableBuilderAddContext* context) override; + bool BlobReferenceLimitReached() const override; + + // Relocation file building. Indirect blob GC writes the live values it copies + // into a relocation file, which is a block-based table (see + // db/blob/blob_relocation_file.h). Create the builder with + // NewBlobRelocationTableBuilder, which also starts the file. + + // Starts a relocation file and writes its 8-byte prefix + // (kBlobRelocationFilePrefix). Also turns off compression sampling. Requires + // a fresh builder (nothing added yet) created with embedded blob support + // (TableBuilderOptions::embedded_blob_options set, as + // NewBlobRelocationTableBuilder does), a persisted file identity + // (orig_file_number and db_session_id), and the bytewise comparator without + // timestamps; otherwise InvalidArgument. + Status StartBlobRelocationFile(); + + // Registers an old file number that may be routed to this file. The file + // stores this served-file list in its directory (see + // db/blob/blob_relocation_file.h). `file_number` must be smaller than this + // file's number; otherwise InvalidArgument. Duplicate calls are allowed and + // removed in Finish(). + Status AddBlobRelocationServedFile(uint64_t file_number); + + // Adds one stored blob payload under its stable identity (origin, original + // offset). The payload is already compressed with `compression` and is + // written as is. `origin_file_number` must be smaller than this file's + // number, and calls must be in strictly increasing (origin, original offset) + // order. A payload above kMaxSimpleGen2BlobPayloadSize or an unsupported + // compression type is also rejected. All these cases return InvalidArgument. + Status AddBlobRelocation(uint64_t origin_file_number, + uint64_t original_value_offset, + const Slice& stored_payload, + CompressionType compression); + + // Streaming counterpart. The source verifies the exact stored bytes with the + // stored CRC. If the source fails, the caller must abandon the whole output + // and must not publish it. + Status AddBlobRelocation(uint64_t origin_file_number, + uint64_t original_value_offset, + uint64_t stored_value_size, + CompressionType compression, + uint32_t stored_checksum, + const VerifiedBlobStoredSource& source); + + // Returns the encoded directory and tail size after Finish() succeeds. + // Otherwise returns the builder's error status, or InvalidArgument if the + // file is not finished yet. + Status GetBlobRelocationFileMetadata( + BlobRelocationFileMetadata* metadata) const; + // Return non-ok iff some error has been detected. Status status() const override; @@ -122,6 +179,9 @@ class BlockBasedTableBuilder : public TableBuilder { // Get table properties TableProperties GetTableProperties() const override; + std::shared_ptr GetBlobReferenceSstSummary() + const override; + // Get file checksum std::string GetFileChecksum() const override; @@ -141,6 +201,11 @@ class BlockBasedTableBuilder : public TableBuilder { private: bool ok() const; + // Add() with the caller's decode of a kTypeBlobIndex `value`, or nullptr + // (see TableBuilderAddContext::decoded_blob_index). + void AddImpl(const Slice& key, const Slice& value, + const BlobIndex* decoded_blob_index); + // Transition state from buffered to unbuffered if the conditions are met. See // `Rep::State` API comment for details of the states. // REQUIRES: `rep_->state == kBuffered` @@ -163,6 +228,8 @@ class BlockBasedTableBuilder : public TableBuilder { void WriteBlock(const Slice& block_contents, BlockHandle* handle, BlockType block_type, bool* skip_delta_encoding = nullptr); // Directly write data to the file. + // The base signatures. They pass warm_cache = true (base behavior: the + // block may be added to the block cache when the options ask for it). void WriteMaybeCompressedBlock( const Slice& block_contents, CompressionType, BlockHandle* handle, BlockType block_type, const Slice* uncompressed_block_data = nullptr, @@ -174,6 +241,20 @@ class BlockBasedTableBuilder : public TableBuilder { bool* skip_delta_encoding = nullptr, const uint32_t* precomputed_block_contents_crc32c = nullptr); + // `warm_cache` false keeps the block out of the block cache even when the + // options ask to warm it (prepopulate_block_cache). Used for blocks that no + // reader looks up in the block cache. + void WriteMaybeCompressedBlock( + const Slice& block_contents, CompressionType, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data, + bool* skip_delta_encoding, + const uint32_t* precomputed_block_contents_crc32c, bool warm_cache); + IOStatus WriteMaybeCompressedBlockImpl( + const Slice& block_contents, CompressionType, BlockHandle* handle, + BlockType block_type, const Slice* uncompressed_block_data, + bool* skip_delta_encoding, + const uint32_t* precomputed_block_contents_crc32c, bool warm_cache); + void SetupCacheKeyPrefix(const TableBuilderOptions& tbo); template @@ -194,6 +275,7 @@ class BlockBasedTableBuilder : public TableBuilder { void WritePropertiesBlock(MetaIndexBuilder* meta_index_builder); void WriteCompressionDictBlock(MetaIndexBuilder* meta_index_builder); void WriteRangeDelBlock(MetaIndexBuilder* meta_index_builder); + void WriteBlobReferenceMetadata(MetaIndexBuilder* meta_index_builder); void WriteFooter(BlockHandle& metaindex_block_handle, BlockHandle& index_block_handle); @@ -216,11 +298,32 @@ class BlockBasedTableBuilder : public TableBuilder { // original value should be used as-is). Status BuildEmbeddedWideColumnEntity(const Slice& value, bool* rewritten); - // Appends one uncompressed same-file blob record for `payload` at the current - // table offset, advances the offset, lazily allocates and updates the - // embedded-blob state (counters), and encodes a same-file BlobIndex into - // rep_->embedded_blob_state->blob_index_buf. - Status WriteEmbeddedBlobRecord(const Slice& payload); + // Appends one same-file blob record at the current table offset and advances + // the offset. `payload` is already in stored form, compressed with + // `compression`, and is written as is. Also updates the embedded-blob + // counters and encodes a same-file BlobIndex into + // rep_->embedded_blob_state->blob_index_buf. When non-null, `record_offset` + // receives the file offset where the record starts. + Status WriteEmbeddedBlobRecord(const Slice& payload, + CompressionType compression, + uint64_t* record_offset); + + // Relocation files: a location entry covers a group of values of one origin + // (see db/blob/blob_relocation_file.h). Before a record of + // `origin_file_number` is written, this writes the open group's location + // entry if the record starts a new group. The location entry is added only + // after the group's records, so a data block is never written between the + // records of one group. + void PrepareBlobRelocationGroup(uint64_t origin_file_number); + + // Adds a written record to the open group, and writes the group's location + // entry when the group is full. + void AddToBlobRelocationGroup(uint64_t origin_file_number, + uint64_t original_value_offset, + uint64_t stored_value_size, + uint64_t record_offset); + // Writes the location entry of the open group, if any. + void FinishBlobRelocationGroup(); struct Rep; class BlockBasedTablePropertiesCollectorFactory; @@ -264,4 +367,53 @@ class BlockBasedTableBuilder : public TableBuilder { void StopParallelCompression(bool abort); }; +// The identity and the size target of one relocation file, for +// NewBlobRelocationTableBuilder(). Callers set the fields by name, so values of +// the same type (db_id and db_session_id, file_number and target_file_size) +// cannot be swapped by mistake. The strings are copied. +// NewBlobRelocationTableBuilder() runs once per relocation file, so the copies +// cost little. +struct BlobRelocationTableFileInfo { + uint32_t column_family_id = 0; + std::string column_family_name; + TableFileCreationReason reason = TableFileCreationReason::kMisc; + std::string db_id; + std::string db_session_id; + std::string db_name; + uint64_t file_number = 0; + // The file size that the caller aims for; 0 means no target. + uint64_t target_file_size = 0; +}; + +// Creates the builder of relocation file `file_info.file_number` of column +// family `file_info.column_family_id`, writing to `file`, and starts the file +// (StartBlobRelocationFile). Settings of the builder: +// - Table options: MakeBlobRelocationTableOptions(`table_options`). +// - Other settings: from `mutable_cf_options`, with blob indirection and the +// prefix extractor turned off. +// - Data blocks: they hold only location entries (see +// db/blob/blob_relocation_file.h). They use the column family's +// bottommost_compression if it is set, else its compression. Bottommost comes +// first because a relocation file holds values that indirect blob GC moved +// out of old files, and such long-lived data is like bottommost data (a +// design choice). The builder writes each value itself, outside the data +// blocks, already in stored form, as an embedded blob record; values are +// never recompressed. +// - No dictionary and no parallel compression: both hold data blocks back +// (dictionary training buffers them, and parallel compression writes them +// later from other threads), but the records must be written to the file +// between the data blocks in order. +// +// `comparator` must use the bytewise comparator and outlive the builder. If +// starting the file fails, the builder is abandoned and `*builder` is reset, so +// the caller only needs to return the status. +Status NewBlobRelocationTableBuilder( + const ImmutableOptions& ioptions, + const MutableCFOptions& mutable_cf_options, + const BlockBasedTableOptions& table_options, + const ReadOptions& read_options, const WriteOptions& write_options, + const InternalKeyComparator& comparator, + const BlobRelocationTableFileInfo& file_info, WritableFileWriter* file, + std::unique_ptr* builder); + } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index c3dce202c7d5..f74838b573f9 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -639,6 +639,20 @@ BlockBasedTableFactory::BlockBasedTableFactory( table_options_.block_cache), SystemClock::Default().get(), release_delay_micros); } + + const auto blob_reference_charged = + table_options_.cache_usage_options.options_overrides + .at(CacheEntryRole::kBlobReferenceMetadataBuildingBuffer) + .charged; + if (table_options_.block_cache && + (blob_reference_charged == CacheEntryRoleOptions::Decision::kEnabled || + blob_reference_charged == CacheEntryRoleOptions::Decision::kFallback)) { + shared_state_->blob_reference_cache_res_mgr = + std::make_shared( + std::make_shared>( + table_options_.block_cache)); + } } void BlockBasedTableFactory::InitializeOptions() { @@ -814,8 +828,9 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder( WritableFileWriter* file) const { // BlockBasedTableBuilder self-detects embedded-blob mode from // table_builder_options.embedded_blob_options. - return new BlockBasedTableBuilder(table_options_, table_builder_options, - file); + return new BlockBasedTableBuilder( + table_options_, table_builder_options, file, + shared_state_->blob_reference_cache_res_mgr); } Status BlockBasedTableFactory::ValidateOptions( @@ -1014,6 +1029,9 @@ Status BlockBasedTableFactory::ValidateOptions( const CacheEntryRoleOptions options = options_overrides_iter->second; static const std::set kMemoryChargingSupported = { CacheEntryRole::kCompressionDictionaryBuildingBuffer, + CacheEntryRole::kBlobReferenceMetadataBuildingBuffer, + CacheEntryRole::kBlobReferenceCensus, + CacheEntryRole::kIndirectBlobGcWorkingMemory, CacheEntryRole::kFilterConstruction, CacheEntryRole::kBlockBasedTableReader, CacheEntryRole::kFileMetadata, diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index 31ebb2cf0927..54df5d62b89f 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -60,6 +60,15 @@ class BlockBasedTableFactory : public TableFactory { const char* Name() const override { return kBlockBasedTableName(); } + // Returns the cache reservation manager that charges the memory of the exact + // blob reference metadata collectors (see db/blob/blob_reference_metadata.h). + // All builders of this factory share it. Null when there is no block cache or + // charging for kBlobReferenceMetadataBuildingBuffer is disabled. + std::shared_ptr + GetBlobReferenceCacheReservationManager() const { + return shared_state_->blob_reference_cache_res_mgr; + } + using TableFactory::NewTableReader; Status NewTableReader( const ReadOptions& ro, const TableReaderOptions& table_reader_options, @@ -110,6 +119,9 @@ class BlockBasedTableFactory : public TableFactory { struct SharedState { std::shared_ptr table_reader_cache_res_mgr; std::shared_ptr readahead_cache_res_mgr; + // See GetBlobReferenceCacheReservationManager(). Each collector charges + // through its own reservation handle. + std::shared_ptr blob_reference_cache_res_mgr; TailPrefetchStats tail_prefetch_stats; }; std::shared_ptr shared_state_; diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc index e81ca03e947e..110c9a8aa6ea 100644 --- a/table/block_based/block_based_table_reader.cc +++ b/table/block_based/block_based_table_reader.cc @@ -9,12 +9,15 @@ #include "table/block_based/block_based_table_reader.h" #include +#include #include #include #include #include #include #include +#include +#include #include #include #include @@ -22,8 +25,10 @@ #include "block_cache.h" #include "cache/cache_entry_roles.h" #include "cache/cache_key.h" +#include "db/blob/blob_contents.h" #include "db/blob/blob_gen2_format.h" #include "db/blob/blob_index.h" +#include "db/blob/blob_reference_metadata.h" #include "db/blob/blob_source.h" #include "db/compaction/compaction_picker.h" #include "db/dbformat.h" @@ -81,6 +86,7 @@ #include "test_util/sync_point.h" #include "util/coding.h" #include "util/crc32c.h" +#include "util/defer.h" #include "util/stop_watch.h" #include "util/string_util.h" @@ -94,11 +100,20 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) { return heap_buf; } -// Cleanup function for a Cleanable/PinnableSlice that owns a `new char[]` -// buffer (used to pin embedded blob payloads without copying). -void DeleteCharArray(void* arg1, void* /*arg2*/) { - delete[] static_cast(arg1); +// Pins all of `contents` into `value`, which takes ownership. +Status PinBlobContents(std::unique_ptr contents, + PinnableSlice* value) { + const Slice result = contents->data(); + BlobContents* const owned = contents.release(); + value->PinSlice( + result, + [](void* arg1, void* /*arg2*/) { + delete static_cast(arg1); + }, + owned, nullptr); + return Status::OK(); } + } // namespace Status AllocateReadScopedBlockBuffer( @@ -993,6 +1008,9 @@ Status BlockBasedTable::Open( rep->persistent_cache_options = PersistentCacheOptions(rep->table_options.persistent_cache, rep->base_cache_key, rep->ioptions.stats); + rep->metadata_persistent_cache_options = avoid_shared_metadata_cache + ? PersistentCacheOptions::kEmpty + : rep->persistent_cache_options; s = new_table->ReadRangeDelBlock(ro, prefetch_buffer.get(), metaindex_iter.get(), internal_comparator, @@ -1193,6 +1211,10 @@ Status BlockBasedTable::ReadPropertiesBlock( // diagnostic only; only presence matters for correctness. rep_->has_embedded_blobs = props.find(kEmbeddedBlobSstStatsPropertyName) != props.end(); + if (rep_->has_embedded_blobs) { + rep_->embedded_blob_decompressor = + GetBuiltinV2CompressionManager()->GetDecompressor(); + } auto index_type_pos = props.find(BlockBasedTablePropertyNames::kIndexType); if (index_type_pos == props.end()) { return Status::Corruption("Missing index type property"); @@ -1240,10 +1262,10 @@ Status BlockBasedTable::ValidateEmbeddedBlobIndex(const BlobIndex& blob_index, } const uint64_t payload_size_u64 = blob_index.size(); - if (payload_size_u64 > - static_cast(std::numeric_limits::max()) - - kSimpleGen2BlobTrailerSize) { - return Status::Corruption("Embedded blob record is too large"); + if (payload_size_u64 > kMaxSimpleGen2BlobPayloadSize || + !CompressionTypeSupported(blob_index.compression())) { + return Status::Corruption( + "Embedded blob record size or compression is not supported"); } const uint64_t record_size_u64 = payload_size_u64 + kSimpleGen2BlobTrailerSize; @@ -1276,18 +1298,36 @@ Status BlockBasedTable::ResolveEmbeddedBlob(const ReadOptions& read_options, return Status::Incomplete( "Cannot read embedded blob in block-cache-only mode"); } - // Read the record (payload + trailer) directly into `value`, then drop the - // trailer with a shrink (no extra copy on the common non-mmap path). - value->resize(record_size); + if (blob_index.compression() == kNoCompression) { + value->resize(record_size); + s = ReadAndVerifySimpleGen2BlobRecord( + read_options, rep_->file.get(), blob_index.offset(), payload_size, + record_size, rep_->footer.checksum_type(), + rep_->footer.base_context_checksum(), blob_index.compression(), + &(*value)[0]); + if (s.ok()) { + value->resize(payload_size); + } + return s; + } + CacheAllocationPtr stored_record = AllocateBlock(record_size, nullptr); s = ReadAndVerifySimpleGen2BlobRecord( read_options, rep_->file.get(), blob_index.offset(), payload_size, record_size, rep_->footer.checksum_type(), rep_->footer.base_context_checksum(), blob_index.compression(), - &(*value)[0]); + stored_record.get()); + if (!s.ok()) { + return s; + } + std::unique_ptr contents; + s = UncompressBlobContentsIfNeeded( + Slice(stored_record.get(), payload_size), blob_index.compression(), + rep_->embedded_blob_decompressor.get(), /*allocator=*/nullptr, + rep_->ioptions.clock, rep_->ioptions.stats, &contents); if (!s.ok()) { return s; } - value->resize(payload_size); + value->assign(contents->data().data(), contents->data().size()); return Status::OK(); } @@ -1306,21 +1346,53 @@ Status BlockBasedTable::ResolveEmbeddedBlobPinned( return Status::Incomplete( "Cannot read embedded blob in block-cache-only mode"); } - // Read into a heap buffer owned by `value` via a registered cleanup, so the - // payload can be pinned (no copy) and outlive this reader. The trailer just - // sits unused at the tail of the buffer. - std::unique_ptr buf(new char[record_size]); + CacheAllocationPtr stored_record = AllocateBlock(record_size, nullptr); s = ReadAndVerifySimpleGen2BlobRecord( read_options, rep_->file.get(), blob_index.offset(), payload_size, record_size, rep_->footer.checksum_type(), rep_->footer.base_context_checksum(), blob_index.compression(), - buf.get()); + stored_record.get()); if (!s.ok()) { return s; } - char* raw = buf.release(); - value->PinSlice(Slice(raw, payload_size), &DeleteCharArray, raw, nullptr); - return Status::OK(); + return DecompressAndPinEmbeddedBlob(std::move(stored_record), payload_size, + blob_index.compression(), value); +} + +void BlockBasedTable::DropUserCollectedProperty(const std::string& name) { + // It replaces rep_->table_properties, which readers share, so it must run + // before any read (see MarkReadStarted()). + assert(!rep_->read_started.load(std::memory_order_relaxed)); + if (rep_->table_properties == nullptr || + rep_->table_properties->user_collected_properties.count(name) == 0) { + return; + } + auto properties = std::make_shared(*rep_->table_properties); + properties->user_collected_properties.erase(name); + rep_->table_properties = std::move(properties); +} + +Status BlockBasedTable::DecompressEmbeddedBlob( + CacheAllocationPtr stored, size_t payload_size, CompressionType compression, + MemoryAllocator* allocator, std::unique_ptr* contents) const { + if (compression == kNoCompression) { + contents->reset(new BlobContents(std::move(stored), payload_size)); + return Status::OK(); + } + return UncompressBlobContentsIfNeeded( + Slice(stored.get(), payload_size), compression, + rep_->embedded_blob_decompressor.get(), allocator, rep_->ioptions.clock, + rep_->ioptions.stats, contents); +} + +Status BlockBasedTable::DecompressAndPinEmbeddedBlob( + CacheAllocationPtr stored, size_t payload_size, CompressionType compression, + PinnableSlice* value) const { + std::unique_ptr contents; + Status s = + DecompressEmbeddedBlob(std::move(stored), payload_size, compression, + /*allocator=*/nullptr, &contents); + return s.ok() ? PinBlobContents(std::move(contents), value) : s; } Status BlockBasedTable::ResolveEmbeddedBlobCached( @@ -1352,23 +1424,52 @@ Status BlockBasedTable::GetSameFileBlob(const ReadOptions& read_options, size_t range_length, BlobVerifyPolicy verify_policy, PinnableSlice* value) const { + MarkReadStarted(); if (range_length != kWholeBlobLength) { - // Byte-range embedded read: value bytes only, no whole-record checksum, no - // cache fill (see ResolveEmbeddedBlobRangeCached). Only issued while - // resolving a lazy result, which always runs against an open column family - // (so a BlobSource is wired) and never forces verification of a strict - // sub-range. Reaching here without a BlobSource means a same-file reference - // was routed from an unexpected context (a corrupt index can produce the - // file_number 0 same-file sentinel), so report corruption -- mirroring - // Version::GetBlobRange's "Invalid blob file number". + if (blob_index.compression() != kNoCompression) { + // As in base RocksDB, a range of a compressed value is not supported: a + // byte range cannot be decompressed alone. Callers read the whole value + // and slice it. + return Status::Corruption("Cannot range-read a compressed blob"); + } + // An uncompressed range reads only the range bytes, with no checksum check + // and no cache fill. Callers send only strict parts of a value (see + // ReadPathBlobResolver::ResolveColumnRange), and a range is never turned + // into a whole read. A DB-owned SST reads the range through BlobSource. A + // reader without a BlobSource, such as a relocation table (opened as a + // standalone block-based reader), reads the range directly with + // ReadSimpleGen2BlobRange. assert(verify_policy != BlobVerifyPolicy::kVerifyIfPresent); - assert(rep_->blob_source_ != nullptr); - if (rep_->blob_source_ == nullptr) { - return Status::Corruption( - "Invalid same-file blob reference (no BlobSource for range read)"); + if (rep_->blob_source_ != nullptr) { + return ResolveEmbeddedBlobRangeCached(read_options, blob_index, + range_offset, range_length, value); } - return ResolveEmbeddedBlobRangeCached(read_options, blob_index, - range_offset, range_length, value); + size_t payload_size = 0; + size_t record_size = 0; + Status status = + ValidateEmbeddedBlobIndex(blob_index, &payload_size, &record_size); + if (!status.ok()) { + return status; + } + if (range_offset > payload_size || + range_length > payload_size - range_offset) { + return Status::InvalidArgument("Embedded blob range is out of bounds"); + } + if (read_options.read_tier == kBlockCacheTier) { + return Status::Incomplete( + "Cannot read embedded blob in block-cache-only mode"); + } + CacheAllocationPtr buffer = + AllocateBlock(range_length, /*allocator=*/nullptr); + status = ReadSimpleGen2BlobRange( + read_options, rep_->file.get(), blob_index.offset(), payload_size, + range_offset, range_length, blob_index.compression(), buffer.get()); + if (!status.ok()) { + return status; + } + value->Reset(); + return PinBlobContents( + std::make_unique(std::move(buffer), range_length), value); } // Whole-value read: prefer the blob-cache-backed read (BLOB_DB_* stats) when @@ -1413,14 +1514,384 @@ Status BlockBasedTable::ResolveEmbeddedBlobRangeCached( range_offset, range_length, value, /*bytes_read=*/nullptr); } +void BlockBasedTable::RetrieveDataBlocks( + const ReadOptions& read_options, + const autovector& handles, + BlockCacheLookupContext* lookup_context, CachableEntry* blocks, + Status* statuses) const { + assert(!handles.empty()); + CachableEntry dict; + Status dict_status; + if (rep_->uncompression_dict_reader) { + dict_status = + rep_->uncompression_dict_reader->GetOrReadUncompressionDictionary( + /*prefetch_buffer=*/nullptr, read_options, + /*get_context=*/nullptr, lookup_context, &dict); + } + TEST_SYNC_POINT_CALLBACK("BlockBasedTable::MultiGetRawEntries:DictStatus", + &dict_status); + if (!dict_status.ok()) { + std::fill_n(statuses, handles.size(), dict_status); + } else { + const UnownedPtr decomp = + dict.GetValue() ? dict.GetValue()->decompressor_.get() + : rep_->decompressor.get(); + // Look up each block in the block cache. A hit, a cache error, or a + // miss without I/O (kBlockCacheTier, which fails the block with + // Incomplete) needs no read: its handle becomes null. The other handles + // are the missing blocks. + autovector missing = handles; + size_t missing_bytes = 0; + size_t missing_count = 0; + std::array cache_hit{}; + BlockCacheInterface block_cache{ + rep_->table_options.block_cache.get()}; + for (size_t b = 0; b < handles.size(); ++b) { + if (block_cache) { + const CacheKey cache_key = + GetCacheKey(rep_->base_cache_key, handles[b]); + statuses[b] = + GetDataBlockFromCache(cache_key.AsSlice(), block_cache, &blocks[b], + /*get_context=*/nullptr, decomp); + if (!statuses[b].ok() || blocks[b].GetValue() != nullptr) { + cache_hit[b] = statuses[b].ok(); + missing[b] = BlockHandle::NullBlockHandle(); + continue; + } + } + if (read_options.read_tier == kBlockCacheTier) { + statuses[b] = Status::Incomplete("no blocking io"); + missing[b] = BlockHandle::NullBlockHandle(); + continue; + } + missing_bytes += BlockSizeWithTrailer(handles[b]); + ++missing_count; + } + // Read the missing blocks, verify and decompress them, and insert them + // into the block cache. Two or more missing blocks are read with one + // batched read (RetrieveMultipleBlocks). One missing block (for example, + // one Get of a relocated value) takes the normal block read path, a + // plain positional read, without a second cache lookup (the lookup above + // already missed). + std::array read_status; + if (missing_count > 1) { + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTable::MultiGetRawEntries:RetrieveMultipleBlocks", + &missing); + char* scratch = nullptr; + std::unique_ptr scratch_owner; + bool use_fs_scratch = false; + if (!rep_->file->use_direct_io()) { + use_fs_scratch = CheckFSFeatureSupport(rep_->ioptions.fs.get(), + FSSupportedOps::kFSBuffer); + } + if (!use_fs_scratch && !rep_->file->use_direct_io() && + rep_->decompressor) { + scratch_owner.reset(new char[missing_bytes]); + scratch = scratch_owner.get(); + } + RetrieveMultipleBlocks(read_options, /*batch=*/nullptr, &missing, + read_status.data(), blocks, scratch, decomp, + use_fs_scratch); + } + for (size_t b = 0; b < handles.size(); ++b) { + if (missing[b].IsNull()) { + continue; + } + if (missing_count == 1) { + blocks[b].Reset(); + statuses[b] = RetrieveBlock( + /*prefetch_buffer=*/nullptr, read_options, missing[b], decomp, + &blocks[b], /*get_context=*/nullptr, lookup_context, + /*for_compaction=*/false, /*use_cache=*/true, + /*async_read=*/false, /*use_block_cache_for_lookup=*/false); + } else { + statuses[b] = read_status[b]; + } + } + // The entries of blocks that were not read in the batch stay unused. + for (Status& status : read_status) { + status.PermitUncheckedError(); + } + // Write a block cache trace record for each cache hit and each block of + // the batched read. A single missing block went through RetrieveBlock(), + // which writes its own record. + if (block_cache_tracer_ && block_cache_tracer_->is_tracing_enabled() && + lookup_context) { + for (size_t b = 0; b < handles.size(); ++b) { + const bool traced_by_read = missing_count == 1 && !missing[b].IsNull(); + if (traced_by_read || !statuses[b].ok() || + blocks[b].GetValue() == nullptr) { + continue; + } + const CacheKey cache_key = + GetCacheKey(rep_->base_cache_key, handles[b]); + SaveLookupContextOrTraceRecord(cache_key.AsSlice(), cache_hit[b], + read_options, blocks[b].GetValue(), + lookup_context); + } + } + } +} + +void BlockBasedTable::MultiGetRawEntriesImpl(const ReadOptions& read_options, + size_t num_keys, const Slice* keys, + RawEntryCallback on_entry_fn, + const void* on_entry_arg) const { + MarkReadStarted(); + const auto on_entry = [&](size_t key_index, const Status& status, + const Slice& internal_key, const Slice& value) { + on_entry_fn(on_entry_arg, key_index, status, internal_key, value); + }; + assert(num_keys <= MultiGetContext::MAX_BATCH_SIZE); + if (num_keys == 0) { + return; + } + // Raw reads of a blob relocation file. The relocation reader checks the raw + // values itself. + BlockCacheLookupContext lookup_context{ + TableReaderCaller::kBlobRelocationFileRead}; + IndexBlockIter iiter_on_stack; + InternalIteratorBase* const iiter = NewIndexIterator( + read_options, /*need_upper_bound_check=*/false, &iiter_on_stack, + /*get_context=*/nullptr, &lookup_context); + std::unique_ptr> iiter_owner; + if (iiter != &iiter_on_stack) { + iiter_owner.reset(iiter); + } + + // 1. Find the data block of each key. Sorted keys in one block are adjacent, + // so each distinct block is listed once. + constexpr size_t kNoBlock = std::numeric_limits::max(); + std::array key_block; + std::array key_status; + autovector handles; + for (size_t i = 0; i < num_keys; ++i) { + assert(i == 0 || + rep_->internal_comparator.Compare(keys[i - 1], keys[i]) <= 0); + key_block[i] = kNoBlock; + iiter->Seek(keys[i]); + if (!iiter->Valid()) { + key_status[i] = + iiter->status().ok() ? Status::NotFound() : iiter->status(); + continue; + } + const BlockHandle handle = iiter->value().handle; + if (handles.empty() || handles.back().offset() != handle.offset()) { + handles.push_back(handle); + } + key_block[i] = handles.size() - 1; + } + + // 2. Look up each block in the block cache, and read the missing blocks. + std::array, MultiGetContext::MAX_BATCH_SIZE> + blocks; + std::array block_status; + if (!handles.empty()) { + RetrieveDataBlocks(read_options, handles, &lookup_context, blocks.data(), + block_status.data()); + } + + // 3. Search each key in its block. + DataBlockIter biter; + size_t iter_block = kNoBlock; + for (size_t i = 0; i < num_keys; ++i) { + const size_t b = key_block[i]; + if (b == kNoBlock) { + on_entry(i, key_status[i], Slice(), Slice()); + continue; + } + if (!block_status[b].ok()) { + on_entry(i, block_status[b], Slice(), Slice()); + continue; + } + if (iter_block != b) { + biter.Invalidate(Status::OK()); + NewDataBlockIterator(read_options, blocks[b].As(), + &biter, block_status[b]); + iter_block = b; + } + // SeekForGet uses the data block hash index when the table has one; + // without it, it finds the first key at or after the lookup key (blob + // relocation files look up values inside a location entry this way). + if (!biter.SeekForGet(keys[i]) || !biter.Valid()) { + on_entry(i, biter.status().ok() ? Status::NotFound() : biter.status(), + Slice(), Slice()); + continue; + } + on_entry(i, Status::OK(), biter.key(), biter.value()); + } + // The callbacks get these statuses by reference, and the entries past the + // keys and blocks of this batch are never set. + for (size_t i = 0; i < key_status.size(); ++i) { + key_status[i].PermitUncheckedError(); + block_status[i].PermitUncheckedError(); + } +} + void BlockBasedTable::MultiGetSameFileBlob( const ReadOptions& read_options, size_t num_reads, SameFileBlobReadRequest* reqs) const { - // Partition the requests into coalesceable whole-record and sub-range groups - // (dispatched to BlobSource as one MultiRead each). Force-verify reads - // (kVerifyIfPresent) and the no-BlobSource fallback go through the scalar - // GetSameFileBlob path since they cannot share the batch's single verify flag - // / BlobSource-backed cache path. + MarkReadStarted(); + if (rep_->blob_source_ == nullptr) { + struct WholePending { + size_t request_index; + size_t payload_size; + size_t record_size; + CacheAllocationPtr buffer; + }; + struct RangePending { + size_t request_index; + size_t range_length; + CacheAllocationPtr buffer; + }; + + std::vector whole_pending; + std::vector range_pending; + std::vector whole_reads; + std::vector verified_whole_reads; + std::vector range_reads; + whole_pending.reserve(num_reads); + range_pending.reserve(num_reads); + whole_reads.reserve(num_reads); + verified_whole_reads.reserve(num_reads); + range_reads.reserve(num_reads); + + for (size_t i = 0; i < num_reads; ++i) { + SameFileBlobReadRequest& req = reqs[i]; + assert(req.blob_index != nullptr); + assert(req.result != nullptr); + assert(req.status != nullptr); + req.result->Reset(); + *req.status = Status::OK(); + + MemoryAllocator* const allocator = + req.standalone != nullptr ? req.standalone->allocator : nullptr; + SimpleGen2ReadExtras* const gen2_extras = + req.standalone != nullptr ? &req.standalone->gen2 : nullptr; + if (req.range_length != kWholeBlobLength && + req.blob_index->compression() != kNoCompression) { + // A range of a compressed value is not supported, as in + // GetSameFileBlob(). This check comes first, so the result does not + // depend on the read tier. + *req.status = Status::Corruption("Cannot range-read a compressed blob"); + continue; + } + size_t payload_size = 0; + size_t record_size = 0; + *req.status = ValidateEmbeddedBlobIndex(*req.blob_index, &payload_size, + &record_size); + if (!req.status->ok()) { + continue; + } + if (read_options.read_tier == kBlockCacheTier) { + *req.status = Status::Incomplete( + "Cannot read embedded blob in block-cache-only mode"); + continue; + } + // An uncompressed range reads only its bytes and checks no checksum. It + // is never turned into a whole read, as in GetSameFileBlob(). + if (req.range_length != kWholeBlobLength) { + assert(req.verify_policy != BlobVerifyPolicy::kVerifyIfPresent); + if (req.range_offset > payload_size || + req.range_length > payload_size - req.range_offset) { + *req.status = + Status::InvalidArgument("Embedded blob range is out of bounds"); + continue; + } + CacheAllocationPtr buffer = AllocateBlock(req.range_length, allocator); + range_reads.push_back(SimpleGen2RangeReadRequest{ + req.blob_index->offset(), payload_size, req.range_offset, + req.range_length, req.blob_index->compression(), buffer.get(), + req.status, gen2_extras}); + range_pending.push_back( + RangePending{i, req.range_length, std::move(buffer)}); + continue; + } + + CacheAllocationPtr buffer = AllocateBlock(record_size, allocator); + SimpleGen2RecordReadRequest read{req.blob_index->offset(), + payload_size, + record_size, + req.blob_index->compression(), + buffer.get(), + req.status, + gen2_extras}; + if (req.verify_policy == BlobVerifyPolicy::kVerifyIfPresent && + !read_options.verify_checksums) { + verified_whole_reads.push_back(read); + } else { + whole_reads.push_back(read); + } + whole_pending.push_back( + WholePending{i, payload_size, record_size, std::move(buffer)}); + } + + if (!whole_reads.empty()) { + ReadAndVerifySimpleGen2BlobRecords( + read_options, rep_->file.get(), rep_->footer.checksum_type(), + rep_->footer.base_context_checksum(), whole_reads.size(), + whole_reads.data()); + } + if (!verified_whole_reads.empty()) { + ReadOptions verified_options = read_options; + verified_options.verify_checksums = true; + ReadAndVerifySimpleGen2BlobRecords( + verified_options, rep_->file.get(), rep_->footer.checksum_type(), + rep_->footer.base_context_checksum(), verified_whole_reads.size(), + verified_whole_reads.data()); + } + if (!range_reads.empty()) { + ReadSimpleGen2BlobRanges(read_options, rep_->file.get(), + range_reads.size(), range_reads.data()); + } + + for (WholePending& pending : whole_pending) { + SameFileBlobReadRequest& req = reqs[pending.request_index]; + // The payload CRC is the value CRC only for an uncompressed value. + if (req.standalone != nullptr && + req.blob_index->compression() != kNoCompression) { + req.standalone->gen2.payload_crc32c.reset(); + } + if (!req.status->ok()) { + continue; + } + MemoryAllocator* const allocator = + req.standalone != nullptr ? req.standalone->allocator : nullptr; + std::unique_ptr contents; + *req.status = DecompressEmbeddedBlob( + std::move(pending.buffer), pending.payload_size, + req.blob_index->compression(), allocator, &contents); + if (!req.status->ok()) { + continue; + } + if (req.standalone == nullptr) { + *req.status = PinBlobContents(std::move(contents), req.result); + } else { + req.standalone->contents = std::move(contents); + } + } + for (RangePending& pending : range_pending) { + SameFileBlobReadRequest& req = reqs[pending.request_index]; + if (!req.status->ok()) { + continue; + } + auto contents = std::make_unique(std::move(pending.buffer), + pending.range_length); + if (req.standalone != nullptr) { + req.standalone->contents = std::move(contents); + } else { + *req.status = PinBlobContents(std::move(contents), req.result); + } + } + return; + } + + // Split the requests into a whole-record group and a sub-range group. Each + // group goes to one BlobSource batched call, which reads its cache misses in + // MultiRead batches. Force-verify reads use the scalar path because they + // cannot share the batch's read options. A range of a compressed value is + // not supported (see GetSameFileBlob()). std::vector whole_reqs; std::vector range_reqs; @@ -1430,8 +1901,12 @@ void BlockBasedTable::MultiGetSameFileBlob( assert(req.result); assert(req.status); - if (req.verify_policy == BlobVerifyPolicy::kVerifyIfPresent || - rep_->blob_source_ == nullptr) { + if (req.range_length != kWholeBlobLength && + req.blob_index->compression() != kNoCompression) { + *req.status = Status::Corruption("Cannot range-read a compressed blob"); + continue; + } + if (req.verify_policy == BlobVerifyPolicy::kVerifyIfPresent) { *req.status = GetSameFileBlob(read_options, *req.blob_index, req.range_offset, req.range_length, req.verify_policy, req.result); @@ -1472,6 +1947,162 @@ void BlockBasedTable::MultiGetSameFileBlob( } } +void BlockBasedTable::MultiGetStoredSameFileBlob( + const ReadOptions& read_options, uint64_t coalesce_bytes, size_t num_reads, + StoredSameFileBlobReadRequest* reqs, uint64_t max_stored_bytes, + CoalescedReadBuffers* buffers, uint64_t* bytes_read) const { + if (bytes_read != nullptr) { + *bytes_read = 0; + } + if (num_reads == 0) { + return; + } + if (reqs == nullptr || rep_ == nullptr) { + return; + } + if (buffers == nullptr) { + for (size_t i = 0; i < num_reads; ++i) { + reqs[i].status = Status::InvalidArgument("Missing blob GC read buffers"); + } + return; + } + const Rep& rep = *rep_; + + struct Pending { + size_t request_index = 0; + size_t payload_size = 0; + uint64_t record_offset = 0; + size_t record_size = 0; + }; + std::vector pending; + pending.reserve(num_reads); + uint64_t total_stored_bytes = 0; + for (size_t i = 0; i < num_reads; ++i) { + StoredSameFileBlobReadRequest& req = reqs[i]; + if (req.stored_payload != nullptr) { + *req.stored_payload = Slice(); + } + req.status = Status::OK(); + size_t payload_size = 0; + size_t record_size = 0; + req.status = + ValidateEmbeddedBlobIndex(req.blob_index, &payload_size, &record_size); + if (!req.status.ok()) { + continue; + } + if (payload_size > + max_stored_bytes - std::min(total_stored_bytes, max_stored_bytes)) { + req.status = + Status::MemoryLimit("Stored blob relocation batch exceeds limit"); + continue; + } + total_stored_bytes += payload_size; + pending.push_back( + Pending{i, payload_size, req.blob_index.offset(), record_size}); + } + if (pending.empty()) { + return; + } + // Records follow key order, so this sort is usually a no-op. + std::sort(pending.begin(), pending.end(), + [](const Pending& lhs, const Pending& rhs) { + return lhs.record_offset < rhs.record_offset; + }); + std::vector offsets(pending.size()); + std::vector lengths(pending.size()); + uint64_t total_record_bytes = 0; + for (size_t i = 0; i < pending.size(); ++i) { + offsets[i] = pending[i].record_offset; + lengths[i] = pending[i].record_size; + total_record_bytes += pending[i].record_size; + } + std::vector records(pending.size()); + std::vector statuses(pending.size()); + // The buffer limit is the requested bytes plus max_stored_bytes for the + // merged gaps (see ReadCoalescedRanges()). Reads past it are deferred + // (Incomplete). + ReadCoalescedRanges( + read_options, rep.file.get(), coalesce_bytes, max_stored_bytes, + total_record_bytes + + std::min(max_stored_bytes, + std::numeric_limits::max() - total_record_bytes), + pending.size(), offsets.data(), lengths.data(), records.data(), + statuses.data(), buffers, + /*num_reads=*/nullptr, /*read_bytes=*/nullptr); + + uint64_t successful_bytes = 0; + for (size_t i = 0; i < pending.size(); ++i) { + const Pending& entry = pending[i]; + StoredSameFileBlobReadRequest& req = reqs[entry.request_index]; + req.status = statuses[i]; + if (req.status.ok()) { + req.status = VerifySimpleGen2BlobRecord( + rep.footer.checksum_type(), rep.footer.base_context_checksum(), + records[i], entry.payload_size, entry.record_offset, + req.blob_index.compression(), rep.file->file_name()); + } + if (!req.status.ok()) { + continue; + } + const Slice stored(records[i].data(), entry.payload_size); + if (BlobIndex::ComputeStoredChecksum( + stored, req.blob_index.compression()) != req.stored_checksum) { + req.status = + Status::Corruption("Stored checksum mismatch while relocating blob"); + continue; + } + if (req.stored_payload != nullptr) { + *req.stored_payload = stored; + } + successful_bytes += entry.record_size; + } + if (bytes_read != nullptr) { + *bytes_read = successful_bytes; + } +} + +Status BlockBasedTable::StreamStoredSameFileBlob( + const ReadOptions& read_options, const BlobIndex& blob_index, + uint32_t stored_checksum, const BlobChunkConsumer& consumer) const { + size_t payload_size = 0; + size_t record_size = 0; + Status status = + ValidateEmbeddedBlobIndex(blob_index, &payload_size, &record_size); + if (!status.ok()) { + return status; + } + // Validate the trailer before delivery. The stored CRC, checked while + // streaming, verifies the payload whatever checksum type the file uses. + std::array trailer; + IOOptions options; + IODebugContext debug_context; + status = rep_->file->PrepareIOOptions(read_options, options, &debug_context); + Slice trailer_slice; + if (status.ok()) { + status = rep_->file->Read(blob_index.offset() + payload_size, + trailer.size(), options, &trailer_slice, + trailer.data(), &debug_context); + } + if (!status.ok()) { + return status; + } + if (trailer_slice.size() != trailer.size() || + static_cast(trailer_slice[0]) != + blob_index.compression()) { + return Status::Corruption("Invalid streaming blob trailer"); + } + if (rep_->footer.checksum_type() == kCRC32c && + DecodeFixed32(trailer_slice.data() + 1) != + crc32c::Mask(stored_checksum) + + ChecksumModifierForContext(rep_->footer.base_context_checksum(), + blob_index.offset())) { + return Status::Corruption("Streaming blob trailer checksum mismatch"); + } + return StreamAndVerifyBlobStoredPayload( + read_options, rep_->file.get(), blob_index.offset(), payload_size, + blob_index.compression(), stored_checksum, consumer); +} + Status BlockBasedTable::MaybeResolveEmbeddedValue( const ReadOptions& read_options, const Slice& internal_key, const Slice& value, std::string* resolved_internal_key, @@ -2087,6 +2718,538 @@ std::shared_ptr BlockBasedTable::GetTableProperties() return rep_->table_properties; } +namespace { + +// Reads one metadata block with checksum verification, without filling the +// block cache and without the persistent cache. Data blocks may be +// compressed; the index is not. +// Blob reference metadata blocks have no block type of their own. They use +// BlockType::kUserDefinedIndex, the type that GetBlockTypeForMetaBlockByName() +// gives the reference index block (kBlobReferenceIndexBlockName). +Status ReadBlobReferenceMetadataBlock(const BlockBasedTable::Rep& rep, + const ReadOptions& caller_read_options, + const BlockHandle& handle, + bool maybe_compressed, + BlockContents* contents) { + // No persistent cache, also for a reader from the table cache, so census + // reads and indirect blob GC reads never fill it (as fill_cache=false does + // for the block cache). + const PersistentCacheOptions no_persistent_cache; + ReadOptions read_options(caller_read_options); + read_options.fill_cache = false; + read_options.verify_checksums = true; + IOStatus io_status = + BlockFetcher(rep.file.get(), nullptr /* prefetch_buffer */, rep.footer, + read_options, handle, contents, rep.ioptions, + maybe_compressed /* do_uncompress */, maybe_compressed, + BlockType::kUserDefinedIndex, + maybe_compressed ? rep.decompressor.get() : nullptr, + no_persistent_cache) + .ReadBlockContents(); + return io_status; +} + +// Checks the descriptor against the table property, then reads and decodes +// the reference index. +Status ReadBlobReferenceIndex(const BlockBasedTable::Rep& rep, + const ReadOptions& read_options, + const Slice& expected_descriptor, + BlobReferenceMetadataDescriptor* descriptor, + BlobReferenceIndex* index) { + if (rep.table_properties == nullptr) { + return Status::Corruption("Missing table properties"); + } + const auto property = rep.table_properties->user_collected_properties.find( + kBlobReferenceDescriptorPropertyName); + if (property == rep.table_properties->user_collected_properties.end()) { + return Status::Corruption("Missing required blob reference descriptor"); + } + if (Slice(property->second) != expected_descriptor) { + return Status::Corruption("Blob reference descriptor mismatch"); + } + Status status = BlobReferenceMetadataDescriptor::DecodeFrom( + expected_descriptor, descriptor); + if (!status.ok()) { + return status; + } + BlockContents index_contents; + status = ReadBlobReferenceMetadataBlock( + rep, read_options, descriptor->index_handle, + /*maybe_compressed=*/false, &index_contents); + if (!status.ok()) { + return status; + } + return DecodeBlobReferenceIndex(index_contents.data, *descriptor, index); +} + +// Helper of BlockBasedTable::ReadBlobReferences(). It reads the reference data +// blocks `blocks` (block ordinals in increasing order) and calls `visit` for +// each one in order with its uncompressed contents. The contents stay valid +// only during the call. +// - Batches: the reads go through MultiRead in batches of at most +// kBlobReferenceReadBatchBytes (256 KB) of on-disk bytes (block plus +// trailer). A block larger than the batch budget returns Corruption. With +// mmap reads or direct I/O, the blocks are read one at a time instead. +// - Requests: each run of adjacent blocks in a batch is one read request. +// - Checks: each block is checked with its trailer checksum. A block that +// fails its checksum or decompression is passed to `visit` with an empty +// Slice and its Corruption status. Other read errors are returned. +Status ReadBlobReferenceDataBlocks( + const BlockBasedTable::Rep& rep, const ReadOptions& read_options, + const BlobReferenceIndex& index, const std::vector& blocks, + const std::function& visit) { + RandomAccessFileReader* const file = rep.file.get(); + if (rep.ioptions.allow_mmap_reads || file->use_direct_io()) { + // With mmap reads or direct I/O, read the same blocks one at a time + // through the normal block read path (ReadBlobReferenceMetadataBlock()) + // instead of the batched MultiRead path below. + for (size_t block : blocks) { + BlockContents contents; + Status status = + ReadBlobReferenceMetadataBlock(rep, read_options, index.blocks[block], + /*maybe_compressed=*/true, &contents); + if (!status.ok() && !status.IsCorruption()) { + return status; + } + status = visit(block, status.ok() ? contents.data : Slice(), status); + if (!status.ok()) { + return status; + } + } + return Status::OK(); + } + const auto on_disk_size = [&](size_t block) { + return static_cast( + BlockBasedTable::BlockSizeWithTrailer(index.blocks[block])); + }; + size_t batch_budget = kBlobReferenceReadBatchBytes; + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTable::ReadBlobReferenceDataBlocks:BatchBytes", &batch_budget); + std::unique_ptr buffer(new char[batch_budget]); + size_t next = 0; + while (next < blocks.size()) { + // One batch: as many blocks as fit in the buffer, at least one. + const size_t batch_begin = next; + size_t batch_bytes = 0; + while (next < blocks.size() && + batch_bytes + on_disk_size(blocks[next]) <= batch_budget) { + batch_bytes += on_disk_size(blocks[next]); + ++next; + } + if (next == batch_begin) { + return Status::Corruption("Blob reference data block is too large"); + } + // One read request per run of adjacent blocks. + std::vector requests; + std::vector request_of_block; + std::vector buffer_offset_of_block; + size_t buffer_offset = 0; + for (size_t i = batch_begin; i < next; ++i) { + const BlockHandle& handle = index.blocks[blocks[i]]; + if (requests.empty() || + requests.back().offset + requests.back().len != handle.offset()) { + FSReadRequest request; + request.offset = handle.offset(); + request.len = 0; + request.scratch = buffer.get() + buffer_offset; + requests.push_back(std::move(request)); + } + requests.back().len += on_disk_size(blocks[i]); + request_of_block.push_back(requests.size() - 1); + buffer_offset_of_block.push_back(buffer_offset); + buffer_offset += on_disk_size(blocks[i]); + } + TEST_SYNC_POINT_CALLBACK( + "BlockBasedTable::ReadBlobReferenceDataBlocks:MultiRead", &requests); + IOOptions io_options; + IODebugContext dbg; + IOStatus io_status = file->PrepareIOOptions(read_options, io_options, &dbg); + if (io_status.ok()) { + io_status = file->MultiRead(io_options, requests.data(), requests.size(), + /*aligned_buf=*/nullptr, &dbg); + } + if (!io_status.ok()) { + return io_status; + } + for (size_t i = batch_begin; i < next; ++i) { + const size_t block = blocks[i]; + const BlockHandle& handle = index.blocks[block]; + const FSReadRequest& request = + requests[request_of_block[i - batch_begin]]; + if (!request.status.ok()) { + return request.status; + } + const size_t offset_in_request = + buffer_offset_of_block[i - batch_begin] - + static_cast(request.scratch - buffer.get()); + Status status; + if (request.result.size() != request.len) { + status = Status::Corruption("Truncated blob reference block read"); + } + const char* const data = request.result.data() + offset_in_request; + const size_t size = static_cast(handle.size()); + if (status.ok()) { + status = + VerifyBlockChecksum(rep.footer, data, size, file->file_name(), + handle.offset(), BlockType::kUserDefinedIndex); + } + BlockContents contents; + if (status.ok()) { + const CompressionType type = static_cast( + static_cast(data[size])); + if (type == kNoCompression) { + contents = BlockContents(Slice(data, size)); + } else if (rep.decompressor == nullptr) { + status = Status::Corruption( + "Compressed blob reference block without a decompressor"); + } else { + status = DecompressSerializedBlock( + data, size, type, *rep.decompressor, &contents, rep.ioptions); + } + } + if (!status.ok() && !status.IsCorruption()) { + return status; + } + status = visit(block, status.ok() ? contents.data : Slice(), status); + if (!status.ok()) { + return status; + } + } + } + return Status::OK(); +} + +// Decode state of one request of a batch read. +struct BlobReferenceGroupReadState { + // The first and last data block that hold this group, as positions in + // index.blocks. + size_t first_block = 0; + size_t last_block = 0; + // The output size when this request started to append, valid once + // `started` is true. + size_t output_begin = 0; + std::optional decoder; + bool started = false; + // True when the request has finished, with success or with an error. The + // block visitor skips it. + bool done = false; +}; + +} // namespace + +Status BlockBasedTable::ReadBlobReferenceGroups( + const ReadOptions& read_options, const Slice& expected_descriptor, + const std::function& before_allocation, + std::vector* groups) const { + if (groups == nullptr) { + return Status::InvalidArgument("Null blob reference group output"); + } + if (rep_ == nullptr) { + return Status::Corruption("Missing block-based table reader state"); + } + // The index holds every group total, so no data block is read. The decoder + // checks that the totals add up to the descriptor. + BlobReferenceMetadataDescriptor descriptor; + BlobReferenceIndex index; + Status status = ReadBlobReferenceIndex( + *rep_, read_options, expected_descriptor, &descriptor, &index); + return status.ok() + ? CopyIndirectBlobReferenceGroups(index, before_allocation, groups) + : status; +} + +namespace { +// Appends to a reserved vector, for requests without a sink. +class VectorBlobReferenceSink : public BlobReferenceSink { + public: + explicit VectorBlobReferenceSink(std::vector* output) + : output_(output) {} + size_t size() const override { return output_->size(); } + size_t Room() const override { return output_->capacity() - output_->size(); } + Status Append(const BlobReference& reference) override { + if (output_->size() == output_->capacity()) { + return Status::Corruption("Blob reference group count mismatch"); + } + output_->push_back(reference); + return Status::OK(); + } + void Truncate(size_t size) override { output_->resize(size); } + const void* identity() const override { return output_; } + + private: + std::vector* output_; +}; + +// Reads the exact groups `reads` (sorted by key, unique) from the decoded +// `index` into `sinks`. +Status ReadIndexedBlobReferenceGroups( + const BlockBasedTable::Rep& rep, const ReadOptions& read_options, + const BlobReferenceIndex& index, std::vector& reads, + const std::vector& sinks) { + // Merge the sorted requests with the sorted index groups, and collect the + // blocks that the found groups need. A request whose group is absent fails + // with NotFound. A request whose group does not fit its output fails with + // Aborted. A group ends in the block where the next group starts. If the next + // group starts at the beginning of its block (run_offset 0), the group ends + // in the block before. The last group ends in the last data block. + // (start_block and run_offset are fields of BlobReferenceIndexEntry in + // db/blob/blob_reference_metadata.h.) + // + // For example, group G starts in block 3, and the next group H starts in + // block 4. There are two cases: + // + // (a) H starts inside block 4 (b) H starts at the first byte of + // (run_offset > 0): block 4 (run_offset 0): + // G ends in block 4. G ends in block 3. + // + // | block 3 | block 4 | | block 3 | block 4 | + // | [G=======|===][H== | | [G======]|[H=========| + std::vector states(reads.size()); + std::vector blocks; + // Several requests can share one sink: a whole-generation read becomes one + // request per group, and indirect blob GC reads all generation numbers of one + // input into one live set. Requests that share one sink share its room. + // Before any decode, each request is checked against the room left after the + // earlier requests of that sink. A request that does not fit gets Aborted and + // adds nothing. + UnorderedMap planned_by_sink; + size_t group = 0; + for (size_t i = 0; i < reads.size(); ++i) { + const BlobReferenceGroup& requested = *reads[i].group; + while (group < index.groups.size() && + index.groups[group].group.group_key() < requested.group_key()) { + ++group; + } + BlobReferenceGroupReadState& state = states[i]; + state.done = true; + if (group == index.groups.size() || + !(index.groups[group].group.group_key() == requested.group_key())) { + reads[i].status = Status::NotFound("Blob reference group is absent"); + continue; + } + const BlobReferenceIndexEntry& entry = index.groups[group]; + reads[i].found_reference_count = entry.group.reference_count; + reads[i].found_stored_value_bytes = entry.group.stored_value_bytes; + reads[i].found_record_bytes = entry.group.record_bytes; + uint64_t& planned = planned_by_sink[sinks[i]->identity()]; + const uint64_t room = sinks[i]->Room(); + if (entry.group.reference_count > + room - std::min(planned, room)) { + reads[i].status = + Status::Aborted("Blob reference group exceeds the reserved output"); + continue; + } + planned += entry.group.reference_count; + state.decoder.emplace(index, group); + state.first_block = entry.start_block; + if (group + 1 < index.groups.size()) { + const BlobReferenceIndexEntry& next = index.groups[group + 1]; + state.last_block = + next.run_offset == 0 ? next.start_block - 1 : next.start_block; + } else { + state.last_block = index.blocks.size() - 1; + } + state.done = false; + for (size_t block = state.first_block; block <= state.last_block; ++block) { + if (blocks.empty() || blocks.back() < block) { + blocks.push_back(block); + } + } + } + + // Fails one request. A request that started to append drops only what it + // appended (its output goes back to output_begin). A request that fails + // before it appended anything leaves the output unchanged. Several requests, + // or several calls, can share one output, so the references that earlier + // requests or the caller appended always stay. A request that is cut short + // fails in its last block (see below), before any later request of the same + // output starts. So a failure never removes a later request's references. + const auto fail = [&](size_t i, const Status& error) { + reads[i].status = error; + if (states[i].started) { + sinks[i]->Truncate(states[i].output_begin); + } + states[i].done = true; + }; + size_t first_open = 0; + Status status = ReadBlobReferenceDataBlocks( + rep, read_options, index, blocks, + [&](size_t block, const Slice& contents, const Status& block_status) { + // Requests are sorted by group key, so their first blocks do not + // decrease. first_open is the first request that has not finished. The + // loop stops at the first request that starts after this block. + for (size_t i = first_open; + i < reads.size() && + (states[i].done || states[i].first_block <= block); + ++i) { + BlobReferenceGroupReadState& state = states[i]; + if (state.done || state.last_block < block) { + continue; + } + if (!block_status.ok()) { + fail(i, block_status); + continue; + } + if (block == state.first_block) { + // Earlier requests or calls may have appended to the same output; + // this request starts after them. + state.output_begin = sinks[i]->size(); + state.started = true; + } + BlobReferenceSink& output = *sinks[i]; + // BlobReferenceGroupDecoder does all format checks of the group. This + // loop only checks that the decoder expects this block and that the + // output has room. + Status run_status = + state.decoder->next_block() != block + ? Status::Corruption("Blob reference group is cut short") + : state.decoder->DecodeRun( + contents, [&](const BlobReference& reference) { + if (output.Room() == 0) { + return Status::Corruption( + "Blob reference group count mismatch"); + } + return output.Append(reference); + }); + if (run_status.ok() && !state.decoder->done() && + block == state.last_block) { + // The group's last block ended while references are still + // missing. Fail the request at its last block, so that it is + // marked done and first_open can move past it. The loop at the end + // of this function would fail it the same way. + run_status = + Status::Corruption("Blob reference group is cut short"); + } + if (run_status.ok() && state.decoder->done()) { + state.done = true; + } + if (!run_status.ok()) { + fail(i, run_status); + } + } + while (first_open < reads.size() && states[first_open].done) { + ++first_open; + } + return Status::OK(); + }); + if (!status.ok()) { + for (size_t i = 0; i < reads.size(); ++i) { + if (!states[i].done) { + fail(i, status); + } + } + return status; + } + for (size_t i = 0; i < reads.size(); ++i) { + if (!states[i].done) { + fail(i, Status::Corruption("Blob reference group is cut short")); + } + } + return Status::OK(); +} + +} // namespace + +Status BlockBasedTable::ReadBlobReferences( + const ReadOptions& read_options, const Slice& expected_descriptor, + const std::function& after_index_read, + std::vector* requests) const { + if (requests == nullptr || rep_ == nullptr) { + return Status::InvalidArgument("Missing blob reference batch state"); + } + std::vector& reads = *requests; + std::vector vector_sinks; + vector_sinks.reserve(reads.size()); + std::vector sinks(reads.size()); + const bool whole_generation = !reads.empty() && reads[0].whole_generation; + for (size_t i = 0; i < reads.size(); ++i) { + // Request i - 1 passed this check, so its group is not null. + const bool out_of_order = + i > 0 && reads[i].group != nullptr && + (whole_generation ? reads[i - 1].group->generation_file_number >= + reads[i].group->generation_file_number + : !(reads[i - 1].group->group_key() < + reads[i].group->group_key())); + if (reads[i].group == nullptr || + (reads[i].output == nullptr && reads[i].sink == nullptr) || + reads[i].whole_generation != whole_generation || out_of_order) { + return Status::InvalidArgument( + "Blob reference batch requests must be sorted and unique"); + } + if (reads[i].sink != nullptr) { + sinks[i] = reads[i].sink; + } else { + vector_sinks.emplace_back(reads[i].output); + sinks[i] = &vector_sinks.back(); + } + reads[i].status = Status::OK(); + } + + BlobReferenceMetadataDescriptor descriptor; + BlobReferenceIndex index; + Status status = ReadBlobReferenceIndex( + *rep_, read_options, expected_descriptor, &descriptor, &index); + if (!status.ok()) { + return status; + } + if (after_index_read) { + status = after_index_read( + sizeof(index) + index.blocks.capacity() * sizeof(BlockHandle) + + index.groups.capacity() * sizeof(BlobReferenceIndexEntry)); + if (!status.ok()) { + return status; + } + } + if (!whole_generation) { + return ReadIndexedBlobReferenceGroups(*rep_, read_options, index, reads, + sinks); + } + + // Expand each generation into its exact groups, in index order (sorted by + // key), then add the results up per generation. + std::vector group_reads; + std::vector group_sinks; + std::vector owners; + for (const BlobReferenceIndexEntry& entry : index.groups) { + const auto request = std::lower_bound( + reads.begin(), reads.end(), entry.group.generation_file_number, + [](const BlobReferenceGroupRead& read, uint64_t generation) { + return read.group->generation_file_number < generation; + }); + if (request == reads.end() || request->group->generation_file_number != + entry.group.generation_file_number) { + continue; + } + const size_t owner = static_cast(request - reads.begin()); + BlobReferenceGroupRead group_read; + group_read.group = &entry.group; + group_read.sink = sinks[owner]; + group_reads.push_back(std::move(group_read)); + group_sinks.push_back(sinks[owner]); + owners.push_back(owner); + } + std::vector found(reads.size(), false); + status = ReadIndexedBlobReferenceGroups(*rep_, read_options, index, + group_reads, group_sinks); + for (size_t i = 0; i < group_reads.size(); ++i) { + BlobReferenceGroupRead& read = reads[owners[i]]; + found[owners[i]] = true; + read.found_reference_count += group_reads[i].found_reference_count; + read.found_stored_value_bytes += group_reads[i].found_stored_value_bytes; + read.found_record_bytes += group_reads[i].found_record_bytes; + if (read.status.ok() && !group_reads[i].status.ok()) { + read.status = group_reads[i].status; + } + } + for (size_t i = 0; i < reads.size(); ++i) { + if (!found[i]) { + reads[i].status = Status::NotFound("Blob reference generation is absent"); + } + } + return status; +} + const SeqnoToTimeMapping& BlockBasedTable::GetSeqnoToTimeMapping() const { return rep_->seqno_to_time_mapping; } @@ -2133,8 +3296,9 @@ Status BlockBasedTable::ReadMetaIndexBlock( rep_->file.get(), prefetch_buffer, rep_->footer, ro, rep_->footer.metaindex_handle(), &metaindex, rep_->ioptions, rep_->create_context, true /*maybe_compressed*/, rep_->decompressor.get(), - rep_->persistent_cache_options, GetMemoryAllocator(rep_->table_options), - false /* for_compaction */, false /* async_read */); + rep_->metadata_persistent_cache_options, + GetMemoryAllocator(rep_->table_options), false /* for_compaction */, + false /* async_read */); if (!s.ok()) { ROCKS_LOG_ERROR(rep_->ioptions.logger, @@ -2811,6 +3975,7 @@ InternalIterator* BlockBasedTable::NewIteratorImpl( Arena* arena, bool skip_filters, TableReaderCaller caller, size_t compaction_readahead_size, bool allow_unprepared_value, const Slice* seek_target) { + MarkReadStarted(); BlockCacheLookupContext lookup_context{caller}; bool need_upper_bound_check = read_options.auto_prefix_mode || PrefixExtractorChanged(prefix_extractor); @@ -2846,9 +4011,12 @@ InternalIterator* BlockBasedTable::NewIteratorImpl( /*input_iter=*/nullptr, /*get_context=*/nullptr, &lookup_context)); // Same-file ("embedded") blob references are resolved by a wrapper iterator // so the block-based iterator never exposes an unresolved BlobIndex. The SST - // dump tool must keep seeing raw BlobIndex values, so it is excluded. + // dump tool and the blob relocation reader must keep seeing raw values, so + // they are excluded. const bool resolve_embedded_values = - caller != TableReaderCaller::kSSTDumpTool && HasEmbeddedBlobRecords(); + caller != TableReaderCaller::kSSTDumpTool && + caller != TableReaderCaller::kBlobRelocationFileRead && + HasEmbeddedBlobRecords(); // When wrapping, the inner iterator must materialize each data block (it must // not defer to the index's first key), so force allow_unprepared_value off; // the wrapper re-introduces value laziness above it. @@ -3294,6 +4462,15 @@ BlockType BlockBasedTable::GetBlockTypeForMetaBlockByName( return BlockType::kUserDefinedIndex; } + // The blob reference index block has no BlockType of its own. + // VerifyChecksumInMetaBlocks() reads every block in the meta-index and needs + // a type for each (an unknown name fails the assert below). + // kUserDefinedIndex blocks are kept as plain bytes (Block_kUserDefinedIndex + // in block_cache.h is a BlockContents), so no code parses them as a Block. + if (meta_block_name == kBlobReferenceIndexBlockName) { + return BlockType::kUserDefinedIndex; + } + if (meta_block_name.starts_with(kObsoleteFilterBlockPrefix)) { // Obsolete but possible in old files return BlockType::kInvalid; @@ -3334,12 +4511,12 @@ Status BlockBasedTable::VerifyChecksumInMetaBlocks( // if it was checked on open. } else { // FIXME? Need to verify checksums of index and filter partitions? - s = BlockFetcher(rep_->file.get(), nullptr /* prefetch buffer */, - rep_->footer, read_options, handle, &contents, - rep_->ioptions, false /* decompress */, - false /*maybe_compressed*/, - GetBlockTypeForMetaBlockByName(meta_block_name), - nullptr /*decompressor*/, rep_->persistent_cache_options) + s = BlockFetcher( + rep_->file.get(), nullptr /* prefetch buffer */, rep_->footer, + read_options, handle, &contents, rep_->ioptions, + false /* decompress */, false /*maybe_compressed*/, + GetBlockTypeForMetaBlockByName(meta_block_name), + nullptr /*decompressor*/, rep_->metadata_persistent_cache_options) .ReadBlockContents(); } if (!s.ok()) { @@ -4028,4 +5205,139 @@ void BlockBasedTable::MarkObsolete(uint32_t uncache_aggressiveness) { rep_->uncache_aggressiveness.StoreRelaxed(uncache_aggressiveness); } +// After the InitBlockIterator specializations, which it uses. +Status BlockBasedTable::PinDataBlocks(const ReadOptions& read_options, + PinnedDataBlocks* pinned) const { + assert(pinned != nullptr); + // The caller's options say how to do I/O (checksums, rate limiter, I/O + // activity). The options below must not depend on the caller, because the + // blocks stay pinned after this call returns: + // - fill_cache and kReadAllTier: every block is read and charged to the + // block cache, even when the caller's read does not fill the cache or + // reads only cached blocks. + // - No deadline or I/O timeout: the reader stays in the blob file cache, so + // a pin that fails on one caller's time limit would leave the reader + // without kept blocks for as long as it is open. + // - No read-scoped buffer provider: such a buffer is freed when the + // caller's read ends, but a pinned block must live as long as the table. + ReadOptions pin_options = read_options; + pin_options.fill_cache = true; + pin_options.read_tier = kReadAllTier; + pin_options.deadline = std::chrono::microseconds::zero(); + pin_options.io_timeout = std::chrono::microseconds::zero(); + pin_options.read_scoped_block_buffer_provider = nullptr; + // The same caller as in MultiGetRawEntries. + BlockCacheLookupContext lookup_context{ + TableReaderCaller::kBlobRelocationFileRead}; + IndexBlockIter iiter_on_stack; + InternalIteratorBase* const iiter = NewIndexIterator( + pin_options, /*need_upper_bound_check=*/false, &iiter_on_stack, + /*get_context=*/nullptr, &lookup_context); + std::unique_ptr> iiter_owner; + if (iiter != &iiter_on_stack) { + iiter_owner.reset(iiter); + } + // Read the blocks in batches of up to MultiGetContext::MAX_BATCH_SIZE (32) + // blocks (see RetrieveDataBlocks()), so a cold open does not do one read per + // block. + const bool has_block_cache = rep_->table_options.block_cache != nullptr; + autovector handles; + std::array statuses; + const auto retrieve_batch = [&]() { + const size_t first = pinned->blocks.size(); + pinned->blocks.resize(first + handles.size()); + RetrieveDataBlocks(pin_options, handles, &lookup_context, + pinned->blocks.data() + first, statuses.data()); + // The loop below stops at the first failed block, so the statuses after it + // can stay unread. + for (const Status& status : statuses) { + status.PermitUncheckedError(); + } + for (size_t b = 0; b < handles.size(); ++b) { + if (!statuses[b].ok()) { + return statuses[b]; + } + const CachableEntry& block = pinned->blocks[first + b]; + if (block.GetValue() == nullptr) { + return Status::Corruption("Data block to pin has no contents"); + } + // A block that the block cache did not take is not charged. A block + // that does not own its bytes points into the memory-mapped file + // (allow_mmap_reads and no compression). The block cache never takes + // such a block. Its bytes are in the mapped file, not on the heap, and + // the mapping lives as long as the table, so the block is kept. + if (has_block_cache && !block.IsCached() && + block.GetValue()->own_bytes()) { + return Status::Incomplete( + "Block cache cannot hold a pinned data block"); + } + pinned->memory_bytes += block.GetValue()->ApproximateMemoryUsage(); + } + handles.clear(); + return Status::OK(); + }; + for (iiter->SeekToFirst(); iiter->Valid(); iiter->Next()) { + const Slice index_key = iiter->key(); + pinned->last_user_keys.emplace_back( + rep_->index_key_includes_seq ? ExtractUserKey(index_key).ToString() + : index_key.ToString()); + handles.push_back(iiter->value().handle); + if (handles.size() == MultiGetContext::MAX_BATCH_SIZE) { + const Status status = retrieve_batch(); + if (!status.ok()) { + return status; + } + } + } + if (!iiter->status().ok()) { + return iiter->status(); + } + return handles.empty() ? Status::OK() : retrieve_batch(); +} + +void BlockBasedTable::MultiGetRawEntriesPinnedImpl( + const PinnedDataBlocks& pinned, size_t num_keys, const Slice* keys, + RawEntryCallback on_entry, const void* on_entry_arg) const { + MarkReadStarted(); + assert(pinned.last_user_keys.size() == pinned.blocks.size()); + const Comparator* const user_comparator = + rep_->internal_comparator.user_comparator(); + constexpr size_t kNoBlock = std::numeric_limits::max(); + size_t iter_block = kNoBlock; + DataBlockIter biter; + for (size_t i = 0; i < num_keys; ++i) { + assert(i == 0 || + rep_->internal_comparator.Compare(keys[i - 1], keys[i]) <= 0); + const Slice user_key = ExtractUserKey(keys[i]); + // The first block whose last key is at or after the key holds the first + // entry at or after it. + const auto found = std::lower_bound( + pinned.last_user_keys.begin(), pinned.last_user_keys.end(), user_key, + [user_comparator](const std::string& last_key, const Slice& key) { + return user_comparator->Compare(last_key, key) < 0; + }); + if (found == pinned.last_user_keys.end()) { + on_entry(on_entry_arg, i, Status::NotFound(), Slice(), Slice()); + continue; + } + const size_t b = static_cast(found - pinned.last_user_keys.begin()); + if (b != iter_block) { + // The block stays pinned by `pinned`, so the iterator must not take its + // cache handle (NewDataBlockIterator() would move it). + biter.Invalidate(Status::OK()); + InitBlockIterator(rep_, pinned.blocks[b].GetValue(), + BlockType::kData, &biter, + /*block_contents_pinned=*/true); + iter_block = b; + } + if (!biter.SeekForGet(keys[i]) || !biter.Valid()) { + on_entry(on_entry_arg, i, + biter.status().ok() ? Status::NotFound() : biter.status(), + Slice(), Slice()); + continue; + } + on_entry(on_entry_arg, i, Status::OK(), biter.key(), biter.value()); + } +} + } // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_reader.h b/table/block_based/block_based_table_reader.h index de7c9d657e1c..812b22a8d79e 100644 --- a/table/block_based/block_based_table_reader.h +++ b/table/block_based/block_based_table_reader.h @@ -9,13 +9,16 @@ #pragma once +#include #include #include #include +#include #include "cache/cache_entry_roles.h" #include "cache/cache_key.h" #include "cache/cache_reservation_manager.h" +#include "db/blob/blob_gen2_format.h" #include "db/blob/same_file_blob_reader.h" #include "db/range_tombstone_fragmenter.h" #include "db/seqno_to_time_mapping.h" @@ -152,8 +155,8 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { // this table charge their memory to it. // @param force_direct_prefetch if true, always prefetching to RocksDB // buffer, rather than calling RandomAccessFile::Prefetch(). - // @param avoid_shared_metadata_cache if true, open-time index/filter/ - // dictionary reads must not insert into the shared block cache. + // @param avoid_shared_metadata_cache if true, metadata reads avoid shared + // block and persistent caches. static Status Open( const ReadOptions& ro, const ImmutableOptions& ioptions, const EnvOptions& env_options, @@ -228,7 +231,21 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { // by the embedded blob stats property. bool HasEmbeddedBlobRecords() const; - // Reads a single same-file blob record referenced by `blob_index`. + // Removes one user-collected property from this reader's table properties, to + // save memory when the caller keeps a decoded copy. It replaces the shared + // properties object with a copy, so call it right after open, before the + // reader is shared. Debug builds assert that no read has started. No-op when + // the property is not present. + void DropUserCollectedProperty(const std::string& name); + + // Remembers that a read has started, in debug builds only, so that + // DropUserCollectedProperty() can assert that it runs before any read. The + // read entry points call it. Release builds do nothing. + void MarkReadStarted() const; + + // Reads a single same-file blob record referenced by `blob_index` and copies + // the value into `value`. Decompresses it when the index says it is + // compressed. Incomplete for a kBlockCacheTier read. Status ResolveEmbeddedBlob(const ReadOptions& read_options, const BlobIndex& blob_index, std::string* value) const; @@ -236,10 +253,25 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { // Like ResolveEmbeddedBlob, but reads the payload into a heap buffer owned by // `value` (via a registered cleanup) and pins it, avoiding a copy. Use on the // Get()/MultiGet() path so the resolved value can be pinned into the output. + // Decompresses into a new buffer when the index says it is compressed. Status ResolveEmbeddedBlobPinned(const ReadOptions& read_options, const BlobIndex& blob_index, PinnableSlice* value) const; + // Takes the stored payload in `stored` and returns the value in `*contents`: + // `stored` itself when uncompressed (the trailer stays unused at its end), + // else a new buffer from `allocator` with the decompressed value. + Status DecompressEmbeddedBlob(CacheAllocationPtr stored, size_t payload_size, + CompressionType compression, + MemoryAllocator* allocator, + std::unique_ptr* contents) const; + + // DecompressEmbeddedBlob, then pins the whole value into `value`. + Status DecompressAndPinEmbeddedBlob(CacheAllocationPtr stored, + size_t payload_size, + CompressionType compression, + PinnableSlice* value) const; + // Like ResolveEmbeddedBlobPinned, but routes the read through the CFD's // BlobSource so the payload is served from / inserted into the blob value // cache and BLOB_DB_* statistics are recorded. BlobSource derives the cache @@ -255,9 +287,8 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { // [range_offset, range_offset + range_length) of an uncompressed same-file // payload (skipping the trailer + checksum verification, and never inserting // into the blob cache). The caller must ensure range_offset + range_length <= - // blob_index.size(). Requires a BlobSource (rep_->blob_source_ != nullptr): - // byte-range embedded reads are only issued on the lazy read path, which - // always has one, so there is no direct-pinned range variant. + // blob_index.size(). Requires a BlobSource (rep_->blob_source_ != nullptr); + // readers without one use ReadSimpleGen2BlobRange in GetSameFileBlob. Status ResolveEmbeddedBlobRangeCached(const ReadOptions& read_options, const BlobIndex& blob_index, uint64_t range_offset, @@ -266,23 +297,134 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { // SameFileBlobReader: resolve a same-file blob reference (whole value or a // sub-range) for GetContext's zero-copy wide-column resolution on the - // Get()/MultiGet() path and for the lazy read path. A whole-value read routes - // through the blob cache (ResolveEmbeddedBlobCached) when a BlobSource is - // wired, else a direct pinned read (ResolveEmbeddedBlobPinned); a sub-range - // read routes through ResolveEmbeddedBlobRangeCached (requires a BlobSource). + // Get()/MultiGet() path and for the lazy read path. Cases: + // - Whole value: through the blob cache (ResolveEmbeddedBlobCached) when a + // BlobSource is wired, else a direct pinned read + // (ResolveEmbeddedBlobPinned). + // - Uncompressed range with a BlobSource: ResolveEmbeddedBlobRangeCached. + // - Uncompressed range without a BlobSource: a raw ReadSimpleGen2BlobRange, + // no checksum, no cache. + // - Compressed range: returns Corruption, as in base RocksDB. A byte range of + // a compressed record cannot be decompressed alone, so callers read the + // whole value and slice it. // See BlobVerifyPolicy / kWholeBlobLength for the range + verify semantics. Status GetSameFileBlob(const ReadOptions& read_options, const BlobIndex& blob_index, uint64_t range_offset, size_t range_length, BlobVerifyPolicy verify_policy, PinnableSlice* value) const override; - // Batched counterpart of GetSameFileBlob: coalesces the cache-miss reads of - // multiple embedded (same-file) records in this SST into a single MultiRead - // (one for whole records, one for sub-ranges). Force-verify reads - // (kVerifyIfPresent) fall back to per-record GetSameFileBlob. + // Batched counterpart of GetSameFileBlob. + // - With a BlobSource (DB-owned SSTs): forced-verify requests + // (kVerifyIfPresent) take the scalar GetSameFileBlob path. The other whole + // records and uncompressed sub-ranges go to BlobSource's batched reads, + // which use the blob cache. + // - Without a BlobSource (standalone readers, such as relocation tables): + // whole records are read in at most two batches, one with the caller's + // checksum setting and one with forced verification. Uncompressed ranges + // are read raw, without a checksum check, and a range is never turned into + // a whole read. Callers send only strict parts of a value (see + // ReadPathBlobResolver::ResolveColumnRange). A kBlockCacheTier read returns + // Incomplete. + // In both cases a compressed range fails with Corruption, as in + // GetSameFileBlob(), before any other check (also for a kBlockCacheTier + // read). void MultiGetSameFileBlob(const ReadOptions& read_options, size_t num_reads, SameFileBlobReadRequest* reqs) const override; + // Raw lookups of `num_keys` internal keys, for relocation tables. The keys + // must be sorted. The caller (BlobRelocationFileReader) checks the + // entries. For each key, calls `on_entry(i, status, internal_key, value)` + // with one of: + // - the entry that DataBlockIter::SeekForGet finds in the key's data block + // (without a data block hash index, the first entry at or after keys[i]); + // - NotFound, when the key is past the last entry or SeekForGet finds no + // entry; + // - the error of the index or of the key's data block. + // The data blocks are first looked up in the block cache. The missing blocks + // are read with one read (a batched read when more than one block is missing) + // and inserted into the block cache, like the normal block read path. Keys in + // the same block share it. Values are raw: same-file blob indexes are not + // resolved. Requires num_keys <= MultiGetContext::MAX_BATCH_SIZE. `on_entry` + // is called through a plain function pointer, so a capturing lambda needs no + // heap allocation. + template + void MultiGetRawEntries(const ReadOptions& read_options, size_t num_keys, + const Slice* keys, const OnEntry& on_entry) const { + MultiGetRawEntriesImpl( + read_options, num_keys, keys, + [](const void* arg, size_t key_index, const Status& status, + const Slice& internal_key, const Slice& value) { + (*static_cast(arg))(key_index, status, internal_key, + value); + }, + &on_entry); + } + + // Like MultiGetSameFileBlob(), but used only by indirect blob GC: it returns + // the exact stored bytes. It checks the stored CRC and, unless the file uses + // kNoChecksum, the record checksum. It reads whole records with + // ReadCoalescedRanges() (gap `coalesce_bytes`, one read at most + // `max_stored_bytes`). Payloads point into `buffers`. Requests past + // `max_stored_bytes` of total stored bytes get MemoryLimit. A request past + // the buffer limit (see ReadCoalescedRanges()) is not read and gets + // Status::Incomplete. The caller retries it later. + void MultiGetStoredSameFileBlob(const ReadOptions& read_options, + uint64_t coalesce_bytes, size_t num_reads, + StoredSameFileBlobReadRequest* reqs, + uint64_t max_stored_bytes, + CoalescedReadBuffers* buffers, + uint64_t* bytes_read) const; + Status StreamStoredSameFileBlob(const ReadOptions& read_options, + const BlobIndex& blob_index, + uint32_t stored_checksum, + const BlobChunkConsumer& consumer) const; + + // Every data block of a table, read once and held in memory, with the last + // user key of each block. The relocation file reader keeps one when + // pin_blob_relocation_location_blocks is on, so its lookups need no index + // or block cache lookup (see PinDataBlocks() and MultiGetRawEntriesPinned()). + struct PinnedDataBlocks { + // The index key of each block. Relocation tables use no index key + // shortening, so it is the user key of the block's last entry. + std::vector last_user_keys; + // Each block, held through its block cache handle (so it is charged to + // the block cache and not evicted), or owned when the table has no block + // cache. With allow_mmap_reads and no compression, a block points into the + // memory-mapped file instead. + std::vector> blocks; + // The memory that the blocks use. + size_t memory_bytes = 0; + }; + + // Reads every data block of this table into `*pinned`. With a block cache, + // each block is held through its cache handle. `read_options` gives only + // the I/O settings: the blocks are always read and charged, with no time + // limit (see the comment in the function). Returns the error of a failed + // block read or block cache insert (for example MemoryLimit with + // strict_capacity_limit), and Incomplete if the block cache did not take a + // block that owns its bytes. On an error, the caller drops `*pinned` and + // uses MultiGetRawEntries() instead. + Status PinDataBlocks(const ReadOptions& read_options, + PinnedDataBlocks* pinned) const; + + // MultiGetRawEntries() over the blocks in `pinned`: a binary search over the + // last keys finds the block of each key, and a seek in that block finds the + // entry. It does no index lookup, block cache lookup or I/O. The keys must + // be sorted, and the results are the same as MultiGetRawEntries(). + template + void MultiGetRawEntriesPinned(const PinnedDataBlocks& pinned, size_t num_keys, + const Slice* keys, + const OnEntry& on_entry) const { + MultiGetRawEntriesPinnedImpl( + pinned, num_keys, keys, + [](const void* arg, size_t key_index, const Status& status, + const Slice& internal_key, const Slice& value) { + (*static_cast(arg))(key_index, status, internal_key, + value); + }, + &on_entry); + } + // If `value` is a same-file BlobIndex, materializes the referenced payload // and updates `resolved_internal_key` to the corresponding value type. Leaves // `resolved`. When `pinned_value` is non-null, a whole-value same-file blob @@ -392,6 +534,15 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { std::shared_ptr GetTableProperties() const override; + Status ReadBlobReferenceGroups( + const ReadOptions& read_options, const Slice& expected_descriptor, + const std::function& before_allocation, + std::vector* groups) const override; + Status ReadBlobReferences( + const ReadOptions& read_options, const Slice& expected_descriptor, + const std::function& after_index_read, + std::vector* requests) const override; + const SeqnoToTimeMapping& GetSeqnoToTimeMapping() const; size_t ApproximateMemoryUsage() const override; @@ -544,6 +695,32 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { void operator=(const TableReader&) = delete; private: + // MultiGetRawEntries calls `on_entry(on_entry_arg, ...)` for each key. + using RawEntryCallback = void (*)(const void* arg, size_t key_index, + const Status& status, + const Slice& internal_key, + const Slice& value); + void MultiGetRawEntriesImpl(const ReadOptions& read_options, size_t num_keys, + const Slice* keys, RawEntryCallback on_entry, + const void* on_entry_arg) const; + // Reads the data blocks `handles` (at most MultiGetContext::MAX_BATCH_SIZE) + // into `blocks`, with one status per block in `statuses`. Each block is + // looked up in the block cache first. Two or more missing blocks are read + // with one batched read (RetrieveMultipleBlocks()); one missing block takes + // the normal block read path. With allow_mmap_reads, RetrieveMultipleBlocks() + // reads the blocks one at a time. A kBlockCacheTier read fails a missing + // block with Incomplete. If the compression dictionary cannot be read, every + // block gets that error. MultiGetRawEntries() and PinDataBlocks() use it. + void RetrieveDataBlocks( + const ReadOptions& read_options, + const autovector& handles, + BlockCacheLookupContext* lookup_context, + CachableEntry* blocks, Status* statuses) const; + void MultiGetRawEntriesPinnedImpl(const PinnedDataBlocks& pinned, + size_t num_keys, const Slice* keys, + RawEntryCallback on_entry, + const void* on_entry_arg) const; + friend class MockedBlockBasedTable; friend class BlockBasedTableReaderTestVerifyChecksum_ChecksumMismatch_Test; BlockCacheTracer* const block_cache_tracer_; @@ -558,7 +735,6 @@ class BlockBasedTable : public TableReader, public SameFileBlobReader { size_t compaction_readahead_size, bool allow_unprepared_value, const Slice* seek_target); - void UpdateCacheHitMetrics(BlockType block_type, GetContext* get_context, size_t usage) const; void UpdateCacheMissMetrics(BlockType block_type, @@ -846,6 +1022,18 @@ struct BlockBasedTable::Rep { OffsetableCacheKey base_cache_key; PersistentCacheOptions persistent_cache_options; + // Persistent cache options for metadata blocks (index, filter, metaindex and + // the other non-data blocks). They are empty when the table was opened with + // avoid_shared_metadata_cache, so such a reader never adds metadata blocks to + // the persistent cache. Data blocks always use persistent_cache_options. + PersistentCacheOptions metadata_persistent_cache_options; + + // Returns the persistent cache options for a block of type `type`. + const PersistentCacheOptions& PersistentCacheFor(BlockType type) const { + return type == BlockType::kData ? persistent_cache_options + : metadata_persistent_cache_options; + } + // Footer contains the fixed table information Footer footer; @@ -912,6 +1100,11 @@ struct BlockBasedTable::Rep { bool index_key_includes_seq = true; bool index_value_is_full = true; +#ifndef NDEBUG + // Debug only: set by the first read (see BlockBasedTable::MarkReadStarted()). + mutable std::atomic read_started{false}; +#endif // NDEBUG + // Restart intervals read from table properties (0 if not available) uint32_t data_block_restart_interval = 0; uint32_t index_block_restart_interval = 0; @@ -930,6 +1123,9 @@ struct BlockBasedTable::Rep { // repair, external-file ingestion prevalidation, etc.); in that case // embedded reads fall back to a direct (uncached) read. BlobSource* blob_source_ = nullptr; + // Decompressor for compressed embedded blob records, created once at open + // when the SST has embedded blobs. + std::shared_ptr embedded_blob_decompressor; // Whether block checksums in metadata blocks were verified on open. // This is only to mostly maintain current dubious behavior of VerifyChecksum @@ -1023,6 +1219,12 @@ struct BlockBasedTable::Rep { } }; +inline void BlockBasedTable::MarkReadStarted() const { +#ifndef NDEBUG + rep_->read_started.store(true, std::memory_order_relaxed); +#endif // NDEBUG +} + // This is an adapter class for `WritableFile` to be used for `std::ostream`. // The adapter wraps a `WritableFile`, which can be passed to a `std::ostream` // constructor for storing streaming data. diff --git a/table/block_based/block_based_table_reader_sync_and_async.h b/table/block_based/block_based_table_reader_sync_and_async.h index 5dc212bb5fc1..deb5d5278efd 100644 --- a/table/block_based/block_based_table_reader_sync_and_async.h +++ b/table/block_based/block_based_table_reader_sync_and_async.h @@ -148,7 +148,8 @@ DEFINE_SYNC_AND_ASYNC(BlocklikeStatus, BlockFetcher block_fetcher( rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &tmp_contents, rep_->ioptions, do_uncompress, maybe_compressed, - TBlocklike::kBlockType, decomp, rep_->persistent_cache_options, + TBlocklike::kBlockType, decomp, + rep_->PersistentCacheFor(TBlocklike::kBlockType), GetMemoryAllocator(rep_->table_options), /*allocator=*/nullptr); @@ -287,7 +288,7 @@ DEFINE_SYNC_AND_ASYNC(BlocklikeStatus, s = CO_AWAIT(ReadAndParseBlockFromFile, rep_->file.get(), prefetch_buffer, rep_->footer, ro, handle, &block, rep_->ioptions, rep_->create_context, maybe_compressed, decomp, - rep_->persistent_cache_options, + rep_->PersistentCacheFor(TBlocklike::kBlockType), GetMemoryAllocator(rep_->table_options), for_compaction, async_read, block_buffer_provider); @@ -509,6 +510,7 @@ DEFINE_SYNC_AND_ASYNC(TBlockIter*, BlockBasedTable::NewDataBlockIterator)( DEFINE_SYNC_AND_ASYNC(Status, BlockBasedTable::Get) (const ReadOptions& read_options, const Slice& key, GetContext* get_context, const SliceTransform* prefix_extractor, bool skip_filters) { + MarkReadStarted(); // Similar to Bloom filter !may_match // If timestamp is beyond the range of the table, skip if (!TimestampMayMatch(read_options)) { @@ -723,7 +725,9 @@ DEFINE_SYNC_AND_ASYNC(Status, BlockBasedTable::Get) // If compression is enabled and also there is no compressed block cache, // the adjacent blocks are read out in one IO (combined read) // batch - A MultiGetRange with only those keys with unique data blocks not -// found in cache +// found in cache, one key per entry of `handles`. May be null for +// callers that have no MultiGetRange. Then all reads are synchronous, +// and no GetContext receives the statistics. // handles - A vector of block handles. Some of them me be NULL handles // scratch - An optional contiguous buffer to read compressed blocks into DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) @@ -736,9 +740,19 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) const ImmutableOptions& ioptions = rep_->ioptions; if (ioptions.allow_mmap_reads) { - size_t idx_in_batch = 0; - for (auto mget_iter = batch->begin(); mget_iter != batch->end(); - ++mget_iter, ++idx_in_batch) { + // No lambda here: in the coroutine version of this function, a lambda + // becomes a frame field whose type has no linkage, which gcc rejects + // (-Wsubobject-linkage). So the get contexts are collected first. + autovector get_contexts; + if (batch != nullptr) { + for (auto mget_iter = batch->begin(); mget_iter != batch->end(); + ++mget_iter) { + get_contexts.push_back(mget_iter->get_context); + } + assert(get_contexts.size() == handles->size()); + } + for (size_t idx_in_batch = 0; idx_in_batch < handles->size(); + ++idx_in_batch) { const BlockHandle& handle = (*handles)[idx_in_batch]; if (handle.IsNull()) { continue; @@ -747,12 +761,12 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) // XXX: use_cache=true means double cache query? statuses[idx_in_batch] = RetrieveBlock( nullptr, options, handle, decomp, - &results[idx_in_batch].As(), mget_iter->get_context, + &results[idx_in_batch].As(), + batch == nullptr ? nullptr : get_contexts[idx_in_batch], /* lookup_context */ nullptr, /* for_compaction */ false, /* use_cache */ true, /* async_read */ false, /* use_block_cache_for_lookup */ true); } - assert(idx_in_batch == handles->size()); CO_RETURN; } @@ -777,8 +791,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) size_t prev_len = 0; autovector req_idx_for_block; autovector req_offset_for_block; - for (auto mget_iter = batch->begin(); mget_iter != batch->end(); - ++mget_iter, ++idx_in_batch) { + for (; idx_in_batch < handles->size(); ++idx_in_batch) { const BlockHandle& handle = (*handles)[idx_in_batch]; if (handle.IsNull()) { continue; @@ -853,7 +866,10 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) // own, so adjacent blocks can read (and charge the rate limiter for) a // shared aligned page twice. Each request keeps its aligned buffer in // fs_scratch until the blocks below are parsed. - if (batch->context()->use_coro_read()) { + if (batch == nullptr) { + s = file->MultiRead(opts, &read_reqs[0], read_reqs.size(), + &direct_io_context, &dbg); + } else if (batch->context()->use_coro_read()) { s = co_await file->MultiReadCoroutine( opts, &read_reqs[0], read_reqs.size(), &direct_io_context, &dbg); } else { @@ -874,10 +890,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks) } } - idx_in_batch = 0; size_t valid_batch_idx = 0; - for (auto mget_iter = batch->begin(); mget_iter != batch->end(); - ++mget_iter, ++idx_in_batch) { + for (idx_in_batch = 0; idx_in_batch < handles->size(); ++idx_in_batch) { const BlockHandle& handle = (*handles)[idx_in_batch]; if (handle.IsNull()) { @@ -1003,6 +1017,7 @@ using MultiGetRange = MultiGetContext::Range; DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::MultiGet) (const ReadOptions& read_options, const MultiGetRange* mget_range, const SliceTransform* prefix_extractor, bool skip_filters) { + MarkReadStarted(); if (mget_range->empty()) { // Caller should ensure non-empty (performance bug) assert(false); diff --git a/table/block_based/block_based_table_reader_test.cc b/table/block_based/block_based_table_reader_test.cc index dee402407551..9d731387b11b 100644 --- a/table/block_based/block_based_table_reader_test.cc +++ b/table/block_based/block_based_table_reader_test.cc @@ -10,6 +10,7 @@ #include #include "cache/cache_reservation_manager.h" +#include "db/blob/blob_index.h" #include "db/db_test_util.h" #include "db/table_properties_collector.h" #include "file/file_util.h" @@ -266,6 +267,119 @@ class BlockBasedTableReaderFormatVersion8Test } }; +// Fixture with the default table options. +class BlockBasedTableReaderDefaultOptionsTest + : public BlockBasedTableReaderBaseTest { + protected: + void ConfigureTableFactory() override { + options_.table_factory.reset( + NewBlockBasedTableFactory(BlockBasedTableOptions())); + } +}; + +TEST_F(BlockBasedTableReaderDefaultOptionsTest, DropOnlyBeforeFirstRead) { + // DropUserCollectedProperty() replaces the table properties that readers + // share, so it may run only before the first read. This test checks that a + // drop right after open is fine and that reads then work. Debug builds + // assert when a drop comes after a read. A death test does not check that + // assert: it forks a process that has background threads, and the child + // can hang. + Options options; + const ImmutableOptions ioptions(options); + const InternalKeyComparator comparator(options.comparator); + CreateTable("DropOnlyBeforeFirstRead", ioptions, kNoCompression, + GenerateKVMap(/*num_block=*/2)); + std::unique_ptr table; + NewBlockBasedTableReader(FileOptions(), ioptions, comparator, + "DropOnlyBeforeFirstRead", &table); + table->DropUserCollectedProperty("not-present"); + // The iterator keeps a reference to the ReadOptions, so it must outlive the + // iterator. + const ReadOptions read_options; + std::unique_ptr iter(table->NewIterator( + read_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, + /*skip_filters=*/false, TableReaderCaller::kUncategorized)); + iter->SeekToFirst(); + ASSERT_TRUE(iter->Valid()); +} + +TEST_F(BlockBasedTableReaderDefaultOptionsTest, RawLookupDictFailureFailsKeys) { + // When the compression dictionary cannot be read, every key of a raw lookup + // gets that error, also keys in different data blocks. A sync point injects + // the failure, because a test table has no dictionary to break. + Options options; + const ImmutableOptions ioptions(options); + const InternalKeyComparator comparator(options.comparator); + const std::vector> kv = + GenerateKVMap(/*num_block=*/2); + CreateTable("RawLookupDictFailure", ioptions, kNoCompression, kv); + std::unique_ptr table; + NewBlockBasedTableReader(FileOptions(), ioptions, comparator, + "RawLookupDictFailure", &table); + SyncPoint::GetInstance()->SetCallBack( + "BlockBasedTable::MultiGetRawEntries:DictStatus", [](void* arg) { + *static_cast(arg) = Status::IOError("injected"); + }); + SyncPoint::GetInstance()->EnableProcessing(); + // The first key of each of the two blocks. + const std::array keys{ + {kv.front().first, kv[kEntriesPerBlock].first}}; + std::array statuses; + table->MultiGetRawEntries(ReadOptions(), keys.size(), keys.data(), + [&](size_t i, const Status& status, const Slice&, + const Slice&) { statuses[i] = status; }); + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + for (const Status& status : statuses) { + EXPECT_TRUE(status.IsIOError()) << status.ToString(); + } +} + +TEST_F(BlockBasedTableReaderDefaultOptionsTest, + CompressedSameFileRangeIsRejectedFirst) { + // As in base RocksDB, a range of a compressed value is not supported. A table + // without a BlobSource (a standalone reader) returns Corruption for one, + // from the scalar and the batched read, before any other check, so a + // kBlockCacheTier read gets Corruption too, not Incomplete. The check needs + // no blob record in the table, so the test uses a made-up same-file index. + Options options; + const ImmutableOptions ioptions(options); + const InternalKeyComparator comparator(options.comparator); + CreateTable("CompressedSameFileRange", ioptions, kNoCompression, + GenerateKVMap(/*num_block=*/1)); + std::unique_ptr table; + NewBlockBasedTableReader(FileOptions(), ioptions, comparator, + "CompressedSameFileRange", &table); + std::string encoded; + BlobIndex::EncodeBlob(&encoded, kCurrentFileBlobIndexFileNumber, + /*offset=*/0, /*size=*/16, kSnappyCompression); + BlobIndex index; + ASSERT_OK(index.DecodeFrom(encoded)); + ASSERT_TRUE(index.IsSameFile()); + + for (const ReadTier read_tier : {kReadAllTier, kBlockCacheTier}) { + ReadOptions read_options; + read_options.read_tier = read_tier; + PinnableSlice value; + EXPECT_TRUE(table + ->GetSameFileBlob( + read_options, index, /*range_offset=*/1, + /*range_length=*/4, + BlobVerifyPolicy::kVerifyIfNoAmplification, &value) + .IsCorruption()); + PinnableSlice batch_value; + Status batch_status; + SameFileBlobReadRequest request; + request.blob_index = &index; + request.range_offset = 1; + request.range_length = 4; + request.result = &batch_value; + request.status = &batch_status; + table->MultiGetSameFileBlob(read_options, /*num_reads=*/1, &request); + EXPECT_TRUE(batch_status.IsCorruption()) << batch_status.ToString(); + } +} + // format_version 8 records, in the footer, the gap (in bytes) between the end // of the metaindex block and the start of the footer, so the metaindex can be // located even when it is not adjacent to the footer (space reserved for future diff --git a/table/block_based/hash_index_reader.cc b/table/block_based/hash_index_reader.cc index 4b96e262366d..3b2b9534d6c2 100644 --- a/table/block_based/hash_index_reader.cc +++ b/table/block_based/hash_index_reader.cc @@ -67,7 +67,8 @@ Status HashIndexReader::Create(const BlockBasedTable* table, RandomAccessFileReader* const file = rep->file.get(); const Footer& footer = rep->footer; const ImmutableOptions& ioptions = rep->ioptions; - const PersistentCacheOptions& cache_options = rep->persistent_cache_options; + const PersistentCacheOptions& cache_options = + rep->metadata_persistent_cache_options; MemoryAllocator* const memory_allocator = GetMemoryAllocator(rep->table_options); diff --git a/table/sst_file_dumper.cc b/table/sst_file_dumper.cc index f6a2158211be..7641dacaa121 100644 --- a/table/sst_file_dumper.cc +++ b/table/sst_file_dumper.cc @@ -486,6 +486,9 @@ Status SstFileDumper::PrimeCompression(CompressionType compress_type, compress_type, compress_opt, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, column_family_name, unknown_level, kUnknownNewestKeyTime); + // The table built here only measures sizes and is thrown away, so it does + // not collect exact blob reference metadata. + tb_opts.collect_blob_reference_metadata = false; std::unique_ptr env(NewMemEnv(options_.env)); std::unique_ptr dest_writer; @@ -740,6 +743,9 @@ Status SstFileDumper::ShowCompressionSize( &block_based_table_factories, compress_type, compress_opt, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, column_family_name, unknown_level, kUnknownNewestKeyTime); + // The table built here only measures sizes and is thrown away, so it does + // not collect exact blob reference metadata. + tb_opts.collect_blob_reference_metadata = false; TableProperties props; std::chrono::microseconds write_time; std::chrono::microseconds read_time; diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index 131e65db5af1..c0bcd891a4c3 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -579,6 +579,10 @@ Status SstFileWriter::Open(const std::string& file_path, Temperature temp) { "SST Writer" /* db_id */, r->db_session_id, 0 /* target_file_size */, r->next_file_number, kMaxSequenceNumber, Slice() /* db_name */, false /* is_remote_compaction */, r->embedded_blob_options.get()); + // External files have fake file numbers and may have no known CF, so they + // cannot carry exact reference metadata. They store no blob reference + // metadata and no classification property. + table_builder_options.collect_blob_reference_metadata = false; // External SST files used to each get a unique session id. Now for // slightly better uniqueness probability in constructing cache keys, we // assign fake file numbers to each file (into table properties) and keep diff --git a/table/table_builder.h b/table/table_builder.h index 52515808ac9f..8db8749d3e88 100644 --- a/table/table_builder.h +++ b/table/table_builder.h @@ -33,7 +33,9 @@ namespace ROCKSDB_NAMESPACE { class Slice; +class BlobIndex; class BlobSource; +struct BlobReferenceSstSummary; struct TableReaderOptions { // @param skip_filters Disables loading/accessing the filter block @@ -113,8 +115,7 @@ struct TableReaderOptions { // Whether the key in the table contains user-defined timestamps. bool user_defined_timestamps_persisted; - // Open-time metadata reads should not insert index/filter/dictionary blocks - // into the shared block cache. + // Metadata reads should avoid shared block and persistent caches. bool avoid_shared_metadata_cache; // Column family that owns this table. Empty outside DB-owned table opens. @@ -169,6 +170,7 @@ struct TableBuilderOptions : public TablePropertiesCollectorFactory::Context { db_name(_db_name), is_remote_compaction(_is_remote_compaction), cur_file_num(_cur_file_num), + collect_blob_reference_metadata(_moptions.enable_blob_indirection), embedded_blob_options(_embedded_blob_options) {} const ImmutableOptions& ioptions; @@ -193,6 +195,33 @@ struct TableBuilderOptions : public TablePropertiesCollectorFactory::Context { // END for FilterBuildingContext const uint64_t cur_file_num; + // Whether the SST gets exact blob reference metadata. The default is + // enable_blob_indirection. Flush sets false (coarse output). Compaction sets + // Compaction::CollectExactBlobReferences(), which is also true when the + // option is off but the DB has used it before (HasBlobIndirectionFormat). + // SstFileWriter sets false. The table builder does not check the column + // family, so a caller with an unknown column family that keeps the default + // collects exact metadata when enable_blob_indirection is on. + bool collect_blob_reference_metadata; + // Set by flush, WAL recovery flush and compaction when the DB uses or has + // used enable_blob_indirection. The two fields together give three cases: + // - collect_blob_reference_metadata true: exact metadata. This field is then + // ignored. + // - collect_blob_reference_metadata false, this field true: a coarse SST. The + // builder stores only the small classification property (flush, WAL + // recovery flush, and a compaction to L0 that has no exact input and is + // not bottommost; see Compaction::CollectExactBlobReferences()). + // - Both false: no blob reference metadata (the DB never used the option, + // and SstFileWriter). + bool track_blob_reference_classification = false; + // The relocation-state fingerprint (a 128-bit hash of the relocation files + // and routes) of the compaction's input Version (see + // VersionStorageInfo::ComputeBlobRelocationStateFingerprint()). It is stored + // in the exact-metadata descriptor and records which relocation state the + // output was built from. Only compaction sets it; every other builder + // leaves 0. + uint64_t blob_route_snapshot_fingerprint_high = 0; + uint64_t blob_route_snapshot_fingerprint_low = 0; // Non-null only for table builders that should write eligible large values as // same-file ("embedded") blob records. Currently only honored by // BlockBasedTableBuilder. @@ -216,6 +245,7 @@ enum class TableBuilderNewTableReason : uint8_t { kSchemaVersionIncompatible, kUnsupportedEntryType, kBuilderPolicy, + kBlobReferenceLimit, }; inline const char* TableBuilderNewTableReasonToString( @@ -233,6 +263,8 @@ inline const char* TableBuilderNewTableReasonToString( return "UnsupportedEntryType"; case TableBuilderNewTableReason::kBuilderPolicy: return "BuilderPolicy"; + case TableBuilderNewTableReason::kBlobReferenceLimit: + return "BlobReferenceLimit"; } assert(false); return "Invalid"; @@ -241,6 +273,12 @@ inline const char* TableBuilderNewTableReasonToString( struct TableBuilderAddContext { TableBuilderNewTableReason new_table_reason = TableBuilderNewTableReason::kNone; + // Input, optional. When the value is a kTypeBlobIndex, the caller can pass + // its own decode of the value here, so that the builder does not decode it + // again. Compaction decodes each output BlobIndex once and shares it with + // the builder, the blob garbage meter and the file metadata update. + // nullptr: the builder decodes the value itself. + const BlobIndex* decoded_blob_index = nullptr; }; // TableBuilder provides the interface used to build a Table @@ -299,6 +337,10 @@ class TableBuilder { // Number of calls to Add() so far. virtual uint64_t NumEntries() const = 0; + // True once the table holds blob_indirection_max_references_per_sst exact + // blob references. Compaction then starts a new output at the next user key. + virtual bool BlobReferenceLimitReached() const { return false; } + // Whether the output file is completely empty. It has neither entries // or tombstones. virtual bool IsEmpty() const { @@ -331,6 +373,17 @@ class TableBuilder { // Returns table properties virtual TableProperties GetTableProperties() const = 0; + // Returns the immutable summary of the exact blob references this builder + // collected (see BlobReferenceSstSummary). Returns null if exact metadata was + // not enabled, Finish() has not run, or the build failed. BuildTable() (used + // by flush, WAL recovery and RepairDB) and compaction put it in the output's + // FileMetaData, so the MANIFEST install can add the SST to the census without + // reopening the new SST. + virtual std::shared_ptr + GetBlobReferenceSstSummary() const { + return nullptr; + } + // Return file checksum virtual std::string GetFileChecksum() const = 0; diff --git a/table/table_reader.h b/table/table_reader.h index 54438194ceb6..f4d248c647e6 100644 --- a/table/table_reader.h +++ b/table/table_reader.h @@ -8,7 +8,9 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #pragma once +#include #include +#include #include "db/range_tombstone_fragmenter.h" #if USE_COROUTINES @@ -24,6 +26,83 @@ namespace ROCKSDB_NAMESPACE { +struct BlobReference; +struct BlobReferenceGroup; + +// Receives the references of requested groups in a form the caller chooses, for +// example a compact copy (see BlobReferenceGroupRead::sink). +class BlobReferenceSink { + public: + virtual ~BlobReferenceSink() = default; + virtual size_t size() const = 0; + + // How many more references Append() accepts. + virtual size_t Room() const = 0; + virtual Status Append(const BlobReference& reference) = 0; + + // Drops the references after the first `size`. + virtual void Truncate(size_t size) = 0; + + // A key for the storage that Append() writes to. ReadBlobReferences() + // checks all requests with the same identity against one shared Room(). + // Override it only when two sink objects write to the same storage. + virtual const void* identity() const { return this; } +}; + +// One request of a batch read of exact blob references from one SST: a group +// key and the vector (or sink) its references are appended to. +struct BlobReferenceGroupRead { + // Only the group key (encoding class, generation number, origin) is used. + // With `whole_generation`, only the generation number is used. + const BlobReferenceGroup* group = nullptr; + // Reads every group of the SST with this generation number (any class, any + // origin) into one output. The found totals are the sum over those groups. + // The status is NotFound if there is no such group. Each group is read like + // its own request. On an error, only the failed group's references are + // dropped. The output keeps the references of the other groups of the + // generation number, earlier and later, and the status is the error of the + // first failed group. Requests of a batch are either all whole-generation + // (sorted by generation number, unique) or all single-group requests (each + // names one class, generation number and origin). + bool whole_generation = false; + // Must have room for the group's references. Used when `sink` is null. + std::vector* output = nullptr; + // Must have room for the group's references. Before the read, each group is + // checked against Room() (or the free capacity of `output`), minus the + // references of earlier requests with the same sink. A group with more + // references gets Aborted, and nothing is appended for it. An Append() error + // fails the group like a decode error. + BlobReferenceSink* sink = nullptr; + // Set by the read when the SST has the group: the group's totals from this + // SST's reference index. + uint64_t found_reference_count = 0; + uint64_t found_stored_value_bytes = 0; + uint64_t found_record_bytes = 0; + // Set by the read: OK, or the error of this group only. See + // TableReader::ReadBlobReferences() for the error codes. + Status status; + + // The destructor marks `status` as checked. A caller may skip the request + // statuses after the whole call fails, or after it stops at the first failed + // request (indirect blob GC does this in ReadLiveSets()). Those skipped + // statuses are not ignored errors. + BlobReferenceGroupRead() = default; + BlobReferenceGroupRead(const BlobReferenceGroupRead&) = default; + BlobReferenceGroupRead(BlobReferenceGroupRead&&) = default; + BlobReferenceGroupRead& operator=(const BlobReferenceGroupRead&) = default; + BlobReferenceGroupRead& operator=(BlobReferenceGroupRead&&) = default; + ~BlobReferenceGroupRead() { status.PermitUncheckedError(); } +}; + +// Largest number of on-disk bytes of blob reference data blocks that one +// MultiRead batch holds in memory (256 KB). It is the size of the read buffer +// that ReadBlobReferences() allocates. 256 KB is an estimate (not measured): it +// holds many data blocks per read but keeps the buffer small. With MultiRead, a +// single data block whose on-disk size (with its trailer) is larger than this +// fails the read with Corruption. With mmap or direct I/O, blocks are read one +// at a time and this limit does not apply. +inline constexpr size_t kBlobReferenceReadBatchBytes = 256 * 1024; + class Iterator; struct ParsedInternalKey; class Slice; @@ -133,6 +212,92 @@ class TableReader { virtual std::shared_ptr GetTableProperties() const = 0; + // The blob reference census uses this to learn which blob files an SST + // references when it has no summary in memory. It reads the exact blob + // reference index block of this SST and sets `*groups` to the groups that are + // not kDirect, in key order. The census does not track kDirect references. + // Only the index block is read: it holds the totals of every group, so no + // data block is needed. + // - `expected_descriptor` is the descriptor that the MANIFEST holds for this + // SST. A different descriptor in the table properties is Corruption. + // - When set, `before_allocation` gets the size in bytes of the group buffer + // before the buffer is allocated, so the caller can charge the memory + // first. If it returns non-OK, this function returns that status. + // + // The default returns NotSupported, because enable_blob_indirection requires + // a block-based table. + virtual Status ReadBlobReferenceGroups( + const ReadOptions& read_options, const Slice& expected_descriptor, + const std::function& before_allocation, + std::vector* groups) const { + (void)read_options; + (void)expected_descriptor; + (void)before_allocation; + (void)groups; + return Status::NotSupported( + "Exact blob reference metadata is not supported"); + } + + // Reads the references of several groups of this SST from its exact reference + // metadata. Each request is a BlobReferenceGroupRead. The batch form lets one + // call read several groups that share data blocks, so that each block is + // read and decompressed only once. Indirect blob GC reads, from each SST, + // all generation numbers of its inputs that the SST holds, in one batch of + // whole-generation requests. + // + // Input: single-group requests must be sorted by group key (encoding class, + // generation number, origin). Whole-generation requests must be sorted by + // generation number. A batch has no repeats and does not mix the two kinds; + // otherwise InvalidArgument. Each output must already have capacity for its + // group, so that its owner can charge the memory before this call. + // + // Reads: it decodes the reference index once and reads each needed data block + // once. Adjacent blocks are read together, in MultiRead batches of at most + // kBlobReferenceReadBatchBytes (one block at a time with mmap or direct I/O). + // + // Index memory report: when set, `after_index_read` gets the memory in bytes + // of the decoded reference index, right after it is decoded and before any + // data block is read. The caller can charge it there. The size grows with the + // number of groups and blocks of the SST, so a fixed estimate can be too + // small. If the callback returns non-OK, this function returns that status. + // + // Results: each found group reports its index totals in the found_* fields. + // A request gets an error status in the cases below, and the other requests + // continue: + // - NotFound: the SST does not have the group. + // - Aborted: the group does not fit its output. + // - Corruption: a block of the group is corrupt, or the references read for + // the group do not match its index totals. + // Errors of the whole call: if the requests are not sorted, or the + // descriptor or the reference index cannot be read or decoded, the call + // returns the error before it reads any group. The request statuses are + // then not meaningful, and every output is unchanged. If a data block read + // fails (for example with an I/O error), the call returns that error and + // also sets it on every request that has not finished. A request that has + // already finished keeps its status and its output. Callers must check the + // return value before they read a request status. + // + // A failed request drops only the references it appended. A request that + // fails before it appends anything leaves the output unchanged. So when + // several requests or calls share one output, the references that earlier + // requests or the caller appended stay. For a whole-generation request, this + // holds for each group, not for the whole request (see + // BlobReferenceGroupRead::whole_generation). + // + // The default returns NotSupported, because SSTs with exact reference + // metadata must be block-based tables. + virtual Status ReadBlobReferences( + const ReadOptions& read_options, const Slice& expected_descriptor, + const std::function& after_index_read, + std::vector* requests) const { + (void)read_options; + (void)expected_descriptor; + (void)after_index_read; + (void)requests; + return Status::NotSupported( + "Exact blob reference metadata is not supported"); + } + // Prepare work that can be done before the real Get() virtual void Prepare(const Slice& /*target*/) {} diff --git a/table/table_test.cc b/table/table_test.cc index e4268cd5fe39..0ebfc7e94275 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -8414,6 +8414,19 @@ TEST_F(CacheUsageOptionsOverridesTest, SanitizeAndValidateOptions) { } Destroy(options); + // The exact blob reference metadata collector can use a large build buffer. + // Its cache role, kBlobReferenceMetadataBuildingBuffer, accepts an explicit + // charging override like the other build buffers. + table_options = BlockBasedTableOptions(); + table_options.cache_usage_options.options_overrides.insert( + {CacheEntryRole::kBlobReferenceMetadataBuildingBuffer, + {/*.charged = */ CacheEntryRoleOptions::Decision::kDisabled}}); + options.table_factory.reset(NewBlockBasedTableFactory(table_options)); + Destroy(options); + s = TryReopen(options); + EXPECT_OK(s); + Destroy(options); + // To test option validation on unsupported CacheEntryRole table_options = BlockBasedTableOptions(); table_options.cache_usage_options.options_overrides.insert( diff --git a/tools/block_cache_analyzer/block_cache_trace_analyzer.cc b/tools/block_cache_analyzer/block_cache_trace_analyzer.cc index 3cd9ec68bb72..168a202cb8b2 100644 --- a/tools/block_cache_analyzer/block_cache_trace_analyzer.cc +++ b/tools/block_cache_analyzer/block_cache_trace_analyzer.cc @@ -252,6 +252,8 @@ std::string caller_to_string(TableReaderCaller caller) { return "SSTFileReader"; case kUncategorized: return "Uncategorized"; + case kBlobRelocationFileRead: + return "BlobRelocationFileRead"; case kMaxBlockCacheLookupCaller: default: break; @@ -289,6 +291,8 @@ TableReaderCaller string_to_caller(const std::string& caller_str) { return kSSTFileReader; } else if (caller_str == "Uncategorized") { return kUncategorized; + } else if (caller_str == "BlobRelocationFileRead") { + return kBlobRelocationFileRead; } return TableReaderCaller::kMaxBlockCacheLookupCaller; } @@ -310,6 +314,10 @@ bool is_user_access(TableReaderCaller caller) { case kFlush: case kSSTFileReader: case kUncategorized: + // A relocation file read has this caller whether a user Get, an iterator, + // a compaction or indirect blob GC caused it, so it cannot count as a user + // access. + case kBlobRelocationFileRead: case kMaxBlockCacheLookupCaller: default: break; diff --git a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc index 5f71477cdb04..24a4c0c0aafb 100644 --- a/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc +++ b/tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc @@ -17,6 +17,7 @@ int main() { #include #include +#include "file/file_util.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/status.h" @@ -790,6 +791,132 @@ TEST_F(BlockCacheTracerTest, MultiGetWithNullReferenceKey) { } } +TEST_F(BlockCacheTracerTest, BlobRelocationFileReadCaller) { + // Reads of blob relocation files use the caller kBlobRelocationFileRead. + // This test checks that the analyzer knows the caller: + // - caller_to_string names it "BlobRelocationFileRead" in the outputs. + // - string_to_caller accepts "BlobRelocationFileRead" in -analyze_callers. + // - is_user_access is false for it, because GC and compaction read + // relocation files too, so it is left out of the user-access-only outputs. + // Blocks 0 to 9 are read by Get and blocks 10 to 19 by the relocation reader. + const std::string result_dir = test_path_ + "/relocation_caller"; + // A failed earlier run can leave the folder behind. + DestroyDir(env_, result_dir).PermitUncheckedError(); + ASSERT_OK(env_->CreateDirIfMissing(result_dir)); + { + std::unique_ptr trace_writer; + ASSERT_OK(NewFileTraceWriter(env_, env_options_, trace_file_path_, + &trace_writer)); + std::unique_ptr writer = NewBlockCacheTraceWriter( + env_->GetSystemClock().get(), BlockCacheTraceWriterOptions(), + std::move(trace_writer)); + ASSERT_NE(writer, nullptr); + ASSERT_OK(writer->WriteHeader()); + for (uint32_t key_id = 0; key_id < 20; key_id++) { + BlockCacheTraceRecord record; + record.block_type = TraceType::kBlockTraceDataBlock; + record.block_size = kBlockSize; + record.block_key = kBlockKeyPrefix + std::to_string(key_id); + record.access_timestamp = (key_id + 1) * kMicrosInSecond; + record.cf_id = kCFId; + record.cf_name = kDefaultColumnFamilyName; + record.caller = key_id < 10 ? TableReaderCaller::kUserGet + : TableReaderCaller::kBlobRelocationFileRead; + record.level = kLevel; + record.sst_fd_number = kSSTStoringEvenKeys; + record.referenced_key = + kRefKeyPrefix + std::to_string(key_id) + std::string(8, 0); + record.referenced_key_exist_in_block = true; + record.num_keys_in_block = kNumKeysInBlock; + ASSERT_OK(writer->WriteBlockAccess( + record, record.block_key, record.cf_name, record.referenced_key)); + } + } + // Returns the first column of each row after the header of `file_name` in + // result_dir, or an empty set if the file does not exist. + const auto read_first_column = [&](const std::string& file_name) { + std::set values; + std::ifstream infile(result_dir + "/" + file_name); + std::string line; + if (!getline(infile, line)) { + return values; + } + while (getline(infile, line)) { + values.insert(line.substr(0, line.find(','))); + } + return values; + }; + + // Run the tool. Every flag is given, so values left by earlier tests in this + // process do not change the run. + { + std::vector params = { + "./block_cache_trace_analyzer", + "-block_cache_trace_path=" + trace_file_path_, + "-block_cache_sim_config_path=", + "-block_cache_analysis_result_dir=" + result_dir, + "-print_block_size_stats=false", + "-print_access_count_stats=false", + "-print_data_block_access_count_stats=false", + "-cache_sim_warmup_seconds=0", + "-analyze_bottom_k_access_count_blocks=0", + "-analyze_top_k_access_count_blocks=0", + "-analyze_blocks_reuse_k_reuse_window=0", + "-timeline_labels=", + "-reuse_distance_labels=", + "-reuse_distance_buckets=", + "-reuse_interval_labels=", + "-reuse_interval_buckets=", + "-reuse_lifetime_labels=", + "-reuse_lifetime_buckets=", + "-analyze_callers=BlobRelocationFileRead", + "-access_count_buckets=", + "-analyze_get_spatial_locality_labels=", + "-analyze_get_spatial_locality_buckets=", + "-analyze_correlation_coefficients_labels=", + "-skew_labels=", + "-skew_buckets=", + "-mrc_only=false", + "-human_readable_trace_file_path="}; + std::vector argv; + for (auto& param : params) { + argv.push_back(¶m[0]); + } + ASSERT_EQ(0, ROCKSDB_NAMESPACE::block_cache_trace_analyzer_tool( + static_cast(argv.size()), argv.data())); + } + ASSERT_EQ(read_first_column("percentage_of_accesses_summary"), + (std::set{"Get", "BlobRelocationFileRead"})); + // The per-caller file has the name of the parsed caller. An unknown caller + // name would give "InvalidCaller". + ASSERT_EQ(read_first_column( + "BlobRelocationFileRead_level_percentage_of_accesses_summary"), + (std::set{std::to_string(kLevel)})); + + // The caller timeline of all accesses has both callers. The timeline of user + // accesses only has Get. + { + BlockCacheTraceAnalyzer analyzer(trace_file_path_, result_dir, + /*human_readable_trace_file_path=*/"", + /*compute_reuse_distance=*/false, + /*mrc_only=*/false, + /*is_human_readable_trace_file=*/false, + /*cache_simulator=*/nullptr); + // Analyze() returns Incomplete when it reaches the end of the trace. + const Status s = analyzer.Analyze(); + ASSERT_TRUE(s.ok() || s.IsIncomplete()) << s.ToString(); + analyzer.WriteAccessTimeline("caller", /*time_unit=*/1, + /*user_access_only=*/false); + analyzer.WriteAccessTimeline("caller", /*time_unit=*/1, + /*user_access_only=*/true); + } + ASSERT_EQ(read_first_column("all_access_caller_1_access_timeline"), + (std::set{"Get", "BlobRelocationFileRead"})); + ASSERT_EQ(read_first_column("user_access_only_caller_1_access_timeline"), + (std::set{"Get"})); + ASSERT_OK(DestroyDir(env_, result_dir)); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index f13caeb93f6a..cd2104a38b27 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -1318,6 +1318,15 @@ DEFINE_int32( ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions().blob_file_starting_level, "[Integrated BlobDB] The starting level for blob files."); +DEFINE_bool(pin_blob_relocation_location_blocks, + ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions() + .pin_blob_relocation_location_blocks, + "[Integrated BlobDB] Keep all location blocks of a relocation " + "file in memory while its reader is open, so lookups of relocated " + "values skip the index and block cache lookups (see " + "pin_blob_relocation_location_blocks in " + "include/rocksdb/advanced_options.h)."); + DEFINE_bool(use_blob_cache, false, "[Integrated BlobDB] Enable blob cache."); DEFINE_bool( @@ -6547,6 +6556,8 @@ class Benchmark { options.blob_compaction_readahead_size = FLAGS_blob_compaction_readahead_size; options.blob_file_starting_level = FLAGS_blob_file_starting_level; + options.pin_blob_relocation_location_blocks = + FLAGS_pin_blob_relocation_location_blocks; options.read_triggered_compaction_threshold = FLAGS_read_triggered_compaction_threshold; diff --git a/tools/ldb_cmd_test.cc b/tools/ldb_cmd_test.cc index 2620652ac8a4..5befdbfa5c77 100644 --- a/tools/ldb_cmd_test.cc +++ b/tools/ldb_cmd_test.cc @@ -10,6 +10,9 @@ #include #include +#include "db/blob/blob_relocation_file.h" +#include "db/blob/blob_relocation_manifest.h" +#include "db/column_family.h" #include "db/db_test_util.h" #include "db/version_edit.h" #include "db/version_set.h" @@ -23,6 +26,7 @@ #include "rocksdb/file_checksum.h" #include "rocksdb/file_system.h" #include "rocksdb/utilities/options_util.h" +#include "table/block_based/block_based_table_builder.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" #include "test_util/testutil.h" @@ -1254,6 +1258,142 @@ TEST_F(LdbCmdTest, CustomComparator) { LDBCommandRunner::RunCommand(4, argv, opts, LDBOptions(), &cfds)); } +TEST_F(LdbCmdTest, ManifestDumpPrintsBlobIndirectionSummary) { + Options options; + options.env = TryLoadCustomOrDefaultEnv(); + options.create_if_missing = true; + options.enable_blob_files = true; + const std::string dbname = + test::PerThreadDBPath(options.env, "ldb_blob_indirection_summary"); + ASSERT_OK(DestroyDB(dbname, options)); + std::unique_ptr db; + ASSERT_OK(DB::Open(options, dbname, &db)); + DBImpl* const db_impl = static_cast_with_check(db.get()); + VersionSet* const version_set = db_impl->GetVersionSet(); + ColumnFamilyData* const cfd = version_set->GetColumnFamilySet()->GetDefault(); + + // Writes a relocation file with one value. File number 1 is an example old + // file: the relocation file lists it as a served file, so a route can send + // its reads to the relocation file. + constexpr uint64_t kOldFileNumber = 1; + db_impl->TEST_LockMutex(); + const uint64_t file_number = version_set->NewFileNumber(); + db_impl->TEST_UnlockMutex(); + Options relocation_options = options; + relocation_options.cf_paths.emplace_back(dbname, /*target_size=*/0); + const BlockBasedTableOptions table_options; + const ImmutableOptions immutable_options(relocation_options); + const MutableCFOptions mutable_options(relocation_options); + const std::string path = BlobFileName(dbname, file_number); + std::unique_ptr writer; + ASSERT_OK(WritableFileWriter::Create(options.env->GetFileSystem(), path, + FileOptions(), &writer, + /*dbg=*/nullptr)); + const InternalKeyComparator comparator(BytewiseComparator()); + BlobRelocationTableFileInfo file_info; + file_info.column_family_id = cfd->GetID(); + file_info.column_family_name = kDefaultColumnFamilyName; + file_info.db_id = "test-db"; + file_info.db_session_id = "test-session"; + file_info.file_number = file_number; + std::unique_ptr builder; + ASSERT_OK(NewBlobRelocationTableBuilder( + immutable_options, mutable_options, table_options, ReadOptions(), + WriteOptions(), comparator, file_info, writer.get(), &builder)); + ASSERT_OK(builder->AddBlobRelocation(kOldFileNumber, + /*original_value_offset=*/11, "payload", + kNoCompression)); + ASSERT_OK(builder->AddBlobRelocationServedFile(kOldFileNumber)); + ASSERT_OK(builder->Finish()); + BlobRelocationFileMetadata metadata; + ASSERT_OK(builder->GetBlobRelocationFileMetadata(&metadata)); + ASSERT_OK(writer->Close(IOOptions(), /*dbg=*/nullptr)); + uint64_t file_size = 0; + ASSERT_OK(options.env->GetFileSize(path, &file_size)); + + VersionEdit edit; + edit.SetColumnFamily(cfd->GetID()); + edit.SetBlobIndirectionFormatVersion(kBlobIndirectionManifestFormatVersion); + edit.AddBlobRelocationFile(BlobRelocationFileAddition( + file_number, file_size, /*checksum_method=*/std::string(), + /*checksum_value=*/std::string(), metadata.encoded_directory, + metadata.tail_size)); + edit.SetBlobRelocationRoute(BlobRelocationRoute(kOldFileNumber, file_number)); + db_impl->TEST_LockMutex(); + ASSERT_OK(version_set->LogAndApply( + cfd, ReadOptions(), WriteOptions(), &edit, db_impl->TEST_Mutex(), + /*dir_contains_current_file=*/nullptr, /*new_descriptor_log=*/false, + /*column_family_options=*/nullptr, /*manifest_wcb=*/{}, + /*pre_cb=*/{})); + db_impl->TEST_UnlockMutex(); + db.reset(); + + const std::string db_argument = "--db=" + dbname; + const char* arguments[] = {"./ldb", db_argument.c_str(), "manifest_dump"}; + testing::internal::CaptureStdout(); + const int result = + LDBCommandRunner::RunCommand(3, arguments, options, LDBOptions(), + /*column_families=*/nullptr); + const std::string output = testing::internal::GetCapturedStdout(); + ASSERT_EQ(result, 0); + const std::string expected = + "--- blob indirection --- format version: " + + std::to_string(kBlobIndirectionManifestFormatVersion) + + ", relocation files: 1 (" + std::to_string(file_size) + + " bytes, 1 values), routes: 1"; + EXPECT_NE(output.find(expected), std::string::npos) << output; + EXPECT_NE(output.find("db blob indirection format version " + + std::to_string(kBlobIndirectionManifestFormatVersion)), + std::string::npos) + << output; + ASSERT_OK(DestroyDB(dbname, options)); +} + +TEST_F(LdbCmdTest, ManifestDumpPrintsBlobReferenceMetadata) { + ColumnFamilyData::TEST_SetAllowBlobIndirection(true); + Options options; + options.env = TryLoadCustomOrDefaultEnv(); + options.create_if_missing = true; + options.enable_blob_files = true; + options.enable_blob_indirection = true; + options.min_blob_size = 0; + const std::string dbname = + test::PerThreadDBPath(options.env, "ldb_blob_reference_metadata"); + ASSERT_OK(DestroyDB(dbname, options)); + std::unique_ptr db; + ASSERT_OK(DB::Open(options, dbname, &db)); + ASSERT_OK(db->Put(WriteOptions(), "key1", "value1")); + ASSERT_OK(db->Put(WriteOptions(), "key2", "value2")); + ASSERT_OK(db->Flush(FlushOptions())); + // A compaction to L1 always writes exact reference metadata. + CompactRangeOptions compact_options; + compact_options.bottommost_level_compaction = + BottommostLevelCompaction::kForce; + ASSERT_OK(db->CompactRange(compact_options, nullptr, nullptr)); + db.reset(); + + const std::string db_argument = "--db=" + dbname; + const char* arguments[] = {"./ldb", db_argument.c_str(), "manifest_dump", + "--verbose"}; + testing::internal::CaptureStdout(); + const int result = + LDBCommandRunner::RunCommand(4, arguments, options, LDBOptions(), + /*column_families=*/nullptr); + const std::string output = testing::internal::GetCapturedStdout(); + ASSERT_EQ(result, 0); + // The AddFile record of the compaction output. + EXPECT_NE( + output.find( + " blob_references: exact (2 references, 12 stored value bytes)"), + std::string::npos) + << output; + EXPECT_NE(output.find("--- blob reference metadata --- exact SSTs: 1, coarse " + "SSTs: 0 (lowest frontier 0), legacy SSTs: 0"), + std::string::npos) + << output; + ASSERT_OK(DestroyDB(dbname, options)); +} + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/tools/sst_dump_test.cc b/tools/sst_dump_test.cc index ee2caf40fe2a..90c523d390ba 100644 --- a/tools/sst_dump_test.cc +++ b/tools/sst_dump_test.cc @@ -10,6 +10,7 @@ #include #include +#include "db/blob/blob_reference_metadata.h" #include "db/wide/wide_column_serialization.h" #include "env/composite_env_wrapper.h" #include "file/random_access_file_reader.h" @@ -433,6 +434,117 @@ TEST_F(SSTDumpToolTest, ListMetaBlocks) { "--list_meta_blocks"); } +namespace { +// Adds fixed user properties to every SST it collects, so a test can write the +// table properties of a feature without the feature's table builder. +class FixedPropertiesCollector : public TablePropertiesCollector { + public: + explicit FixedPropertiesCollector(UserCollectedProperties properties) + : properties_(std::move(properties)) {} + Status AddUserKey(const Slice& /*key*/, const Slice& /*value*/, + EntryType /*type*/, SequenceNumber /*seq*/, + uint64_t /*file_size*/) override { + return Status::OK(); + } + Status Finish(UserCollectedProperties* properties) override { + properties->insert(properties_.begin(), properties_.end()); + return Status::OK(); + } + UserCollectedProperties GetReadableProperties() const override { return {}; } + const char* Name() const override { return "FixedPropertiesCollector"; } + + private: + const UserCollectedProperties properties_; +}; + +class FixedPropertiesCollectorFactory : public TablePropertiesCollectorFactory { + public: + explicit FixedPropertiesCollectorFactory(UserCollectedProperties properties) + : properties_(std::move(properties)) {} + TablePropertiesCollector* CreateTablePropertiesCollector( + TablePropertiesCollectorFactory::Context /*context*/) override { + return new FixedPropertiesCollector(properties_); + } + const char* Name() const override { + return "FixedPropertiesCollectorFactory"; + } + + private: + const UserCollectedProperties properties_; +}; +} // namespace + +TEST_F(SSTDumpToolTest, ShowBlobReferenceProperties) { + // The descriptor values are an example; DecodeFrom() checks only their form, + // so the index handle does not need to point to a real block. + BlobReferenceMetadataDescriptor descriptor; + descriptor.index_handle = BlockHandle(/*offset=*/4096, /*size=*/64); + descriptor.reference_count = 3; + descriptor.stored_value_bytes = 300; + descriptor.column_family_id = 0; + descriptor.sst_file_number = 12; + std::string encoded_descriptor; + descriptor.EncodeTo(&encoded_descriptor); + + Options opts; + opts.env = env(); + opts.table_properties_collector_factories.emplace_back( + std::make_shared(UserCollectedProperties{ + {kBlobReferenceDescriptorPropertyName, encoded_descriptor}})); + const std::string file_path = MakeFilePath("blob_reference_properties.sst"); + SstFileWriter writer(EnvOptions(), opts); + ASSERT_OK(writer.Open(file_path)); + ASSERT_OK(writer.Put("key", "value")); + ASSERT_OK(writer.Finish()); + + char* usage[3]; + auto cleanup_usage = CleanupUsage{usage}; + PopulateCommandArgs(file_path, "--show_properties", usage); + ROCKSDB_NAMESPACE::SSTDumpTool tool; + testing::internal::CaptureStdout(); + const int result = tool.Run(3, usage, opts); + const std::string output = testing::internal::GetCapturedStdout(); + ASSERT_EQ(result, 0); + EXPECT_NE(output.find("Blob reference metadata"), std::string::npos); + EXPECT_NE(output.find("exact metadata: 3 references, 300 stored value bytes"), + std::string::npos); + EXPECT_NE(output.find("SST file number when built: 12"), std::string::npos); + EXPECT_NE(output.find("index block: offset 4096, size 64"), + std::string::npos); + cleanup(opts, file_path); +} + +TEST_F(SSTDumpToolTest, ShowBlobReferenceClassification) { + // Frontier 12 is an example blob file number. The frontier is the oldest + // indirect-origin file number a coarse SST may reference (see "Coarse SST" in + // db/blob/blob_reference_metadata.h). + std::string encoded_classification; + EncodeBlobReferenceClassification(/*frontier=*/12, &encoded_classification); + Options opts; + opts.env = env(); + opts.table_properties_collector_factories.emplace_back( + std::make_shared(UserCollectedProperties{ + {kBlobReferenceClassificationPropertyName, encoded_classification}})); + const std::string file_path = MakeFilePath("blob_reference_coarse.sst"); + SstFileWriter writer(EnvOptions(), opts); + ASSERT_OK(writer.Open(file_path)); + ASSERT_OK(writer.Put("key", "value")); + ASSERT_OK(writer.Finish()); + + char* usage[3]; + auto cleanup_usage = CleanupUsage{usage}; + PopulateCommandArgs(file_path, "--show_properties", usage); + ROCKSDB_NAMESPACE::SSTDumpTool tool; + testing::internal::CaptureStdout(); + const int result = tool.Run(3, usage, opts); + const std::string output = testing::internal::GetCapturedStdout(); + ASSERT_EQ(result, 0); + EXPECT_NE(output.find("coarse classification: frontier 12"), + std::string::npos); + EXPECT_EQ(output.find("exact metadata"), std::string::npos); + cleanup(opts, file_path); +} + namespace { using Compressor8A = test::CompressorCustomAlg; class MyManager : public CompressionManager { diff --git a/tools/sst_dump_tool.cc b/tools/sst_dump_tool.cc index 70d9310d0010..7ebcc617e35d 100644 --- a/tools/sst_dump_tool.cc +++ b/tools/sst_dump_tool.cc @@ -11,6 +11,7 @@ #include #include +#include "db/blob/blob_reference_metadata.h" #include "db_stress_tool/db_stress_compression_manager.h" #include "options/options_helper.h" #include "port/port.h" @@ -231,6 +232,68 @@ void PrintTableProperties(const char* label, label, tp->ToString("\n ", ": ").c_str()); } +// Prints the blob reference table properties of an SST written with blob +// indirection as readable text: the descriptor of exact metadata, or the +// coarse classification. The formats are described in +// db/blob/blob_reference_metadata.h. The raw bytes are also in the "Raw user +// collected properties" list. Prints nothing for an SST without them. +void PrintBlobReferenceProperties(const TableProperties& tp) { + const UserCollectedProperties& properties = tp.user_collected_properties; + const auto descriptor = properties.find(kBlobReferenceDescriptorPropertyName); + const auto classification = + properties.find(kBlobReferenceClassificationPropertyName); + if (descriptor == properties.end() && classification == properties.end()) { + return; + } + fprintf(stdout, + "Blob reference metadata\n" + "------------------------------\n"); + if (descriptor != properties.end()) { + BlobReferenceMetadataDescriptor decoded; + const Status s = BlobReferenceMetadataDescriptor::DecodeFrom( + descriptor->second, &decoded); + if (!s.ok()) { + fprintf(stdout, " exact metadata: cannot decode the descriptor: %s\n", + s.ToString().c_str()); + } else { + fprintf(stdout, + " exact metadata: %" PRIu64 " references, %" PRIu64 + " stored value bytes\n", + decoded.reference_count, decoded.stored_value_bytes); + fprintf(stdout, " column family ID: %" PRIu32 "\n", + decoded.column_family_id); + fprintf(stdout, " SST file number when built: %" PRIu64 "\n", + decoded.sst_file_number); + fprintf(stdout, " index block: offset %" PRIu64 ", size %" PRIu64 "\n", + decoded.index_handle.offset(), decoded.index_handle.size()); + } + } + if (classification != properties.end()) { + uint64_t frontier = kInvalidBlobFileNumber; + const Status s = + DecodeBlobReferenceClassification(classification->second, &frontier); + if (!s.ok()) { + fprintf(stdout, + " coarse classification: cannot decode the property: %s\n", + s.ToString().c_str()); + } else if (frontier == kInvalidBlobFileNumber) { + // Frontier 0: the SST has no indirect reference and is stored as a legacy + // SST. The frontier is the oldest indirect-origin file number a coarse + // SST may reference (see "Coarse SST" in + // db/blob/blob_reference_metadata.h). + fprintf(stdout, + " coarse classification: no indirect references (legacy " + "SST)\n"); + } else { + fprintf(stdout, + " coarse classification: frontier %" PRIu64 + " (indirect blob GC does not collect a blob file numbered at " + "or above it)\n", + frontier); + } + } +} + // Appends a JSON string literal (with surrounding quotes) for `s` to `out`. // Control characters and any non-ASCII bytes (0x7f-0xff) are emitted as \u00XX // escapes so the result is always valid JSON even when `s` contains non-UTF-8 @@ -1305,6 +1368,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) { fprintf(stdout, " # %s: 0x%s\n", prop_name.c_str(), prop_val.c_str()); } + PrintBlobReferenceProperties(*table_properties); } } else { fprintf(stderr, "Reader unexpectedly returned null properties\n"); diff --git a/util/coding.h b/util/coding.h index 58382048ca0f..6c2c66319c54 100644 --- a/util/coding.h +++ b/util/coding.h @@ -61,6 +61,11 @@ bool GetFixed32(Slice* input, uint32_t* value); bool GetFixed16(Slice* input, uint16_t* value); bool GetVarint32(Slice* input, uint32_t* value); bool GetVarint64(Slice* input, uint64_t* value); +// Like GetVarint32/64, but rejects overlong (non-minimal) encodings, so each +// value has exactly one encoding. On failure `input` may be partly consumed; +// callers treat that as corruption. +bool GetCanonicalVarint32(Slice* input, uint32_t* value); +bool GetCanonicalVarint64(Slice* input, uint64_t* value); bool GetVarsignedint64(Slice* input, int64_t* value); bool GetLengthPrefixedSlice(Slice* input, Slice* result); // This function assumes data is well-formed. @@ -294,6 +299,18 @@ inline bool GetVarint64(Slice* input, uint64_t* value) { } } +inline bool GetCanonicalVarint32(Slice* input, uint32_t* value) { + const size_t before = input->size(); + return GetVarint32(input, value) && + before - input->size() == VarintLength(*value); +} + +inline bool GetCanonicalVarint64(Slice* input, uint64_t* value) { + const size_t before = input->size(); + return GetVarint64(input, value) && + before - input->size() == VarintLength(*value); +} + inline bool GetVarsignedint64(Slice* input, int64_t* value) { const char* p = input->data(); const char* limit = p + input->size(); diff --git a/util/coding_test.cc b/util/coding_test.cc index cb8939f9cbb2..a4dc3601cd7b 100644 --- a/util/coding_test.cc +++ b/util/coding_test.cc @@ -117,6 +117,30 @@ TEST(Coding, Varint32) { ASSERT_EQ(p, s.data() + s.size()); } +TEST(Coding, CanonicalVarint) { + // 0x85 0x00 is an overlong encoding of 5; the minimal one is 0x05. + const std::string overlong("\x85\x00", 2); + const std::string minimal("\x05", 1); + { + Slice input(overlong); + uint32_t value = 0; + ASSERT_FALSE(GetCanonicalVarint32(&input, &value)); + input = Slice(minimal); + ASSERT_TRUE(GetCanonicalVarint32(&input, &value)); + ASSERT_EQ(value, 5U); + ASSERT_TRUE(input.empty()); + } + { + Slice input(overlong); + uint64_t value = 0; + ASSERT_FALSE(GetCanonicalVarint64(&input, &value)); + input = Slice(minimal); + ASSERT_TRUE(GetCanonicalVarint64(&input, &value)); + ASSERT_EQ(value, 5U); + ASSERT_TRUE(input.empty()); + } +} + TEST(Coding, Varint64) { // Construct the list of values to check std::vector values;