Skip to content

Commit 40dae01

Browse files
ttyusupovttyusupov
committed
[yugabyte#27594] docdb: sst_dump output and logging improvements for check_data_blocks command
Summary: Implemented the following improvements for check_data_blocks command output and logging: - Include data block number - Include data block index key - For generated `dd` commands distinguish between data blocks which are corrupt vs ones which are before/after corrupted region but consistent themselves Jira: DB-17170 Test Plan: Tested manually on artificially corrupted SST file: ``` [timur@dev-server-timur tablet-6e27439238ea4f4aa23c013d0ab55931]$ ./sst_dump --command=check_data_blocks --file=000046.sst 2>&1 | tee sst_dump.out WARNING: Logging before InitGoogleLogging() is written to STDERR I0610 18:13:52.752576 204553 sst_dump_tool.cc:452] Checking data block #0 handle: BlockHandle { offset: 0 size: 3927 } W0610 18:13:52.756827 204553 file_reader_writer.cc:101] Read attempt #1 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. W0610 18:13:52.756884 204553 file_reader_writer.cc:101] Read attempt #2 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. W0610 18:13:52.756889 204553 format.cc:466] ReadBlockContents: Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. W0610 18:13:52.756894 204553 sst_dump_tool.cc:460] Failed to read data block yugabyte#247 (index_key: 305482E7904980000000000024D107FFFFFFFFFFFFFF), block handle: BlockHandle { offset: 996452 size: 4024 }. Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 996452 size: 4024 }, expected checksum: 1118200703, actual checksum: 1023486239. from [] to [] Process 000046.sst Sst file format: block-based dd if="000046.sst.sblock.0" bs=1 skip=992433 count=4014 of="000046.sst.sblock.0.block_246.offset_992433.size_4014.part.prev" # Data block yugabyte#246 index_key: 305482E7904980000000000024AB07FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=996452 count=4024 of="000046.sst.sblock.0.block_247.offset_996452.size_4024.part.corrupt" # Data block yugabyte#247 index_key: 305482E7904980000000000024D107FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=1000481 count=4019 of="000046.sst.sblock.0.block_248.offset_1000481.size_4019.part.next" # Data block yugabyte#248 index_key: 305482E7904980000000000024F707FFFFFFFFFFFFFF W0610 18:13:53.201548 204553 file_reader_writer.cc:101] Read attempt #1 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119695806 size: 11770 }, expected checksum: 2324967102, actual checksum: 2577758987. W0610 18:13:53.201601 204553 file_reader_writer.cc:101] Read attempt #2 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119695806 size: 11770 }, expected checksum: 2324967102, actual checksum: 2577758987. W0610 18:13:53.201606 204553 format.cc:466] ReadBlockContents: Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119695806 size: 11770 }, expected checksum: 2324967102, actual checksum: 2577758987. W0610 18:13:53.201610 204553 sst_dump_tool.cc:460] Failed to read data block yugabyte#27462 (index_key: 30C33B7F8D4980000000000F30EC07FFFFFFFFFFFFFF), block handle: BlockHandle { offset: 119695806 size: 11770 }. Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119695806 size: 11770 }, expected checksum: 2324967102, actual checksum: 2577758987. dd if="000046.sst.sblock.0" bs=1 skip=119684040 count=11761 of="000046.sst.sblock.0.block_27461.offset_119684040.size_11761.part.prev" # Data block yugabyte#27461 index_key: 30C33B7F8D4980000000000F2D8707FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119695806 count=11770 of="000046.sst.sblock.0.block_27462.offset_119695806.size_11770.part.corrupt" # Data block yugabyte#27462 index_key: 30C33B7F8D4980000000000F30EC07FFFFFFFFFFFFFF W0610 18:13:53.201637 204553 file_reader_writer.cc:101] Read attempt #1 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119707581 size: 11779 }, expected checksum: 2324967102, actual checksum: 4201726259. W0610 18:13:53.201644 204553 file_reader_writer.cc:101] Read attempt #2 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119707581 size: 11779 }, expected checksum: 2324967102, actual checksum: 4201726259. W0610 18:13:53.201648 204553 format.cc:466] ReadBlockContents: Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119707581 size: 11779 }, expected checksum: 2324967102, actual checksum: 4201726259. W0610 18:13:53.201651 204553 sst_dump_tool.cc:460] Failed to read data block yugabyte#27463 (index_key: 30C33B7F8D4980000000000F345207FFFFFFFFFFFFFF), block handle: BlockHandle { offset: 119707581 size: 11779 }. Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119707581 size: 11779 }, expected checksum: 2324967102, actual checksum: 4201726259. dd if="000046.sst.sblock.0" bs=1 skip=119707581 count=11779 of="000046.sst.sblock.0.block_27463.offset_119707581.size_11779.part.corrupt" # Data block yugabyte#27463 index_key: 30C33B7F8D4980000000000F345207FFFFFFFFFFFFFF W0610 18:13:53.201665 204553 file_reader_writer.cc:101] Read attempt #1 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119719365 size: 11745 }, expected checksum: 632490478, actual checksum: 639140170. W0610 18:13:53.201674 204553 file_reader_writer.cc:101] Read attempt #2 failed in file 000046.sst.sblock.0 : Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119719365 size: 11745 }, expected checksum: 632490478, actual checksum: 639140170. W0610 18:13:53.201679 204553 format.cc:466] ReadBlockContents: Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119719365 size: 11745 }, expected checksum: 632490478, actual checksum: 639140170. W0610 18:13:53.201684 204553 sst_dump_tool.cc:460] Failed to read data block yugabyte#27464 (index_key: 30C33B7F8D4980000000000F37B707FFFFFFFFFFFFFF), block handle: BlockHandle { offset: 119719365 size: 11745 }. Corruption (yb/rocksdb/table/format.cc:345): Block checksum mismatch in file: 000046.sst.sblock.0, block handle: BlockHandle { offset: 119719365 size: 11745 }, expected checksum: 632490478, actual checksum: 639140170. dd if="000046.sst.sblock.0" bs=1 skip=119719365 count=11745 of="000046.sst.sblock.0.block_27464.offset_119719365.size_11745.part.corrupt" # Data block yugabyte#27464 index_key: 30C33B7F8D4980000000000F37B707FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119731115 count=11781 of="000046.sst.sblock.0.block_27465.offset_119731115.size_11781.part.next" # Data block yugabyte#27465 index_key: 30C33B7F8D4980000000000F3B1C07FFFFFFFFFFFFFF ``` ``` [timur@dev-server-timur tablet-6e27439238ea4f4aa23c013d0ab55931]$ cat sst_dump.out | grep dd dd if="000046.sst.sblock.0" bs=1 skip=992433 count=4014 of="000046.sst.sblock.0.block_246.offset_992433.size_4014.part.prev" # Data block yugabyte#246 index_key: 305482E7904980000000000024AB07FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=996452 count=4024 of="000046.sst.sblock.0.block_247.offset_996452.size_4024.part.corrupt" # Data block yugabyte#247 index_key: 305482E7904980000000000024D107FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=1000481 count=4019 of="000046.sst.sblock.0.block_248.offset_1000481.size_4019.part.next" # Data block yugabyte#248 index_key: 305482E7904980000000000024F707FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119684040 count=11761 of="000046.sst.sblock.0.block_27461.offset_119684040.size_11761.part.prev" # Data block yugabyte#27461 index_key: 30C33B7F8D4980000000000F2D8707FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119695806 count=11770 of="000046.sst.sblock.0.block_27462.offset_119695806.size_11770.part.corrupt" # Data block yugabyte#27462 index_key: 30C33B7F8D4980000000000F30EC07FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119707581 count=11779 of="000046.sst.sblock.0.block_27463.offset_119707581.size_11779.part.corrupt" # Data block yugabyte#27463 index_key: 30C33B7F8D4980000000000F345207FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119719365 count=11745 of="000046.sst.sblock.0.block_27464.offset_119719365.size_11745.part.corrupt" # Data block yugabyte#27464 index_key: 30C33B7F8D4980000000000F37B707FFFFFFFFFFFFFF dd if="000046.sst.sblock.0" bs=1 skip=119731115 count=11781 of="000046.sst.sblock.0.block_27465.offset_119731115.size_11781.part.next" # Data block yugabyte#27465 index_key: 30C33B7F8D4980000000000F3B1C07FFFFFFFFFFFFFF ``` Reviewers: arybochkin Reviewed By: arybochkin Subscribers: ybase Tags: #jenkins-ready Differential Revision: https://phorge.dev.yugabyte.com/D44692
1 parent 59d6f53 commit 40dae01

1 file changed

Lines changed: 42 additions & 22 deletions

File tree

‎src/yb/rocksdb/tools/sst_dump_tool.cc‎

Lines changed: 42 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
#include "yb/docdb/docdb_debug.h"
5555

5656
#include "yb/util/format.h"
57+
#include "yb/util/kv_util.h"
5758
#include "yb/util/status_log.h"
5859

5960
using yb::docdb::StorageDbType;
@@ -337,6 +338,7 @@ Status SstFileReader::ReadSequential(bool print_kv,
337338
iter->SeekToFirst();
338339
}
339340
for (; iter->Valid(); iter->Next()) {
341+
WARN_NOT_OK(iter->status(), "Iterator error: ");
340342
Slice key = iter->key();
341343
Slice value = iter->value();
342344
++i;
@@ -386,10 +388,25 @@ Status SstFileReader::ReadSequential(bool print_kv,
386388

387389
namespace {
388390

389-
void PrintSaveBlockCommand(const std::string& data_file_path, const BlockHandle& block_handle) {
390-
std::cout << "dd if=\"" << data_file_path << "\" bs=1 skip=" << block_handle.offset()
391-
<< " count=" << block_handle.size() << " of=\"" << data_file_path << ".offset_"
392-
<< block_handle.offset() << ".size_" << block_handle.size() << ".part\"" << std::endl;
391+
struct DataBlockInfo {
392+
yb::KeyBuffer index_key;
393+
BlockHandle handle;
394+
size_t index_entry_pos = 0;
395+
bool saved = false;
396+
};
397+
398+
void PrintSaveBlockCommand(
399+
const std::string& data_file_path, const std::string& ext, DataBlockInfo* info) {
400+
if (info->saved) {
401+
return;
402+
}
403+
std::cout << "dd if=\"" << data_file_path << "\" bs=1 skip=" << info->handle.offset()
404+
<< " count=" << info->handle.size() << " of=\"" << data_file_path << ".block_"
405+
<< info->index_entry_pos << ".offset_" << info->handle.offset() << ".size_"
406+
<< info->handle.size() << ".part" << ext << "\" # Data block #" << info->index_entry_pos
407+
<< " index_key: "
408+
<< info->index_key.AsSlice().PrefixNoLongerThan(256).ToDebugHexString() << std::endl;
409+
info->saved = true;
393410
}
394411

395412
} // namespace
@@ -411,43 +428,46 @@ Status SstFileReader::CheckDataBlocks(DoUncompress do_uncompress) {
411428
std::unique_ptr<RandomAccessFileReader> data_file_reader(
412429
new RandomAccessFileReader(std::move(data_file)));
413430

414-
size_t index_entry_pos = 0;
415-
BlockHandle prev_block_handle;
416-
BlockHandle block_handle;
431+
DataBlockInfo prev_block_info;
432+
DataBlockInfo block_info;
417433
bool save_block = false;
418434
for (index_iterator->SeekToFirst(); index_iterator->Valid();
419-
index_iterator->Next(), ++index_entry_pos) {
420-
prev_block_handle = block_handle;
435+
index_iterator->Next(), prev_block_info = block_info, ++block_info.index_entry_pos) {
436+
block_info.index_key = index_iterator->Entry().key;
437+
block_info.saved = false;
421438
{
422439
auto index_value_slice = index_iterator->Entry().value;
423-
auto status = block_handle.DecodeFrom(&index_value_slice);
440+
auto status = block_info.handle.DecodeFrom(&index_value_slice);
424441
if (!status.ok()) {
425-
LOG(WARNING) << "Failed to decode SST index entry #" << index_entry_pos << ": "
426-
<< index_iterator->Entry().value.ToDebugHexString() << ". " << status;
442+
LOG(WARNING) << "Failed to decode SST index entry #" << block_info.index_entry_pos
443+
<< " (index_key: "
444+
<< block_info.index_key.AsSlice().PrefixNoLongerThan(256).ToDebugHexString()
445+
<< "): " << index_iterator->Entry().value.ToDebugHexString() << ". " << status;
427446
continue;
428447
}
429448
LOG_IF(WARNING, index_value_slice.size() > 0)
430449
<< "Extra bytes (" << index_value_slice.size()
431450
<< ") in index entry: " << index_iterator->Entry().value.ToDebugHexString();
432451
}
433-
YB_LOG_EVERY_N_SECS(INFO, 30) << "Checking data block #" << index_entry_pos
434-
<< " handle: " << block_handle.ToDebugString();
452+
YB_LOG_EVERY_N_SECS(INFO, 30) << "Checking data block #" << block_info.index_entry_pos
453+
<< " handle: " << block_info.handle.ToDebugString();
435454

436455
BlockContents block_contents;
437456
auto status = ReadBlockContents(
438-
data_file_reader.get(), footer_, read_options, block_handle, &block_contents, options_.env,
439-
/* mem_tracker = */ nullptr, do_uncompress);
457+
data_file_reader.get(), footer_, read_options, block_info.handle, &block_contents,
458+
options_.env, /* mem_tracker = */ nullptr, do_uncompress);
440459
if (!status.ok()) {
441-
LOG(WARNING) << "Failed to read block with handle: " << block_handle.ToDebugString() << ". "
442-
<< status;
443-
if (prev_block_handle.IsSet()) {
444-
PrintSaveBlockCommand(data_file_path, prev_block_handle);
460+
LOG(WARNING) << "Failed to read data block #" << block_info.index_entry_pos << " (index_key: "
461+
<< block_info.index_key.AsSlice().PrefixNoLongerThan(256).ToDebugHexString()
462+
<< "), block handle: " << block_info.handle.ToDebugString() << ". " << status;
463+
if (prev_block_info.handle.IsSet()) {
464+
PrintSaveBlockCommand(data_file_path, ".prev", &prev_block_info);
445465
}
446-
PrintSaveBlockCommand(data_file_path, block_handle);
466+
PrintSaveBlockCommand(data_file_path, ".corrupt", &block_info);
447467
// Save next block as well.
448468
save_block = true;
449469
} else if (save_block) {
450-
PrintSaveBlockCommand(data_file_path, block_handle);
470+
PrintSaveBlockCommand(data_file_path, ".next", &block_info);
451471
save_block = false;
452472
}
453473
}

0 commit comments

Comments
 (0)