\u6807\u7b7e\uff08\u7a7a\u683c\u5206\u9694\uff09\uff1a leveldb log<\/p>\n
leveldb\u4e2d\uff0c\u6240\u6709\u5bf9\u6570\u636e\u5e93\u7684\u64cd\u4f5c\u90fd\u4f1a\u8bb0\u5f55\u5230log\u4e2d\uff0c\u5f53log\u6587\u4ef6\u8fbe\u5230\u9884\u5b9a\u4e49\u7684\u5927\u5c0f\u540e\uff0c\u5c31\u4f1a\u88ab\u8f6c\u6362\u6210\u6570\u636e\u8868(sstable)\uff0c\u6240\u4ee5log\u6587\u4ef6\u7684\u751f\u6210\u548c\u8bfb\u53d6\u7b97\u662fleveldb\u6838\u5fc3\u90e8\u5206\u7684\u7b2c\u4e00\u6b65\u3002
\n\u8fd9\u7bc7\u6587\u7ae0\u6d89\u53ca\u5230\u7684leveldb\u7684\u6e90\u4ee3\u7801\u6587\u4ef6\u6709:<\/p>\n
db\/log_format.h\ndb\/log_reader.h\ndb\/log_reader.cc\ndb\/log_writer.h\ndb\/log_writer.cc\nleveldb\/slice.h\nleveldb\/status.h\n<\/code><\/pre>\n\u4e8c\u3001\u65e5\u5fd7\u5b58\u50a8\u683c\u5f0f\u2014\u2014log_format.h<\/h2>\n
\u5728\u4e4b\u524d\u7684\u6587\u7ae0\u4e2d\uff0c\u4e86\u89e3\u5230log\u662f\u7531\u4e00\u4e2a\u4e2ablock<\/code>\u7ec4\u6210\uff0c\u6bcf\u4e2ablock\u662f32KB<\/code>\u7684\u5927\u5c0f\u3002<\/p>\nblock := record* trailer?\nrecord :=\n checksum: uint32 \/\/ crc32c of type and data[] ; little-endian\n length: uint16 \/\/ little-endian\n type: uint8 \/\/ One of FULL, FIRST, MIDDLE, LAST\n data: uint8[length]\n<\/code><\/pre>\n\u6bcf\u4e2ablock<\/code>\u4e2d\u5b58\u50a8\u7684\u662f\u4e00\u6761\u6761\u7684\u8bb0\u5f55<\/code>,\u5982\u679c\u4e00\u4e2ablock<\/code>\u7684\u672b\u5c3e\u6700\u540e\u5269\u4e0b\u5c0f\u4e8e\u7b49\u4e8e6bytes\u7684\u7a7a\u95f4\uff0c\u5c31\u4f1a\u4f7f\u7528\u4e00\u4e2atrailer<\/code>\u586b\u5145\u800c\u4e0d\u662f\u7528\u65b0\u7684\u8bb0\u5f55\u586b\u5145\uff0c\u8fd9\u4e2atrailer<\/code>\u90fd\u662f\u75310\u5b57\u8282\u7ec4\u6210\u3002\u4e00\u6761\u8bb0\u5f55\u5305\u62ec\u4e00\u4e2acrc32c\u751f\u6210\u7684\u6821\u9a8c\u7801checksum\u3001\u8bb0\u5f55\u7684\u957f\u5ea6length\u3001\u8bb0\u5f55\u7684\u7c7b\u578btype\u3001\u6570\u636e\u3002<\/p>\n\u5982\u679c\u4e00\u6761\u8bb0\u5f55\u8de8\u8d8a\u4e86\u591a\u4e2ablock<\/code>,\u5c31\u4f1a\u88ab\u5206\u6bb5\uff0c\u5206\u6bb5\u7c7b\u578b\u67093\u79cd\u2014\u2014First\u3001Middle\u3001Last<\/p>\n\u5173\u4e8e\u8fd9\u90e8\u5206\u7684\u5b9e\u73b0\u662f\u5728log_format.h\u4e2d<\/p>\n
enum RecordType {\n \/\/0\u662f\u4fdd\u7559\u5b57\uff0c\u7528\u6765\u8868\u793a\u9884\u5206\u914d\u7684\u6587\u4ef6\n kZeroType = 0,\n\n kFullType = 1,\n\n \/\/ \u4e00\u6761\u8bb0\u5f55\u88ab\u5206\u6bb5\u76843\u79cd\u7c7b\u578b\n kFirstType = 2,\n kMiddleType = 3,\n kLastType = 4\n};\n\/\/\u5b9a\u4e49\u4e86\u6700\u5927\u7684\u8bb0\u5f55\u7c7b\u578b\u503c\nstatic const int kMaxRecordType = kLastType;\n\/\/\u5b9a\u4e49\u4e86\u6700\u5927\u7684`block`\u5927\u5c0f\nstatic const int kBlockSize = 32768;\n\n\/\/ \u8bb0\u5f55\u5934\u662f checksum (4 bytes), length (2 bytes), type (1 byte).\u603b\u51717bytes\nstatic const int kHeaderSize = 4 + 2 + 1;\n<\/code><\/pre>\n\u4e09\u3001\u65e5\u5fd7\u7684\u5199\u5165<\/h2>\n
\u5728log_writer\u4ee3\u7801\u4e2d\uff0c\u5b9e\u73b0\u4e86Writer\u7c7b\uff0c\u6700\u4e3b\u8981\u7684\u5c31\u662f\u4e00\u4e2aAddRecord(const Slice& slice)<\/code>\u3002<\/p>\nWriter\u7c7b\u6709\u4e24\u4e2a\u6784\u9020\u51fd\u6570\uff0c\u5bf9\u4e8e\u53ea\u4f20\u4e86\u4e00\u4e2a\u53c2\u6570\u2014\u2014WritableFile\u6307\u9488\u7684\u6784\u9020\u51fd\u6570\uff0c\u4f7f\u7528\u4e86explicit\u663e\u793a\u6784\u9020\uff0c\u8fd9\u6837\u53ef\u4ee5\u9632\u6b62\u9690\u5f0f\u8f6c\u6362\u3002\u53e6\u5916\u5c06Writer(const Writer&)<\/code>\u548cvoid operator=(const Writer&)<\/code>\u653e\u5728private\u57df\uff0c\u9632\u6b62\u62f7\u8d1d\u8d4b\u503c\u3002<\/p>\nWriter\u6709\u4e09\u4e2a\u6210\u5458\u53d8\u91cf WritableFile* dest_<\/code>\u7528\u6765\u4fdd\u5b58\u53ef\u5199\u7684\u6587\u4ef6\uff0cint block_offset_<\/code>\u7528\u6765\u4fdd\u5b58\u5f53\u524d\u5757\u7684\u504f\u79fb\u91cf, uint32_t type_crc_[kMaxRecordType + 1]<\/code>\u7528\u6765\u4fdd\u5b585\u79cd\u8bb0\u5f55\u7c7b\u578b\u7684crc32c\u8ba1\u7b97\u7ed3\u679c\u3002\u8fd9\u91cc\u4f1a\u63d0\u524d\u8ba1\u7b97\u597d\u4fdd\u5b58\u8fdb\u8fd9\u4e2a\u6570\u7ec4\u6765\u51cf\u5c11\u5b58\u50a8\u65f6\u8ba1\u7b97\u7684\u8d1f\u8f7d\u3002<\/p>\n\u63a5\u4e0b\u6765\u5c31\u662f\u6700\u91cd\u8981\u7684AddRecord(const Slice& slice)<\/code>\u4e86\u3002Slice\u662fleveldb\u4e2d\u7528\u6765\u4fdd\u5b58key\u6216\u8005value\u7684\u503c\u7684\uff0c\u5305\u542b\u4e00\u4e2a\u6307\u9488\uff0c\u4e00\u4e2a\u957f\u5ea6\u3002<\/p>\nStatus Writer::AddRecord(const Slice& slice) {\n const char* ptr = slice.data(); \/\/\u53d6\u51fa\u6570\u636e\u7684\u6307\u9488\n size_t left = slice.size(); \/\/\u53d6\u51fa\u6570\u636e\u7684\u957f\u5ea6\n\n \/\/ \u5982\u679c\u6709\u5fc5\u8981\u7684\u8bdd\uff0c\u5bf9\u8bb0\u5f55\u5206\u6bb5\uff0c\u5e76\u4fdd\u5b58.\n \/\/ \u6ce8\u610f\u5982\u679cslice\u662f\u7a7a\u7684\uff0c\u6211\u4eec\u4e5f\u4f1a\u8fed\u4ee3\u4e00\u6b21\uff0c\u4fdd\u5b58\u4e00\u4e2a\u957f\u5ea6\u4e3a0\u7684\u8bb0\u5f55\n Status s; \/\/\u7528\u6765\u4fdd\u5b58\u8fd9\u6b21\u64cd\u4f5c\u7684\u7ed3\u679c\n bool begin = true; \/\/\u662f\u5426\u662f\u4e00\u6761\u8bb0\u5f55\u7684\u5f00\u59cb\n do {\n \/\/\u8ba1\u7b97\u51fa\u5f53\u524d\u5757\u5269\u4e0b\u7684\u7a7a\u95f4\n const int leftover = kBlockSize - block_offset_;\n \/\/\u4f7f\u7528assert\u8fdb\u884c\u65ad\u8a00\uff0c\u8fd9\u8fb9\u662f\u4e00\u5b9a\u4f1a>=0\u7684\uff0cassert\u8fd9\u91cc\u53ea\u662f\u4e3a\u4e86\u5e2e\u52a9\u8c03\u8bd5\u7a0b\u5e8f\uff0c\u5b9e\u9645\u4f7f\u7528\u65f6\u5728\u7f16\u8bd1\u65f6\u901a\u8fc7\u53c2\u6570\u53bb\u9664assert\u8bed\u53e5\n assert(leftover >= 0);\n\n \/\/\u5982\u679c\u5269\u4e0b\u7684\u7a7a\u95f4\u5df2\u7ecf\u4e0d\u8db3\u4e00\u6761\u8bb0\u5f55\u7684header\u4e86\n if (leftover < kHeaderSize) {\n \/\/ \u5207\u6362\u5230\u65b0\u7684\u5757\n if (leftover > 0) {\n \/\/ \u5982\u679c\u5269\u4e0b\u7684\u7a7a\u95f4>0\uff0c\u9700\u8981\u4f7f\u7528\\x00\u586b\u5145\n assert(kHeaderSize == 7);\n dest_->Append(Slice(\"\\x00\\x00\\x00\\x00\\x00\\x00\", leftover));\n }\n \/\/\u65b0\u7684\u5757\u504f\u79fb\u91cf\u662f0\n block_offset_ = 0;\n }\n\n \/\/ \u4e0d\u53d8\u7684\u9053\u7406\uff1a\u6211\u4eec\u7edd\u5bf9\u4e0d\u4f1a\u8ba9\u5269\u4e0b\u7684\u7a7a\u95f4<kHeaderSize.\n assert(kBlockSize - block_offset_ - kHeaderSize >= 0);\n\n \/\/\u5f53\u524d\u8fd8\u53ef\u7528\u7684\u7a7a\u95f4\n const size_t avail = kBlockSize - block_offset_ - kHeaderSize;\n\n \/\/\u8bb0\u5f55\u7247\u6bb5\u7684\u5927\u5c0f\uff0c\u53d6\u53ef\u7528\u7684\u7a7a\u95f4\u5927\u5c0f\u548c\u8bb0\u5f55\u7684\u5927\u5c0f\u7684\u6700\u5c0f\u503c\n const size_t fragment_length = (left < avail) ? left : avail;\n\n \/\/\u5f53\u524d\u7684\u8bb0\u5f55\u7c7b\u578b\n RecordType type;\n\n \/\/\u5982\u679c\u8bb0\u5f55\u7684\u5927\u5c0f\u548c\u7247\u6bb5\u7684\u957f\u5ea6\u662f\u4e00\u6837\u7684\uff0c\u8868\u793a\u5230\u8fbe\u4e86\u8bb0\u5f55\u7684\u672b\u5c3e\u4e3atrue\uff0c\u5426\u5219false\n const bool end = (left == fragment_length);\n\n \/\/\u8bb0\u5f55\u7c7b\u578b\u7684\u5224\u65ad\n if (begin && end) {\n type = kFullType;\n } else if (begin) {\n type = kFirstType;\n } else if (end) {\n type = kLastType;\n } else {\n type = kMiddleType;\n }\n\n \/\/\u5199\u5165\u5230\u7269\u7406\u5b58\u50a8\u4e2d\uff0c\n s = EmitPhysicalRecord(type, ptr, fragment_length);\n ptr += fragment_length; \/\/\u6570\u636e\u6307\u9488\u53f3\u79fb\u4e00\u4e2a\u7247\u6bb5\u957f\u5ea6\n left -= fragment_length; \/\/\u6570\u636e\u7684\u957f\u5ea6\u51cf\u53bb\u4e00\u4e2a\u7247\u6bb5\u7684\u957f\u5ea6\n begin = false; \/\/\u4e0d\u518d\u662f\u5f00\u59cb\u4e86\n } while (s.ok() && left > 0); \/\/\u5982\u679c\u6ca1\u6709\u51fa\u9519\uff0c\u5e76\u4e14\u6570\u636e\u8fd8\u5269\u5c31\u518d\u5faa\u73af\n\n return s;\n}\n<\/code><\/pre>\nAddRecord(const Slice& slice)<\/code>\u8c03\u7528\u4e86\u4e00\u4e2a\u79c1\u6709\u6210\u5458\u51fd\u6570EmitPhysicalRecord(RecordType t, const char* ptr, size_t n)<\/code>,\u770b\u4e00\u770b\u8fd9\u4e2a\u5c06\u8bb0\u5f55\u5199\u5165\u6301\u4e45\u5316\u5b58\u50a8\u662f\u5982\u4f55\u5b9e\u73b0\u7684\u3002<\/p>\nStatus Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {\n assert(n <= 0xffff); \/\/ n\u5fc5\u987b\u53ef\u4ee5\u75282KB\u8868\u793a\uff0c\u56e0\u4e3a\u4e00\u6761\u8bb0\u5f55\u7684length\u662fuint16\u5927\u5c0f\u7684\uff0clittle-endian\n assert(block_offset_ + kHeaderSize + n <= kBlockSize);\n\n \/\/ \u5bf9\u8bb0\u5f55\u7684header\u8fdb\u884c\u683c\u5f0f\u5316\n char buf[kHeaderSize];\n \/\/\u53d6\u4f4e\u4f4d\u5b57\u8282\n buf[4] = static_cast<char>(n & 0xff);\n \/\/\u53d6\u9ad8\u4f4d\u5b57\u8282\n buf[5] = static_cast<char>(n >> 8);\n \/\/\u4e00\u4f4d\u7684\u8bb0\u5f55\u7c7b\u578b\n buf[6] = static_cast<char>(t);\n\n \/\/ \u8ba1\u7b97\u8bb0\u5f55\u7c7b\u578b\u7684crc\u503c\u548c\u6709\u6548\u8f7d\u8377\n uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n);\n crc = crc32c::Mask(crc); \/\/ Adjust for storage\n \/\/\u586b\u51454\u4f4d\u7684\u6821\u9a8c\u503c\n EncodeFixed32(buf, crc);\n\n \/\/ \u5c06\u8bb0\u5f55\u7684header\u5199\u5165\n Status s = dest_->Append(Slice(buf, kHeaderSize));\n if (s.ok()) {\n \/\/\u5c06\u8bb0\u5f55\u7684\u6570\u636e\u5199\u5165\n s = dest_->Append(Slice(ptr, n));\n if (s.ok()) {\n \/\/flush\u4e00\u4e0b\n s = dest_->Flush();\n }\n }\n\n \/\/\u5757\u7684\u504f\u79fb\u91cf\u53f3\u79fbheader\u548c\u6570\u636e\u7684\u957f\u5ea6\u548c\n block_offset_ += kHeaderSize + n;\n return s;\n}\n<\/code><\/pre>\n\u4e0a\u9762\u7684\u4ee3\u7801\u5206\u6790\u5176\u5b9e\u5c31\u662flog\u90e8\u5206\u903b\u8f91\u4e0a\u7684\u5b9e\u73b0\uff0c\u5c06\u4e00\u6761\u8bb0\u5f55\u683c\u5f0f\u5316\u6210\u8bbe\u8ba1\u597d\u7684\u683c\u5f0f\u3002\u8fd9\u91cc\u6709\u4e00\u4e2a\u7591\u95ee\u5c31\u662fdest_->Append(Slice& slice)<\/code>\u548cdest_->Flush()<\/code>\u662f\u5982\u4f55\u5de5\u4f5c\u7684\u3002\u6240\u4ee5\u7ee7\u7eed\u8ffd\u8e2a\u4e0b\u53bb\uff0c\u5230include\/leveldb\/env.h<\/code>\u4e2d\uff1a<\/p>\n\/\/ A file abstraction for sequential writing. The implementation\n\/\/ must provide buffering since callers may append small fragments\n\/\/ at a time to the file.\nclass WritableFile {\n public:\n WritableFile() { }\n virtual ~WritableFile();\n\n virtual Status Append(const Slice& data) = 0;\n virtual Status Close() = 0;\n virtual Status Flush() = 0;\n virtual Status Sync() = 0;\n\n private:\n \/\/ No copying allowed\n WritableFile(const WritableFile&);\n void operator=(const WritableFile&);\n};\n<\/code><\/pre>\n\u8fd9\u90e8\u5206\u662fWritableFile\u7684\u62bd\u8c61\u90e8\u5206\uff0cAppend\u3001Close\u3001Flush\u3001Sync\u90fd\u662f\u7eaf\u865a\u51fd\u6570\uff0c\u5177\u4f53\u5b9e\u73b0\u5fc5\u987b\u7531\u7528\u6237\u81ea\u5df1\u53bb\u505a\u3002\u5e76\u4e14\u5b9e\u73b0\u65f6\u5fc5\u987b\u63d0\u4f9b\u7f13\u51b2\u533a\uff0c\u8fd9\u6837\u8c03\u7528\u8005\u53ef\u4ee5append\u5c0f\u7684\u6570\u636e\u6bb5\u5230\u6587\u4ef6\u4e2d\u3002<\/p>\n
\u5b9e\u9645\u4e0aleveldb\u4e2d\u4e5f\u6709\u9ed8\u8ba4(posix)\u7684\u5b9e\u73b0\u3002\u5728util\/env\/env_posix.cc<\/code>\u4e2d\u3002\u76ee\u524d\u53ea\u770bAppend\u548cFlush\u5177\u4f53\u5b9e\u73b0\uff1a<\/p>\n virtual Status Append(const Slice& data) {\n size_t r = fwrite_unlocked(data.data(), 1, data.size(), file_);\n if (r != data.size()) {\n return IOError(filename_, errno);\n }\n return Status::OK();\n }\n<\/code><\/pre>\n virtual Status Flush() {\n if (fflush_unlocked(file_) != 0) {\n return IOError(filename_, errno);\n }\n return Status::OK();\n }\n<\/code><\/pre>\nfwrite_unlocked\u548cfflush_unlocked\u90fd\u662fdefine\u7684fwrite\u548cfflush\u65b9\u6cd5\u3002<\/p>\n
\u6240\u4ee5\u8fd9\u91cc\u7684\u8bbe\u8ba1\u662f\u5f88\u597d\u7684\u6848\u4f8b\u3002\u901a\u8fc7\u5c06WritableFile\u62bd\u8c61\u5316\uff0c\u63d0\u4f9b\u7ed9\u7528\u6237\u81ea\u5df1\u6765\u5b9e\u73b0\uff0c\u4ece\u800c\u5f88\u8f7b\u677e\u7684\u5c31\u53ef\u4ee5\u5728\u591a\u5e73\u53f0\u95f4\u79fb\u690d\u3002<\/p>\n
\u56db\u3001\u65e5\u5fd7\u7684\u8bfb\u53d6<\/h2>\n
\u5728log_reader\u4e2d\uff0c\u5b9e\u73b0\u4e86Reader\u7c7b\uff0c\u7528\u6765\u8bfb\u53d6\u65e5\u5fd7\u6587\u4ef6\u3002
\nReader\u7684\u6784\u9020\u51fd\u6570\u5b9a\u4e49\u5982\u4e0b:<\/p>\n
Reader(SequentialFile* file, Reporter* reporter, bool checksum,\n uint64_t initial_offset);\n<\/code><\/pre>\nfile\u662f\u4ee3\u8868\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\uff0creporter\u662f\u7528\u6765\u62a5\u544a\u65e5\u5fd7\u6587\u4ef6\u4e2d\u53d1\u73b0\u9519\u8bef\u7684\uff0cchecksum\u662f\u5f53\u524d\u6587\u4ef6\u7684\u6821\u9a8c\u7801\uff0cinitial_offset\u662f\u5f53\u524d\u6587\u4ef6\u4e2d\u5f00\u59cb\u8bfb\u53d6\u7684\u7269\u7406\u4f4d\u7f6e\u3002<\/p>\n
Reader\u7c7b\u4e2d\u79c1\u6709\u7684\u6210\u5458\u53d8\u91cf\u548c\u51fd\u6570\u9700\u8981\u6ce8\u610f\u7684\u6709:<\/p>\n
SequentialFile* const file_; \/\/\u8981\u8bfb\u53d6\u7684\u6587\u4ef6\n Reporter* const reporter_; \/\/\u62a5\u544a\u9519\u8bef\u7684\u5bf9\u8c61\n bool const checksum_; \/\/\u6821\u9a8c\u503c\n char* const backing_store_; \/\/\u5907\u4efd\u5b58\u50a8\n Slice buffer_; \/\/\u7f13\u51b2\n bool eof_; \/\/ Last Read() indicated EOF by returning < kBlockSize\n\n \/\/ Offset of the last record returned by ReadRecord.\n uint64_t last_record_offset_; \/\/\u6700\u540e\u4e00\u4e2aRecord\u7684\u504f\u79fb\u91cf\n \/\/ Offset of the first location past the end of buffer_.\n uint64_t end_of_buffer_offset_; \/\/\n\n \/\/ Offset at which to start looking for the first record to return\n uint64_t const initial_offset_; \/\/\u7b2c\u4e00\u4e2a\u8bb0\u5f55\u7684\u521d\u59cb\u504f\u79fb\u91cf\n\n \/\/ True if we are resynchronizing after a seek (initial_offset_ > 0). In\n \/\/ particular, a run of kMiddleType and kLastType records can be silently\n \/\/ skipped in this mode\n bool resyncing_; \/\/\u5728\u4e00\u6b21\u67e5\u627e\u540e\u6211\u4eec\u518d\u540c\u6b65\u5219\u662fTrue\u3002\u7279\u522b\u7684\uff0ckMiddleType\u548ckLastType\u4f1a\u88ab\u8df3\u8fc7 \n\n \/\/ Extend record types with the following special values\n enum {\n kEof = kMaxRecordType + 1,\n \/\/ \u65e0\u6548\u7684\u8bb0\u5f55\u503c\uff0c\u53ef\u80fd\u6709\u4ee5\u4e0b\u60c5\u51b5\n \/\/ * \u5177\u6709\u65e0\u6548\u7684CRC\u503c(checksum\u4e0d\u6b63\u786e)(ReadPhysicalRecord reports a drop)\n \/\/ * \u957f\u5ea6\u4e3a0\u7684\u8bb0\u5f55 (No drop is reported)\n \/\/ * The record is below constructor's initial_offset (No drop is reported)\n kBadRecord = kMaxRecordType + 2\n };\n\n bool SkipToInitialBlock(); \/\/\u8df3\u8fc7\u76f4\u5230initial_offset\u7684\u4f4d\u7f6e,\u6210\u529f\u5219\u8fd4\u56detrue\n unsigned int ReadPhysicalRecord(Slice* result); \/\/\u8bfb\u53d6Record\uff0c\u8fd4\u56de\u8bb0\u5f55\u7c7b\u578b\uff0c\u6216\u8005kEof\u3001kBadRecord\n\n void ReportCorruption(uint64_t bytes, const char* reason); \/\/\u6c47\u62a5\u4e22\u5f03\u7684\u5b57\u8282\uff0c\u4ee5\u53ca\u539f\u56e0\n void ReportDrop(uint64_t bytes, const Status& reason); \/\/\u6c47\u62a5\u4e22\u5f03\u7684\u5b57\u8282\uff0c\u4ee5\u53ca\u539f\u56e0\n<\/code><\/pre>\n\u8fd9\u4e9b\u6210\u5458\u51fd\u6570\u4e2d\u7740\u91cd\u770b\u4e00\u4e0bReadPhysicalRecord<\/code><\/p>\nunsigned int Reader::ReadPhysicalRecord(Slice* result) {\n while (true) {\n if (buffer_.size() < kHeaderSize) {\n if (!eof_) {\n \/\/ Last read was a full read, so this is a trailer to skip\n buffer_.clear();\n Status status = file_->Read(kBlockSize, &buffer_, backing_store_);\n end_of_buffer_offset_ += buffer_.size();\n if (!status.ok()) {\n buffer_.clear();\n ReportDrop(kBlockSize, status);\n eof_ = true;\n return kEof;\n } else if (buffer_.size() < kBlockSize) {\n eof_ = true;\n }\n continue;\n } else {\n \/\/ Note that if buffer_ is non-empty, we have a truncated header at the\n \/\/ end of the file, which can be caused by the writer crashing in the\n \/\/ middle of writing the header. Instead of considering this an error,\n \/\/ just report EOF.\n buffer_.clear();\n return kEof;\n }\n }\n\n \/\/ Parse the header\n const char* header = buffer_.data();\n const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;\n const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;\n const unsigned int type = header[6];\n const uint32_t length = a | (b << 8);\n if (kHeaderSize + length > buffer_.size()) {\n size_t drop_size = buffer_.size();\n buffer_.clear();\n if (!eof_) {\n ReportCorruption(drop_size, \"bad record length\");\n return kBadRecord;\n }\n \/\/ If the end of the file has been reached without reading |length| bytes\n \/\/ of payload, assume the writer died in the middle of writing the record.\n \/\/ Don't report a corruption.\n return kEof;\n }\n\n if (type == kZeroType && length == 0) {\n \/\/ Skip zero length record without reporting any drops since\n \/\/ such records are produced by the mmap based writing code in\n \/\/ env_posix.cc that preallocates file regions.\n buffer_.clear();\n return kBadRecord;\n }\n\n \/\/ Check crc\n if (checksum_) {\n uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header));\n uint32_t actual_crc = crc32c::Value(header + 6, 1 + length);\n if (actual_crc != expected_crc) {\n \/\/ Drop the rest of the buffer since \"length\" itself may have\n \/\/ been corrupted and if we trust it, we could find some\n \/\/ fragment of a real log record that just happens to look\n \/\/ like a valid log record.\n size_t drop_size = buffer_.size();\n buffer_.clear();\n ReportCorruption(drop_size, \"checksum mismatch\");\n return kBadRecord;\n }\n }\n\n buffer_.remove_prefix(kHeaderSize + length);\n\n \/\/ Skip physical record that started before initial_offset_\n if (end_of_buffer_offset_ - buffer_.size() - kHeaderSize - length <\n initial_offset_) {\n result->clear();\n return kBadRecord;\n }\n\n *result = Slice(header + kHeaderSize, length);\n return type;\n }\n}\n<\/code><\/pre>\nReader\u7c7b\u4e2d\u516c\u5171\u7684\u6210\u5458\u51fd\u6570\u9700\u8981\u6ce8\u610f\u7684\u6709:
\nbool ReadRecord(Slice* record, std::string* scratch);<\/code>
\nuint64_t LastRecordOffset();<\/code><\/p>\nReadRecord\u7528\u6765\u8bfb\u53d6\u4e0b\u4e00\u4e2a\u8981\u8bfb\u53d6\u7684Record\uff0c\u5982\u679c\u8bfb\u53d6\u6210\u529f\uff0c\u8fd4\u56detrue,\u5b58\u50a8\u5728record\u4e2d\uff0c\u5982\u679c\u8bfb\u5230\u4e86\u7ed3\u5c3e\u6ca1\u6709\u4e00\u4e2a\u5b8c\u6574\u7684Record\uff0c\u5219\u5b58\u50a8\u5728scratch\u4e2d<\/p>\n
bool Reader::ReadRecord(Slice* record, std::string* scratch) {\n if (last_record_offset_ < initial_offset_) {\n if (!SkipToInitialBlock()) { \/\/\u8df3\u5230\u521d\u59cb\u7684\u504f\u79fb\u4f4d\u7f6e\n return false;\n }\n }\n\n scratch->clear();\n record->clear();\n bool in_fragmented_record = false;\n \/\/ Record offset of the logical record that we're reading\n \/\/ 0 is a dummy value to make compilers happy\n uint64_t prospective_record_offset = 0;\n\n Slice fragment;\n while (true) {\n const unsigned int record_type = ReadPhysicalRecord(&fragment);\n\n \/\/ ReadPhysicalRecord may have only had an empty trailer remaining in its\n \/\/ internal buffer. Calculate the offset of the next physical record now\n \/\/ that it has returned, properly accounting for its header size.\n uint64_t physical_record_offset =\n end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();\n\n if (resyncing_) {\n if (record_type == kMiddleType) {\n continue;\n } else if (record_type == kLastType) {\n resyncing_ = false;\n continue;\n } else {\n resyncing_ = false;\n }\n }\n\n switch (record_type) {\n case kFullType:\n if (in_fragmented_record) {\n \/\/ Handle bug in earlier versions of log::Writer where\n \/\/ it could emit an empty kFirstType record at the tail end\n \/\/ of a block followed by a kFullType or kFirstType record\n \/\/ at the beginning of the next block.\n if (scratch->empty()) {\n in_fragmented_record = false;\n } else {\n ReportCorruption(scratch->size(), \"partial record without end(1)\");\n }\n }\n prospective_record_offset = physical_record_offset;\n scratch->clear();\n *record = fragment;\n last_record_offset_ = prospective_record_offset;\n return true;\n\n case kFirstType:\n if (in_fragmented_record) {\n \/\/ Handle bug in earlier versions of log::Writer where\n \/\/ it could emit an empty kFirstType record at the tail end\n \/\/ of a block followed by a kFullType or kFirstType record\n \/\/ at the beginning of the next block.\n if (scratch->empty()) {\n in_fragmented_record = false;\n } else {\n ReportCorruption(scratch->size(), \"partial record without end(2)\");\n }\n }\n prospective_record_offset = physical_record_offset;\n scratch->assign(fragment.data(), fragment.size());\n in_fragmented_record = true;\n break;\n\n case kMiddleType:\n if (!in_fragmented_record) {\n ReportCorruption(fragment.size(),\n \"missing start of fragmented record(1)\");\n } else {\n scratch->append(fragment.data(), fragment.size());\n }\n break;\n\n case kLastType:\n if (!in_fragmented_record) {\n ReportCorruption(fragment.size(),\n \"missing start of fragmented record(2)\");\n } else {\n scratch->append(fragment.data(), fragment.size());\n *record = Slice(*scratch);\n last_record_offset_ = prospective_record_offset;\n return true;\n }\n break;\n\n case kEof:\n if (in_fragmented_record) {\n \/\/ This can be caused by the writer dying immediately after\n \/\/ writing a physical record but before completing the next; don't\n \/\/ treat it as a corruption, just ignore the entire logical record.\n scratch->clear();\n }\n return false;\n\n case kBadRecord:\n if (in_fragmented_record) {\n ReportCorruption(scratch->size(), \"error in middle of record\");\n in_fragmented_record = false;\n scratch->clear();\n }\n break;\n\n default: {\n char buf[40];\n snprintf(buf, sizeof(buf), \"unknown record type %u\", record_type);\n ReportCorruption(\n (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),\n buf);\n in_fragmented_record = false;\n scratch->clear();\n break;\n }\n }\n }\n return false;\n}\n\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"