From a652877bd6a142cd98f704f73b1c3fd472ca57ab Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 4 Sep 2020 15:47:04 -0700 Subject: [PATCH 1/2] libsnapshot: Add support for brotli compression. Bug: 162274240 Test: cow_api_test Change-Id: I0b0ceec3c3041a6aea4b1e6c4d01ed0a8860d7e8 --- fs_mgr/libsnapshot/Android.bp | 10 ++-- fs_mgr/libsnapshot/cow_api_test.cpp | 12 +++-- fs_mgr/libsnapshot/cow_decompress.cpp | 53 +++++++++++++++++++ fs_mgr/libsnapshot/cow_decompress.h | 1 + fs_mgr/libsnapshot/cow_reader.cpp | 3 ++ fs_mgr/libsnapshot/cow_writer.cpp | 23 ++++++++ .../include/libsnapshot/cow_format.h | 1 + 7 files changed, 96 insertions(+), 7 deletions(-) diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index bdf1da682..95fbab8ba 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -152,6 +152,7 @@ cc_library_static { "liblog", ], static_libs: [ + "libbrotli", "libz", ], ramdisk_available: true, @@ -362,10 +363,11 @@ cc_defaults { static_libs: [ "libbase", + "libbrotli", "liblog", "libdm", - "libz", - "libsnapshot_cow", + "libz", + "libsnapshot_cow", ], } @@ -403,6 +405,7 @@ cc_test { "libz", ], static_libs: [ + "libbrotli", "libgtest", "libsnapshot_cow", ], @@ -494,11 +497,12 @@ cc_test { shared_libs: [ "libbase", "liblog", - "libz", ], static_libs: [ + "libbrotli", "libgtest", "libsnapshot_cow", + "libz", ], header_libs: [ "libstorage_literals_headers", diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp index d98fe59c1..ec206ad9a 100644 --- a/fs_mgr/libsnapshot/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/cow_api_test.cpp @@ -30,12 +30,12 @@ namespace snapshot { class CowTest : public ::testing::Test { protected: - void SetUp() override { + virtual void SetUp() override { cow_ = std::make_unique(); ASSERT_GE(cow_->fd, 0) << strerror(errno); } - void TearDown() override { cow_ = nullptr; } + virtual void TearDown() override { cow_ = nullptr; } std::unique_ptr cow_; }; @@ -211,9 +211,11 @@ class HorribleStringSink : public StringSink { void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); } }; -TEST_F(CowTest, HorribleSink) { +class CompressionTest : public CowTest, public testing::WithParamInterface {}; + +TEST_P(CompressionTest, HorribleSink) { CowOptions options; - options.compression = "gz"; + options.compression = GetParam(); CowWriter writer(options); ASSERT_TRUE(writer.Initialize(cow_->fd)); @@ -239,6 +241,8 @@ TEST_F(CowTest, HorribleSink) { ASSERT_EQ(sink.stream(), data); } +INSTANTIATE_TEST_SUITE_P(CowApi, CompressionTest, testing::Values("none", "gz", "brotli")); + TEST_F(CowTest, GetSize) { CowOptions options; CowWriter writer(options); diff --git a/fs_mgr/libsnapshot/cow_decompress.cpp b/fs_mgr/libsnapshot/cow_decompress.cpp index f480b85df..faceafe17 100644 --- a/fs_mgr/libsnapshot/cow_decompress.cpp +++ b/fs_mgr/libsnapshot/cow_decompress.cpp @@ -19,6 +19,7 @@ #include #include +#include #include namespace android { @@ -207,5 +208,57 @@ std::unique_ptr IDecompressor::Gz() { return std::unique_ptr(new GzDecompressor()); } +class BrotliDecompressor final : public StreamDecompressor { + public: + ~BrotliDecompressor(); + + bool Init() override; + bool DecompressInput(const uint8_t* data, size_t length) override; + bool Done() override { return BrotliDecoderIsFinished(decoder_); } + + private: + BrotliDecoderState* decoder_ = nullptr; +}; + +bool BrotliDecompressor::Init() { + decoder_ = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + return true; +} + +BrotliDecompressor::~BrotliDecompressor() { + if (decoder_) { + BrotliDecoderDestroyInstance(decoder_); + } +} + +bool BrotliDecompressor::DecompressInput(const uint8_t* data, size_t length) { + size_t available_in = length; + const uint8_t* next_in = data; + + bool needs_more_output = false; + while (available_in || needs_more_output) { + if (!output_buffer_remaining_ && !GetFreshBuffer()) { + return false; + } + + auto output_buffer = output_buffer_; + auto r = BrotliDecoderDecompressStream(decoder_, &available_in, &next_in, + &output_buffer_remaining_, &output_buffer_, nullptr); + if (r == BROTLI_DECODER_RESULT_ERROR) { + LOG(ERROR) << "brotli decode failed"; + return false; + } + if (!sink_->ReturnData(output_buffer, output_buffer_ - output_buffer)) { + return false; + } + needs_more_output = (r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); + } + return true; +} + +std::unique_ptr IDecompressor::Brotli() { + return std::unique_ptr(new BrotliDecompressor()); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/cow_decompress.h b/fs_mgr/libsnapshot/cow_decompress.h index 1c8c40d91..f4852561f 100644 --- a/fs_mgr/libsnapshot/cow_decompress.h +++ b/fs_mgr/libsnapshot/cow_decompress.h @@ -40,6 +40,7 @@ class IDecompressor { // Factory methods for decompression methods. static std::unique_ptr Uncompressed(); static std::unique_ptr Gz(); + static std::unique_ptr Brotli(); // |output_bytes| is the expected total number of bytes to sink. virtual bool Decompress(size_t output_bytes) = 0; diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp index 1aea3a92d..720ee3b45 100644 --- a/fs_mgr/libsnapshot/cow_reader.cpp +++ b/fs_mgr/libsnapshot/cow_reader.cpp @@ -233,6 +233,9 @@ bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) { case kCowCompressGz: decompressor = IDecompressor::Gz(); break; + case kCowCompressBrotli: + decompressor = IDecompressor::Brotli(); + break; default: LOG(ERROR) << "Unknown compression type: " << op.compression; return false; diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp index 76238c262..f05f9ba4b 100644 --- a/fs_mgr/libsnapshot/cow_writer.cpp +++ b/fs_mgr/libsnapshot/cow_writer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -63,6 +64,10 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) { if (options_.compression == "gz") { compression_ = kCowCompressGz; + } else if (options_.compression == "brotli") { + compression_ = kCowCompressBrotli; + } else if (options_.compression == "none") { + compression_ = kCowCompressNone; } else if (!options_.compression.empty()) { LOG(ERROR) << "unrecognized compression: " << options_.compression; return false; @@ -171,6 +176,24 @@ std::basic_string CowWriter::Compress(const void* data, size_t length) } return std::basic_string(buffer.get(), dest_len); } + case kCowCompressBrotli: { + auto bound = BrotliEncoderMaxCompressedSize(length); + if (!bound) { + LOG(ERROR) << "BrotliEncoderMaxCompressedSize returned 0"; + return {}; + } + auto buffer = std::make_unique(bound); + + size_t encoded_size = bound; + auto rv = BrotliEncoderCompress( + BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, length, + reinterpret_cast(data), &encoded_size, buffer.get()); + if (!rv) { + LOG(ERROR) << "BrotliEncoderCompress failed"; + return {}; + } + return std::basic_string(buffer.get(), encoded_size); + } default: LOG(ERROR) << "unhandled compression type: " << compression_; break; diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index 6d500e7e5..0adce4867 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -98,6 +98,7 @@ static constexpr uint8_t kCowZeroOp = 3; static constexpr uint8_t kCowCompressNone = 0; static constexpr uint8_t kCowCompressGz = 1; +static constexpr uint8_t kCowCompressBrotli = 2; } // namespace snapshot } // namespace android From a889c87b0f6633e4f6167aaa7af3977c0285cc18 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 14 Sep 2020 17:05:04 -0700 Subject: [PATCH 2/2] libsnapshot: Add an append mode to CowWriter. When in append mode, CowWriter will re-open the existing COW and resume writing at the end of the old data position. All existing operations will be reimported and buffered in memory. The size calculation has been simplified to make this work. We now advance ops_offset and no longer track the number of bytes written. Additionally, a "header_size" field has been added to the header. This was missing from the original format and is useful for introducing forward compatibility later. Finally, Finalize has been renamed to Flush. It's still mandatory, but it can be called multiple times to continue appending data without reopening. Bug: 168554689 Test: cow_api_test gtest Change-Id: I637e99ae08a4db5b273c06318e6db523ea8ec7c5 --- fs_mgr/libsnapshot/Android.bp | 1 + fs_mgr/libsnapshot/cow_api_test.cpp | 68 ++++++++- fs_mgr/libsnapshot/cow_reader.cpp | 5 + fs_mgr/libsnapshot/cow_snapuserd_test.cpp | 2 +- fs_mgr/libsnapshot/cow_writer.cpp | 138 ++++++++++++------ .../estimate_cow_from_nonab_ota.cpp | 2 +- .../include/libsnapshot/cow_format.h | 3 + .../include/libsnapshot/cow_writer.h | 24 +-- fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp | 2 +- 9 files changed, 178 insertions(+), 67 deletions(-) diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index 95fbab8ba..0bb1b8719 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -412,6 +412,7 @@ cc_test { test_min_api_level: 30, auto_gen_config: true, require_root: false, + host_supported: true, } cc_binary { diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp index ec206ad9a..40d5efbeb 100644 --- a/fs_mgr/libsnapshot/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/cow_api_test.cpp @@ -70,7 +70,7 @@ TEST_F(CowTest, ReadWrite) { ASSERT_TRUE(writer.AddCopy(10, 20)); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); ASSERT_TRUE(writer.AddZeroBlocks(51, 2)); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -105,7 +105,7 @@ TEST_F(CowTest, ReadWrite) { ASSERT_EQ(op->compression, kCowCompressNone); ASSERT_EQ(op->data_length, 4096); ASSERT_EQ(op->new_block, 50); - ASSERT_EQ(op->source, 104); + ASSERT_EQ(op->source, 106); ASSERT_TRUE(reader.ReadData(*op, &sink)); ASSERT_EQ(sink.stream(), data); @@ -145,7 +145,7 @@ TEST_F(CowTest, CompressGz) { data.resize(options.block_size, '\0'); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -163,7 +163,7 @@ TEST_F(CowTest, CompressGz) { ASSERT_EQ(op->compression, kCowCompressGz); ASSERT_EQ(op->data_length, 56); // compressed! ASSERT_EQ(op->new_block, 50); - ASSERT_EQ(op->source, 104); + ASSERT_EQ(op->source, 106); ASSERT_TRUE(reader.ReadData(*op, &sink)); ASSERT_EQ(sink.stream(), data); @@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) { data.resize(options.block_size * 2, '\0'); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -224,7 +224,7 @@ TEST_P(CompressionTest, HorribleSink) { data.resize(options.block_size, '\0'); ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -259,7 +259,7 @@ TEST_F(CowTest, GetSize) { ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); ASSERT_TRUE(writer.AddZeroBlocks(51, 2)); auto size_before = writer.GetCowSize(); - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); auto size_after = writer.GetCowSize(); ASSERT_EQ(size_before, size_after); struct stat buf; @@ -271,6 +271,60 @@ TEST_F(CowTest, GetSize) { ASSERT_EQ(buf.st_size, writer.GetCowSize()); } +TEST_F(CowTest, Append) { + CowOptions options; + auto writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer->Flush()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + writer = std::make_unique(options); + ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND)); + + std::string data2 = "More data!"; + data2.resize(options.block_size, '\0'); + ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size())); + ASSERT_TRUE(writer->Flush()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + struct stat buf; + ASSERT_EQ(fstat(cow_->fd, &buf), 0); + ASSERT_EQ(buf.st_size, writer->GetCowSize()); + + // Read back both operations. + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + StringSink sink; + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + + ASSERT_FALSE(iter->Done()); + auto op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data); + + iter->Next(); + sink.Reset(); + + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data2); + + iter->Next(); + ASSERT_TRUE(iter->Done()); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp index 720ee3b45..60093ab68 100644 --- a/fs_mgr/libsnapshot/cow_reader.cpp +++ b/fs_mgr/libsnapshot/cow_reader.cpp @@ -78,6 +78,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { << "Expected: " << kCowMagicNumber; return false; } + if (header_.header_size != sizeof(CowHeader)) { + LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected " + << sizeof(CowHeader); + return false; + } if ((header_.major_version != kCowVersionMajor) || (header_.minor_version != kCowVersionMinor)) { diff --git a/fs_mgr/libsnapshot/cow_snapuserd_test.cpp b/fs_mgr/libsnapshot/cow_snapuserd_test.cpp index d767022fb..4c8ff5f2a 100644 --- a/fs_mgr/libsnapshot/cow_snapuserd_test.cpp +++ b/fs_mgr/libsnapshot/cow_snapuserd_test.cpp @@ -128,7 +128,7 @@ TEST_F(SnapuserdTest, ReadWrite) { ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_2.get(), size)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer.Flush()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp index f05f9ba4b..4cf2119a7 100644 --- a/fs_mgr/libsnapshot/cow_writer.cpp +++ b/fs_mgr/libsnapshot/cow_writer.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -40,17 +41,48 @@ void CowWriter::SetupHeaders() { header_.magic = kCowMagicNumber; header_.major_version = kCowVersionMajor; header_.minor_version = kCowVersionMinor; + header_.header_size = sizeof(CowHeader); header_.block_size = options_.block_size; } -bool CowWriter::Initialize(android::base::unique_fd&& fd) { - owned_fd_ = std::move(fd); - return Initialize(android::base::borrowed_fd{owned_fd_}); +bool CowWriter::ParseOptions() { + if (options_.compression == "gz") { + compression_ = kCowCompressGz; + } else if (options_.compression == "brotli") { + compression_ = kCowCompressBrotli; + } else if (options_.compression == "none") { + compression_ = kCowCompressNone; + } else if (!options_.compression.empty()) { + LOG(ERROR) << "unrecognized compression: " << options_.compression; + return false; + } + return true; } -bool CowWriter::Initialize(android::base::borrowed_fd fd) { +bool CowWriter::Initialize(android::base::unique_fd&& fd, OpenMode mode) { + owned_fd_ = std::move(fd); + return Initialize(android::base::borrowed_fd{owned_fd_}, mode); +} + +bool CowWriter::Initialize(android::base::borrowed_fd fd, OpenMode mode) { fd_ = fd; + if (!ParseOptions()) { + return false; + } + + switch (mode) { + case OpenMode::WRITE: + return OpenForWrite(); + case OpenMode::APPEND: + return OpenForAppend(); + default: + LOG(ERROR) << "Unknown open mode in CowWriter"; + return false; + } +} + +bool CowWriter::OpenForWrite() { // This limitation is tied to the data field size in CowOperation. if (header_.block_size > std::numeric_limits::max()) { LOG(ERROR) << "Block size is too large"; @@ -62,35 +94,52 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) { return false; } - if (options_.compression == "gz") { - compression_ = kCowCompressGz; - } else if (options_.compression == "brotli") { - compression_ = kCowCompressBrotli; - } else if (options_.compression == "none") { - compression_ = kCowCompressNone; - } else if (!options_.compression.empty()) { - LOG(ERROR) << "unrecognized compression: " << options_.compression; + // Headers are not complete, but this ensures the file is at the right + // position. + if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { + PLOG(ERROR) << "write failed"; return false; } - // Headers are not complete, but this ensures the file is at the right - // position. - if (!WriteFully(fd_, &header_, sizeof(header_))) { - PLOG(ERROR) << "write failed"; + header_.ops_offset = header_.header_size; + return true; +} + +bool CowWriter::OpenForAppend() { + auto reader = std::make_unique(); + if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) { + return false; + } + options_.block_size = header_.block_size; + + // Reset this, since we're going to reimport all operations. + header_.num_ops = 0; + + auto iter = reader->GetOpIter(); + while (!iter->Done()) { + auto& op = iter->Get(); + AddOperation(op); + + iter->Next(); + } + + // Free reader so we own the descriptor position again. + reader = nullptr; + + // Seek to the end of the data section. + if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek failed"; return false; } return true; } bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) { - header_.num_ops++; - CowOperation op = {}; op.type = kCowCopyOp; op.new_block = new_block; op.source = old_block; - ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); - + AddOperation(op); return true; } @@ -108,8 +157,6 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t const uint8_t* iter = reinterpret_cast(data); for (size_t i = 0; i < size / header_.block_size; i++) { - header_.num_ops++; - CowOperation op = {}; op.type = kCowReplaceOp; op.new_block = new_block_start + i; @@ -125,7 +172,7 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes"; return false; } - if (!WriteFully(fd_, data.data(), data.size())) { + if (!WriteRawData(data.data(), data.size())) { PLOG(ERROR) << "AddRawBlocks: write failed"; return false; } @@ -137,11 +184,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t pos += header_.block_size; } - ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + AddOperation(op); iter += header_.block_size; } - if (!compression_ && !WriteFully(fd_, data, size)) { + if (!compression_ && !WriteRawData(data, size)) { PLOG(ERROR) << "AddRawBlocks: write failed"; return false; } @@ -150,13 +197,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { for (uint64_t i = 0; i < num_blocks; i++) { - header_.num_ops++; - CowOperation op = {}; op.type = kCowZeroOp; op.new_block = new_block_start + i; op.source = 0; - ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + AddOperation(op); } return true; } @@ -212,17 +257,7 @@ static void SHA256(const void*, size_t, uint8_t[]) { #endif } -bool CowWriter::Finalize() { - // If both fields are set then Finalize is already called. - if (header_.ops_offset > 0 && header_.ops_size > 0) { - return true; - } - auto offs = lseek(fd_.get(), 0, SEEK_CUR); - if (offs < 0) { - PLOG(ERROR) << "lseek failed"; - return false; - } - header_.ops_offset = offs; +bool CowWriter::Flush() { header_.ops_size = ops_.size(); memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32); @@ -235,8 +270,6 @@ bool CowWriter::Finalize() { PLOG(ERROR) << "lseek failed"; return false; } - // Header is already written, calling WriteFully will increment - // bytes_written_. So use android::base::WriteFully() here. if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { PLOG(ERROR) << "write header failed"; return false; @@ -250,13 +283,16 @@ bool CowWriter::Finalize() { return false; } - // clear ops_ so that subsequent calls to GetSize() still works. - ops_.clear(); + // Re-position for any subsequent writes. + if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek ops failed"; + return false; + } return true; } size_t CowWriter::GetCowSize() { - return bytes_written_ + ops_.size() * sizeof(ops_[0]); + return header_.ops_offset + header_.num_ops * sizeof(CowOperation); } bool CowWriter::GetDataPos(uint64_t* pos) { @@ -269,9 +305,17 @@ bool CowWriter::GetDataPos(uint64_t* pos) { return true; } -bool CowWriter::WriteFully(base::borrowed_fd fd, const void* data, size_t size) { - bytes_written_ += size; - return android::base::WriteFully(fd, data, size); +void CowWriter::AddOperation(const CowOperation& op) { + header_.num_ops++; + ops_.insert(ops_.size(), reinterpret_cast(&op), sizeof(op)); +} + +bool CowWriter::WriteRawData(const void* data, size_t size) { + if (!android::base::WriteFully(fd_, data, size)) { + return false; + } + header_.ops_offset += size; + return true; } } // namespace snapshot diff --git a/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp b/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp index 45833e121..2a0136b6d 100644 --- a/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp +++ b/fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp @@ -375,7 +375,7 @@ bool NonAbEstimator::AnalyzePartition(const std::string& partition_name) { } } - if (!writer->Finalize()) { + if (!writer->Flush()) { return false; } diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index 0adce4867..4a6bd4ed0 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -45,6 +45,9 @@ struct CowHeader { uint16_t major_version; uint16_t minor_version; + // Size of this struct. + uint16_t header_size; + // Offset to the location of the operation sequence, and size of the // operation sequence buffer. |ops_offset| is also the end of the // raw data region. diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h index 8826b7a02..8569161fb 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -47,12 +47,11 @@ class ICowWriter { // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size. virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; - // Finalize all COW operations and flush pending writes. - // Return true if successful. - virtual bool Finalize() = 0; + // Flush all pending writes. This must be called before closing the writer + // to ensure that the correct headers and footers are written. + virtual bool Flush() = 0; - // Return 0 if failed, on success return number of bytes the cow image would be - // after calling Finalize(); + // Return number of bytes the cow image occupies on disk. virtual size_t GetCowSize() = 0; protected: @@ -61,24 +60,30 @@ class ICowWriter { class CowWriter : public ICowWriter { public: + enum class OpenMode { WRITE, APPEND }; + explicit CowWriter(const CowOptions& options); // Set up the writer. - bool Initialize(android::base::unique_fd&& fd); - bool Initialize(android::base::borrowed_fd fd); + bool Initialize(android::base::unique_fd&& fd, OpenMode mode = OpenMode::WRITE); + bool Initialize(android::base::borrowed_fd fd, OpenMode mode = OpenMode::WRITE); bool AddCopy(uint64_t new_block, uint64_t old_block) override; bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; - bool Finalize() override; + bool Flush() override; size_t GetCowSize() override; private: void SetupHeaders(); + bool ParseOptions(); + bool OpenForWrite(); + bool OpenForAppend(); bool GetDataPos(uint64_t* pos); - bool WriteFully(base::borrowed_fd fd, const void* data, size_t size); + bool WriteRawData(const void* data, size_t size); + void AddOperation(const CowOperation& op); std::basic_string Compress(const void* data, size_t length); private: @@ -90,7 +95,6 @@ class CowWriter : public ICowWriter { // :TODO: this is not efficient, but stringstream ubsan aborts because some // bytes overflow a signed char. std::basic_string ops_; - std::atomic bytes_written_ = 0; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp b/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp index d0b8f52e0..f76107709 100644 --- a/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp +++ b/fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp @@ -204,7 +204,7 @@ bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) { } } - if (!writer_->Finalize()) { + if (!writer_->Flush()) { LOG(ERROR) << "Unable to finalize COW for " << partition_name; return false; }