libsnapshot: add CowSizeInfo struct

Adding a cow size info struct as writer will now need to know the op buffer
size at the time of initialization. The sequence of events is as follows
(same as estimate_cow_size but putting down here for clarity)

1. ota_from_target_files does dry run to determine cow size + ops buffer
   size
2. data is passed through delta archive manifest
3. snapshot.cpp parses these fields and confgiures cowoptions struct to
   pass to writer initialization
4. cow is initialized with correct sizing. Data is incrementally added
   at the ends of the cow ops buffer (which is why we need to know the
   sizing ahead of time)

Test: ota
Change-Id: I950e5ef82c9bd7e9bd9603b0599c930767ee3f0d
This commit is contained in:
Daniel Zheng 2023-11-13 15:56:00 -08:00
parent 75d3663d6d
commit d0c3a04cb0
8 changed files with 31 additions and 23 deletions

View file

@ -33,7 +33,10 @@
namespace android {
namespace snapshot {
struct CowSizeInfo {
uint64_t cow_size;
uint64_t op_count_max;
};
struct CowOptions {
uint32_t block_size = 4096;
std::string compression;
@ -92,8 +95,9 @@ class ICowWriter {
// to ensure that the correct headers and footers are written.
virtual bool Finalize() = 0;
// Return number of bytes the cow image occupies on disk.
virtual uint64_t GetCowSize() = 0;
// Return number of bytes the cow image occupies on disk + the size of sequence && ops buffer
// The latter two fields are used in v3 cow format and left as 0 for v2 cow format
virtual CowSizeInfo GetCowSizeInfo() const = 0;
virtual uint32_t GetBlockSize() const = 0;
virtual std::optional<uint32_t> GetMaxBlocks() const = 0;

View file

@ -24,8 +24,7 @@ class MockCowWriter : public ICowWriter {
using FileDescriptor = chromeos_update_engine::FileDescriptor;
MOCK_METHOD(bool, Finalize, (), (override));
MOCK_METHOD(uint64_t, GetCowSize, (), (override));
MOCK_METHOD(CowSizeInfo, GetCowSizeInfo, (), (const, override));
MOCK_METHOD(bool, AddCopy, (uint64_t, uint64_t, uint64_t), (override));
MOCK_METHOD(bool, AddRawBlocks, (uint64_t, const void*, size_t), (override));

View file

@ -601,14 +601,14 @@ TEST_F(CowTest, GetSize) {
ASSERT_TRUE(writer.AddCopy(10, 20));
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
auto size_before = writer.GetCowSize();
auto size_before = writer.GetCowSizeInfo().cow_size;
ASSERT_TRUE(writer.Finalize());
auto size_after = writer.GetCowSize();
auto size_after = writer.GetCowSizeInfo().cow_size;
ASSERT_EQ(size_before, size_after);
struct stat buf;
ASSERT_GE(fstat(cow_->fd, &buf), 0) << strerror(errno);
ASSERT_EQ(buf.st_size, writer.GetCowSize());
ASSERT_EQ(buf.st_size, writer.GetCowSizeInfo().cow_size);
}
TEST_F(CowTest, AppendLabelSmall) {
@ -637,7 +637,7 @@ TEST_F(CowTest, AppendLabelSmall) {
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
ASSERT_EQ(buf.st_size, writer->GetCowSizeInfo().cow_size);
// Read back both operations, and label.
CowReader reader;
@ -690,7 +690,7 @@ TEST_F(CowTest, AppendLabelMissing) {
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->AddLabel(1));
// Drop the tail end of the last op header, corrupting it.
ftruncate(cow_->fd, writer->GetCowSize() - sizeof(CowFooter) - 3);
ftruncate(cow_->fd, writer->GetCowSizeInfo().cow_size - sizeof(CowFooter) - 3);
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -705,7 +705,7 @@ TEST_F(CowTest, AppendLabelMissing) {
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
ASSERT_EQ(buf.st_size, writer->GetCowSizeInfo().cow_size);
// Read back both operations.
CowReader reader;
@ -763,7 +763,7 @@ TEST_F(CowTest, AppendExtendedCorrupted) {
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
ASSERT_EQ(buf.st_size, writer->GetCowSizeInfo().cow_size);
// Read back all valid operations
CowReader reader;
@ -812,7 +812,7 @@ TEST_F(CowTest, AppendbyLabel) {
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
ASSERT_EQ(buf.st_size, writer->GetCowSizeInfo().cow_size);
// Read back all ops
CowReader reader;
@ -989,7 +989,7 @@ TEST_F(CowTest, ClusterAppendTest) {
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
ASSERT_EQ(buf.st_size, writer->GetCowSizeInfo().cow_size);
// Read back both operations, plus cluster op at end
CowReader reader;

View file

@ -658,14 +658,14 @@ TEST_F(CowTestV3, CowSizeEstimate) {
options.compression = "none";
auto estimator = android::snapshot::CreateCowEstimator(3, options);
ASSERT_TRUE(estimator->AddZeroBlocks(0, 1024 * 1024));
const auto cow_size = estimator->GetCowSize();
const auto cow_size = estimator->GetCowSizeInfo().cow_size;
options.op_count_max = 1024 * 1024;
options.max_blocks = 1024 * 1024;
CowWriterV3 writer(options, GetCowFd());
ASSERT_TRUE(writer.Initialize());
ASSERT_TRUE(writer.AddZeroBlocks(0, 1024 * 1024));
ASSERT_LE(writer.GetCowSize(), cow_size);
ASSERT_LE(writer.GetCowSizeInfo().cow_size, cow_size);
}
TEST_F(CowTestV3, CopyOpMany) {

View file

@ -576,12 +576,14 @@ bool CowWriterV2::Finalize() {
return Sync();
}
uint64_t CowWriterV2::GetCowSize() {
CowSizeInfo CowWriterV2::GetCowSizeInfo() const {
CowSizeInfo info;
if (current_data_size_ > 0) {
return next_data_pos_ + sizeof(footer_);
info.cow_size = next_data_pos_ + sizeof(footer_);
} else {
return next_op_pos_ + sizeof(footer_);
info.cow_size = next_op_pos_ + sizeof(footer_);
}
return info;
}
bool CowWriterV2::GetDataPos(uint64_t* pos) {

View file

@ -27,7 +27,7 @@ class CowWriterV2 : public CowWriterBase {
bool Initialize(std::optional<uint64_t> label = {}) override;
bool Finalize() override;
uint64_t GetCowSize() override;
CowSizeInfo GetCowSizeInfo() const override;
protected:
virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

View file

@ -491,8 +491,11 @@ bool CowWriterV3::Finalize() {
return Sync();
}
uint64_t CowWriterV3::GetCowSize() {
return next_data_pos_;
CowSizeInfo CowWriterV3::GetCowSizeInfo() const {
CowSizeInfo info;
info.cow_size = next_data_pos_;
info.op_count_max = header_.op_count_max;
return info;
}
} // namespace snapshot

View file

@ -30,7 +30,7 @@ class CowWriterV3 : public CowWriterBase {
bool Initialize(std::optional<uint64_t> label = {}) override;
bool Finalize() override;
uint64_t GetCowSize() override;
CowSizeInfo GetCowSizeInfo() const override;
protected:
virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;