diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index cba684467..e931bec39 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -174,12 +174,13 @@ cc_library_static { "libsnapshot_cow_defaults", ], srcs: [ - "libsnapshot_cow/cow_decompress.cpp", - "libsnapshot_cow/cow_reader.cpp", - "libsnapshot_cow/cow_writer.cpp", - "libsnapshot_cow/cow_format.cpp", "libsnapshot_cow/cow_compress.cpp", + "libsnapshot_cow/cow_decompress.cpp", + "libsnapshot_cow/cow_format.cpp", + "libsnapshot_cow/cow_reader.cpp", "libsnapshot_cow/parser_v2.cpp", + "libsnapshot_cow/writer_base.cpp", + "libsnapshot_cow/writer_v2.cpp", ], host_supported: true, recovery_available: true, @@ -371,7 +372,7 @@ cc_test { "libsnapshot_cow_defaults", ], srcs: [ - "libsnapshot_cow/cow_api_test.cpp", + "libsnapshot_cow/test_v2.cpp", ], cflags: [ "-D_FILE_OFFSET_BITS=64", diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index b228dff8d..dd626bc46 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -31,6 +31,10 @@ static constexpr uint32_t kCowVersionManifest = 2; static constexpr uint32_t kMinCowVersion = 1; static constexpr uint32_t kMaxCowVersion = 2; +// Normally, this should be kMaxCowVersion. When a new version is under testing +// it may be the previous value of kMaxCowVersion. +static constexpr uint32_t kDefaultCowVersion = 2; + // This header appears as the first sequence of bytes in the COW. All fields // in the layout are little-endian encoded. The on-disk layout is: // diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h index 7881f3509..af2d3efc5 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -1,16 +1,16 @@ -// Copyright (C) 2019 The Android Open Source Project +// copyright (c) 2019 the android open source project // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// licensed under the apache license, version 2.0 (the "license"); +// you may not use this file except in compliance with the license. +// you may obtain a copy of the license at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/license-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// unless required by applicable law or agreed to in writing, software +// distributed under the license is distributed on an "as is" basis, +// without warranties or conditions of any kind, either express or implied. +// see the license for the specific language governing permissions and +// limitations under the license. #pragma once @@ -61,30 +61,28 @@ struct CowOptions { // will occur in the sequence they were added to the COW. class ICowWriter { public: - explicit ICowWriter(const CowOptions& options) : options_(options) {} - virtual ~ICowWriter() {} // Encode an operation that copies the contents of |old_block| to the // location of |new_block|. 'num_blocks' is the number of contiguous // COPY operations from |old_block| to |new_block|. - bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1); + virtual bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0; // Encode a sequence of raw blocks. |size| must be a multiple of the block size. - bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size); + virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; // Add a sequence of xor'd blocks. |size| must be a multiple of the block size. - bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, - uint16_t offset); + virtual bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) = 0; // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size. - bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks); + virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; // Add a label to the op sequence. - bool AddLabel(uint64_t label); + virtual bool AddLabel(uint64_t label) = 0; // Add sequence data for op merging. Data is a list of the destination block numbers. - bool AddSequenceData(size_t num_ops, const uint32_t* data); + virtual bool AddSequenceData(size_t num_ops, const uint32_t* data) = 0; // Flush all pending writes. This must be called before closing the writer // to ensure that the correct headers and footers are written. @@ -93,21 +91,8 @@ class ICowWriter { // Return number of bytes the cow image occupies on disk. virtual uint64_t GetCowSize() = 0; - const CowOptions& options() { return options_; } - - protected: - virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0; - virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; - virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) = 0; - virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; - virtual bool EmitLabel(uint64_t label) = 0; - virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0; - - bool ValidateNewBlock(uint64_t new_block); - - protected: - CowOptions options_; + virtual uint32_t GetBlockSize() const = 0; + virtual std::optional GetMaxBlocks() const = 0; }; class CompressWorker { @@ -146,96 +131,15 @@ class CompressWorker { std::vector>* compressed_data); }; -class CowWriter : public ICowWriter { - public: - explicit CowWriter(const CowOptions& options); - ~CowWriter(); +// Create an ICowWriter not backed by any file. This is useful for estimating +// the final size of a cow file. +std::unique_ptr CreateCowEstimator(uint32_t version, const CowOptions& options); - // Set up the writer. - // The file starts from the beginning. - // - // If fd is < 0, the CowWriter will be opened against /dev/null. This is for - // computing COW sizes without using storage space. - bool Initialize(android::base::unique_fd&& fd); - bool Initialize(android::base::borrowed_fd fd); - // Set up a writer, assuming that the given label is the last valid label. - // This will result in dropping any labels that occur after the given on, and will fail - // if the given label does not appear. - bool InitializeAppend(android::base::unique_fd&&, uint64_t label); - bool InitializeAppend(android::base::borrowed_fd fd, uint64_t label); - - bool Finalize() override; - - uint64_t GetCowSize() override; - - protected: - virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; - virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; - virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) override; - virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; - virtual bool EmitLabel(uint64_t label) override; - virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override; - - private: - bool EmitCluster(); - bool EmitClusterIfNeeded(); - bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block, - uint16_t offset, uint8_t type); - void SetupHeaders(); - void SetupWriteOptions(); - bool ParseOptions(); - bool OpenForWrite(); - bool OpenForAppend(uint64_t label); - bool GetDataPos(uint64_t* pos); - bool WriteRawData(const void* data, size_t size); - bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0); - void AddOperation(const CowOperation& op); - void InitPos(); - void InitBatchWrites(); - void InitWorkers(); - bool FlushCluster(); - - bool CompressBlocks(size_t num_blocks, const void* data); - bool SetFd(android::base::borrowed_fd fd); - bool Sync(); - bool Truncate(off_t length); - bool EnsureSpaceAvailable(const uint64_t bytes_needed) const; - - private: - android::base::unique_fd owned_fd_; - android::base::borrowed_fd fd_; - CowHeader header_{}; - CowFooter footer_{}; - CowCompressionAlgorithm compression_ = kCowCompressNone; - uint64_t current_op_pos_ = 0; - uint64_t next_op_pos_ = 0; - uint64_t next_data_pos_ = 0; - uint64_t current_data_pos_ = 0; - ssize_t total_data_written_ = 0; - uint32_t cluster_size_ = 0; - uint32_t current_cluster_size_ = 0; - uint64_t current_data_size_ = 0; - bool is_dev_null_ = false; - bool merge_in_progress_ = false; - bool is_block_device_ = false; - uint64_t cow_image_size_ = INT64_MAX; - - int num_compress_threads_ = 1; - std::vector> compress_threads_; - std::vector> threads_; - std::vector> compressed_buf_; - std::vector>::iterator buf_iter_; - - std::vector> opbuffer_vec_; - std::vector> databuffer_vec_; - std::unique_ptr cowop_vec_; - int op_vec_index_ = 0; - - std::unique_ptr data_vec_; - int data_vec_index_ = 0; - bool batch_write_ = false; -}; +// Create an ICowWriter of the given version and options. If a label is given, +// the writer is opened in append mode. +std::unique_ptr CreateCowWriter(uint32_t version, const CowOptions& options, + android::base::unique_fd&& fd, + std::optional label = {}); } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h index d798e25e3..52e3a9c4e 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h @@ -23,30 +23,23 @@ class MockSnapshotWriter : public ISnapshotWriter { public: using FileDescriptor = ISnapshotWriter::FileDescriptor; - explicit MockSnapshotWriter(const CowOptions& options) : ISnapshotWriter(options) {} - MockSnapshotWriter() : ISnapshotWriter({}) {} - MOCK_METHOD(bool, Finalize, (), (override)); // Return number of bytes the cow image occupies on disk. MOCK_METHOD(uint64_t, GetCowSize, (), (override)); - MOCK_METHOD(bool, EmitCopy, (uint64_t, uint64_t, uint64_t), (override)); - MOCK_METHOD(bool, EmitRawBlocks, (uint64_t, const void*, size_t), (override)); - MOCK_METHOD(bool, EmitXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t), + MOCK_METHOD(bool, AddCopy, (uint64_t, uint64_t, uint64_t), (override)); + MOCK_METHOD(bool, AddRawBlocks, (uint64_t, const void*, size_t), (override)); + MOCK_METHOD(bool, AddXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t), (override)); - MOCK_METHOD(bool, EmitZeroBlocks, (uint64_t, uint64_t), (override)); - MOCK_METHOD(bool, EmitLabel, (uint64_t), (override)); - MOCK_METHOD(bool, EmitSequenceData, (size_t, const uint32_t*), (override)); - - // Open the writer in write mode (no append). + MOCK_METHOD(bool, AddZeroBlocks, (uint64_t, uint64_t), (override)); + MOCK_METHOD(bool, AddLabel, (uint64_t), (override)); + MOCK_METHOD(bool, AddSequenceData, (size_t, const uint32_t*), (override)); MOCK_METHOD(bool, Initialize, (), (override)); + MOCK_METHOD(bool, InitializeAppend, (uint64_t), (override)); MOCK_METHOD(bool, VerifyMergeOps, (), (override, const, noexcept)); - - // Open the writer in append mode, with the last label to resume - // from. See CowWriter::InitializeAppend. - MOCK_METHOD(bool, InitializeAppend, (uint64_t label), (override)); - MOCK_METHOD(std::unique_ptr, OpenReader, (), (override)); + MOCK_METHOD(uint32_t, GetBlockSize, (), (override, const)); + MOCK_METHOD(std::optional, GetMaxBlocks, (), (override, const)); }; } // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h index 8f6344c3b..2653a6053 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h @@ -31,13 +31,7 @@ class ISnapshotWriter : public ICowWriter { public: using FileDescriptor = chromeos_update_engine::FileDescriptor; - explicit ISnapshotWriter(const CowOptions& options); - - // Set the source device. This is used for AddCopy() operations, if the - // underlying writer needs the original bytes (for example if backed by - // dm-snapshot or if writing directly to an unsnapshotted region). The - // device is only opened on the first operation that requires it. - void SetSourceDevice(const std::string& source_device); + virtual ~ISnapshotWriter() {} // Open the writer in write mode (no append). virtual bool Initialize() = 0; @@ -47,15 +41,8 @@ class ISnapshotWriter : public ICowWriter { virtual bool InitializeAppend(uint64_t label) = 0; virtual std::unique_ptr OpenReader() = 0; + virtual bool VerifyMergeOps() const noexcept = 0; - - protected: - android::base::borrowed_fd GetSourceFd(); - - std::optional source_device_; - - private: - android::base::unique_fd source_fd_; }; // Send writes to a COW or a raw device directly, based on a threshold. @@ -63,6 +50,8 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { public: CompressedSnapshotWriter(const CowOptions& options); + void SetSourceDevice(const std::string& source_device); + // Sets the COW device; this is required. bool SetCowDevice(android::base::unique_fd&& cow_device); @@ -70,23 +59,34 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { bool InitializeAppend(uint64_t label) override; bool Finalize() override; uint64_t GetCowSize() override; + uint32_t GetBlockSize() const override; + std::optional GetMaxBlocks() const override; std::unique_ptr OpenReader() override; bool VerifyMergeOps() const noexcept; - protected: - bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; - bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; - bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, - uint16_t offset) override; - bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; - bool EmitLabel(uint64_t label) override; - bool EmitSequenceData(size_t num_ops, const uint32_t* data) override; + bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; + bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, + uint16_t offset) override; + bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + bool AddLabel(uint64_t label) override; + bool AddSequenceData(size_t num_ops, const uint32_t* data) override; private: std::unique_ptr OpenCowReader() const; - android::base::unique_fd cow_device_; + android::base::borrowed_fd GetSourceFd(); - std::unique_ptr cow_; + CowOptions options_; + + // Set the source device. This is used for AddCopy() operations, if the + // underlying writer needs the original bytes (for example if backed by + // dm-snapshot or if writing directly to an unsnapshotted region). The + // device is only opened on the first operation that requires it. + std::optional source_device_; + android::base::unique_fd source_fd_; + + android::base::unique_fd cow_device_; + std::unique_ptr cow_; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp index 2157d0fa9..ff3ccecb8 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp @@ -19,10 +19,13 @@ #include #include +#include "writer_v2.h" namespace android { namespace snapshot { +using android::base::unique_fd; + std::ostream& operator<<(std::ostream& os, CowOperation const& op) { os << "CowOperation(type:"; if (op.type == kCowCopyOp) @@ -103,5 +106,27 @@ bool IsOrderedOp(const CowOperation& op) { } } +std::unique_ptr CreateCowWriter(uint32_t version, const CowOptions& options, + unique_fd&& fd, std::optional label) { + std::unique_ptr base; + switch (version) { + case 1: + case 2: + base = std::make_unique(options, std::move(fd)); + break; + default: + LOG(ERROR) << "Cannot create unknown cow version: " << version; + return nullptr; + } + if (!base->Initialize(label)) { + return nullptr; + } + return base; +} + +std::unique_ptr CreateCowEstimator(uint32_t version, const CowOptions& options) { + return CreateCowWriter(version, options, unique_fd{-1}, std::nullopt); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp similarity index 91% rename from fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp rename to fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp index 8f80bb345..120b2c062 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp @@ -25,7 +25,9 @@ #include #include #include "cow_decompress.h" +#include "writer_v2.h" +using android::base::unique_fd; using testing::AssertionFailure; using testing::AssertionResult; using testing::AssertionSuccess; @@ -42,6 +44,8 @@ class CowTest : public ::testing::Test { virtual void TearDown() override { cow_ = nullptr; } + unique_fd GetCowFd() { return unique_fd{dup(cow_->fd)}; } + std::unique_ptr cow_; }; @@ -53,9 +57,9 @@ static inline bool ReadData(CowReader& reader, const CowOperation* op, void* buf TEST_F(CowTest, CopyContiguous) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddCopy(10, 1000, 100)); ASSERT_TRUE(writer.Finalize()); @@ -96,9 +100,9 @@ TEST_F(CowTest, CopyContiguous) { TEST_F(CowTest, ReadWrite) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -175,9 +179,9 @@ TEST_F(CowTest, ReadWrite) { TEST_F(CowTest, ReadWriteXor) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -256,9 +260,9 @@ TEST_F(CowTest, CompressGz) { CowOptions options; options.cluster_ops = 0; options.compression = "gz"; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -296,9 +300,9 @@ TEST_P(CompressionTest, ThreadedBatchWrites) { options.compression = GetParam(); options.num_compress_threads = 2; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string xor_data = "This is test data-1. Testing xor"; xor_data.resize(options.block_size, '\0'); @@ -374,9 +378,9 @@ TEST_P(CompressionTest, NoBatchWrites) { options.num_compress_threads = 1; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "Testing replace ops without batch writes"; data.resize(options.block_size * 1024, '\0'); @@ -497,9 +501,9 @@ TEST_F(CowTest, ClusterCompressGz) { CowOptions options; options.compression = "gz"; options.cluster_ops = 2; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -562,9 +566,9 @@ TEST_F(CowTest, CompressTwoBlocks) { CowOptions options; options.compression = "gz"; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size * 2, '\0'); @@ -595,12 +599,12 @@ TEST_F(CowTest, CompressTwoBlocks) { TEST_F(CowTest, GetSize) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); if (ftruncate(cow_->fd, 0) < 0) { perror("Fails to set temp file size"); FAIL(); } - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -621,8 +625,8 @@ TEST_F(CowTest, GetSize) { TEST_F(CowTest, AppendLabelSmall) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -632,8 +636,8 @@ TEST_F(CowTest, AppendLabelSmall) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 3)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({3})); std::string data2 = "More data!"; data2.resize(options.block_size, '\0'); @@ -688,8 +692,8 @@ TEST_F(CowTest, AppendLabelSmall) { TEST_F(CowTest, AppendLabelMissing) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddLabel(0)); std::string data = "This is some data, believe it"; @@ -701,9 +705,9 @@ TEST_F(CowTest, AppendLabelMissing) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 1)); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 0)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_FALSE(writer->Initialize({1})); + ASSERT_TRUE(writer->Initialize({0})); ASSERT_TRUE(writer->AddZeroBlocks(51, 1)); ASSERT_TRUE(writer->Finalize()); @@ -740,8 +744,8 @@ TEST_F(CowTest, AppendLabelMissing) { TEST_F(CowTest, AppendExtendedCorrupted) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddLabel(5)); @@ -763,8 +767,8 @@ TEST_F(CowTest, AppendExtendedCorrupted) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({5})); ASSERT_TRUE(writer->Finalize()); @@ -791,8 +795,8 @@ TEST_F(CowTest, AppendExtendedCorrupted) { TEST_F(CowTest, AppendbyLabel) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size * 2, '\0'); @@ -810,9 +814,9 @@ TEST_F(CowTest, AppendbyLabel) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 12)); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_FALSE(writer->Initialize({12})); + ASSERT_TRUE(writer->Initialize({5})); // This should drop label 6 ASSERT_TRUE(writer->Finalize()); @@ -879,8 +883,8 @@ TEST_F(CowTest, AppendbyLabel) { TEST_F(CowTest, ClusterTest) { CowOptions options; options.cluster_ops = 4; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -976,16 +980,16 @@ TEST_F(CowTest, ClusterTest) { TEST_F(CowTest, ClusterAppendTest) { CowOptions options; options.cluster_ops = 3; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddLabel(50)); ASSERT_TRUE(writer->Finalize()); // Adds a cluster op, should be dropped on append ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 50)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({50})); std::string data2 = "More data!"; data2.resize(options.block_size, '\0'); @@ -1037,8 +1041,8 @@ TEST_F(CowTest, ClusterAppendTest) { TEST_F(CowTest, AppendAfterFinalize) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -1058,8 +1062,8 @@ TEST_F(CowTest, AppendAfterFinalize) { ASSERT_TRUE(reader.Parse(cow_->fd)); } -AssertionResult WriteDataBlock(CowWriter* writer, uint64_t new_block, std::string data) { - data.resize(writer->options().block_size, '\0'); +AssertionResult WriteDataBlock(ICowWriter* writer, uint64_t new_block, std::string data) { + data.resize(writer->GetBlockSize(), '\0'); if (!writer->AddRawBlocks(new_block, data.data(), data.size())) { return AssertionFailure() << "Failed to add raw block"; } @@ -1088,8 +1092,8 @@ AssertionResult CompareDataBlock(CowReader* reader, const CowOperation* op, TEST_F(CowTest, ResumeMidCluster) { CowOptions options; options.cluster_ops = 7; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1")); ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2")); @@ -1099,8 +1103,8 @@ TEST_F(CowTest, ResumeMidCluster) { ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4")); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4")); ASSERT_TRUE(WriteDataBlock(writer.get(), 5, "Block 5")); ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6")); @@ -1145,8 +1149,8 @@ TEST_F(CowTest, ResumeEndCluster) { CowOptions options; int cluster_ops = 5; options.cluster_ops = cluster_ops; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1")); ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2")); @@ -1160,8 +1164,8 @@ TEST_F(CowTest, ResumeEndCluster) { ASSERT_TRUE(WriteDataBlock(writer.get(), 8, "Block 8")); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4")); ASSERT_TRUE(WriteDataBlock(writer.get(), 5, "Block 5")); ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6")); @@ -1205,8 +1209,8 @@ TEST_F(CowTest, ResumeEndCluster) { TEST_F(CowTest, DeleteMidCluster) { CowOptions options; options.cluster_ops = 7; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1")); ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2")); @@ -1218,8 +1222,8 @@ TEST_F(CowTest, DeleteMidCluster) { ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6")); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(writer->Finalize()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -1255,14 +1259,14 @@ TEST_F(CowTest, DeleteMidCluster) { TEST_F(CowTest, BigSeqOp) { CowOptions options; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); const int seq_len = std::numeric_limits::max() / sizeof(uint32_t) + 1; uint32_t sequence[seq_len]; for (int i = 0; i < seq_len; i++) { sequence[i] = i + 1; } - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddSequenceData(seq_len, sequence)); ASSERT_TRUE(writer.AddZeroBlocks(1, seq_len)); @@ -1287,14 +1291,14 @@ TEST_F(CowTest, BigSeqOp) { TEST_F(CowTest, MissingSeqOp) { CowOptions options; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); const int seq_len = 10; uint32_t sequence[seq_len]; for (int i = 0; i < seq_len; i++) { sequence[i] = i + 1; } - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddSequenceData(seq_len, sequence)); ASSERT_TRUE(writer.AddZeroBlocks(1, seq_len - 1)); @@ -1308,14 +1312,14 @@ TEST_F(CowTest, MissingSeqOp) { TEST_F(CowTest, ResumeSeqOp) { CowOptions options; - auto writer = std::make_unique(options); + auto writer = std::make_unique(options, GetCowFd()); const int seq_len = 10; uint32_t sequence[seq_len]; for (int i = 0; i < seq_len; i++) { sequence[i] = i + 1; } - ASSERT_TRUE(writer->Initialize(cow_->fd)); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddSequenceData(seq_len, sequence)); ASSERT_TRUE(writer->AddZeroBlocks(1, seq_len / 2)); @@ -1328,8 +1332,8 @@ TEST_F(CowTest, ResumeSeqOp) { auto itr = reader->GetRevMergeOpIter(); ASSERT_TRUE(itr->AtEnd()); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(writer->AddZeroBlocks(1 + seq_len / 2, seq_len / 2)); ASSERT_TRUE(writer->Finalize()); @@ -1358,10 +1362,10 @@ TEST_F(CowTest, RevMergeOpItrTest) { CowOptions options; options.cluster_ops = 5; options.num_merge_ops = 1; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); uint32_t sequence[] = {2, 10, 6, 7, 3, 5}; - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddSequenceData(6, sequence)); ASSERT_TRUE(writer.AddCopy(6, 13)); @@ -1408,9 +1412,9 @@ TEST_F(CowTest, LegacyRevMergeOpItrTest) { CowOptions options; options.cluster_ops = 5; options.num_merge_ops = 1; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddCopy(2, 11)); ASSERT_TRUE(writer.AddCopy(10, 12)); @@ -1459,10 +1463,10 @@ TEST_F(CowTest, InvalidMergeOrderTest) { options.num_merge_ops = 1; std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); - auto writer = std::make_unique(options); + auto writer = std::make_unique(options, GetCowFd()); CowReader reader; - ASSERT_TRUE(writer->Initialize(cow_->fd)); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddCopy(3, 2)); ASSERT_TRUE(writer->AddCopy(2, 1)); @@ -1471,14 +1475,14 @@ TEST_F(CowTest, InvalidMergeOrderTest) { ASSERT_TRUE(reader.Parse(cow_->fd)); ASSERT_TRUE(reader.VerifyMergeOps()); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(writer->AddCopy(4, 2)); ASSERT_TRUE(writer->Finalize()); ASSERT_TRUE(reader.Parse(cow_->fd)); ASSERT_FALSE(reader.VerifyMergeOps()); - writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddCopy(2, 1)); ASSERT_TRUE(writer->AddXorBlocks(3, &data, data.size(), 1, 1)); ASSERT_TRUE(writer->Finalize()); diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp new file mode 100644 index 000000000..22e63d0a3 --- /dev/null +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp @@ -0,0 +1,163 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "writer_base.h" + +#include +#include +#include +#include +#include + +#include + +// The info messages here are spammy, but as useful for update_engine. Disable +// them when running on the host. +#ifdef __ANDROID__ +#define LOG_INFO LOG(INFO) +#else +#define LOG_INFO LOG(VERBOSE) +#endif + +namespace android { +namespace snapshot { + +using android::base::borrowed_fd; +using android::base::unique_fd; + +namespace { +std::string GetFdPath(borrowed_fd fd) { + const auto fd_path = "/proc/self/fd/" + std::to_string(fd.get()); + std::string file_path(512, '\0'); + const auto err = readlink(fd_path.c_str(), file_path.data(), file_path.size()); + if (err <= 0) { + PLOG(ERROR) << "Failed to determine path for fd " << fd.get(); + file_path.clear(); + } else { + file_path.resize(err); + } + return file_path; +} +} // namespace + +CowWriterBase::CowWriterBase(const CowOptions& options, unique_fd&& fd) + : options_(options), fd_(std::move(fd)) {} + +bool CowWriterBase::InitFd() { + if (fd_.get() < 0) { + fd_.reset(open("/dev/null", O_RDWR | O_CLOEXEC)); + if (fd_ < 0) { + PLOG(ERROR) << "open /dev/null failed"; + return false; + } + is_dev_null_ = true; + return true; + } + + struct stat stat {}; + if (fstat(fd_.get(), &stat) < 0) { + PLOG(ERROR) << "fstat failed"; + return false; + } + const auto file_path = GetFdPath(fd_); + is_block_device_ = S_ISBLK(stat.st_mode); + if (is_block_device_) { + uint64_t size_in_bytes = 0; + if (ioctl(fd_.get(), BLKGETSIZE64, &size_in_bytes)) { + PLOG(ERROR) << "Failed to get total size for: " << fd_.get(); + return false; + } + cow_image_size_ = size_in_bytes; + LOG_INFO << "COW image " << file_path << " has size " << size_in_bytes; + } else { + LOG_INFO << "COW image " << file_path + << " is not a block device, assuming unlimited space."; + } + return true; +} + +bool CowWriterBase::AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { + CHECK(num_blocks != 0); + + for (size_t i = 0; i < num_blocks; i++) { + if (!ValidateNewBlock(new_block + i)) { + return false; + } + } + + return EmitCopy(new_block, old_block, num_blocks); +} + +bool CowWriterBase::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) { + if (size % options_.block_size != 0) { + LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " + << options_.block_size; + return false; + } + + uint64_t num_blocks = size / options_.block_size; + uint64_t last_block = new_block_start + num_blocks - 1; + if (!ValidateNewBlock(last_block)) { + return false; + } + return EmitRawBlocks(new_block_start, data, size); +} + +bool CowWriterBase::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) { + if (size % options_.block_size != 0) { + LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " + << options_.block_size; + return false; + } + + uint64_t num_blocks = size / options_.block_size; + uint64_t last_block = new_block_start + num_blocks - 1; + if (!ValidateNewBlock(last_block)) { + return false; + } + if (offset >= options_.block_size) { + LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than " + << options_.block_size; + } + return EmitXorBlocks(new_block_start, data, size, old_block, offset); +} + +bool CowWriterBase::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { + uint64_t last_block = new_block_start + num_blocks - 1; + if (!ValidateNewBlock(last_block)) { + return false; + } + return EmitZeroBlocks(new_block_start, num_blocks); +} + +bool CowWriterBase::AddLabel(uint64_t label) { + return EmitLabel(label); +} + +bool CowWriterBase::AddSequenceData(size_t num_ops, const uint32_t* data) { + return EmitSequenceData(num_ops, data); +} + +bool CowWriterBase::ValidateNewBlock(uint64_t new_block) { + if (options_.max_blocks && new_block >= options_.max_blocks.value()) { + LOG(ERROR) << "New block " << new_block << " exceeds maximum block count " + << options_.max_blocks.value(); + return false; + } + return true; +} + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h new file mode 100644 index 000000000..8fa90656f --- /dev/null +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h @@ -0,0 +1,71 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace android { +namespace snapshot { + +class CowWriterBase : public ICowWriter { + public: + CowWriterBase(const CowOptions& options, android::base::unique_fd&& fd); + virtual ~CowWriterBase() {} + + // Set up the writer. + // The file starts from the beginning. + // + // If fd is < 0, the CowWriter will be opened against /dev/null. This is for + // computing COW sizes without using storage space. + // + // If a label is given, any operations after the given label will be dropped. + // If the given label is not found, Initialize will fail. + virtual bool Initialize(std::optional label = {}) = 0; + + bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; + bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, + uint16_t offset) override; + bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + bool AddLabel(uint64_t label) override; + bool AddSequenceData(size_t num_ops, const uint32_t* data) override; + uint32_t GetBlockSize() const override { return options_.block_size; } + std::optional GetMaxBlocks() const override { return options_.max_blocks; } + + const CowOptions& options() const { return options_; } + + protected: + virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0; + virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; + virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) = 0; + virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; + virtual bool EmitLabel(uint64_t label) = 0; + virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0; + + bool InitFd(); + bool ValidateNewBlock(uint64_t new_block); + + CowOptions options_; + CowHeader header_{}; + + android::base::unique_fd fd_; + bool is_dev_null_ = false; + bool is_block_device_ = false; + uint64_t cow_image_size_ = INT64_MAX; +}; + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp similarity index 75% rename from fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp rename to fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp index 1eaa0385e..b6603da52 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp @@ -37,6 +37,8 @@ #include #include +#include "writer_v2.h" + // The info messages here are spammy, but as useful for update_engine. Disable // them when running on the host. #ifdef __ANDROID__ @@ -48,104 +50,17 @@ namespace android { namespace snapshot { -namespace { -std::string GetFdPath(int fd) { - const auto fd_path = "/proc/self/fd/" + std::to_string(fd); - std::string file_path(512, '\0'); - const auto err = readlink(fd_path.c_str(), file_path.data(), file_path.size()); - if (err <= 0) { - PLOG(ERROR) << "Failed to determine path for fd " << fd; - file_path.clear(); - } else { - file_path.resize(err); - } - return file_path; -} -} // namespace - static_assert(sizeof(off_t) == sizeof(uint64_t)); -using android::base::borrowed_fd; using android::base::unique_fd; -bool ICowWriter::AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { - CHECK(num_blocks != 0); - - for (size_t i = 0; i < num_blocks; i++) { - if (!ValidateNewBlock(new_block + i)) { - return false; - } - } - - return EmitCopy(new_block, old_block, num_blocks); -} - -bool ICowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) { - if (size % options_.block_size != 0) { - LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " - << options_.block_size; - return false; - } - - uint64_t num_blocks = size / options_.block_size; - uint64_t last_block = new_block_start + num_blocks - 1; - if (!ValidateNewBlock(last_block)) { - return false; - } - return EmitRawBlocks(new_block_start, data, size); -} - -bool ICowWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) { - if (size % options_.block_size != 0) { - LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " - << options_.block_size; - return false; - } - - uint64_t num_blocks = size / options_.block_size; - uint64_t last_block = new_block_start + num_blocks - 1; - if (!ValidateNewBlock(last_block)) { - return false; - } - if (offset >= options_.block_size) { - LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than " - << options_.block_size; - } - return EmitXorBlocks(new_block_start, data, size, old_block, offset); -} - -bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { - uint64_t last_block = new_block_start + num_blocks - 1; - if (!ValidateNewBlock(last_block)) { - return false; - } - return EmitZeroBlocks(new_block_start, num_blocks); -} - -bool ICowWriter::AddLabel(uint64_t label) { - return EmitLabel(label); -} - -bool ICowWriter::AddSequenceData(size_t num_ops, const uint32_t* data) { - return EmitSequenceData(num_ops, data); -} - -bool ICowWriter::ValidateNewBlock(uint64_t new_block) { - if (options_.max_blocks && new_block >= options_.max_blocks.value()) { - LOG(ERROR) << "New block " << new_block << " exceeds maximum block count " - << options_.max_blocks.value(); - return false; - } - return true; -} - -CowWriter::CowWriter(const CowOptions& options) : ICowWriter(options), fd_(-1) { +CowWriterV2::CowWriterV2(const CowOptions& options, unique_fd&& fd) + : CowWriterBase(options, std::move(fd)) { SetupHeaders(); SetupWriteOptions(); } -CowWriter::~CowWriter() { +CowWriterV2::~CowWriterV2() { for (size_t i = 0; i < compress_threads_.size(); i++) { CompressWorker* worker = compress_threads_[i].get(); if (worker) { @@ -164,7 +79,7 @@ CowWriter::~CowWriter() { compress_threads_.clear(); } -void CowWriter::SetupWriteOptions() { +void CowWriterV2::SetupWriteOptions() { num_compress_threads_ = options_.num_compress_threads; if (!num_compress_threads_) { @@ -184,7 +99,7 @@ void CowWriter::SetupWriteOptions() { } } -void CowWriter::SetupHeaders() { +void CowWriterV2::SetupHeaders() { header_ = {}; header_.prefix.magic = kCowMagicNumber; header_.prefix.major_version = kCowVersionMajor; @@ -201,7 +116,7 @@ void CowWriter::SetupHeaders() { footer_.op.type = kCowFooterOp; } -bool CowWriter::ParseOptions() { +bool CowWriterV2::ParseOptions() { auto algorithm = CompressionAlgorithmFromString(options_.compression); if (!algorithm) { LOG(ERROR) << "unrecognized compression: " << options_.compression; @@ -216,42 +131,7 @@ bool CowWriter::ParseOptions() { return true; } -bool CowWriter::SetFd(android::base::borrowed_fd fd) { - if (fd.get() < 0) { - owned_fd_.reset(open("/dev/null", O_RDWR | O_CLOEXEC)); - if (owned_fd_ < 0) { - PLOG(ERROR) << "open /dev/null failed"; - return false; - } - fd_ = owned_fd_; - is_dev_null_ = true; - } else { - fd_ = fd; - - struct stat stat {}; - if (fstat(fd.get(), &stat) < 0) { - PLOG(ERROR) << "fstat failed"; - return false; - } - const auto file_path = GetFdPath(fd.get()); - is_block_device_ = S_ISBLK(stat.st_mode); - if (is_block_device_) { - uint64_t size_in_bytes = 0; - if (ioctl(fd.get(), BLKGETSIZE64, &size_in_bytes)) { - PLOG(ERROR) << "Failed to get total size for: " << fd.get(); - return false; - } - cow_image_size_ = size_in_bytes; - LOG_INFO << "COW image " << file_path << " has size " << size_in_bytes; - } else { - LOG_INFO << "COW image " << file_path - << " is not a block device, assuming unlimited space."; - } - } - return true; -} - -void CowWriter::InitBatchWrites() { +void CowWriterV2::InitBatchWrites() { if (batch_write_) { cowop_vec_ = std::make_unique(header_.cluster_ops); data_vec_ = std::make_unique(header_.cluster_ops); @@ -277,7 +157,7 @@ void CowWriter::InitBatchWrites() { LOG_INFO << "Batch writes: " << batch_write; } -void CowWriter::InitWorkers() { +void CowWriterV2::InitWorkers() { if (num_compress_threads_ <= 1) { LOG_INFO << "Not creating new threads for compression."; return; @@ -291,44 +171,27 @@ void CowWriter::InitWorkers() { LOG_INFO << num_compress_threads_ << " thread used for compression"; } -bool CowWriter::Initialize(unique_fd&& fd) { - owned_fd_ = std::move(fd); - return Initialize(borrowed_fd{owned_fd_}); -} - -bool CowWriter::Initialize(borrowed_fd fd) { - if (!SetFd(fd) || !ParseOptions()) { +bool CowWriterV2::Initialize(std::optional label) { + if (!InitFd() || !ParseOptions()) { return false; } - - if (!OpenForWrite()) { - return false; + if (!label) { + if (!OpenForWrite()) { + return false; + } + } else { + if (!OpenForAppend(*label)) { + return false; + } } - InitWorkers(); + if (!compress_threads_.size()) { + InitWorkers(); + } return true; } -bool CowWriter::InitializeAppend(android::base::unique_fd&& fd, uint64_t label) { - owned_fd_ = std::move(fd); - return InitializeAppend(android::base::borrowed_fd{owned_fd_}, label); -} - -bool CowWriter::InitializeAppend(android::base::borrowed_fd fd, uint64_t label) { - if (!SetFd(fd) || !ParseOptions()) { - return false; - } - - bool ret = OpenForAppend(label); - - if (ret && !compress_threads_.size()) { - InitWorkers(); - } - - return ret; -} - -void CowWriter::InitPos() { +void CowWriterV2::InitPos() { next_op_pos_ = sizeof(header_) + header_.buffer_size; cluster_size_ = header_.cluster_ops * sizeof(CowOperation); if (header_.cluster_ops) { @@ -340,7 +203,7 @@ void CowWriter::InitPos() { current_data_size_ = 0; } -bool CowWriter::OpenForWrite() { +bool CowWriterV2::OpenForWrite() { // This limitation is tied to the data field size in CowOperation. if (header_.block_size > std::numeric_limits::max()) { LOG(ERROR) << "Block size is too large"; @@ -388,7 +251,7 @@ bool CowWriter::OpenForWrite() { return true; } -bool CowWriter::OpenForAppend(uint64_t label) { +bool CowWriterV2::OpenForAppend(uint64_t label) { auto reader = std::make_unique(); std::queue toAdd; @@ -424,7 +287,7 @@ bool CowWriter::OpenForAppend(uint64_t label) { return EmitClusterIfNeeded(); } -bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { +bool CowWriterV2::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { CHECK(!merge_in_progress_); for (size_t i = 0; i < num_blocks; i++) { @@ -440,16 +303,16 @@ bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_bl return true; } -bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) { +bool CowWriterV2::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) { return EmitBlocks(new_block_start, data, size, 0, 0, kCowReplaceOp); } -bool CowWriter::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) { +bool CowWriterV2::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) { return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp); } -bool CowWriter::CompressBlocks(size_t num_blocks, const void* data) { +bool CowWriterV2::CompressBlocks(size_t num_blocks, const void* data) { size_t num_threads = (num_blocks == 1) ? 1 : num_compress_threads_; size_t num_blocks_per_thread = num_blocks / num_threads; const uint8_t* iter = reinterpret_cast(data); @@ -483,8 +346,8 @@ bool CowWriter::CompressBlocks(size_t num_blocks, const void* data) { return true; } -bool CowWriter::EmitBlocks(uint64_t new_block_start, const void* data, size_t size, - uint64_t old_block, uint16_t offset, uint8_t type) { +bool CowWriterV2::EmitBlocks(uint64_t new_block_start, const void* data, size_t size, + uint64_t old_block, uint16_t offset, uint8_t type) { CHECK(!merge_in_progress_); const uint8_t* iter = reinterpret_cast(data); @@ -558,7 +421,7 @@ bool CowWriter::EmitBlocks(uint64_t new_block_start, const void* data, size_t si return true; } -bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { +bool CowWriterV2::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { CHECK(!merge_in_progress_); for (uint64_t i = 0; i < num_blocks; i++) { CowOperation op = {}; @@ -570,7 +433,7 @@ bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { return true; } -bool CowWriter::EmitLabel(uint64_t label) { +bool CowWriterV2::EmitLabel(uint64_t label) { CHECK(!merge_in_progress_); CowOperation op = {}; op.type = kCowLabelOp; @@ -578,7 +441,7 @@ bool CowWriter::EmitLabel(uint64_t label) { return WriteOperation(op) && Sync(); } -bool CowWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) { +bool CowWriterV2::EmitSequenceData(size_t num_ops, const uint32_t* data) { CHECK(!merge_in_progress_); size_t to_add = 0; size_t max_ops = (header_.block_size * 2) / sizeof(uint32_t); @@ -598,7 +461,7 @@ bool CowWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) { return true; } -bool CowWriter::EmitCluster() { +bool CowWriterV2::EmitCluster() { CowOperation op = {}; op.type = kCowClusterOp; // Next cluster starts after remainder of current cluster and the next data block. @@ -606,7 +469,7 @@ bool CowWriter::EmitCluster() { return WriteOperation(op); } -bool CowWriter::EmitClusterIfNeeded() { +bool CowWriterV2::EmitClusterIfNeeded() { // If there isn't room for another op and the cluster end op, end the current cluster if (cluster_size_ && cluster_size_ < current_cluster_size_ + 2 * sizeof(CowOperation)) { if (!EmitCluster()) return false; @@ -614,7 +477,7 @@ bool CowWriter::EmitClusterIfNeeded() { return true; } -bool CowWriter::Finalize() { +bool CowWriterV2::Finalize() { if (!FlushCluster()) { LOG(ERROR) << "Finalize: FlushCluster() failed"; return false; @@ -688,7 +551,7 @@ bool CowWriter::Finalize() { return Sync(); } -uint64_t CowWriter::GetCowSize() { +uint64_t CowWriterV2::GetCowSize() { if (current_data_size_ > 0) { return next_data_pos_ + sizeof(footer_); } else { @@ -696,7 +559,7 @@ uint64_t CowWriter::GetCowSize() { } } -bool CowWriter::GetDataPos(uint64_t* pos) { +bool CowWriterV2::GetDataPos(uint64_t* pos) { off_t offs = lseek(fd_.get(), 0, SEEK_CUR); if (offs < 0) { PLOG(ERROR) << "lseek failed"; @@ -706,7 +569,7 @@ bool CowWriter::GetDataPos(uint64_t* pos) { return true; } -bool CowWriter::EnsureSpaceAvailable(const uint64_t bytes_needed) const { +bool CowWriterV2::EnsureSpaceAvailable(const uint64_t bytes_needed) const { if (bytes_needed > cow_image_size_) { LOG(ERROR) << "No space left on COW device. Required: " << bytes_needed << ", available: " << cow_image_size_; @@ -716,7 +579,7 @@ bool CowWriter::EnsureSpaceAvailable(const uint64_t bytes_needed) const { return true; } -bool CowWriter::FlushCluster() { +bool CowWriterV2::FlushCluster() { ssize_t ret; if (op_vec_index_) { @@ -745,7 +608,7 @@ bool CowWriter::FlushCluster() { return true; } -bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t size) { +bool CowWriterV2::WriteOperation(const CowOperation& op, const void* data, size_t size) { if (!EnsureSpaceAvailable(next_op_pos_ + sizeof(op))) { return false; } @@ -793,7 +656,7 @@ bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t return EmitClusterIfNeeded(); } -void CowWriter::AddOperation(const CowOperation& op) { +void CowWriterV2::AddOperation(const CowOperation& op) { footer_.op.num_ops++; if (op.type == kCowClusterOp) { @@ -808,14 +671,14 @@ void CowWriter::AddOperation(const CowOperation& op) { next_op_pos_ += sizeof(CowOperation) + GetNextOpOffset(op, header_.cluster_ops); } -bool CowWriter::WriteRawData(const void* data, const size_t size) { +bool CowWriterV2::WriteRawData(const void* data, const size_t size) { if (!android::base::WriteFullyAtOffset(fd_, data, size, next_data_pos_)) { return false; } return true; } -bool CowWriter::Sync() { +bool CowWriterV2::Sync() { if (is_dev_null_) { return true; } @@ -826,7 +689,7 @@ bool CowWriter::Sync() { return true; } -bool CowWriter::Truncate(off_t length) { +bool CowWriterV2::Truncate(off_t length) { if (is_dev_null_ || is_block_device_) { return true; } diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h new file mode 100644 index 000000000..809ae5747 --- /dev/null +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h @@ -0,0 +1,94 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "writer_base.h" + +namespace android { +namespace snapshot { + +class CowWriterV2 : public CowWriterBase { + public: + explicit CowWriterV2(const CowOptions& options, android::base::unique_fd&& fd); + ~CowWriterV2() override; + + bool Initialize(std::optional label = {}) override; + bool Finalize() override; + uint64_t GetCowSize() override; + + protected: + virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; + virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) override; + virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + virtual bool EmitLabel(uint64_t label) override; + virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override; + + private: + bool EmitCluster(); + bool EmitClusterIfNeeded(); + bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block, + uint16_t offset, uint8_t type); + void SetupHeaders(); + void SetupWriteOptions(); + bool ParseOptions(); + bool OpenForWrite(); + bool OpenForAppend(uint64_t label); + bool GetDataPos(uint64_t* pos); + bool WriteRawData(const void* data, size_t size); + bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0); + void AddOperation(const CowOperation& op); + void InitPos(); + void InitBatchWrites(); + void InitWorkers(); + bool FlushCluster(); + + bool CompressBlocks(size_t num_blocks, const void* data); + bool Sync(); + bool Truncate(off_t length); + bool EnsureSpaceAvailable(const uint64_t bytes_needed) const; + + private: + CowFooter footer_{}; + CowCompressionAlgorithm compression_ = kCowCompressNone; + uint64_t current_op_pos_ = 0; + uint64_t next_op_pos_ = 0; + uint64_t next_data_pos_ = 0; + uint64_t current_data_pos_ = 0; + ssize_t total_data_written_ = 0; + uint32_t cluster_size_ = 0; + uint32_t current_cluster_size_ = 0; + uint64_t current_data_size_ = 0; + bool merge_in_progress_ = false; + + int num_compress_threads_ = 1; + std::vector> compress_threads_; + std::vector> threads_; + std::vector> compressed_buf_; + std::vector>::iterator buf_iter_; + + std::vector> opbuffer_vec_; + std::vector> databuffer_vec_; + std::unique_ptr cowop_vec_; + int op_vec_index_ = 0; + + std::unique_ptr data_vec_; + int data_vec_index_ = 0; + bool batch_write_ = false; +}; + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp index e114d2556..5920bc22f 100644 --- a/fs_mgr/libsnapshot/snapshot.cpp +++ b/fs_mgr/libsnapshot/snapshot.cpp @@ -45,6 +45,7 @@ #include #include #include "device_info.h" +#include "libsnapshot_cow/writer_v2.h" #include "partition_cow_creator.h" #include "snapshot_metadata_updater.h" #include "snapshot_reader.h" @@ -3557,8 +3558,8 @@ Return SnapshotManager::InitializeUpdateSnapshots( } options.compression = it->second.compression_algorithm(); - CowWriter writer(options); - if (!writer.Initialize(fd) || !writer.Finalize()) { + CowWriterV2 writer(options, std::move(fd)); + if (!writer.Initialize(std::nullopt) || !writer.Finalize()) { LOG(ERROR) << "Could not initialize COW device for " << target_partition->name(); return Return::Error(); } diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp index 757f6f181..dac1b771b 100644 --- a/fs_mgr/libsnapshot/snapshot_test.cpp +++ b/fs_mgr/libsnapshot/snapshot_test.cpp @@ -1220,13 +1220,13 @@ class SnapshotUpdateTest : public SnapshotTest { SHA256_CTX ctx; SHA256_Init(&ctx); - if (!writer->options().max_blocks) { + if (!writer->GetMaxBlocks()) { LOG(ERROR) << "CowWriter must specify maximum number of blocks"; return false; } - const auto num_blocks = writer->options().max_blocks.value(); + const auto num_blocks = writer->GetMaxBlocks().value(); - const auto block_size = writer->options().block_size; + const auto block_size = writer->GetBlockSize(); std::string block(block_size, '\0'); for (uint64_t i = 0; i < num_blocks; i++) { if (!ReadFully(rand, block.data(), block.size())) { @@ -1254,13 +1254,13 @@ class SnapshotUpdateTest : public SnapshotTest { if (auto res = MapUpdateSnapshot(name, &writer); !res) { return res; } - if (!writer->options().max_blocks || !*writer->options().max_blocks) { + if (!writer->GetMaxBlocks() || !*writer->GetMaxBlocks()) { return AssertionFailure() << "No max blocks set for " << name << " writer"; } - uint64_t src_block = (old_size / writer->options().block_size) - 1; + uint64_t src_block = (old_size / writer->GetBlockSize()) - 1; uint64_t dst_block = 0; - uint64_t max_blocks = *writer->options().max_blocks; + uint64_t max_blocks = *writer->GetMaxBlocks(); while (dst_block < max_blocks && dst_block < src_block) { if (!writer->AddCopy(dst_block, src_block)) { return AssertionFailure() << "Unable to add copy for " << name << " for blocks " diff --git a/fs_mgr/libsnapshot/snapshot_writer.cpp b/fs_mgr/libsnapshot/snapshot_writer.cpp index 6a3906ead..0ea424c7e 100644 --- a/fs_mgr/libsnapshot/snapshot_writer.cpp +++ b/fs_mgr/libsnapshot/snapshot_writer.cpp @@ -19,6 +19,7 @@ #include #include #include +#include "libsnapshot_cow/writer_v2.h" #include "snapshot_reader.h" namespace android { @@ -28,13 +29,11 @@ using android::base::borrowed_fd; using android::base::unique_fd; using chromeos_update_engine::FileDescriptor; -ISnapshotWriter::ISnapshotWriter(const CowOptions& options) : ICowWriter(options) {} - -void ISnapshotWriter::SetSourceDevice(const std::string& source_device) { +void CompressedSnapshotWriter::SetSourceDevice(const std::string& source_device) { source_device_ = {source_device}; } -borrowed_fd ISnapshotWriter::GetSourceFd() { +borrowed_fd CompressedSnapshotWriter::GetSourceFd() { if (!source_device_) { LOG(ERROR) << "Attempted to read from source device but none was set"; return borrowed_fd{-1}; @@ -50,12 +49,10 @@ borrowed_fd ISnapshotWriter::GetSourceFd() { return source_fd_; } -CompressedSnapshotWriter::CompressedSnapshotWriter(const CowOptions& options) - : ISnapshotWriter(options) {} +CompressedSnapshotWriter::CompressedSnapshotWriter(const CowOptions& options) : options_(options) {} bool CompressedSnapshotWriter::SetCowDevice(android::base::unique_fd&& cow_device) { cow_device_ = std::move(cow_device); - cow_ = std::make_unique(options_); return true; } @@ -106,47 +103,76 @@ std::unique_ptr CompressedSnapshotWriter::OpenReader() { reader->SetSourceDevice(*source_device_); } - const auto& cow_options = options(); - if (cow_options.max_blocks) { - reader->SetBlockDeviceSize(*cow_options.max_blocks * cow_options.block_size); + if (options_.max_blocks) { + reader->SetBlockDeviceSize(*options_.max_blocks * options_.block_size); } return reader; } -bool CompressedSnapshotWriter::EmitCopy(uint64_t new_block, uint64_t old_block, - uint64_t num_blocks) { +bool CompressedSnapshotWriter::AddCopy(uint64_t new_block, uint64_t old_block, + uint64_t num_blocks) { return cow_->AddCopy(new_block, old_block, num_blocks); } -bool CompressedSnapshotWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, - size_t size) { +bool CompressedSnapshotWriter::AddRawBlocks(uint64_t new_block_start, const void* data, + size_t size) { return cow_->AddRawBlocks(new_block_start, data, size); } -bool CompressedSnapshotWriter::EmitXorBlocks(uint32_t new_block_start, const void* data, - size_t size, uint32_t old_block, uint16_t offset) { +bool CompressedSnapshotWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) { return cow_->AddXorBlocks(new_block_start, data, size, old_block, offset); } -bool CompressedSnapshotWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { +bool CompressedSnapshotWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { return cow_->AddZeroBlocks(new_block_start, num_blocks); } -bool CompressedSnapshotWriter::EmitLabel(uint64_t label) { +bool CompressedSnapshotWriter::AddLabel(uint64_t label) { return cow_->AddLabel(label); } -bool CompressedSnapshotWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) { +bool CompressedSnapshotWriter::AddSequenceData(size_t num_ops, const uint32_t* data) { return cow_->AddSequenceData(num_ops, data); } bool CompressedSnapshotWriter::Initialize() { - return cow_->Initialize(cow_device_); + unique_fd cow_fd(dup(cow_device_.get())); + if (cow_fd < 0) { + PLOG(ERROR) << "dup COW device"; + return false; + } + + auto cow = std::make_unique(options_, std::move(cow_fd)); + if (!cow->Initialize(std::nullopt)) { + return false; + } + cow_ = std::move(cow); + return true; } bool CompressedSnapshotWriter::InitializeAppend(uint64_t label) { - return cow_->InitializeAppend(cow_device_, label); + unique_fd cow_fd(dup(cow_device_.get())); + if (cow_fd < 0) { + PLOG(ERROR) << "dup COW device"; + return false; + } + + auto cow = std::make_unique(options_, std::move(cow_fd)); + if (!cow->Initialize(label)) { + return false; + } + cow_ = std::move(cow); + return true; +} + +uint32_t CompressedSnapshotWriter::GetBlockSize() const { + return cow_->GetBlockSize(); +} + +std::optional CompressedSnapshotWriter::GetMaxBlocks() const { + return cow_->GetMaxBlocks(); } } // namespace snapshot diff --git a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp index 3c4ab2ef7..737c480a8 100644 --- a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp +++ b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp @@ -122,6 +122,7 @@ class CowSnapuserdTest final { void SimulateDaemonRestart(); void StartMerge(); + std::unique_ptr CreateCowDeviceInternal(); void CreateCowDevice(); void CreateCowDeviceOrderedOps(); void CreateCowDeviceOrderedOpsInverted(); @@ -164,6 +165,7 @@ class CowSnapuserdMetadataTest final { private: void InitMetadata(); + std::unique_ptr CreateCowDeviceInternal(); void CreateCowDevice(); void CreateCowPartialFilledArea(); @@ -258,6 +260,19 @@ void CowSnapuserdTest::StartSnapuserdDaemon() { } } +std::unique_ptr CowSnapuserdTest::CreateCowDeviceInternal() { + std::string path = android::base::GetExecutableDirectory(); + cow_system_ = std::make_unique(path); + + CowOptions options; + options.compression = "gz"; + + unique_fd fd(cow_system_->fd); + cow_system_->fd = -1; + + return CreateCowWriter(kDefaultCowVersion, options, std::move(fd)); +} + void CowSnapuserdTest::ReadLastBlock() { unique_fd rnd_fd; total_base_size_ = BLOCK_SZ * 2; @@ -280,9 +295,6 @@ void CowSnapuserdTest::ReadLastBlock() { base_loop_ = std::make_unique(base_fd_, 10s); ASSERT_TRUE(base_loop_->valid()); - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); - std::unique_ptr random_buffer_1_ = std::make_unique(total_base_size_); loff_t offset = 0; @@ -294,16 +306,13 @@ void CowSnapuserdTest::ReadLastBlock() { offset += BLOCK_SZ; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); + ASSERT_TRUE(writer->AddRawBlocks(0, random_buffer_1_.get(), BLOCK_SZ)); + ASSERT_TRUE(writer->AddRawBlocks(1, (char*)random_buffer_1_.get() + BLOCK_SZ, BLOCK_SZ)); - ASSERT_TRUE(writer.AddRawBlocks(0, random_buffer_1_.get(), BLOCK_SZ)); - ASSERT_TRUE(writer.AddRawBlocks(1, (char*)random_buffer_1_.get() + BLOCK_SZ, BLOCK_SZ)); - - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); SetDeviceControlName(); @@ -381,22 +390,16 @@ void CowSnapuserdTest::ReadSnapshotDeviceAndValidate() { } void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = 0; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy, blk_src_copy + 1)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy, blk_src_copy + 1)); x -= 1; if (x == 1) { break; @@ -405,7 +408,7 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -433,22 +436,16 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = num_blocks - 1; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy + 1, blk_src_copy)); x -= 1; if (x == 0) { ASSERT_EQ(blk_src_copy, 0); @@ -458,7 +455,7 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -468,10 +465,11 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { true); // Merged operations - ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0), + ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), writer->GetBlockSize(), + 0), true); ASSERT_EQ(android::base::ReadFullyAtOffset( - base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0), + base_fd_, (char*)orig_buffer_.get() + writer->GetBlockSize(), size_, 0), true); } @@ -479,8 +477,8 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -495,13 +493,7 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 3; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -509,7 +501,7 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -519,12 +511,12 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { } for (size_t i = num_blocks; i > 0; i--) { - ASSERT_TRUE(writer.AddXorBlocks(num_blocks + i - 1, - &random_buffer_1_.get()[options.block_size * (i - 1)], - options.block_size, 2 * num_blocks + i - 1, xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks( + num_blocks + i - 1, &random_buffer_1_.get()[writer->GetBlockSize() * (i - 1)], + writer->GetBlockSize(), 2 * num_blocks + i - 1, xor_offset)); } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -542,8 +534,8 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOps() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -559,20 +551,14 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOps() { } memset(random_buffer_1_.get(), 0, size_); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t source_blk = 0; size_t blk_src_copy = 2 * num_blocks; uint16_t xor_offset = 5; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { @@ -582,10 +568,10 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOps() { blk_src_copy += 1; } - ASSERT_TRUE(writer.AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -603,8 +589,8 @@ void CowSnapuserdTest::CreateCowDevice() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -619,13 +605,7 @@ void CowSnapuserdTest::CreateCowDevice() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 2; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -639,11 +619,11 @@ void CowSnapuserdTest::CreateCowDevice() { for (int i = 0; i < num_blocks; i++) { sequence[num_blocks + i] = 5 * num_blocks - 1 - i; } - ASSERT_TRUE(writer.AddSequenceData(2 * num_blocks, sequence)); + ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence)); size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -655,24 +635,24 @@ void CowSnapuserdTest::CreateCowDevice() { source_blk = num_blocks; blk_src_copy = blk_end_copy; - ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); size_t blk_zero_copy_start = source_blk + num_blocks; size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks; - ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks)); + ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks)); size_t blk_random2_replace_start = blk_zero_copy_end; - ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); size_t blk_xor_start = blk_random2_replace_start + num_blocks; size_t xor_offset = BLOCK_SZ / 2; - ASSERT_TRUE(writer.AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); std::string zero_buffer(size_, 0); @@ -902,29 +882,36 @@ void CowSnapuserdTest::MergeInterrupt() { ASSERT_TRUE(Merge()); } -void CowSnapuserdMetadataTest::CreateCowPartialFilledArea() { +std::unique_ptr CowSnapuserdMetadataTest::CreateCowDeviceInternal() { std::string path = android::base::GetExecutableDirectory(); cow_system_ = std::make_unique(path); CowOptions options; options.compression = "gz"; - CowWriter writer(options); - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); + unique_fd fd(cow_system_->fd); + cow_system_->fd = -1; + + return CreateCowWriter(kDefaultCowVersion, options, std::move(fd)); +} + +void CowSnapuserdMetadataTest::CreateCowPartialFilledArea() { + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); // Area 0 is completely filled with 256 exceptions for (int i = 0; i < 256; i++) { - ASSERT_TRUE(writer.AddCopy(i, 256 + i)); + ASSERT_TRUE(writer->AddCopy(i, 256 + i)); } // Area 1 is partially filled with 2 copy ops and 10 zero ops - ASSERT_TRUE(writer.AddCopy(500, 1000)); - ASSERT_TRUE(writer.AddCopy(501, 1001)); + ASSERT_TRUE(writer->AddCopy(500, 1000)); + ASSERT_TRUE(writer->AddCopy(501, 1001)); - ASSERT_TRUE(writer.AddZeroBlocks(300, 10)); + ASSERT_TRUE(writer->AddZeroBlocks(300, 10)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); } void CowSnapuserdMetadataTest::ValidatePartialFilledArea() { @@ -956,8 +943,8 @@ void CowSnapuserdMetadataTest::CreateCowDevice() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -972,50 +959,44 @@ void CowSnapuserdMetadataTest::CreateCowDevice() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); // Overlapping region. This has to be split // into two batch operations - ASSERT_TRUE(writer.AddCopy(23, 20)); - ASSERT_TRUE(writer.AddCopy(22, 19)); - ASSERT_TRUE(writer.AddCopy(21, 18)); - ASSERT_TRUE(writer.AddCopy(20, 17)); - ASSERT_TRUE(writer.AddCopy(19, 16)); - ASSERT_TRUE(writer.AddCopy(18, 15)); + ASSERT_TRUE(writer->AddCopy(23, 20)); + ASSERT_TRUE(writer->AddCopy(22, 19)); + ASSERT_TRUE(writer->AddCopy(21, 18)); + ASSERT_TRUE(writer->AddCopy(20, 17)); + ASSERT_TRUE(writer->AddCopy(19, 16)); + ASSERT_TRUE(writer->AddCopy(18, 15)); // Contiguous region but blocks in ascending order // Daemon has to ensure that these blocks are merged // in a batch - ASSERT_TRUE(writer.AddCopy(50, 75)); - ASSERT_TRUE(writer.AddCopy(51, 76)); - ASSERT_TRUE(writer.AddCopy(52, 77)); - ASSERT_TRUE(writer.AddCopy(53, 78)); + ASSERT_TRUE(writer->AddCopy(50, 75)); + ASSERT_TRUE(writer->AddCopy(51, 76)); + ASSERT_TRUE(writer->AddCopy(52, 77)); + ASSERT_TRUE(writer->AddCopy(53, 78)); // Dis-contiguous region - ASSERT_TRUE(writer.AddCopy(110, 130)); - ASSERT_TRUE(writer.AddCopy(105, 125)); - ASSERT_TRUE(writer.AddCopy(100, 120)); + ASSERT_TRUE(writer->AddCopy(110, 130)); + ASSERT_TRUE(writer->AddCopy(105, 125)); + ASSERT_TRUE(writer->AddCopy(100, 120)); // Overlap - ASSERT_TRUE(writer.AddCopy(25, 30)); - ASSERT_TRUE(writer.AddCopy(30, 31)); + ASSERT_TRUE(writer->AddCopy(25, 30)); + ASSERT_TRUE(writer->AddCopy(30, 31)); size_t source_blk = num_blocks; - ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); size_t blk_zero_copy_start = source_blk + num_blocks; - ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks)); + ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); } void CowSnapuserdMetadataTest::InitMetadata() { diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp index 57f9e7ac1..efe0c1443 100644 --- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp +++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp @@ -125,6 +125,7 @@ class SnapuserdTest : public ::testing::Test { void SimulateDaemonRestart(); + std::unique_ptr CreateCowDeviceInternal(); void CreateCowDevice(); void CreateCowDeviceOrderedOps(); void CreateCowDeviceOrderedOpsInverted(); @@ -277,23 +278,30 @@ void SnapuserdTest::ReadSnapshotDeviceAndValidate() { ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0); } -void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { +std::unique_ptr SnapuserdTest::CreateCowDeviceInternal() { std::string path = android::base::GetExecutableDirectory(); cow_system_ = std::make_unique(path); CowOptions options; options.compression = "gz"; - CowWriter writer(options); - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); + unique_fd fd(cow_system_->fd); + cow_system_->fd = -1; - size_t num_blocks = size_ / options.block_size; + return CreateCowWriter(kDefaultCowVersion, options, std::move(fd)); +} + +void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); + + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = 0; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy, blk_src_copy + 1)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy, blk_src_copy + 1)); x -= 1; if (x == 1) { break; @@ -302,7 +310,7 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -330,22 +338,16 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = num_blocks - 1; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy + 1, blk_src_copy)); x -= 1; if (x == 0) { ASSERT_EQ(blk_src_copy, 0); @@ -355,7 +357,7 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -365,10 +367,11 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { true); // Merged operations - ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0), + ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), writer->GetBlockSize(), + 0), true); ASSERT_EQ(android::base::ReadFullyAtOffset( - base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0), + base_fd_, (char*)orig_buffer_.get() + writer->GetBlockSize(), size_, 0), true); } @@ -376,8 +379,8 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -392,13 +395,7 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 3; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -406,7 +403,7 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -416,12 +413,12 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { } for (size_t i = num_blocks; i > 0; i--) { - ASSERT_TRUE(writer.AddXorBlocks(num_blocks + i - 1, - &random_buffer_1_.get()[options.block_size * (i - 1)], - options.block_size, 2 * num_blocks + i - 1, xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks( + num_blocks + i - 1, &random_buffer_1_.get()[writer->GetBlockSize() * (i - 1)], + writer->GetBlockSize(), 2 * num_blocks + i - 1, xor_offset)); } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -439,8 +436,8 @@ void SnapuserdTest::CreateCowDeviceOrderedOps() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -456,20 +453,14 @@ void SnapuserdTest::CreateCowDeviceOrderedOps() { } memset(random_buffer_1_.get(), 0, size_); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t source_blk = 0; size_t blk_src_copy = 2 * num_blocks; uint16_t xor_offset = 5; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { @@ -479,10 +470,10 @@ void SnapuserdTest::CreateCowDeviceOrderedOps() { blk_src_copy += 1; } - ASSERT_TRUE(writer.AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -500,8 +491,8 @@ void SnapuserdTest::CreateCowDevice() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -516,13 +507,7 @@ void SnapuserdTest::CreateCowDevice() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 2; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -536,11 +521,11 @@ void SnapuserdTest::CreateCowDevice() { for (int i = 0; i < num_blocks; i++) { sequence[num_blocks + i] = 5 * num_blocks - 1 - i; } - ASSERT_TRUE(writer.AddSequenceData(2 * num_blocks, sequence)); + ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence)); size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -552,24 +537,24 @@ void SnapuserdTest::CreateCowDevice() { source_blk = num_blocks; blk_src_copy = blk_end_copy; - ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); size_t blk_zero_copy_start = source_blk + num_blocks; size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks; - ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks)); + ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks)); size_t blk_random2_replace_start = blk_zero_copy_end; - ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); size_t blk_xor_start = blk_random2_replace_start + num_blocks; size_t xor_offset = BLOCK_SZ / 2; - ASSERT_TRUE(writer.AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); std::string zero_buffer(size_, 0);