From 874fdaed435009dfba8255bae4a2dbdc1900b2a4 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 8 May 2023 21:17:32 -0700 Subject: [PATCH] libsnapshot: Add CowWriterBase, clean up CowWriter. To support multiple implementations of CowWriter, we will need to move direct usage of CowWriter to ICowWriter. This CL does this while also adding some small cleanups: - Move ICowWriter implementation methods to a new CowWriterBase class. This keeps ICowWriter as a clean interface. - Make the "Add" methods pure virtual, move the "Emit" methods to CowWriterBase as an implementation detail. - Simplify Initialize/InitializeAppend so they can be shared. - Rename CowWriter to CowWriterV2. - Rename cow_writer.cpp to writer_v2.cpp. - Rename cow_api_test.cpp to test_v2.cpp. - Remove ICowWriter::options, replace with GetBlockSize. - Add a CreateCowWriter helper to avoid implementation details in update_engine. Bug: 280529365 Test: builds Change-Id: If50faf03b292c6c8b23a6170e3f37329fb759ff6 --- fs_mgr/libsnapshot/Android.bp | 11 +- .../include/libsnapshot/cow_format.h | 4 + .../include/libsnapshot/cow_writer.h | 150 ++--------- .../libsnapshot/mock_snapshot_writer.h | 25 +- .../include/libsnapshot/snapshot_writer.h | 50 ++-- .../libsnapshot_cow/cow_format.cpp | 25 ++ .../{cow_api_test.cpp => test_v2.cpp} | 154 ++++++------ .../libsnapshot_cow/writer_base.cpp | 163 ++++++++++++ .../libsnapshot/libsnapshot_cow/writer_base.h | 71 ++++++ .../{cow_writer.cpp => writer_v2.cpp} | 233 ++++-------------- .../libsnapshot/libsnapshot_cow/writer_v2.h | 94 +++++++ fs_mgr/libsnapshot/snapshot.cpp | 5 +- fs_mgr/libsnapshot/snapshot_test.cpp | 12 +- fs_mgr/libsnapshot/snapshot_writer.cpp | 68 +++-- .../dm-snapshot-merge/cow_snapuserd_test.cpp | 209 +++++++--------- .../user-space-merge/snapuserd_test.cpp | 111 ++++----- 16 files changed, 750 insertions(+), 635 deletions(-) rename fs_mgr/libsnapshot/libsnapshot_cow/{cow_api_test.cpp => test_v2.cpp} (91%) create mode 100644 fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp create mode 100644 fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h rename fs_mgr/libsnapshot/libsnapshot_cow/{cow_writer.cpp => writer_v2.cpp} (75%) create mode 100644 fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index cba684467..e931bec39 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -174,12 +174,13 @@ cc_library_static { "libsnapshot_cow_defaults", ], srcs: [ - "libsnapshot_cow/cow_decompress.cpp", - "libsnapshot_cow/cow_reader.cpp", - "libsnapshot_cow/cow_writer.cpp", - "libsnapshot_cow/cow_format.cpp", "libsnapshot_cow/cow_compress.cpp", + "libsnapshot_cow/cow_decompress.cpp", + "libsnapshot_cow/cow_format.cpp", + "libsnapshot_cow/cow_reader.cpp", "libsnapshot_cow/parser_v2.cpp", + "libsnapshot_cow/writer_base.cpp", + "libsnapshot_cow/writer_v2.cpp", ], host_supported: true, recovery_available: true, @@ -371,7 +372,7 @@ cc_test { "libsnapshot_cow_defaults", ], srcs: [ - "libsnapshot_cow/cow_api_test.cpp", + "libsnapshot_cow/test_v2.cpp", ], cflags: [ "-D_FILE_OFFSET_BITS=64", diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h index b228dff8d..dd626bc46 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -31,6 +31,10 @@ static constexpr uint32_t kCowVersionManifest = 2; static constexpr uint32_t kMinCowVersion = 1; static constexpr uint32_t kMaxCowVersion = 2; +// Normally, this should be kMaxCowVersion. When a new version is under testing +// it may be the previous value of kMaxCowVersion. +static constexpr uint32_t kDefaultCowVersion = 2; + // This header appears as the first sequence of bytes in the COW. All fields // in the layout are little-endian encoded. The on-disk layout is: // diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h index 7881f3509..af2d3efc5 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -1,16 +1,16 @@ -// Copyright (C) 2019 The Android Open Source Project +// copyright (c) 2019 the android open source project // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// licensed under the apache license, version 2.0 (the "license"); +// you may not use this file except in compliance with the license. +// you may obtain a copy of the license at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/license-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// unless required by applicable law or agreed to in writing, software +// distributed under the license is distributed on an "as is" basis, +// without warranties or conditions of any kind, either express or implied. +// see the license for the specific language governing permissions and +// limitations under the license. #pragma once @@ -61,30 +61,28 @@ struct CowOptions { // will occur in the sequence they were added to the COW. class ICowWriter { public: - explicit ICowWriter(const CowOptions& options) : options_(options) {} - virtual ~ICowWriter() {} // Encode an operation that copies the contents of |old_block| to the // location of |new_block|. 'num_blocks' is the number of contiguous // COPY operations from |old_block| to |new_block|. - bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1); + virtual bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0; // Encode a sequence of raw blocks. |size| must be a multiple of the block size. - bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size); + virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; // Add a sequence of xor'd blocks. |size| must be a multiple of the block size. - bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, - uint16_t offset); + virtual bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) = 0; // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size. - bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks); + virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; // Add a label to the op sequence. - bool AddLabel(uint64_t label); + virtual bool AddLabel(uint64_t label) = 0; // Add sequence data for op merging. Data is a list of the destination block numbers. - bool AddSequenceData(size_t num_ops, const uint32_t* data); + virtual bool AddSequenceData(size_t num_ops, const uint32_t* data) = 0; // Flush all pending writes. This must be called before closing the writer // to ensure that the correct headers and footers are written. @@ -93,21 +91,8 @@ class ICowWriter { // Return number of bytes the cow image occupies on disk. virtual uint64_t GetCowSize() = 0; - const CowOptions& options() { return options_; } - - protected: - virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0; - virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; - virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) = 0; - virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; - virtual bool EmitLabel(uint64_t label) = 0; - virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0; - - bool ValidateNewBlock(uint64_t new_block); - - protected: - CowOptions options_; + virtual uint32_t GetBlockSize() const = 0; + virtual std::optional GetMaxBlocks() const = 0; }; class CompressWorker { @@ -146,96 +131,15 @@ class CompressWorker { std::vector>* compressed_data); }; -class CowWriter : public ICowWriter { - public: - explicit CowWriter(const CowOptions& options); - ~CowWriter(); +// Create an ICowWriter not backed by any file. This is useful for estimating +// the final size of a cow file. +std::unique_ptr CreateCowEstimator(uint32_t version, const CowOptions& options); - // Set up the writer. - // The file starts from the beginning. - // - // If fd is < 0, the CowWriter will be opened against /dev/null. This is for - // computing COW sizes without using storage space. - bool Initialize(android::base::unique_fd&& fd); - bool Initialize(android::base::borrowed_fd fd); - // Set up a writer, assuming that the given label is the last valid label. - // This will result in dropping any labels that occur after the given on, and will fail - // if the given label does not appear. - bool InitializeAppend(android::base::unique_fd&&, uint64_t label); - bool InitializeAppend(android::base::borrowed_fd fd, uint64_t label); - - bool Finalize() override; - - uint64_t GetCowSize() override; - - protected: - virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; - virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; - virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) override; - virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; - virtual bool EmitLabel(uint64_t label) override; - virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override; - - private: - bool EmitCluster(); - bool EmitClusterIfNeeded(); - bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block, - uint16_t offset, uint8_t type); - void SetupHeaders(); - void SetupWriteOptions(); - bool ParseOptions(); - bool OpenForWrite(); - bool OpenForAppend(uint64_t label); - bool GetDataPos(uint64_t* pos); - bool WriteRawData(const void* data, size_t size); - bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0); - void AddOperation(const CowOperation& op); - void InitPos(); - void InitBatchWrites(); - void InitWorkers(); - bool FlushCluster(); - - bool CompressBlocks(size_t num_blocks, const void* data); - bool SetFd(android::base::borrowed_fd fd); - bool Sync(); - bool Truncate(off_t length); - bool EnsureSpaceAvailable(const uint64_t bytes_needed) const; - - private: - android::base::unique_fd owned_fd_; - android::base::borrowed_fd fd_; - CowHeader header_{}; - CowFooter footer_{}; - CowCompressionAlgorithm compression_ = kCowCompressNone; - uint64_t current_op_pos_ = 0; - uint64_t next_op_pos_ = 0; - uint64_t next_data_pos_ = 0; - uint64_t current_data_pos_ = 0; - ssize_t total_data_written_ = 0; - uint32_t cluster_size_ = 0; - uint32_t current_cluster_size_ = 0; - uint64_t current_data_size_ = 0; - bool is_dev_null_ = false; - bool merge_in_progress_ = false; - bool is_block_device_ = false; - uint64_t cow_image_size_ = INT64_MAX; - - int num_compress_threads_ = 1; - std::vector> compress_threads_; - std::vector> threads_; - std::vector> compressed_buf_; - std::vector>::iterator buf_iter_; - - std::vector> opbuffer_vec_; - std::vector> databuffer_vec_; - std::unique_ptr cowop_vec_; - int op_vec_index_ = 0; - - std::unique_ptr data_vec_; - int data_vec_index_ = 0; - bool batch_write_ = false; -}; +// Create an ICowWriter of the given version and options. If a label is given, +// the writer is opened in append mode. +std::unique_ptr CreateCowWriter(uint32_t version, const CowOptions& options, + android::base::unique_fd&& fd, + std::optional label = {}); } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h index d798e25e3..52e3a9c4e 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h @@ -23,30 +23,23 @@ class MockSnapshotWriter : public ISnapshotWriter { public: using FileDescriptor = ISnapshotWriter::FileDescriptor; - explicit MockSnapshotWriter(const CowOptions& options) : ISnapshotWriter(options) {} - MockSnapshotWriter() : ISnapshotWriter({}) {} - MOCK_METHOD(bool, Finalize, (), (override)); // Return number of bytes the cow image occupies on disk. MOCK_METHOD(uint64_t, GetCowSize, (), (override)); - MOCK_METHOD(bool, EmitCopy, (uint64_t, uint64_t, uint64_t), (override)); - MOCK_METHOD(bool, EmitRawBlocks, (uint64_t, const void*, size_t), (override)); - MOCK_METHOD(bool, EmitXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t), + MOCK_METHOD(bool, AddCopy, (uint64_t, uint64_t, uint64_t), (override)); + MOCK_METHOD(bool, AddRawBlocks, (uint64_t, const void*, size_t), (override)); + MOCK_METHOD(bool, AddXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t), (override)); - MOCK_METHOD(bool, EmitZeroBlocks, (uint64_t, uint64_t), (override)); - MOCK_METHOD(bool, EmitLabel, (uint64_t), (override)); - MOCK_METHOD(bool, EmitSequenceData, (size_t, const uint32_t*), (override)); - - // Open the writer in write mode (no append). + MOCK_METHOD(bool, AddZeroBlocks, (uint64_t, uint64_t), (override)); + MOCK_METHOD(bool, AddLabel, (uint64_t), (override)); + MOCK_METHOD(bool, AddSequenceData, (size_t, const uint32_t*), (override)); MOCK_METHOD(bool, Initialize, (), (override)); + MOCK_METHOD(bool, InitializeAppend, (uint64_t), (override)); MOCK_METHOD(bool, VerifyMergeOps, (), (override, const, noexcept)); - - // Open the writer in append mode, with the last label to resume - // from. See CowWriter::InitializeAppend. - MOCK_METHOD(bool, InitializeAppend, (uint64_t label), (override)); - MOCK_METHOD(std::unique_ptr, OpenReader, (), (override)); + MOCK_METHOD(uint32_t, GetBlockSize, (), (override, const)); + MOCK_METHOD(std::optional, GetMaxBlocks, (), (override, const)); }; } // namespace android::snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h index 8f6344c3b..2653a6053 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h @@ -31,13 +31,7 @@ class ISnapshotWriter : public ICowWriter { public: using FileDescriptor = chromeos_update_engine::FileDescriptor; - explicit ISnapshotWriter(const CowOptions& options); - - // Set the source device. This is used for AddCopy() operations, if the - // underlying writer needs the original bytes (for example if backed by - // dm-snapshot or if writing directly to an unsnapshotted region). The - // device is only opened on the first operation that requires it. - void SetSourceDevice(const std::string& source_device); + virtual ~ISnapshotWriter() {} // Open the writer in write mode (no append). virtual bool Initialize() = 0; @@ -47,15 +41,8 @@ class ISnapshotWriter : public ICowWriter { virtual bool InitializeAppend(uint64_t label) = 0; virtual std::unique_ptr OpenReader() = 0; + virtual bool VerifyMergeOps() const noexcept = 0; - - protected: - android::base::borrowed_fd GetSourceFd(); - - std::optional source_device_; - - private: - android::base::unique_fd source_fd_; }; // Send writes to a COW or a raw device directly, based on a threshold. @@ -63,6 +50,8 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { public: CompressedSnapshotWriter(const CowOptions& options); + void SetSourceDevice(const std::string& source_device); + // Sets the COW device; this is required. bool SetCowDevice(android::base::unique_fd&& cow_device); @@ -70,23 +59,34 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { bool InitializeAppend(uint64_t label) override; bool Finalize() override; uint64_t GetCowSize() override; + uint32_t GetBlockSize() const override; + std::optional GetMaxBlocks() const override; std::unique_ptr OpenReader() override; bool VerifyMergeOps() const noexcept; - protected: - bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; - bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; - bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, - uint16_t offset) override; - bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; - bool EmitLabel(uint64_t label) override; - bool EmitSequenceData(size_t num_ops, const uint32_t* data) override; + bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; + bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, + uint16_t offset) override; + bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + bool AddLabel(uint64_t label) override; + bool AddSequenceData(size_t num_ops, const uint32_t* data) override; private: std::unique_ptr OpenCowReader() const; - android::base::unique_fd cow_device_; + android::base::borrowed_fd GetSourceFd(); - std::unique_ptr cow_; + CowOptions options_; + + // Set the source device. This is used for AddCopy() operations, if the + // underlying writer needs the original bytes (for example if backed by + // dm-snapshot or if writing directly to an unsnapshotted region). The + // device is only opened on the first operation that requires it. + std::optional source_device_; + android::base::unique_fd source_fd_; + + android::base::unique_fd cow_device_; + std::unique_ptr cow_; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp index 2157d0fa9..ff3ccecb8 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp @@ -19,10 +19,13 @@ #include #include +#include "writer_v2.h" namespace android { namespace snapshot { +using android::base::unique_fd; + std::ostream& operator<<(std::ostream& os, CowOperation const& op) { os << "CowOperation(type:"; if (op.type == kCowCopyOp) @@ -103,5 +106,27 @@ bool IsOrderedOp(const CowOperation& op) { } } +std::unique_ptr CreateCowWriter(uint32_t version, const CowOptions& options, + unique_fd&& fd, std::optional label) { + std::unique_ptr base; + switch (version) { + case 1: + case 2: + base = std::make_unique(options, std::move(fd)); + break; + default: + LOG(ERROR) << "Cannot create unknown cow version: " << version; + return nullptr; + } + if (!base->Initialize(label)) { + return nullptr; + } + return base; +} + +std::unique_ptr CreateCowEstimator(uint32_t version, const CowOptions& options) { + return CreateCowWriter(version, options, unique_fd{-1}, std::nullopt); +} + } // namespace snapshot } // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp similarity index 91% rename from fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp rename to fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp index 8f80bb345..120b2c062 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/test_v2.cpp @@ -25,7 +25,9 @@ #include #include #include "cow_decompress.h" +#include "writer_v2.h" +using android::base::unique_fd; using testing::AssertionFailure; using testing::AssertionResult; using testing::AssertionSuccess; @@ -42,6 +44,8 @@ class CowTest : public ::testing::Test { virtual void TearDown() override { cow_ = nullptr; } + unique_fd GetCowFd() { return unique_fd{dup(cow_->fd)}; } + std::unique_ptr cow_; }; @@ -53,9 +57,9 @@ static inline bool ReadData(CowReader& reader, const CowOperation* op, void* buf TEST_F(CowTest, CopyContiguous) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddCopy(10, 1000, 100)); ASSERT_TRUE(writer.Finalize()); @@ -96,9 +100,9 @@ TEST_F(CowTest, CopyContiguous) { TEST_F(CowTest, ReadWrite) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -175,9 +179,9 @@ TEST_F(CowTest, ReadWrite) { TEST_F(CowTest, ReadWriteXor) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -256,9 +260,9 @@ TEST_F(CowTest, CompressGz) { CowOptions options; options.cluster_ops = 0; options.compression = "gz"; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -296,9 +300,9 @@ TEST_P(CompressionTest, ThreadedBatchWrites) { options.compression = GetParam(); options.num_compress_threads = 2; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string xor_data = "This is test data-1. Testing xor"; xor_data.resize(options.block_size, '\0'); @@ -374,9 +378,9 @@ TEST_P(CompressionTest, NoBatchWrites) { options.num_compress_threads = 1; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "Testing replace ops without batch writes"; data.resize(options.block_size * 1024, '\0'); @@ -497,9 +501,9 @@ TEST_F(CowTest, ClusterCompressGz) { CowOptions options; options.compression = "gz"; options.cluster_ops = 2; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -562,9 +566,9 @@ TEST_F(CowTest, CompressTwoBlocks) { CowOptions options; options.compression = "gz"; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size * 2, '\0'); @@ -595,12 +599,12 @@ TEST_F(CowTest, CompressTwoBlocks) { TEST_F(CowTest, GetSize) { CowOptions options; options.cluster_ops = 0; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); if (ftruncate(cow_->fd, 0) < 0) { perror("Fails to set temp file size"); FAIL(); } - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -621,8 +625,8 @@ TEST_F(CowTest, GetSize) { TEST_F(CowTest, AppendLabelSmall) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -632,8 +636,8 @@ TEST_F(CowTest, AppendLabelSmall) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 3)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({3})); std::string data2 = "More data!"; data2.resize(options.block_size, '\0'); @@ -688,8 +692,8 @@ TEST_F(CowTest, AppendLabelSmall) { TEST_F(CowTest, AppendLabelMissing) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddLabel(0)); std::string data = "This is some data, believe it"; @@ -701,9 +705,9 @@ TEST_F(CowTest, AppendLabelMissing) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 1)); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 0)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_FALSE(writer->Initialize({1})); + ASSERT_TRUE(writer->Initialize({0})); ASSERT_TRUE(writer->AddZeroBlocks(51, 1)); ASSERT_TRUE(writer->Finalize()); @@ -740,8 +744,8 @@ TEST_F(CowTest, AppendLabelMissing) { TEST_F(CowTest, AppendExtendedCorrupted) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddLabel(5)); @@ -763,8 +767,8 @@ TEST_F(CowTest, AppendExtendedCorrupted) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({5})); ASSERT_TRUE(writer->Finalize()); @@ -791,8 +795,8 @@ TEST_F(CowTest, AppendExtendedCorrupted) { TEST_F(CowTest, AppendbyLabel) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size * 2, '\0'); @@ -810,9 +814,9 @@ TEST_F(CowTest, AppendbyLabel) { ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 12)); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_FALSE(writer->Initialize({12})); + ASSERT_TRUE(writer->Initialize({5})); // This should drop label 6 ASSERT_TRUE(writer->Finalize()); @@ -879,8 +883,8 @@ TEST_F(CowTest, AppendbyLabel) { TEST_F(CowTest, ClusterTest) { CowOptions options; options.cluster_ops = 4; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -976,16 +980,16 @@ TEST_F(CowTest, ClusterTest) { TEST_F(CowTest, ClusterAppendTest) { CowOptions options; options.cluster_ops = 3; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddLabel(50)); ASSERT_TRUE(writer->Finalize()); // Adds a cluster op, should be dropped on append ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 50)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({50})); std::string data2 = "More data!"; data2.resize(options.block_size, '\0'); @@ -1037,8 +1041,8 @@ TEST_F(CowTest, ClusterAppendTest) { TEST_F(CowTest, AppendAfterFinalize) { CowOptions options; options.cluster_ops = 0; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); @@ -1058,8 +1062,8 @@ TEST_F(CowTest, AppendAfterFinalize) { ASSERT_TRUE(reader.Parse(cow_->fd)); } -AssertionResult WriteDataBlock(CowWriter* writer, uint64_t new_block, std::string data) { - data.resize(writer->options().block_size, '\0'); +AssertionResult WriteDataBlock(ICowWriter* writer, uint64_t new_block, std::string data) { + data.resize(writer->GetBlockSize(), '\0'); if (!writer->AddRawBlocks(new_block, data.data(), data.size())) { return AssertionFailure() << "Failed to add raw block"; } @@ -1088,8 +1092,8 @@ AssertionResult CompareDataBlock(CowReader* reader, const CowOperation* op, TEST_F(CowTest, ResumeMidCluster) { CowOptions options; options.cluster_ops = 7; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1")); ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2")); @@ -1099,8 +1103,8 @@ TEST_F(CowTest, ResumeMidCluster) { ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4")); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4")); ASSERT_TRUE(WriteDataBlock(writer.get(), 5, "Block 5")); ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6")); @@ -1145,8 +1149,8 @@ TEST_F(CowTest, ResumeEndCluster) { CowOptions options; int cluster_ops = 5; options.cluster_ops = cluster_ops; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1")); ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2")); @@ -1160,8 +1164,8 @@ TEST_F(CowTest, ResumeEndCluster) { ASSERT_TRUE(WriteDataBlock(writer.get(), 8, "Block 8")); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(WriteDataBlock(writer.get(), 4, "Block 4")); ASSERT_TRUE(WriteDataBlock(writer.get(), 5, "Block 5")); ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6")); @@ -1205,8 +1209,8 @@ TEST_F(CowTest, ResumeEndCluster) { TEST_F(CowTest, DeleteMidCluster) { CowOptions options; options.cluster_ops = 7; - auto writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + auto writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(WriteDataBlock(writer.get(), 1, "Block 1")); ASSERT_TRUE(WriteDataBlock(writer.get(), 2, "Block 2")); @@ -1218,8 +1222,8 @@ TEST_F(CowTest, DeleteMidCluster) { ASSERT_TRUE(WriteDataBlock(writer.get(), 6, "Block 6")); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(writer->Finalize()); ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); @@ -1255,14 +1259,14 @@ TEST_F(CowTest, DeleteMidCluster) { TEST_F(CowTest, BigSeqOp) { CowOptions options; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); const int seq_len = std::numeric_limits::max() / sizeof(uint32_t) + 1; uint32_t sequence[seq_len]; for (int i = 0; i < seq_len; i++) { sequence[i] = i + 1; } - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddSequenceData(seq_len, sequence)); ASSERT_TRUE(writer.AddZeroBlocks(1, seq_len)); @@ -1287,14 +1291,14 @@ TEST_F(CowTest, BigSeqOp) { TEST_F(CowTest, MissingSeqOp) { CowOptions options; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); const int seq_len = 10; uint32_t sequence[seq_len]; for (int i = 0; i < seq_len; i++) { sequence[i] = i + 1; } - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddSequenceData(seq_len, sequence)); ASSERT_TRUE(writer.AddZeroBlocks(1, seq_len - 1)); @@ -1308,14 +1312,14 @@ TEST_F(CowTest, MissingSeqOp) { TEST_F(CowTest, ResumeSeqOp) { CowOptions options; - auto writer = std::make_unique(options); + auto writer = std::make_unique(options, GetCowFd()); const int seq_len = 10; uint32_t sequence[seq_len]; for (int i = 0; i < seq_len; i++) { sequence[i] = i + 1; } - ASSERT_TRUE(writer->Initialize(cow_->fd)); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddSequenceData(seq_len, sequence)); ASSERT_TRUE(writer->AddZeroBlocks(1, seq_len / 2)); @@ -1328,8 +1332,8 @@ TEST_F(CowTest, ResumeSeqOp) { auto itr = reader->GetRevMergeOpIter(); ASSERT_TRUE(itr->AtEnd()); - writer = std::make_unique(options); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(writer->AddZeroBlocks(1 + seq_len / 2, seq_len / 2)); ASSERT_TRUE(writer->Finalize()); @@ -1358,10 +1362,10 @@ TEST_F(CowTest, RevMergeOpItrTest) { CowOptions options; options.cluster_ops = 5; options.num_merge_ops = 1; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); uint32_t sequence[] = {2, 10, 6, 7, 3, 5}; - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddSequenceData(6, sequence)); ASSERT_TRUE(writer.AddCopy(6, 13)); @@ -1408,9 +1412,9 @@ TEST_F(CowTest, LegacyRevMergeOpItrTest) { CowOptions options; options.cluster_ops = 5; options.num_merge_ops = 1; - CowWriter writer(options); + CowWriterV2 writer(options, GetCowFd()); - ASSERT_TRUE(writer.Initialize(cow_->fd)); + ASSERT_TRUE(writer.Initialize()); ASSERT_TRUE(writer.AddCopy(2, 11)); ASSERT_TRUE(writer.AddCopy(10, 12)); @@ -1459,10 +1463,10 @@ TEST_F(CowTest, InvalidMergeOrderTest) { options.num_merge_ops = 1; std::string data = "This is some data, believe it"; data.resize(options.block_size, '\0'); - auto writer = std::make_unique(options); + auto writer = std::make_unique(options, GetCowFd()); CowReader reader; - ASSERT_TRUE(writer->Initialize(cow_->fd)); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddCopy(3, 2)); ASSERT_TRUE(writer->AddCopy(2, 1)); @@ -1471,14 +1475,14 @@ TEST_F(CowTest, InvalidMergeOrderTest) { ASSERT_TRUE(reader.Parse(cow_->fd)); ASSERT_TRUE(reader.VerifyMergeOps()); - ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1)); + ASSERT_TRUE(writer->Initialize({1})); ASSERT_TRUE(writer->AddCopy(4, 2)); ASSERT_TRUE(writer->Finalize()); ASSERT_TRUE(reader.Parse(cow_->fd)); ASSERT_FALSE(reader.VerifyMergeOps()); - writer = std::make_unique(options); - ASSERT_TRUE(writer->Initialize(cow_->fd)); + writer = std::make_unique(options, GetCowFd()); + ASSERT_TRUE(writer->Initialize()); ASSERT_TRUE(writer->AddCopy(2, 1)); ASSERT_TRUE(writer->AddXorBlocks(3, &data, data.size(), 1, 1)); ASSERT_TRUE(writer->Finalize()); diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp new file mode 100644 index 000000000..22e63d0a3 --- /dev/null +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.cpp @@ -0,0 +1,163 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "writer_base.h" + +#include +#include +#include +#include +#include + +#include + +// The info messages here are spammy, but as useful for update_engine. Disable +// them when running on the host. +#ifdef __ANDROID__ +#define LOG_INFO LOG(INFO) +#else +#define LOG_INFO LOG(VERBOSE) +#endif + +namespace android { +namespace snapshot { + +using android::base::borrowed_fd; +using android::base::unique_fd; + +namespace { +std::string GetFdPath(borrowed_fd fd) { + const auto fd_path = "/proc/self/fd/" + std::to_string(fd.get()); + std::string file_path(512, '\0'); + const auto err = readlink(fd_path.c_str(), file_path.data(), file_path.size()); + if (err <= 0) { + PLOG(ERROR) << "Failed to determine path for fd " << fd.get(); + file_path.clear(); + } else { + file_path.resize(err); + } + return file_path; +} +} // namespace + +CowWriterBase::CowWriterBase(const CowOptions& options, unique_fd&& fd) + : options_(options), fd_(std::move(fd)) {} + +bool CowWriterBase::InitFd() { + if (fd_.get() < 0) { + fd_.reset(open("/dev/null", O_RDWR | O_CLOEXEC)); + if (fd_ < 0) { + PLOG(ERROR) << "open /dev/null failed"; + return false; + } + is_dev_null_ = true; + return true; + } + + struct stat stat {}; + if (fstat(fd_.get(), &stat) < 0) { + PLOG(ERROR) << "fstat failed"; + return false; + } + const auto file_path = GetFdPath(fd_); + is_block_device_ = S_ISBLK(stat.st_mode); + if (is_block_device_) { + uint64_t size_in_bytes = 0; + if (ioctl(fd_.get(), BLKGETSIZE64, &size_in_bytes)) { + PLOG(ERROR) << "Failed to get total size for: " << fd_.get(); + return false; + } + cow_image_size_ = size_in_bytes; + LOG_INFO << "COW image " << file_path << " has size " << size_in_bytes; + } else { + LOG_INFO << "COW image " << file_path + << " is not a block device, assuming unlimited space."; + } + return true; +} + +bool CowWriterBase::AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { + CHECK(num_blocks != 0); + + for (size_t i = 0; i < num_blocks; i++) { + if (!ValidateNewBlock(new_block + i)) { + return false; + } + } + + return EmitCopy(new_block, old_block, num_blocks); +} + +bool CowWriterBase::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) { + if (size % options_.block_size != 0) { + LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " + << options_.block_size; + return false; + } + + uint64_t num_blocks = size / options_.block_size; + uint64_t last_block = new_block_start + num_blocks - 1; + if (!ValidateNewBlock(last_block)) { + return false; + } + return EmitRawBlocks(new_block_start, data, size); +} + +bool CowWriterBase::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) { + if (size % options_.block_size != 0) { + LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " + << options_.block_size; + return false; + } + + uint64_t num_blocks = size / options_.block_size; + uint64_t last_block = new_block_start + num_blocks - 1; + if (!ValidateNewBlock(last_block)) { + return false; + } + if (offset >= options_.block_size) { + LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than " + << options_.block_size; + } + return EmitXorBlocks(new_block_start, data, size, old_block, offset); +} + +bool CowWriterBase::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { + uint64_t last_block = new_block_start + num_blocks - 1; + if (!ValidateNewBlock(last_block)) { + return false; + } + return EmitZeroBlocks(new_block_start, num_blocks); +} + +bool CowWriterBase::AddLabel(uint64_t label) { + return EmitLabel(label); +} + +bool CowWriterBase::AddSequenceData(size_t num_ops, const uint32_t* data) { + return EmitSequenceData(num_ops, data); +} + +bool CowWriterBase::ValidateNewBlock(uint64_t new_block) { + if (options_.max_blocks && new_block >= options_.max_blocks.value()) { + LOG(ERROR) << "New block " << new_block << " exceeds maximum block count " + << options_.max_blocks.value(); + return false; + } + return true; +} + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h new file mode 100644 index 000000000..8fa90656f --- /dev/null +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_base.h @@ -0,0 +1,71 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace android { +namespace snapshot { + +class CowWriterBase : public ICowWriter { + public: + CowWriterBase(const CowOptions& options, android::base::unique_fd&& fd); + virtual ~CowWriterBase() {} + + // Set up the writer. + // The file starts from the beginning. + // + // If fd is < 0, the CowWriter will be opened against /dev/null. This is for + // computing COW sizes without using storage space. + // + // If a label is given, any operations after the given label will be dropped. + // If the given label is not found, Initialize will fail. + virtual bool Initialize(std::optional label = {}) = 0; + + bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; + bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block, + uint16_t offset) override; + bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + bool AddLabel(uint64_t label) override; + bool AddSequenceData(size_t num_ops, const uint32_t* data) override; + uint32_t GetBlockSize() const override { return options_.block_size; } + std::optional GetMaxBlocks() const override { return options_.max_blocks; } + + const CowOptions& options() const { return options_; } + + protected: + virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0; + virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; + virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) = 0; + virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; + virtual bool EmitLabel(uint64_t label) = 0; + virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0; + + bool InitFd(); + bool ValidateNewBlock(uint64_t new_block); + + CowOptions options_; + CowHeader header_{}; + + android::base::unique_fd fd_; + bool is_dev_null_ = false; + bool is_block_device_ = false; + uint64_t cow_image_size_ = INT64_MAX; +}; + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp similarity index 75% rename from fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp rename to fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp index 1eaa0385e..b6603da52 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/cow_writer.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.cpp @@ -37,6 +37,8 @@ #include #include +#include "writer_v2.h" + // The info messages here are spammy, but as useful for update_engine. Disable // them when running on the host. #ifdef __ANDROID__ @@ -48,104 +50,17 @@ namespace android { namespace snapshot { -namespace { -std::string GetFdPath(int fd) { - const auto fd_path = "/proc/self/fd/" + std::to_string(fd); - std::string file_path(512, '\0'); - const auto err = readlink(fd_path.c_str(), file_path.data(), file_path.size()); - if (err <= 0) { - PLOG(ERROR) << "Failed to determine path for fd " << fd; - file_path.clear(); - } else { - file_path.resize(err); - } - return file_path; -} -} // namespace - static_assert(sizeof(off_t) == sizeof(uint64_t)); -using android::base::borrowed_fd; using android::base::unique_fd; -bool ICowWriter::AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { - CHECK(num_blocks != 0); - - for (size_t i = 0; i < num_blocks; i++) { - if (!ValidateNewBlock(new_block + i)) { - return false; - } - } - - return EmitCopy(new_block, old_block, num_blocks); -} - -bool ICowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) { - if (size % options_.block_size != 0) { - LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " - << options_.block_size; - return false; - } - - uint64_t num_blocks = size / options_.block_size; - uint64_t last_block = new_block_start + num_blocks - 1; - if (!ValidateNewBlock(last_block)) { - return false; - } - return EmitRawBlocks(new_block_start, data, size); -} - -bool ICowWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) { - if (size % options_.block_size != 0) { - LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " - << options_.block_size; - return false; - } - - uint64_t num_blocks = size / options_.block_size; - uint64_t last_block = new_block_start + num_blocks - 1; - if (!ValidateNewBlock(last_block)) { - return false; - } - if (offset >= options_.block_size) { - LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than " - << options_.block_size; - } - return EmitXorBlocks(new_block_start, data, size, old_block, offset); -} - -bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { - uint64_t last_block = new_block_start + num_blocks - 1; - if (!ValidateNewBlock(last_block)) { - return false; - } - return EmitZeroBlocks(new_block_start, num_blocks); -} - -bool ICowWriter::AddLabel(uint64_t label) { - return EmitLabel(label); -} - -bool ICowWriter::AddSequenceData(size_t num_ops, const uint32_t* data) { - return EmitSequenceData(num_ops, data); -} - -bool ICowWriter::ValidateNewBlock(uint64_t new_block) { - if (options_.max_blocks && new_block >= options_.max_blocks.value()) { - LOG(ERROR) << "New block " << new_block << " exceeds maximum block count " - << options_.max_blocks.value(); - return false; - } - return true; -} - -CowWriter::CowWriter(const CowOptions& options) : ICowWriter(options), fd_(-1) { +CowWriterV2::CowWriterV2(const CowOptions& options, unique_fd&& fd) + : CowWriterBase(options, std::move(fd)) { SetupHeaders(); SetupWriteOptions(); } -CowWriter::~CowWriter() { +CowWriterV2::~CowWriterV2() { for (size_t i = 0; i < compress_threads_.size(); i++) { CompressWorker* worker = compress_threads_[i].get(); if (worker) { @@ -164,7 +79,7 @@ CowWriter::~CowWriter() { compress_threads_.clear(); } -void CowWriter::SetupWriteOptions() { +void CowWriterV2::SetupWriteOptions() { num_compress_threads_ = options_.num_compress_threads; if (!num_compress_threads_) { @@ -184,7 +99,7 @@ void CowWriter::SetupWriteOptions() { } } -void CowWriter::SetupHeaders() { +void CowWriterV2::SetupHeaders() { header_ = {}; header_.prefix.magic = kCowMagicNumber; header_.prefix.major_version = kCowVersionMajor; @@ -201,7 +116,7 @@ void CowWriter::SetupHeaders() { footer_.op.type = kCowFooterOp; } -bool CowWriter::ParseOptions() { +bool CowWriterV2::ParseOptions() { auto algorithm = CompressionAlgorithmFromString(options_.compression); if (!algorithm) { LOG(ERROR) << "unrecognized compression: " << options_.compression; @@ -216,42 +131,7 @@ bool CowWriter::ParseOptions() { return true; } -bool CowWriter::SetFd(android::base::borrowed_fd fd) { - if (fd.get() < 0) { - owned_fd_.reset(open("/dev/null", O_RDWR | O_CLOEXEC)); - if (owned_fd_ < 0) { - PLOG(ERROR) << "open /dev/null failed"; - return false; - } - fd_ = owned_fd_; - is_dev_null_ = true; - } else { - fd_ = fd; - - struct stat stat {}; - if (fstat(fd.get(), &stat) < 0) { - PLOG(ERROR) << "fstat failed"; - return false; - } - const auto file_path = GetFdPath(fd.get()); - is_block_device_ = S_ISBLK(stat.st_mode); - if (is_block_device_) { - uint64_t size_in_bytes = 0; - if (ioctl(fd.get(), BLKGETSIZE64, &size_in_bytes)) { - PLOG(ERROR) << "Failed to get total size for: " << fd.get(); - return false; - } - cow_image_size_ = size_in_bytes; - LOG_INFO << "COW image " << file_path << " has size " << size_in_bytes; - } else { - LOG_INFO << "COW image " << file_path - << " is not a block device, assuming unlimited space."; - } - } - return true; -} - -void CowWriter::InitBatchWrites() { +void CowWriterV2::InitBatchWrites() { if (batch_write_) { cowop_vec_ = std::make_unique(header_.cluster_ops); data_vec_ = std::make_unique(header_.cluster_ops); @@ -277,7 +157,7 @@ void CowWriter::InitBatchWrites() { LOG_INFO << "Batch writes: " << batch_write; } -void CowWriter::InitWorkers() { +void CowWriterV2::InitWorkers() { if (num_compress_threads_ <= 1) { LOG_INFO << "Not creating new threads for compression."; return; @@ -291,44 +171,27 @@ void CowWriter::InitWorkers() { LOG_INFO << num_compress_threads_ << " thread used for compression"; } -bool CowWriter::Initialize(unique_fd&& fd) { - owned_fd_ = std::move(fd); - return Initialize(borrowed_fd{owned_fd_}); -} - -bool CowWriter::Initialize(borrowed_fd fd) { - if (!SetFd(fd) || !ParseOptions()) { +bool CowWriterV2::Initialize(std::optional label) { + if (!InitFd() || !ParseOptions()) { return false; } - - if (!OpenForWrite()) { - return false; + if (!label) { + if (!OpenForWrite()) { + return false; + } + } else { + if (!OpenForAppend(*label)) { + return false; + } } - InitWorkers(); + if (!compress_threads_.size()) { + InitWorkers(); + } return true; } -bool CowWriter::InitializeAppend(android::base::unique_fd&& fd, uint64_t label) { - owned_fd_ = std::move(fd); - return InitializeAppend(android::base::borrowed_fd{owned_fd_}, label); -} - -bool CowWriter::InitializeAppend(android::base::borrowed_fd fd, uint64_t label) { - if (!SetFd(fd) || !ParseOptions()) { - return false; - } - - bool ret = OpenForAppend(label); - - if (ret && !compress_threads_.size()) { - InitWorkers(); - } - - return ret; -} - -void CowWriter::InitPos() { +void CowWriterV2::InitPos() { next_op_pos_ = sizeof(header_) + header_.buffer_size; cluster_size_ = header_.cluster_ops * sizeof(CowOperation); if (header_.cluster_ops) { @@ -340,7 +203,7 @@ void CowWriter::InitPos() { current_data_size_ = 0; } -bool CowWriter::OpenForWrite() { +bool CowWriterV2::OpenForWrite() { // This limitation is tied to the data field size in CowOperation. if (header_.block_size > std::numeric_limits::max()) { LOG(ERROR) << "Block size is too large"; @@ -388,7 +251,7 @@ bool CowWriter::OpenForWrite() { return true; } -bool CowWriter::OpenForAppend(uint64_t label) { +bool CowWriterV2::OpenForAppend(uint64_t label) { auto reader = std::make_unique(); std::queue toAdd; @@ -424,7 +287,7 @@ bool CowWriter::OpenForAppend(uint64_t label) { return EmitClusterIfNeeded(); } -bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { +bool CowWriterV2::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks) { CHECK(!merge_in_progress_); for (size_t i = 0; i < num_blocks; i++) { @@ -440,16 +303,16 @@ bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_bl return true; } -bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) { +bool CowWriterV2::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) { return EmitBlocks(new_block_start, data, size, 0, 0, kCowReplaceOp); } -bool CowWriter::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, - uint32_t old_block, uint16_t offset) { +bool CowWriterV2::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) { return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp); } -bool CowWriter::CompressBlocks(size_t num_blocks, const void* data) { +bool CowWriterV2::CompressBlocks(size_t num_blocks, const void* data) { size_t num_threads = (num_blocks == 1) ? 1 : num_compress_threads_; size_t num_blocks_per_thread = num_blocks / num_threads; const uint8_t* iter = reinterpret_cast(data); @@ -483,8 +346,8 @@ bool CowWriter::CompressBlocks(size_t num_blocks, const void* data) { return true; } -bool CowWriter::EmitBlocks(uint64_t new_block_start, const void* data, size_t size, - uint64_t old_block, uint16_t offset, uint8_t type) { +bool CowWriterV2::EmitBlocks(uint64_t new_block_start, const void* data, size_t size, + uint64_t old_block, uint16_t offset, uint8_t type) { CHECK(!merge_in_progress_); const uint8_t* iter = reinterpret_cast(data); @@ -558,7 +421,7 @@ bool CowWriter::EmitBlocks(uint64_t new_block_start, const void* data, size_t si return true; } -bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { +bool CowWriterV2::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { CHECK(!merge_in_progress_); for (uint64_t i = 0; i < num_blocks; i++) { CowOperation op = {}; @@ -570,7 +433,7 @@ bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { return true; } -bool CowWriter::EmitLabel(uint64_t label) { +bool CowWriterV2::EmitLabel(uint64_t label) { CHECK(!merge_in_progress_); CowOperation op = {}; op.type = kCowLabelOp; @@ -578,7 +441,7 @@ bool CowWriter::EmitLabel(uint64_t label) { return WriteOperation(op) && Sync(); } -bool CowWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) { +bool CowWriterV2::EmitSequenceData(size_t num_ops, const uint32_t* data) { CHECK(!merge_in_progress_); size_t to_add = 0; size_t max_ops = (header_.block_size * 2) / sizeof(uint32_t); @@ -598,7 +461,7 @@ bool CowWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) { return true; } -bool CowWriter::EmitCluster() { +bool CowWriterV2::EmitCluster() { CowOperation op = {}; op.type = kCowClusterOp; // Next cluster starts after remainder of current cluster and the next data block. @@ -606,7 +469,7 @@ bool CowWriter::EmitCluster() { return WriteOperation(op); } -bool CowWriter::EmitClusterIfNeeded() { +bool CowWriterV2::EmitClusterIfNeeded() { // If there isn't room for another op and the cluster end op, end the current cluster if (cluster_size_ && cluster_size_ < current_cluster_size_ + 2 * sizeof(CowOperation)) { if (!EmitCluster()) return false; @@ -614,7 +477,7 @@ bool CowWriter::EmitClusterIfNeeded() { return true; } -bool CowWriter::Finalize() { +bool CowWriterV2::Finalize() { if (!FlushCluster()) { LOG(ERROR) << "Finalize: FlushCluster() failed"; return false; @@ -688,7 +551,7 @@ bool CowWriter::Finalize() { return Sync(); } -uint64_t CowWriter::GetCowSize() { +uint64_t CowWriterV2::GetCowSize() { if (current_data_size_ > 0) { return next_data_pos_ + sizeof(footer_); } else { @@ -696,7 +559,7 @@ uint64_t CowWriter::GetCowSize() { } } -bool CowWriter::GetDataPos(uint64_t* pos) { +bool CowWriterV2::GetDataPos(uint64_t* pos) { off_t offs = lseek(fd_.get(), 0, SEEK_CUR); if (offs < 0) { PLOG(ERROR) << "lseek failed"; @@ -706,7 +569,7 @@ bool CowWriter::GetDataPos(uint64_t* pos) { return true; } -bool CowWriter::EnsureSpaceAvailable(const uint64_t bytes_needed) const { +bool CowWriterV2::EnsureSpaceAvailable(const uint64_t bytes_needed) const { if (bytes_needed > cow_image_size_) { LOG(ERROR) << "No space left on COW device. Required: " << bytes_needed << ", available: " << cow_image_size_; @@ -716,7 +579,7 @@ bool CowWriter::EnsureSpaceAvailable(const uint64_t bytes_needed) const { return true; } -bool CowWriter::FlushCluster() { +bool CowWriterV2::FlushCluster() { ssize_t ret; if (op_vec_index_) { @@ -745,7 +608,7 @@ bool CowWriter::FlushCluster() { return true; } -bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t size) { +bool CowWriterV2::WriteOperation(const CowOperation& op, const void* data, size_t size) { if (!EnsureSpaceAvailable(next_op_pos_ + sizeof(op))) { return false; } @@ -793,7 +656,7 @@ bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t return EmitClusterIfNeeded(); } -void CowWriter::AddOperation(const CowOperation& op) { +void CowWriterV2::AddOperation(const CowOperation& op) { footer_.op.num_ops++; if (op.type == kCowClusterOp) { @@ -808,14 +671,14 @@ void CowWriter::AddOperation(const CowOperation& op) { next_op_pos_ += sizeof(CowOperation) + GetNextOpOffset(op, header_.cluster_ops); } -bool CowWriter::WriteRawData(const void* data, const size_t size) { +bool CowWriterV2::WriteRawData(const void* data, const size_t size) { if (!android::base::WriteFullyAtOffset(fd_, data, size, next_data_pos_)) { return false; } return true; } -bool CowWriter::Sync() { +bool CowWriterV2::Sync() { if (is_dev_null_) { return true; } @@ -826,7 +689,7 @@ bool CowWriter::Sync() { return true; } -bool CowWriter::Truncate(off_t length) { +bool CowWriterV2::Truncate(off_t length) { if (is_dev_null_ || is_block_device_) { return true; } diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h new file mode 100644 index 000000000..809ae5747 --- /dev/null +++ b/fs_mgr/libsnapshot/libsnapshot_cow/writer_v2.h @@ -0,0 +1,94 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "writer_base.h" + +namespace android { +namespace snapshot { + +class CowWriterV2 : public CowWriterBase { + public: + explicit CowWriterV2(const CowOptions& options, android::base::unique_fd&& fd); + ~CowWriterV2() override; + + bool Initialize(std::optional label = {}) override; + bool Finalize() override; + uint64_t GetCowSize() override; + + protected: + virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override; + virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) override; + virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + virtual bool EmitLabel(uint64_t label) override; + virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override; + + private: + bool EmitCluster(); + bool EmitClusterIfNeeded(); + bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block, + uint16_t offset, uint8_t type); + void SetupHeaders(); + void SetupWriteOptions(); + bool ParseOptions(); + bool OpenForWrite(); + bool OpenForAppend(uint64_t label); + bool GetDataPos(uint64_t* pos); + bool WriteRawData(const void* data, size_t size); + bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0); + void AddOperation(const CowOperation& op); + void InitPos(); + void InitBatchWrites(); + void InitWorkers(); + bool FlushCluster(); + + bool CompressBlocks(size_t num_blocks, const void* data); + bool Sync(); + bool Truncate(off_t length); + bool EnsureSpaceAvailable(const uint64_t bytes_needed) const; + + private: + CowFooter footer_{}; + CowCompressionAlgorithm compression_ = kCowCompressNone; + uint64_t current_op_pos_ = 0; + uint64_t next_op_pos_ = 0; + uint64_t next_data_pos_ = 0; + uint64_t current_data_pos_ = 0; + ssize_t total_data_written_ = 0; + uint32_t cluster_size_ = 0; + uint32_t current_cluster_size_ = 0; + uint64_t current_data_size_ = 0; + bool merge_in_progress_ = false; + + int num_compress_threads_ = 1; + std::vector> compress_threads_; + std::vector> threads_; + std::vector> compressed_buf_; + std::vector>::iterator buf_iter_; + + std::vector> opbuffer_vec_; + std::vector> databuffer_vec_; + std::unique_ptr cowop_vec_; + int op_vec_index_ = 0; + + std::unique_ptr data_vec_; + int data_vec_index_ = 0; + bool batch_write_ = false; +}; + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp index e114d2556..5920bc22f 100644 --- a/fs_mgr/libsnapshot/snapshot.cpp +++ b/fs_mgr/libsnapshot/snapshot.cpp @@ -45,6 +45,7 @@ #include #include #include "device_info.h" +#include "libsnapshot_cow/writer_v2.h" #include "partition_cow_creator.h" #include "snapshot_metadata_updater.h" #include "snapshot_reader.h" @@ -3557,8 +3558,8 @@ Return SnapshotManager::InitializeUpdateSnapshots( } options.compression = it->second.compression_algorithm(); - CowWriter writer(options); - if (!writer.Initialize(fd) || !writer.Finalize()) { + CowWriterV2 writer(options, std::move(fd)); + if (!writer.Initialize(std::nullopt) || !writer.Finalize()) { LOG(ERROR) << "Could not initialize COW device for " << target_partition->name(); return Return::Error(); } diff --git a/fs_mgr/libsnapshot/snapshot_test.cpp b/fs_mgr/libsnapshot/snapshot_test.cpp index 757f6f181..dac1b771b 100644 --- a/fs_mgr/libsnapshot/snapshot_test.cpp +++ b/fs_mgr/libsnapshot/snapshot_test.cpp @@ -1220,13 +1220,13 @@ class SnapshotUpdateTest : public SnapshotTest { SHA256_CTX ctx; SHA256_Init(&ctx); - if (!writer->options().max_blocks) { + if (!writer->GetMaxBlocks()) { LOG(ERROR) << "CowWriter must specify maximum number of blocks"; return false; } - const auto num_blocks = writer->options().max_blocks.value(); + const auto num_blocks = writer->GetMaxBlocks().value(); - const auto block_size = writer->options().block_size; + const auto block_size = writer->GetBlockSize(); std::string block(block_size, '\0'); for (uint64_t i = 0; i < num_blocks; i++) { if (!ReadFully(rand, block.data(), block.size())) { @@ -1254,13 +1254,13 @@ class SnapshotUpdateTest : public SnapshotTest { if (auto res = MapUpdateSnapshot(name, &writer); !res) { return res; } - if (!writer->options().max_blocks || !*writer->options().max_blocks) { + if (!writer->GetMaxBlocks() || !*writer->GetMaxBlocks()) { return AssertionFailure() << "No max blocks set for " << name << " writer"; } - uint64_t src_block = (old_size / writer->options().block_size) - 1; + uint64_t src_block = (old_size / writer->GetBlockSize()) - 1; uint64_t dst_block = 0; - uint64_t max_blocks = *writer->options().max_blocks; + uint64_t max_blocks = *writer->GetMaxBlocks(); while (dst_block < max_blocks && dst_block < src_block) { if (!writer->AddCopy(dst_block, src_block)) { return AssertionFailure() << "Unable to add copy for " << name << " for blocks " diff --git a/fs_mgr/libsnapshot/snapshot_writer.cpp b/fs_mgr/libsnapshot/snapshot_writer.cpp index 6a3906ead..0ea424c7e 100644 --- a/fs_mgr/libsnapshot/snapshot_writer.cpp +++ b/fs_mgr/libsnapshot/snapshot_writer.cpp @@ -19,6 +19,7 @@ #include #include #include +#include "libsnapshot_cow/writer_v2.h" #include "snapshot_reader.h" namespace android { @@ -28,13 +29,11 @@ using android::base::borrowed_fd; using android::base::unique_fd; using chromeos_update_engine::FileDescriptor; -ISnapshotWriter::ISnapshotWriter(const CowOptions& options) : ICowWriter(options) {} - -void ISnapshotWriter::SetSourceDevice(const std::string& source_device) { +void CompressedSnapshotWriter::SetSourceDevice(const std::string& source_device) { source_device_ = {source_device}; } -borrowed_fd ISnapshotWriter::GetSourceFd() { +borrowed_fd CompressedSnapshotWriter::GetSourceFd() { if (!source_device_) { LOG(ERROR) << "Attempted to read from source device but none was set"; return borrowed_fd{-1}; @@ -50,12 +49,10 @@ borrowed_fd ISnapshotWriter::GetSourceFd() { return source_fd_; } -CompressedSnapshotWriter::CompressedSnapshotWriter(const CowOptions& options) - : ISnapshotWriter(options) {} +CompressedSnapshotWriter::CompressedSnapshotWriter(const CowOptions& options) : options_(options) {} bool CompressedSnapshotWriter::SetCowDevice(android::base::unique_fd&& cow_device) { cow_device_ = std::move(cow_device); - cow_ = std::make_unique(options_); return true; } @@ -106,47 +103,76 @@ std::unique_ptr CompressedSnapshotWriter::OpenReader() { reader->SetSourceDevice(*source_device_); } - const auto& cow_options = options(); - if (cow_options.max_blocks) { - reader->SetBlockDeviceSize(*cow_options.max_blocks * cow_options.block_size); + if (options_.max_blocks) { + reader->SetBlockDeviceSize(*options_.max_blocks * options_.block_size); } return reader; } -bool CompressedSnapshotWriter::EmitCopy(uint64_t new_block, uint64_t old_block, - uint64_t num_blocks) { +bool CompressedSnapshotWriter::AddCopy(uint64_t new_block, uint64_t old_block, + uint64_t num_blocks) { return cow_->AddCopy(new_block, old_block, num_blocks); } -bool CompressedSnapshotWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, - size_t size) { +bool CompressedSnapshotWriter::AddRawBlocks(uint64_t new_block_start, const void* data, + size_t size) { return cow_->AddRawBlocks(new_block_start, data, size); } -bool CompressedSnapshotWriter::EmitXorBlocks(uint32_t new_block_start, const void* data, - size_t size, uint32_t old_block, uint16_t offset) { +bool CompressedSnapshotWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, + uint32_t old_block, uint16_t offset) { return cow_->AddXorBlocks(new_block_start, data, size, old_block, offset); } -bool CompressedSnapshotWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { +bool CompressedSnapshotWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { return cow_->AddZeroBlocks(new_block_start, num_blocks); } -bool CompressedSnapshotWriter::EmitLabel(uint64_t label) { +bool CompressedSnapshotWriter::AddLabel(uint64_t label) { return cow_->AddLabel(label); } -bool CompressedSnapshotWriter::EmitSequenceData(size_t num_ops, const uint32_t* data) { +bool CompressedSnapshotWriter::AddSequenceData(size_t num_ops, const uint32_t* data) { return cow_->AddSequenceData(num_ops, data); } bool CompressedSnapshotWriter::Initialize() { - return cow_->Initialize(cow_device_); + unique_fd cow_fd(dup(cow_device_.get())); + if (cow_fd < 0) { + PLOG(ERROR) << "dup COW device"; + return false; + } + + auto cow = std::make_unique(options_, std::move(cow_fd)); + if (!cow->Initialize(std::nullopt)) { + return false; + } + cow_ = std::move(cow); + return true; } bool CompressedSnapshotWriter::InitializeAppend(uint64_t label) { - return cow_->InitializeAppend(cow_device_, label); + unique_fd cow_fd(dup(cow_device_.get())); + if (cow_fd < 0) { + PLOG(ERROR) << "dup COW device"; + return false; + } + + auto cow = std::make_unique(options_, std::move(cow_fd)); + if (!cow->Initialize(label)) { + return false; + } + cow_ = std::move(cow); + return true; +} + +uint32_t CompressedSnapshotWriter::GetBlockSize() const { + return cow_->GetBlockSize(); +} + +std::optional CompressedSnapshotWriter::GetMaxBlocks() const { + return cow_->GetMaxBlocks(); } } // namespace snapshot diff --git a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp index 3c4ab2ef7..737c480a8 100644 --- a/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp +++ b/fs_mgr/libsnapshot/snapuserd/dm-snapshot-merge/cow_snapuserd_test.cpp @@ -122,6 +122,7 @@ class CowSnapuserdTest final { void SimulateDaemonRestart(); void StartMerge(); + std::unique_ptr CreateCowDeviceInternal(); void CreateCowDevice(); void CreateCowDeviceOrderedOps(); void CreateCowDeviceOrderedOpsInverted(); @@ -164,6 +165,7 @@ class CowSnapuserdMetadataTest final { private: void InitMetadata(); + std::unique_ptr CreateCowDeviceInternal(); void CreateCowDevice(); void CreateCowPartialFilledArea(); @@ -258,6 +260,19 @@ void CowSnapuserdTest::StartSnapuserdDaemon() { } } +std::unique_ptr CowSnapuserdTest::CreateCowDeviceInternal() { + std::string path = android::base::GetExecutableDirectory(); + cow_system_ = std::make_unique(path); + + CowOptions options; + options.compression = "gz"; + + unique_fd fd(cow_system_->fd); + cow_system_->fd = -1; + + return CreateCowWriter(kDefaultCowVersion, options, std::move(fd)); +} + void CowSnapuserdTest::ReadLastBlock() { unique_fd rnd_fd; total_base_size_ = BLOCK_SZ * 2; @@ -280,9 +295,6 @@ void CowSnapuserdTest::ReadLastBlock() { base_loop_ = std::make_unique(base_fd_, 10s); ASSERT_TRUE(base_loop_->valid()); - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); - std::unique_ptr random_buffer_1_ = std::make_unique(total_base_size_); loff_t offset = 0; @@ -294,16 +306,13 @@ void CowSnapuserdTest::ReadLastBlock() { offset += BLOCK_SZ; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); + ASSERT_TRUE(writer->AddRawBlocks(0, random_buffer_1_.get(), BLOCK_SZ)); + ASSERT_TRUE(writer->AddRawBlocks(1, (char*)random_buffer_1_.get() + BLOCK_SZ, BLOCK_SZ)); - ASSERT_TRUE(writer.AddRawBlocks(0, random_buffer_1_.get(), BLOCK_SZ)); - ASSERT_TRUE(writer.AddRawBlocks(1, (char*)random_buffer_1_.get() + BLOCK_SZ, BLOCK_SZ)); - - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); SetDeviceControlName(); @@ -381,22 +390,16 @@ void CowSnapuserdTest::ReadSnapshotDeviceAndValidate() { } void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = 0; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy, blk_src_copy + 1)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy, blk_src_copy + 1)); x -= 1; if (x == 1) { break; @@ -405,7 +408,7 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -433,22 +436,16 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = num_blocks - 1; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy + 1, blk_src_copy)); x -= 1; if (x == 0) { ASSERT_EQ(blk_src_copy, 0); @@ -458,7 +455,7 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -468,10 +465,11 @@ void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { true); // Merged operations - ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0), + ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), writer->GetBlockSize(), + 0), true); ASSERT_EQ(android::base::ReadFullyAtOffset( - base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0), + base_fd_, (char*)orig_buffer_.get() + writer->GetBlockSize(), size_, 0), true); } @@ -479,8 +477,8 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -495,13 +493,7 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 3; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -509,7 +501,7 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -519,12 +511,12 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOpsInverted() { } for (size_t i = num_blocks; i > 0; i--) { - ASSERT_TRUE(writer.AddXorBlocks(num_blocks + i - 1, - &random_buffer_1_.get()[options.block_size * (i - 1)], - options.block_size, 2 * num_blocks + i - 1, xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks( + num_blocks + i - 1, &random_buffer_1_.get()[writer->GetBlockSize() * (i - 1)], + writer->GetBlockSize(), 2 * num_blocks + i - 1, xor_offset)); } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -542,8 +534,8 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOps() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -559,20 +551,14 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOps() { } memset(random_buffer_1_.get(), 0, size_); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t source_blk = 0; size_t blk_src_copy = 2 * num_blocks; uint16_t xor_offset = 5; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { @@ -582,10 +568,10 @@ void CowSnapuserdTest::CreateCowDeviceOrderedOps() { blk_src_copy += 1; } - ASSERT_TRUE(writer.AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -603,8 +589,8 @@ void CowSnapuserdTest::CreateCowDevice() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -619,13 +605,7 @@ void CowSnapuserdTest::CreateCowDevice() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 2; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -639,11 +619,11 @@ void CowSnapuserdTest::CreateCowDevice() { for (int i = 0; i < num_blocks; i++) { sequence[num_blocks + i] = 5 * num_blocks - 1 - i; } - ASSERT_TRUE(writer.AddSequenceData(2 * num_blocks, sequence)); + ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence)); size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -655,24 +635,24 @@ void CowSnapuserdTest::CreateCowDevice() { source_blk = num_blocks; blk_src_copy = blk_end_copy; - ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); size_t blk_zero_copy_start = source_blk + num_blocks; size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks; - ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks)); + ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks)); size_t blk_random2_replace_start = blk_zero_copy_end; - ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); size_t blk_xor_start = blk_random2_replace_start + num_blocks; size_t xor_offset = BLOCK_SZ / 2; - ASSERT_TRUE(writer.AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); std::string zero_buffer(size_, 0); @@ -902,29 +882,36 @@ void CowSnapuserdTest::MergeInterrupt() { ASSERT_TRUE(Merge()); } -void CowSnapuserdMetadataTest::CreateCowPartialFilledArea() { +std::unique_ptr CowSnapuserdMetadataTest::CreateCowDeviceInternal() { std::string path = android::base::GetExecutableDirectory(); cow_system_ = std::make_unique(path); CowOptions options; options.compression = "gz"; - CowWriter writer(options); - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); + unique_fd fd(cow_system_->fd); + cow_system_->fd = -1; + + return CreateCowWriter(kDefaultCowVersion, options, std::move(fd)); +} + +void CowSnapuserdMetadataTest::CreateCowPartialFilledArea() { + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); // Area 0 is completely filled with 256 exceptions for (int i = 0; i < 256; i++) { - ASSERT_TRUE(writer.AddCopy(i, 256 + i)); + ASSERT_TRUE(writer->AddCopy(i, 256 + i)); } // Area 1 is partially filled with 2 copy ops and 10 zero ops - ASSERT_TRUE(writer.AddCopy(500, 1000)); - ASSERT_TRUE(writer.AddCopy(501, 1001)); + ASSERT_TRUE(writer->AddCopy(500, 1000)); + ASSERT_TRUE(writer->AddCopy(501, 1001)); - ASSERT_TRUE(writer.AddZeroBlocks(300, 10)); + ASSERT_TRUE(writer->AddZeroBlocks(300, 10)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); } void CowSnapuserdMetadataTest::ValidatePartialFilledArea() { @@ -956,8 +943,8 @@ void CowSnapuserdMetadataTest::CreateCowDevice() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -972,50 +959,44 @@ void CowSnapuserdMetadataTest::CreateCowDevice() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); // Overlapping region. This has to be split // into two batch operations - ASSERT_TRUE(writer.AddCopy(23, 20)); - ASSERT_TRUE(writer.AddCopy(22, 19)); - ASSERT_TRUE(writer.AddCopy(21, 18)); - ASSERT_TRUE(writer.AddCopy(20, 17)); - ASSERT_TRUE(writer.AddCopy(19, 16)); - ASSERT_TRUE(writer.AddCopy(18, 15)); + ASSERT_TRUE(writer->AddCopy(23, 20)); + ASSERT_TRUE(writer->AddCopy(22, 19)); + ASSERT_TRUE(writer->AddCopy(21, 18)); + ASSERT_TRUE(writer->AddCopy(20, 17)); + ASSERT_TRUE(writer->AddCopy(19, 16)); + ASSERT_TRUE(writer->AddCopy(18, 15)); // Contiguous region but blocks in ascending order // Daemon has to ensure that these blocks are merged // in a batch - ASSERT_TRUE(writer.AddCopy(50, 75)); - ASSERT_TRUE(writer.AddCopy(51, 76)); - ASSERT_TRUE(writer.AddCopy(52, 77)); - ASSERT_TRUE(writer.AddCopy(53, 78)); + ASSERT_TRUE(writer->AddCopy(50, 75)); + ASSERT_TRUE(writer->AddCopy(51, 76)); + ASSERT_TRUE(writer->AddCopy(52, 77)); + ASSERT_TRUE(writer->AddCopy(53, 78)); // Dis-contiguous region - ASSERT_TRUE(writer.AddCopy(110, 130)); - ASSERT_TRUE(writer.AddCopy(105, 125)); - ASSERT_TRUE(writer.AddCopy(100, 120)); + ASSERT_TRUE(writer->AddCopy(110, 130)); + ASSERT_TRUE(writer->AddCopy(105, 125)); + ASSERT_TRUE(writer->AddCopy(100, 120)); // Overlap - ASSERT_TRUE(writer.AddCopy(25, 30)); - ASSERT_TRUE(writer.AddCopy(30, 31)); + ASSERT_TRUE(writer->AddCopy(25, 30)); + ASSERT_TRUE(writer->AddCopy(30, 31)); size_t source_blk = num_blocks; - ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); size_t blk_zero_copy_start = source_blk + num_blocks; - ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks)); + ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); } void CowSnapuserdMetadataTest::InitMetadata() { diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp index 57f9e7ac1..efe0c1443 100644 --- a/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp +++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/snapuserd_test.cpp @@ -125,6 +125,7 @@ class SnapuserdTest : public ::testing::Test { void SimulateDaemonRestart(); + std::unique_ptr CreateCowDeviceInternal(); void CreateCowDevice(); void CreateCowDeviceOrderedOps(); void CreateCowDeviceOrderedOpsInverted(); @@ -277,23 +278,30 @@ void SnapuserdTest::ReadSnapshotDeviceAndValidate() { ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 4), size_), 0); } -void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { +std::unique_ptr SnapuserdTest::CreateCowDeviceInternal() { std::string path = android::base::GetExecutableDirectory(); cow_system_ = std::make_unique(path); CowOptions options; options.compression = "gz"; - CowWriter writer(options); - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); + unique_fd fd(cow_system_->fd); + cow_system_->fd = -1; - size_t num_blocks = size_ / options.block_size; + return CreateCowWriter(kDefaultCowVersion, options, std::move(fd)); +} + +void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); + + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = 0; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy, blk_src_copy + 1)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy, blk_src_copy + 1)); x -= 1; if (x == 1) { break; @@ -302,7 +310,7 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -330,22 +338,16 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_2() { } void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t blk_src_copy = num_blocks - 1; // Create overlapping copy operations while (1) { - ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(blk_src_copy + 1, blk_src_copy)); x -= 1; if (x == 0) { ASSERT_EQ(blk_src_copy, 0); @@ -355,7 +357,7 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); @@ -365,10 +367,11 @@ void SnapuserdTest::CreateCowDeviceWithCopyOverlap_1() { true); // Merged operations - ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0), + ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), writer->GetBlockSize(), + 0), true); ASSERT_EQ(android::base::ReadFullyAtOffset( - base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0), + base_fd_, (char*)orig_buffer_.get() + writer->GetBlockSize(), size_, 0), true); } @@ -376,8 +379,8 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -392,13 +395,7 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 3; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -406,7 +403,7 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -416,12 +413,12 @@ void SnapuserdTest::CreateCowDeviceOrderedOpsInverted() { } for (size_t i = num_blocks; i > 0; i--) { - ASSERT_TRUE(writer.AddXorBlocks(num_blocks + i - 1, - &random_buffer_1_.get()[options.block_size * (i - 1)], - options.block_size, 2 * num_blocks + i - 1, xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks( + num_blocks + i - 1, &random_buffer_1_.get()[writer->GetBlockSize() * (i - 1)], + writer->GetBlockSize(), 2 * num_blocks + i - 1, xor_offset)); } // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -439,8 +436,8 @@ void SnapuserdTest::CreateCowDeviceOrderedOps() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -456,20 +453,14 @@ void SnapuserdTest::CreateCowDeviceOrderedOps() { } memset(random_buffer_1_.get(), 0, size_); - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t x = num_blocks; size_t source_blk = 0; size_t blk_src_copy = 2 * num_blocks; uint16_t xor_offset = 5; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { @@ -479,10 +470,10 @@ void SnapuserdTest::CreateCowDeviceOrderedOps() { blk_src_copy += 1; } - ASSERT_TRUE(writer.AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(num_blocks, random_buffer_1_.get(), size_, 2 * num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); // Read the entire base device @@ -500,8 +491,8 @@ void SnapuserdTest::CreateCowDevice() { unique_fd rnd_fd; loff_t offset = 0; - std::string path = android::base::GetExecutableDirectory(); - cow_system_ = std::make_unique(path); + auto writer = CreateCowDeviceInternal(); + ASSERT_NE(writer, nullptr); rnd_fd.reset(open("/dev/random", O_RDONLY)); ASSERT_TRUE(rnd_fd > 0); @@ -516,13 +507,7 @@ void SnapuserdTest::CreateCowDevice() { offset += 1_MiB; } - CowOptions options; - options.compression = "gz"; - CowWriter writer(options); - - ASSERT_TRUE(writer.Initialize(cow_system_->fd)); - - size_t num_blocks = size_ / options.block_size; + size_t num_blocks = size_ / writer->GetBlockSize(); size_t blk_end_copy = num_blocks * 2; size_t source_blk = num_blocks - 1; size_t blk_src_copy = blk_end_copy - 1; @@ -536,11 +521,11 @@ void SnapuserdTest::CreateCowDevice() { for (int i = 0; i < num_blocks; i++) { sequence[num_blocks + i] = 5 * num_blocks - 1 - i; } - ASSERT_TRUE(writer.AddSequenceData(2 * num_blocks, sequence)); + ASSERT_TRUE(writer->AddSequenceData(2 * num_blocks, sequence)); size_t x = num_blocks; while (1) { - ASSERT_TRUE(writer.AddCopy(source_blk, blk_src_copy)); + ASSERT_TRUE(writer->AddCopy(source_blk, blk_src_copy)); x -= 1; if (x == 0) { break; @@ -552,24 +537,24 @@ void SnapuserdTest::CreateCowDevice() { source_blk = num_blocks; blk_src_copy = blk_end_copy; - ASSERT_TRUE(writer.AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(source_blk, random_buffer_1_.get(), size_)); size_t blk_zero_copy_start = source_blk + num_blocks; size_t blk_zero_copy_end = blk_zero_copy_start + num_blocks; - ASSERT_TRUE(writer.AddZeroBlocks(blk_zero_copy_start, num_blocks)); + ASSERT_TRUE(writer->AddZeroBlocks(blk_zero_copy_start, num_blocks)); size_t blk_random2_replace_start = blk_zero_copy_end; - ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); + ASSERT_TRUE(writer->AddRawBlocks(blk_random2_replace_start, random_buffer_1_.get(), size_)); size_t blk_xor_start = blk_random2_replace_start + num_blocks; size_t xor_offset = BLOCK_SZ / 2; - ASSERT_TRUE(writer.AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, - xor_offset)); + ASSERT_TRUE(writer->AddXorBlocks(blk_xor_start, random_buffer_1_.get(), size_, num_blocks, + xor_offset)); // Flush operations - ASSERT_TRUE(writer.Finalize()); + ASSERT_TRUE(writer->Finalize()); // Construct the buffer required for validation orig_buffer_ = std::make_unique(total_base_size_); std::string zero_buffer(size_, 0);