diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index eaef18037..f8e4b7aec 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -122,6 +122,39 @@ cc_library_static { ], } +cc_defaults { + name: "libsnapshot_cow_defaults", + defaults: [ + "fs_mgr_defaults", + ], + cflags: [ + "-D_FILE_OFFSET_BITS=64", + "-Wall", + "-Werror", + ], + export_include_dirs: ["include"], + srcs: [ + "cow_reader.cpp", + "cow_writer.cpp", + ], +} + +cc_library_static { + name: "libsnapshot_cow", + defaults: [ + "libsnapshot_cow_defaults", + ], + host_supported: true, + shared_libs: [ + "libbase", + "libcrypto", + "liblog", + ], + static_libs: [ + "libz", + ], +} + cc_library_static { name: "libsnapshot_test_helpers", defaults: ["libsnapshot_defaults"], @@ -343,3 +376,30 @@ cc_binary { static_executable: true, system_shared_libs: [], } + +cc_test { + name: "cow_api_test", + defaults: [ + "fs_mgr_defaults", + ], + srcs: [ + "cow_api_test.cpp", + ], + cflags: [ + "-Wall", + "-Werror", + ], + shared_libs: [ + "libbase", + "libcrypto", + "liblog", + "libz", + ], + static_libs: [ + "libgtest", + "libsnapshot_cow", + ], + test_min_api_level: 30, + auto_gen_config: true, + require_root: false, +} diff --git a/fs_mgr/libsnapshot/cow_api_test.cpp b/fs_mgr/libsnapshot/cow_api_test.cpp new file mode 100644 index 000000000..3b3fc4730 --- /dev/null +++ b/fs_mgr/libsnapshot/cow_api_test.cpp @@ -0,0 +1,244 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include +#include +#include +#include + +namespace android { +namespace snapshot { + +class CowTest : public ::testing::Test { + protected: + void SetUp() override { + cow_ = std::make_unique(); + ASSERT_GE(cow_->fd, 0) << strerror(errno); + } + + void TearDown() override { cow_ = nullptr; } + + std::unique_ptr cow_; +}; + +// Sink that always appends to the end of a string. +class StringSink : public IByteSink { + public: + void* GetBuffer(size_t requested, size_t* actual) override { + size_t old_size = stream_.size(); + stream_.resize(old_size + requested, '\0'); + *actual = requested; + return stream_.data() + old_size; + } + bool ReturnData(void*, size_t) override { return true; } + void Reset() { stream_.clear(); } + + std::string& stream() { return stream_; } + + private: + std::string stream_; +}; + +TEST_F(CowTest, ReadWrite) { + CowOptions options; + CowWriter writer(options); + + ASSERT_TRUE(writer.Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size, '\0'); + + ASSERT_TRUE(writer.AddCopy(10, 20)); + ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer.AddZeroBlocks(51, 2)); + ASSERT_TRUE(writer.Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + CowReader reader; + CowHeader header; + ASSERT_TRUE(reader.Parse(cow_->fd)); + ASSERT_TRUE(reader.GetHeader(&header)); + ASSERT_EQ(header.magic, kCowMagicNumber); + ASSERT_EQ(header.major_version, kCowVersionMajor); + ASSERT_EQ(header.minor_version, kCowVersionMinor); + ASSERT_EQ(header.block_size, options.block_size); + ASSERT_EQ(header.num_ops, 4); + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + ASSERT_FALSE(iter->Done()); + auto op = &iter->Get(); + + ASSERT_EQ(op->type, kCowCopyOp); + ASSERT_EQ(op->compression, kCowCompressNone); + ASSERT_EQ(op->data_length, 0); + ASSERT_EQ(op->new_block, 10); + ASSERT_EQ(op->source, 20); + + StringSink sink; + + iter->Next(); + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_EQ(op->compression, kCowCompressNone); + ASSERT_EQ(op->data_length, 4096); + ASSERT_EQ(op->new_block, 50); + ASSERT_EQ(op->source, 104); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data); + + iter->Next(); + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + + // Note: the zero operation gets split into two blocks. + ASSERT_EQ(op->type, kCowZeroOp); + ASSERT_EQ(op->compression, kCowCompressNone); + ASSERT_EQ(op->data_length, 0); + ASSERT_EQ(op->new_block, 51); + ASSERT_EQ(op->source, 0); + + iter->Next(); + ASSERT_FALSE(iter->Done()); + op = &iter->Get(); + + ASSERT_EQ(op->type, kCowZeroOp); + ASSERT_EQ(op->compression, kCowCompressNone); + ASSERT_EQ(op->data_length, 0); + ASSERT_EQ(op->new_block, 52); + ASSERT_EQ(op->source, 0); + + iter->Next(); + ASSERT_TRUE(iter->Done()); +} + +TEST_F(CowTest, CompressGz) { + CowOptions options; + options.compression = "gz"; + CowWriter writer(options); + + ASSERT_TRUE(writer.Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size, '\0'); + + ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer.Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + ASSERT_FALSE(iter->Done()); + auto op = &iter->Get(); + + StringSink sink; + + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_EQ(op->compression, kCowCompressGz); + ASSERT_EQ(op->data_length, 56); // compressed! + ASSERT_EQ(op->new_block, 50); + ASSERT_EQ(op->source, 104); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data); + + iter->Next(); + ASSERT_TRUE(iter->Done()); +} + +TEST_F(CowTest, CompressTwoBlocks) { + CowOptions options; + options.compression = "gz"; + CowWriter writer(options); + + ASSERT_TRUE(writer.Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size * 2, '\0'); + + ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer.Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + ASSERT_FALSE(iter->Done()); + iter->Next(); + ASSERT_FALSE(iter->Done()); + + StringSink sink; + + auto op = &iter->Get(); + ASSERT_EQ(op->type, kCowReplaceOp); + ASSERT_EQ(op->compression, kCowCompressGz); + ASSERT_EQ(op->new_block, 51); + ASSERT_TRUE(reader.ReadData(*op, &sink)); +} + +// Only return 1-byte buffers, to stress test the partial read logic in +// CowReader. +class HorribleStringSink : public StringSink { + public: + void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); } +}; + +TEST_F(CowTest, HorribleSink) { + CowOptions options; + options.compression = "gz"; + CowWriter writer(options); + + ASSERT_TRUE(writer.Initialize(cow_->fd)); + + std::string data = "This is some data, believe it"; + data.resize(options.block_size, '\0'); + + ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size())); + ASSERT_TRUE(writer.Finalize()); + + ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0); + + CowReader reader; + ASSERT_TRUE(reader.Parse(cow_->fd)); + + auto iter = reader.GetOpIter(); + ASSERT_NE(iter, nullptr); + ASSERT_FALSE(iter->Done()); + + HorribleStringSink sink; + auto op = &iter->Get(); + ASSERT_TRUE(reader.ReadData(*op, &sink)); + ASSERT_EQ(sink.stream(), data); +} + +} // namespace snapshot +} // namespace android + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/fs_mgr/libsnapshot/cow_reader.cpp b/fs_mgr/libsnapshot/cow_reader.cpp new file mode 100644 index 000000000..86565c4b5 --- /dev/null +++ b/fs_mgr/libsnapshot/cow_reader.cpp @@ -0,0 +1,264 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include + +#include +#include +#include +#include +#include + +namespace android { +namespace snapshot { + +CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {} + +static void SHA256(const void* data, size_t length, uint8_t out[32]) { + SHA256_CTX c; + SHA256_Init(&c); + SHA256_Update(&c, data, length); + SHA256_Final(out, &c); +} + +bool CowReader::Parse(android::base::unique_fd&& fd) { + owned_fd_ = std::move(fd); + return Parse(android::base::borrowed_fd{owned_fd_}); +} + +bool CowReader::Parse(android::base::borrowed_fd fd) { + fd_ = fd; + + auto pos = lseek(fd_.get(), 0, SEEK_END); + if (pos < 0) { + PLOG(ERROR) << "lseek end failed"; + return false; + } + fd_size_ = pos; + + if (lseek(fd_.get(), 0, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek header failed"; + return false; + } + if (!android::base::ReadFully(fd_, &header_, sizeof(header_))) { + PLOG(ERROR) << "read header failed"; + return false; + } + + // Validity check the ops range. + if (header_.ops_offset >= fd_size_) { + LOG(ERROR) << "ops offset " << header_.ops_offset << " larger than fd size " << fd_size_; + return false; + } + if (fd_size_ - header_.ops_offset < header_.ops_size) { + LOG(ERROR) << "ops size " << header_.ops_size << " is too large"; + return false; + } + + uint8_t header_csum[32]; + { + CowHeader tmp = header_; + memset(&tmp.header_checksum, 0, sizeof(tmp.header_checksum)); + SHA256(&tmp, sizeof(tmp), header_csum); + } + if (memcmp(header_csum, header_.header_checksum, sizeof(header_csum)) != 0) { + LOG(ERROR) << "header checksum is invalid"; + return false; + } + return true; +} + +bool CowReader::GetHeader(CowHeader* header) { + *header = header_; + return true; +} + +class CowOpIter final : public ICowOpIter { + public: + CowOpIter(std::unique_ptr&& ops, size_t len); + + bool Done() override; + const CowOperation& Get() override; + void Next() override; + + private: + bool HasNext(); + + std::unique_ptr ops_; + const uint8_t* pos_; + const uint8_t* end_; + bool done_; +}; + +CowOpIter::CowOpIter(std::unique_ptr&& ops, size_t len) + : ops_(std::move(ops)), pos_(ops_.get()), end_(pos_ + len), done_(!HasNext()) {} + +bool CowOpIter::Done() { + return done_; +} + +bool CowOpIter::HasNext() { + return pos_ < end_ && size_t(end_ - pos_) >= sizeof(CowOperation); +} + +void CowOpIter::Next() { + CHECK(!Done()); + + pos_ += sizeof(CowOperation); + if (!HasNext()) done_ = true; +} + +const CowOperation& CowOpIter::Get() { + CHECK(!Done()); + CHECK(HasNext()); + return *reinterpret_cast(pos_); +} + +std::unique_ptr CowReader::GetOpIter() { + if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek ops failed"; + return nullptr; + } + auto ops_buffer = std::make_unique(header_.ops_size); + if (!android::base::ReadFully(fd_, ops_buffer.get(), header_.ops_size)) { + PLOG(ERROR) << "read ops failed"; + return nullptr; + } + + uint8_t csum[32]; + SHA256(ops_buffer.get(), header_.ops_size, csum); + if (memcmp(csum, header_.ops_checksum, sizeof(csum)) != 0) { + LOG(ERROR) << "ops checksum does not match"; + return nullptr; + } + + return std::make_unique(std::move(ops_buffer), header_.ops_size); +} + +bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len) { + // Validate the offset, taking care to acknowledge possible overflow of offset+len. + if (offset < sizeof(header_) || offset >= header_.ops_offset || len >= fd_size_ || + offset + len > header_.ops_offset) { + LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes"; + return false; + } + if (lseek(fd_.get(), offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek to read raw bytes failed"; + return false; + } + if (!android::base::ReadFully(fd_, buffer, len)) { + PLOG(ERROR) << "read raw bytes failed"; + return false; + } + return true; +} + +bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) { + uint64_t offset = op.source; + + switch (op.compression) { + case kCowCompressNone: { + size_t remaining = op.data_length; + while (remaining) { + size_t amount = remaining; + void* buffer = sink->GetBuffer(amount, &amount); + if (!buffer) { + LOG(ERROR) << "Could not acquire buffer from sink"; + return false; + } + if (!GetRawBytes(offset, buffer, amount)) { + return false; + } + if (!sink->ReturnData(buffer, amount)) { + LOG(ERROR) << "Could not return buffer to sink"; + return false; + } + remaining -= amount; + offset += amount; + } + return true; + } + case kCowCompressGz: { + auto input = std::make_unique(op.data_length); + if (!GetRawBytes(offset, input.get(), op.data_length)) { + return false; + } + + z_stream z = {}; + z.next_in = input.get(); + z.avail_in = op.data_length; + if (int rv = inflateInit(&z); rv != Z_OK) { + LOG(ERROR) << "inflateInit returned error code " << rv; + return false; + } + + while (z.total_out < header_.block_size) { + // If no more output buffer, grab a new buffer. + if (z.avail_out == 0) { + size_t amount = header_.block_size - z.total_out; + z.next_out = reinterpret_cast(sink->GetBuffer(amount, &amount)); + if (!z.next_out) { + LOG(ERROR) << "Could not acquire buffer from sink"; + return false; + } + z.avail_out = amount; + } + + // Remember the position of the output buffer so we can call ReturnData. + auto buffer = z.next_out; + auto avail_out = z.avail_out; + + // Decompress. + int rv = inflate(&z, Z_NO_FLUSH); + if (rv != Z_OK && rv != Z_STREAM_END) { + LOG(ERROR) << "inflate returned error code " << rv; + return false; + } + + // Return the section of the buffer that was updated. + if (z.avail_out < avail_out && !sink->ReturnData(buffer, avail_out - z.avail_out)) { + LOG(ERROR) << "Could not return buffer to sink"; + return false; + } + + if (rv == Z_STREAM_END) { + // Error if the stream has ended, but we didn't fill the entire block. + if (z.total_out != header_.block_size) { + LOG(ERROR) << "Reached gz stream end but did not read a full block of data"; + return false; + } + break; + } + + CHECK(rv == Z_OK); + + // Error if the stream is expecting more data, but we don't have any to read. + if (z.avail_in == 0) { + LOG(ERROR) << "Gz stream ended prematurely"; + return false; + } + } + return true; + } + default: + LOG(ERROR) << "Unknown compression type: " << op.compression; + return false; + } +} + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/cow_writer.cpp b/fs_mgr/libsnapshot/cow_writer.cpp new file mode 100644 index 000000000..ea8e534bf --- /dev/null +++ b/fs_mgr/libsnapshot/cow_writer.cpp @@ -0,0 +1,230 @@ +// +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace android { +namespace snapshot { + +static_assert(sizeof(off_t) == sizeof(uint64_t)); + +CowWriter::CowWriter(const CowOptions& options) : ICowWriter(options), fd_(-1) { + SetupHeaders(); +} + +void CowWriter::SetupHeaders() { + header_ = {}; + header_.magic = kCowMagicNumber; + header_.major_version = kCowVersionMajor; + header_.minor_version = kCowVersionMinor; + header_.block_size = options_.block_size; +} + +bool CowWriter::Initialize(android::base::unique_fd&& fd) { + owned_fd_ = std::move(fd); + return Initialize(android::base::borrowed_fd{owned_fd_}); +} + +bool CowWriter::Initialize(android::base::borrowed_fd fd) { + fd_ = fd; + + // This limitation is tied to the data field size in CowOperation. + if (header_.block_size > std::numeric_limits::max()) { + LOG(ERROR) << "Block size is too large"; + return false; + } + + if (lseek(fd_.get(), 0, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek failed"; + return false; + } + + if (options_.compression == "gz") { + compression_ = kCowCompressGz; + } else if (!options_.compression.empty()) { + LOG(ERROR) << "unrecognized compression: " << options_.compression; + return false; + } + + // Headers are not complete, but this ensures the file is at the right + // position. + if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { + PLOG(ERROR) << "write failed"; + return false; + } + return true; +} + +bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) { + header_.num_ops++; + + CowOperation op = {}; + op.type = kCowCopyOp; + op.new_block = new_block; + op.source = old_block; + ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + + return true; +} + +bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) { + if (size % header_.block_size != 0) { + LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of " + << header_.block_size; + return false; + } + + uint64_t pos; + if (!GetDataPos(&pos)) { + return false; + } + + const uint8_t* iter = reinterpret_cast(data); + for (size_t i = 0; i < size / header_.block_size; i++) { + header_.num_ops++; + + CowOperation op = {}; + op.type = kCowReplaceOp; + op.new_block = new_block_start + i; + op.source = pos; + + if (compression_) { + auto data = Compress(iter, header_.block_size); + if (data.empty()) { + PLOG(ERROR) << "AddRawBlocks: compression failed"; + return false; + } + if (data.size() > std::numeric_limits::max()) { + LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes"; + return false; + } + if (!android::base::WriteFully(fd_, data.data(), data.size())) { + PLOG(ERROR) << "AddRawBlocks: write failed"; + return false; + } + op.compression = compression_; + op.data_length = static_cast(data.size()); + pos += data.size(); + } else { + op.data_length = static_cast(header_.block_size); + pos += header_.block_size; + } + + ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + iter += header_.block_size; + } + + if (!compression_ && !android::base::WriteFully(fd_, data, size)) { + PLOG(ERROR) << "AddRawBlocks: write failed"; + return false; + } + return true; +} + +bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) { + for (uint64_t i = 0; i < num_blocks; i++) { + header_.num_ops++; + + CowOperation op = {}; + op.type = kCowZeroOp; + op.new_block = new_block_start + i; + op.source = 0; + ops_ += std::basic_string(reinterpret_cast(&op), sizeof(op)); + } + return true; +} + +std::basic_string CowWriter::Compress(const void* data, size_t length) { + switch (compression_) { + case kCowCompressGz: { + auto bound = compressBound(length); + auto buffer = std::make_unique(bound); + + uLongf dest_len = bound; + auto rv = compress2(buffer.get(), &dest_len, reinterpret_cast(data), + length, Z_BEST_COMPRESSION); + if (rv != Z_OK) { + LOG(ERROR) << "compress2 returned: " << rv; + return {}; + } + return std::basic_string(buffer.get(), dest_len); + } + default: + LOG(ERROR) << "unhandled compression type: " << compression_; + break; + } + return {}; +} + +static void SHA256(const void* data, size_t length, uint8_t out[32]) { + SHA256_CTX c; + SHA256_Init(&c); + SHA256_Update(&c, data, length); + SHA256_Final(out, &c); +} + +bool CowWriter::Finalize() { + auto offs = lseek(fd_.get(), 0, SEEK_CUR); + if (offs < 0) { + PLOG(ERROR) << "lseek failed"; + return false; + } + header_.ops_offset = offs; + header_.ops_size = ops_.size(); + + SHA256(ops_.data(), ops_.size(), header_.ops_checksum); + SHA256(&header_, sizeof(header_), header_.header_checksum); + + if (lseek(fd_.get(), 0, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek start failed"; + return false; + } + if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) { + PLOG(ERROR) << "write header failed"; + return false; + } + if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) { + PLOG(ERROR) << "lseek ops failed"; + return false; + } + if (!android::base::WriteFully(fd_, ops_.data(), ops_.size())) { + PLOG(ERROR) << "write ops failed"; + return false; + } + return true; +} + +bool CowWriter::GetDataPos(uint64_t* pos) { + off_t offs = lseek(fd_.get(), 0, SEEK_CUR); + if (offs < 0) { + PLOG(ERROR) << "lseek failed"; + return false; + } + *pos = offs; + return true; +} + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h new file mode 100644 index 000000000..6d500e7e5 --- /dev/null +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_format.h @@ -0,0 +1,103 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace android { +namespace snapshot { + +static constexpr uint64_t kCowMagicNumber = 0x436f77634f572121ULL; +static constexpr uint32_t kCowVersionMajor = 1; +static constexpr uint32_t kCowVersionMinor = 0; + +// This header appears as the first sequence of bytes in the COW. All fields +// in the layout are little-endian encoded. The on-disk layout is: +// +// +-----------------------+ +// | Header (fixed) | +// +-----------------------+ +// | Raw Data (variable) | +// +-----------------------+ +// | Operations (variable) | +// +-----------------------+ +// +// The "raw data" occurs immediately after the header, and the operation +// sequence occurs after the raw data. This ordering is intentional. While +// streaming an OTA, we can immediately write compressed data, but store the +// metadata in memory. At the end, we can simply append the metadata and flush +// the file. There is no need to create separate files to store the metadata +// and block data. +struct CowHeader { + uint64_t magic; + uint16_t major_version; + uint16_t minor_version; + + // Offset to the location of the operation sequence, and size of the + // operation sequence buffer. |ops_offset| is also the end of the + // raw data region. + uint64_t ops_offset; + uint64_t ops_size; + uint64_t num_ops; + + // The size of block operations, in bytes. + uint32_t block_size; + + // SHA256 checksums of this header, with this field set to 0. + uint8_t header_checksum[32]; + + // SHA256 of the operation sequence. + uint8_t ops_checksum[32]; +} __attribute__((packed)); + +// Cow operations are currently fixed-size entries, but this may change if +// needed. +struct CowOperation { + // The operation code (see the constants and structures below). + uint8_t type; + + // If this operation reads from the data section of the COW, this contains + // the compression type of that data (see constants below). + uint8_t compression; + + // If this operation reads from the data section of the COW, this contains + // the length. + uint16_t data_length; + + // The block of data in the new image that this operation modifies. + uint64_t new_block; + + // The value of |source| depends on the operation code. + // + // For copy operations, this is a block location in the source image. + // + // For replace operations, this is a byte offset within the COW's data + // section (eg, not landing within the header or metadata). It is an + // absolute position within the image. + // + // For zero operations (replace with all zeroes), this is unused and must + // be zero. + uint64_t source; +} __attribute__((packed)); + +static constexpr uint8_t kCowCopyOp = 1; +static constexpr uint8_t kCowReplaceOp = 2; +static constexpr uint8_t kCowZeroOp = 3; + +static constexpr uint8_t kCowCompressNone = 0; +static constexpr uint8_t kCowCompressGz = 1; + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h new file mode 100644 index 000000000..a3b129177 --- /dev/null +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h @@ -0,0 +1,107 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include + +#include +#include + +namespace android { +namespace snapshot { + +class ICowOpIter; + +// A ByteSink object handles requests for a buffer of a specific size. It +// always owns the underlying buffer. It's designed to minimize potential +// copying as we parse or decompress the COW. +class IByteSink { + public: + virtual ~IByteSink() {} + + // Called when the reader has data. The size of the request is given. The + // sink must return a valid pointer (or null on failure), and return the + // maximum number of bytes that can be written to the returned buffer. + // + // The returned buffer is owned by IByteSink, but must remain valid until + // the ready operation has completed (or the entire buffer has been + // covered by calls to ReturnData). + // + // After calling GetBuffer(), all previous buffers returned are no longer + // valid. + virtual void* GetBuffer(size_t requested, size_t* actual) = 0; + + // Called when a section returned by |GetBuffer| has been filled with data. + virtual bool ReturnData(void* buffer, size_t length) = 0; +}; + +// Interface for reading from a snapuserd COW. +class ICowReader { + public: + virtual ~ICowReader() {} + + // Return the file header. + virtual bool GetHeader(CowHeader* header) = 0; + + // Return an iterator for retrieving CowOperation entries. + virtual std::unique_ptr GetOpIter() = 0; + + // Get raw bytes from the data section. + virtual bool GetRawBytes(uint64_t offset, void* buffer, size_t len) = 0; + + // Get decoded bytes from the data section, handling any decompression. + // All retrieved data is passed to the sink. + virtual bool ReadData(const CowOperation& op, IByteSink* sink) = 0; +}; + +// Iterate over a sequence of COW operations. +class ICowOpIter { + public: + virtual ~ICowOpIter() {} + + // True if there are more items to read, false otherwise. + virtual bool Done() = 0; + + // Read the current operation. + virtual const CowOperation& Get() = 0; + + // Advance to the next item. + virtual void Next() = 0; +}; + +class CowReader : public ICowReader { + public: + CowReader(); + + bool Parse(android::base::unique_fd&& fd); + bool Parse(android::base::borrowed_fd fd); + + bool GetHeader(CowHeader* header) override; + std::unique_ptr GetOpIter() override; + bool GetRawBytes(uint64_t offset, void* buffer, size_t len) override; + bool ReadData(const CowOperation& op, IByteSink* sink) override; + + private: + android::base::unique_fd owned_fd_; + android::base::borrowed_fd fd_; + CowHeader header_; + uint64_t fd_size_; +}; + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h new file mode 100644 index 000000000..5a2cbd6ec --- /dev/null +++ b/fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h @@ -0,0 +1,86 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include + +#include +#include + +namespace android { +namespace snapshot { + +struct CowOptions { + uint32_t block_size = 4096; + std::string compression; +}; + +// Interface for writing to a snapuserd COW. All operations are ordered; merges +// will occur in the sequence they were added to the COW. +class ICowWriter { + public: + explicit ICowWriter(const CowOptions& options) : options_(options) {} + + virtual ~ICowWriter() {} + + // Encode an operation that copies the contents of |old_block| to the + // location of |new_block|. + virtual bool AddCopy(uint64_t new_block, uint64_t old_block) = 0; + + // Encode a sequence of raw blocks. |size| must be a multiple of the block size. + virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0; + + // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size. + virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0; + + protected: + CowOptions options_; +}; + +class CowWriter : public ICowWriter { + public: + explicit CowWriter(const CowOptions& options); + + // Set up the writer. + bool Initialize(android::base::unique_fd&& fd); + bool Initialize(android::base::borrowed_fd fd); + + bool AddCopy(uint64_t new_block, uint64_t old_block) override; + bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override; + bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override; + + // Finalize all COW operations and flush pending writes. + bool Finalize(); + + private: + void SetupHeaders(); + bool GetDataPos(uint64_t* pos); + std::basic_string Compress(const void* data, size_t length); + + private: + android::base::unique_fd owned_fd_; + android::base::borrowed_fd fd_; + CowHeader header_; + int compression_ = 0; + + // :TODO: this is not efficient, but stringstream ubsan aborts because some + // bytes overflow a signed char. + std::basic_string ops_; +}; + +} // namespace snapshot +} // namespace android