ACPR/android_system_core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge changes I3339d552,I126e1583,Iccc6580a am: 323060840a am: 37ee1dc4ac am: 32fd86148d

Browse source 
Original change: https://android-review.googlesource.com/c/platform/system/core/+/1465344

Change-Id: I269cfb1687f4161bf07b7fa75649883cad1b439c
This commit is contained in:
Daniel Rosenberg 2020-10-23 11:51:46 +00:00 • committed by Automerger Merge Worker
commit 5003a5e2ea
14 changed files with 664 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
fs_mgr/libsnapshot/Android.bp
View file

@ -149,6 +149,7 @@ cc_defaults {
"cow_decompress.cpp",
"cow_reader.cpp",
"cow_writer.cpp",
"cow_format.cpp",
],
}

237
fs_mgr/libsnapshot/cow_api_test.cpp
View file

@ -70,19 +70,21 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_TRUE(writer.AddCopy(10, 20));
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
ASSERT_TRUE(writer.Flush());
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
CowHeader header;
CowFooter footer;
ASSERT_TRUE(reader.Parse(cow_->fd));
ASSERT_TRUE(reader.GetHeader(&header));
ASSERT_TRUE(reader.GetFooter(&footer));
ASSERT_EQ(header.magic, kCowMagicNumber);
ASSERT_EQ(header.major_version, kCowVersionMajor);
ASSERT_EQ(header.minor_version, kCowVersionMinor);
ASSERT_EQ(header.block_size, options.block_size);
ASSERT_EQ(header.num_ops, 4);
ASSERT_EQ(footer.op.num_ops, 4);
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
@ -105,7 +107,6 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -145,7 +146,7 @@ TEST_F(CowTest, CompressGz) {
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Flush());
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -163,7 +164,6 @@ TEST_F(CowTest, CompressGz) {
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) {
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Flush());
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -224,7 +224,7 @@ TEST_P(CompressionTest, HorribleSink) {
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Flush());
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -259,7 +259,7 @@ TEST_F(CowTest, GetSize) {
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
auto size_before = writer.GetCowSize();
ASSERT_TRUE(writer.Flush());
ASSERT_TRUE(writer.Finalize());
auto size_after = writer.GetCowSize();
ASSERT_EQ(size_before, size_after);
struct stat buf;
@ -272,6 +272,7 @@ TEST_F(CowTest, GetSize) {
}
TEST_F(CowTest, Append) {
cow_->DoNotRemove();
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
@ -279,7 +280,7 @@ TEST_F(CowTest, Append) {
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->Flush());
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -289,7 +290,7 @@ TEST_F(CowTest, Append) {
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));
ASSERT_TRUE(writer->Flush());
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -299,7 +300,9 @@ TEST_F(CowTest, Append) {
// Read back both operations.
CowReader reader;
uint64_t label;
ASSERT_TRUE(reader.Parse(cow_->fd));
ASSERT_FALSE(reader.GetLastLabel(&label));
StringSink sink;
@ -325,6 +328,220 @@ TEST_F(CowTest, Append) {
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, AppendCorrupted) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddLabel(0));
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->AddLabel(1));
ASSERT_TRUE(writer->AddZeroBlocks(50, 1));
ASSERT_TRUE(writer->Finalize());
// Drop the tail end of the header. Last entry may be corrupted.
ftruncate(cow_->fd, writer->GetCowSize() - 5);
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));
ASSERT_TRUE(writer->AddLabel(2));
ASSERT_TRUE(writer->AddZeroBlocks(50, 1));
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddLabel(3));
ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));
ASSERT_TRUE(writer->Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back all three operations.
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 0);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
iter->Next();
sink.Reset();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 2);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowZeroOp);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 3);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data2);
iter->Next();
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, AppendExtendedCorrupted) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
ASSERT_TRUE(writer->AddLabel(5));
ASSERT_TRUE(writer->AddLabel(6));
std::string data = "This is some data, believe it";
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
// fail to write the footer
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
// Get the last known good label
CowReader label_reader;
uint64_t label;
ASSERT_TRUE(label_reader.Parse(cow_->fd));
ASSERT_TRUE(label_reader.GetLastLabel(&label));
ASSERT_EQ(label, 5);
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));
ASSERT_TRUE(writer->Finalize());
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back all three operations.
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 5);
iter->Next();
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, AppendbyLabel) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
ASSERT_TRUE(writer->AddLabel(4));
ASSERT_TRUE(writer->AddLabel(5));
std::string data = "This is some data, believe it";
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->AddLabel(6));
ASSERT_TRUE(writer->AddZeroBlocks(50, 2));
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_FALSE(writer->InitializeAppend(cow_->fd, 12));
ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 5));
// This should drop label 6
ASSERT_TRUE(writer->Finalize());
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back all ops
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 4);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowLabelOp);
ASSERT_EQ(op->source, 5);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data.substr(0, options.block_size));
iter->Next();
sink.Reset();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data.substr(options.block_size, 2 * options.block_size));
iter->Next();
sink.Reset();
ASSERT_TRUE(iter->Done());
}
} // namespace snapshot
} // namespace android

63
fs_mgr/libsnapshot/cow_format.cpp Normal file
View file

@ -0,0 +1,63 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <libsnapshot/cow_format.h>
#include <sys/types.h>
#include <unistd.h>
#include <android-base/logging.h>
namespace android {
namespace snapshot {
std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
os << "CowOperation(type:";
if (op.type == kCowCopyOp)
os << "kCowCopyOp, ";
else if (op.type == kCowReplaceOp)
os << "kCowReplaceOp, ";
else if (op.type == kCowZeroOp)
os << "kZeroOp, ";
else if (op.type == kCowFooterOp)
os << "kCowFooterOp, ";
else if (op.type == kCowLabelOp)
os << "kCowLabelOp, ";
else
os << (int)op.type << "?,";
os << "compression:";
if (op.compression == kCowCompressNone)
os << "kCowCompressNone, ";
else if (op.compression == kCowCompressGz)
os << "kCowCompressGz, ";
else if (op.compression == kCowCompressBrotli)
os << "kCowCompressBrotli, ";
else
os << (int)op.compression << "?, ";
os << "data_length:" << op.data_length << ",\t";
os << "new_block:" << op.new_block << ",\t";
os << "source:" << op.source << ")";
return os;
}
int64_t GetNextOpOffset(const CowOperation& op) {
if (op.type == kCowReplaceOp)
return op.data_length;
else
return 0;
}
} // namespace snapshot
} // namespace android

168
fs_mgr/libsnapshot/cow_reader.cpp
View file

@ -18,17 +18,26 @@
#include <unistd.h>
#include <limits>
#include <vector>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <libsnapshot/cow_reader.h>
#include <zlib.h>
#include "cow_decompress.h"
namespace android {
namespace snapshot {
CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}
CowReader::CowReader()
: fd_(-1),
header_(),
footer_(),
fd_size_(0),
has_footer_(false),
last_label_(0),
has_last_label_(false) {}
static void SHA256(const void*, size_t, uint8_t[]) {
#if 0
@ -63,16 +72,6 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
return false;
}
// Validity check the ops range.
if (header_.ops_offset >= fd_size_) {
LOG(ERROR) << "ops offset " << header_.ops_offset << " larger than fd size " << fd_size_;
return false;
}
if (fd_size_ - header_.ops_offset < header_.ops_size) {
LOG(ERROR) << "ops size " << header_.ops_size << " is too large";
return false;
}
if (header_.magic != kCowMagicNumber) {
LOG(ERROR) << "Header Magic corrupted. Magic: " << header_.magic
<< "Expected: " << kCowMagicNumber;
@ -83,6 +82,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
<< sizeof(CowHeader);
return false;
}
if (header_.footer_size != sizeof(CowFooter)) {
LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected "
<< sizeof(CowFooter);
return false;
}
if ((header_.major_version != kCowVersionMajor) ||
(header_.minor_version != kCowVersionMinor)) {
@ -94,19 +98,75 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
return false;
}
uint8_t header_csum[32];
{
CowHeader tmp = header_;
memset(&tmp.header_checksum, 0, sizeof(tmp.header_checksum));
memset(header_csum, 0, sizeof(uint8_t) * 32);
SHA256(&tmp, sizeof(tmp), header_csum);
}
if (memcmp(header_csum, header_.header_checksum, sizeof(header_csum)) != 0) {
LOG(ERROR) << "header checksum is invalid";
auto footer_pos = lseek(fd_.get(), -header_.footer_size, SEEK_END);
if (footer_pos != fd_size_ - header_.footer_size) {
LOG(ERROR) << "Failed to read full footer!";
return false;
}
if (!android::base::ReadFully(fd_, &footer_, sizeof(footer_))) {
PLOG(ERROR) << "read footer failed";
return false;
}
has_footer_ = (footer_.op.type == kCowFooterOp);
return ParseOps();
}
bool CowReader::ParseOps() {
uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET);
if (pos != sizeof(header_)) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
uint64_t next_last_label = 0;
bool has_next = false;
auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
if (has_footer_) ops_buffer->reserve(footer_.op.num_ops);
uint64_t current_op_num = 0;
// Look until we reach the last possible non-footer position.
uint64_t last_pos = fd_size_ - (has_footer_ ? sizeof(footer_) : sizeof(CowOperation));
// Alternating op and data
while (pos < last_pos) {
ops_buffer->resize(current_op_num + 1);
if (!android::base::ReadFully(fd_, ops_buffer->data() + current_op_num,
sizeof(CowOperation))) {
PLOG(ERROR) << "read op failed";
return false;
}
auto& current_op = ops_buffer->data()[current_op_num];
pos = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);
if (pos < 0) {
PLOG(ERROR) << "lseek next op failed";
return false;
}
current_op_num++;
if (current_op.type == kCowLabelOp) {
// If we don't have a footer, the last label may be incomplete
if (has_footer_) {
has_last_label_ = true;
last_label_ = current_op.source;
} else {
last_label_ = next_last_label;
if (has_next) has_last_label_ = true;
next_last_label = current_op.source;
has_next = true;
}
}
}
uint8_t csum[32];
memset(csum, 0, sizeof(uint8_t) * 32);
if (has_footer_) {
SHA256(ops_buffer.get()->data(), footer_.op.ops_size, csum);
if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return false;
}
} else {
LOG(INFO) << "No Footer, recovered data";
}
ops_ = ops_buffer;
return true;
}
@ -115,74 +175,58 @@ bool CowReader::GetHeader(CowHeader* header) {
return true;
}
bool CowReader::GetFooter(CowFooter* footer) {
if (!has_footer_) return false;
*footer = footer_;
return true;
}
bool CowReader::GetLastLabel(uint64_t* label) {
if (!has_last_label_) return false;
*label = last_label_;
return true;
}
class CowOpIter final : public ICowOpIter {
public:
CowOpIter(std::unique_ptr<uint8_t[]>&& ops, size_t len);
CowOpIter(std::shared_ptr<std::vector<CowOperation>>& ops);
bool Done() override;
const CowOperation& Get() override;
void Next() override;
private:
bool HasNext();
std::unique_ptr<uint8_t[]> ops_;
const uint8_t* pos_;
const uint8_t* end_;
bool done_;
std::shared_ptr<std::vector<CowOperation>> ops_;
std::vector<CowOperation>::iterator op_iter_;
};
CowOpIter::CowOpIter(std::unique_ptr<uint8_t[]>&& ops, size_t len)
: ops_(std::move(ops)), pos_(ops_.get()), end_(pos_ + len), done_(!HasNext()) {}
bool CowOpIter::Done() {
return done_;
CowOpIter::CowOpIter(std::shared_ptr<std::vector<CowOperation>>& ops) {
ops_ = ops;
op_iter_ = ops_.get()->begin();
}
bool CowOpIter::HasNext() {
return pos_ < end_ && size_t(end_ - pos_) >= sizeof(CowOperation);
bool CowOpIter::Done() {
return op_iter_ == ops_.get()->end();
}
void CowOpIter::Next() {
CHECK(!Done());
pos_ += sizeof(CowOperation);
if (!HasNext()) done_ = true;
op_iter_++;
}
const CowOperation& CowOpIter::Get() {
CHECK(!Done());
CHECK(HasNext());
return *reinterpret_cast<const CowOperation*>(pos_);
return (*op_iter_);
}
std::unique_ptr<ICowOpIter> CowReader::GetOpIter() {
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return nullptr;
}
auto ops_buffer = std::make_unique<uint8_t[]>(header_.ops_size);
if (!android::base::ReadFully(fd_, ops_buffer.get(), header_.ops_size)) {
PLOG(ERROR) << "read ops failed";
return nullptr;
}
uint8_t csum[32];
memset(csum, 0, sizeof(uint8_t) * 32);
SHA256(ops_buffer.get(), header_.ops_size, csum);
if (memcmp(csum, header_.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return nullptr;
}
return std::make_unique<CowOpIter>(std::move(ops_buffer), header_.ops_size);
return std::make_unique<CowOpIter>(ops_);
}
bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
// Validate the offset, taking care to acknowledge possible overflow of offset+len.
if (offset < sizeof(header_) || offset >= header_.ops_offset || len >= fd_size_ ||
offset + len > header_.ops_offset) {
if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(footer_) || len >= fd_size_ ||
offset + len > fd_size_ - sizeof(footer_)) {
LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
return false;
}

2
fs_mgr/libsnapshot/cow_snapuserd_test.cpp
View file

@ -227,7 +227,7 @@ void SnapuserdTest::CreateCowDevice(std::unique_ptr<TemporaryFile>& cow) {
ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_2_.get(), size_));
// Flush operations
ASSERT_TRUE(writer.Flush());
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow->fd, 0, SEEK_SET), 0);
}

196
fs_mgr/libsnapshot/cow_writer.cpp
View file

@ -18,6 +18,7 @@
#include <unistd.h>
#include <limits>
#include <queue>
#include <android-base/file.h>
#include <android-base/logging.h>
@ -65,6 +66,10 @@ bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
return EmitZeroBlocks(new_block_start, num_blocks);
}
bool ICowWriter::AddLabel(uint64_t label) {
return EmitLabel(label);
}
bool ICowWriter::ValidateNewBlock(uint64_t new_block) {
if (options_.max_blocks && new_block >= options_.max_blocks.value()) {
LOG(ERROR) << "New block " << new_block << " exceeds maximum block count "
@ -84,7 +89,11 @@ void CowWriter::SetupHeaders() {
header_.major_version = kCowVersionMajor;
header_.minor_version = kCowVersionMinor;
header_.header_size = sizeof(CowHeader);
header_.footer_size = sizeof(CowFooter);
header_.block_size = options_.block_size;
footer_ = {};
footer_.op.data_length = 64;
footer_.op.type = kCowFooterOp;
}
bool CowWriter::ParseOptions() {
@ -124,6 +133,21 @@ bool CowWriter::Initialize(borrowed_fd fd, OpenMode mode) {
}
}
bool CowWriter::InitializeAppend(android::base::unique_fd&& fd, uint64_t label) {
owned_fd_ = std::move(fd);
return InitializeAppend(android::base::borrowed_fd{owned_fd_}, label);
}
bool CowWriter::InitializeAppend(android::base::borrowed_fd fd, uint64_t label) {
fd_ = fd;
if (!ParseOptions()) {
return false;
}
return OpenForAppend(label);
}
bool CowWriter::OpenForWrite() {
// This limitation is tied to the data field size in CowOperation.
if (header_.block_size > std::numeric_limits<uint16_t>::max()) {
@ -143,33 +167,99 @@ bool CowWriter::OpenForWrite() {
return false;
}
header_.ops_offset = header_.header_size;
next_op_pos_ = sizeof(header_);
return true;
}
bool CowWriter::OpenForAppend() {
auto reader = std::make_unique<CowReader>();
bool incomplete = false;
std::queue<CowOperation> toAdd;
if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
return false;
}
incomplete = !reader->GetFooter(&footer_);
options_.block_size = header_.block_size;
// Reset this, since we're going to reimport all operations.
header_.num_ops = 0;
footer_.op.num_ops = 0;
next_op_pos_ = sizeof(header_);
auto iter = reader->GetOpIter();
while (!iter->Done()) {
auto& op = iter->Get();
AddOperation(op);
CowOperation op = iter->Get();
if (op.type == kCowFooterOp) break;
if (incomplete) {
// Last operation translation may be corrupt. Wait to add it.
if (op.type == kCowLabelOp) {
while (!toAdd.empty()) {
AddOperation(toAdd.front());
toAdd.pop();
}
}
toAdd.push(op);
} else {
AddOperation(op);
}
iter->Next();
}
// Free reader so we own the descriptor position again.
reader = nullptr;
// Seek to the end of the data section.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
// Position for new writing
if (ftruncate(fd_.get(), next_op_pos_) != 0) {
PLOG(ERROR) << "Failed to trim file";
return false;
}
if (lseek(fd_.get(), 0, SEEK_END) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
return true;
}
bool CowWriter::OpenForAppend(uint64_t label) {
auto reader = std::make_unique<CowReader>();
std::queue<CowOperation> toAdd;
if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
return false;
}
options_.block_size = header_.block_size;
bool found_label = false;
// Reset this, since we're going to reimport all operations.
footer_.op.num_ops = 0;
next_op_pos_ = sizeof(header_);
auto iter = reader->GetOpIter();
while (!iter->Done()) {
CowOperation op = iter->Get();
if (op.type == kCowFooterOp) break;
if (op.type == kCowLabelOp) {
if (found_label) break;
if (op.source == label) found_label = true;
}
AddOperation(op);
iter->Next();
}
if (!found_label) {
PLOG(ERROR) << "Failed to find last label";
return false;
}
// Free reader so we own the descriptor position again.
reader = nullptr;
// Position for new writing
if (ftruncate(fd_.get(), next_op_pos_) != 0) {
PLOG(ERROR) << "Failed to trim file";
return false;
}
if (lseek(fd_.get(), 0, SEEK_END) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
@ -181,22 +271,18 @@ bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block) {
op.type = kCowCopyOp;
op.new_block = new_block;
op.source = old_block;
AddOperation(op);
return true;
return WriteOperation(op);
}
bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
uint64_t pos;
if (!GetDataPos(&pos)) {
return false;
}
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
uint64_t pos;
for (size_t i = 0; i < size / header_.block_size; i++) {
CowOperation op = {};
op.type = kCowReplaceOp;
op.new_block = new_block_start + i;
op.source = pos;
GetDataPos(&pos);
op.source = pos + sizeof(op);
if (compression_) {
auto data = Compress(iter, header_.block_size);
@ -208,26 +294,23 @@ bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t
LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes";
return false;
}
if (!WriteRawData(data.data(), data.size())) {
op.compression = compression_;
op.data_length = static_cast<uint16_t>(data.size());
if (!WriteOperation(op, data.data(), data.size())) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
op.compression = compression_;
op.data_length = static_cast<uint16_t>(data.size());
pos += data.size();
} else {
op.data_length = static_cast<uint16_t>(header_.block_size);
pos += header_.block_size;
if (!WriteOperation(op, iter, header_.block_size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
}
AddOperation(op);
iter += header_.block_size;
}
if (!compression_ && !WriteRawData(data, size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
return true;
}
@ -237,11 +320,18 @@ bool CowWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
op.type = kCowZeroOp;
op.new_block = new_block_start + i;
op.source = 0;
AddOperation(op);
WriteOperation(op);
}
return true;
}
bool CowWriter::EmitLabel(uint64_t label) {
CowOperation op = {};
op.type = kCowLabelOp;
op.source = label;
return WriteOperation(op);
}
std::basic_string<uint8_t> CowWriter::Compress(const void* data, size_t length) {
switch (compression_) {
case kCowCompressGz: {
@ -293,34 +383,28 @@ static void SHA256(const void*, size_t, uint8_t[]) {
#endif
}
bool CowWriter::Flush() {
header_.ops_size = ops_.size();
bool CowWriter::Finalize() {
footer_.op.ops_size = ops_.size() + sizeof(footer_.op);
uint64_t pos;
memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32);
memset(header_.header_checksum, 0, sizeof(uint8_t) * 32);
SHA256(ops_.data(), ops_.size(), header_.ops_checksum);
SHA256(&header_, sizeof(header_), header_.header_checksum);
if (lseek(fd_.get(), 0, SEEK_SET)) {
PLOG(ERROR) << "lseek failed";
if (!GetDataPos(&pos)) {
PLOG(ERROR) << "failed to get file position";
return false;
}
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write header failed";
return false;
}
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
if (!WriteFully(fd_, ops_.data(), ops_.size())) {
PLOG(ERROR) << "write ops failed";
memset(&footer_.data.ops_checksum, 0, sizeof(uint8_t) * 32);
memset(&footer_.data.footer_checksum, 0, sizeof(uint8_t) * 32);
SHA256(ops_.data(), ops_.size(), footer_.data.ops_checksum);
SHA256(&footer_.op, sizeof(footer_.op), footer_.data.footer_checksum);
// Write out footer at end of file
if (!android::base::WriteFully(fd_, reinterpret_cast<const uint8_t*>(&footer_),
sizeof(footer_))) {
PLOG(ERROR) << "write footer failed";
return false;
}
// Re-position for any subsequent writes.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
if (lseek(fd_.get(), pos, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
@ -328,7 +412,7 @@ bool CowWriter::Flush() {
}
uint64_t CowWriter::GetCowSize() {
return header_.ops_offset + header_.num_ops * sizeof(CowOperation);
return next_op_pos_ + sizeof(footer_);
}
bool CowWriter::GetDataPos(uint64_t* pos) {
@ -341,8 +425,19 @@ bool CowWriter::GetDataPos(uint64_t* pos) {
return true;
}
bool CowWriter::WriteOperation(const CowOperation& op, const void* data, size_t size) {
if (!android::base::WriteFully(fd_, reinterpret_cast<const uint8_t*>(&op), sizeof(op))) {
return false;
}
if (data != NULL && size > 0)
if (!WriteRawData(data, size)) return false;
AddOperation(op);
return !fsync(fd_.get());
}
void CowWriter::AddOperation(const CowOperation& op) {
header_.num_ops++;
footer_.op.num_ops++;
next_op_pos_ += sizeof(CowOperation) + GetNextOpOffset(op);
ops_.insert(ops_.size(), reinterpret_cast<const uint8_t*>(&op), sizeof(op));
}
@ -350,7 +445,6 @@ bool CowWriter::WriteRawData(const void* data, size_t size) {
if (!android::base::WriteFully(fd_, data, size)) {
return false;
}
header_.ops_offset += size;
return true;
}

2
fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp
View file

@ -375,7 +375,7 @@ bool NonAbEstimator::AnalyzePartition(const std::string& partition_name) {
}
}
if (!writer->Flush()) {
if (!writer->Finalize()) {
return false;
}

70
fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
View file

@ -15,6 +15,7 @@
#pragma once
#include <stdint.h>
#include <string>
namespace android {
namespace snapshot {
@ -29,17 +30,22 @@ static constexpr uint32_t kCowVersionMinor = 0;
// +-----------------------+
// | Header (fixed) |
// +-----------------------+
// | Raw Data (variable) |
// | Operation (variable) |
// | Data (variable) |
// +-----------------------+
// | Operations (variable) |
// | Footer (fixed) |
// +-----------------------+
//
// The "raw data" occurs immediately after the header, and the operation
// sequence occurs after the raw data. This ordering is intentional. While
// streaming an OTA, we can immediately write compressed data, but store the
// metadata in memory. At the end, we can simply append the metadata and flush
// the file. There is no need to create separate files to store the metadata
// and block data.
// The operations begin immediately after the header, and the "raw data"
// immediately follows the operation which refers to it. While streaming
// an OTA, we can immediately write the op and data, syncing after each pair,
// while storing operation metadata in memory. At the end, we compute data and
// hashes for the footer, which is placed at the tail end of the file.
//
// A missing or corrupt footer likely indicates that writing was cut off
// between writing the last operation/data pair, or the footer itself. In this
// case, the safest way to proceed is to assume the last operation is faulty.
struct CowHeader {
uint64_t magic;
uint16_t major_version;
@ -48,18 +54,35 @@ struct CowHeader {
// Size of this struct.
uint16_t header_size;
// Offset to the location of the operation sequence, and size of the
// operation sequence buffer. |ops_offset| is also the end of the
// raw data region.
uint64_t ops_offset;
uint64_t ops_size;
uint64_t num_ops;
// Size of footer struct
uint16_t footer_size;
// The size of block operations, in bytes.
uint32_t block_size;
} __attribute__((packed));
// SHA256 checksums of this header, with this field set to 0.
uint8_t header_checksum[32];
// This structure is the same size of a normal Operation, but is repurposed for the footer.
struct CowFooterOperation {
// The operation code (always kCowFooterOp).
uint8_t type;
// If this operation reads from the data section of the COW, this contains
// the compression type of that data (see constants below).
uint8_t compression;
// Length of Footer Data. Currently 64 for both checksums
uint16_t data_length;
// The amount of file space used by Cow operations
uint64_t ops_size;
// The number of cow operations in the file
uint64_t num_ops;
} __attribute__((packed));
struct CowFooterData {
// SHA256 checksums of Footer op
uint8_t footer_checksum[32];
// SHA256 of the operation sequence.
uint8_t ops_checksum[32];
@ -92,16 +115,31 @@ struct CowOperation {
//
// For zero operations (replace with all zeroes), this is unused and must
// be zero.
//
// For Label operations, this is the value of the applied label.
uint64_t source;
} __attribute__((packed));
static_assert(sizeof(CowOperation) == sizeof(CowFooterOperation));
static constexpr uint8_t kCowCopyOp = 1;
static constexpr uint8_t kCowReplaceOp = 2;
static constexpr uint8_t kCowZeroOp = 3;
static constexpr uint8_t kCowLabelOp = 4;
static constexpr uint8_t kCowFooterOp = -1;
static constexpr uint8_t kCowCompressNone = 0;
static constexpr uint8_t kCowCompressGz = 1;
static constexpr uint8_t kCowCompressBrotli = 2;
struct CowFooter {
CowFooterOperation op;
CowFooterData data;
} __attribute__((packed));
std::ostream& operator<<(std::ostream& os, CowOperation const& arg);
int64_t GetNextOpOffset(const CowOperation& op);
} // namespace snapshot
} // namespace android

20
fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h
View file

@ -58,6 +58,12 @@ class ICowReader {
// Return the file header.
virtual bool GetHeader(CowHeader* header) = 0;
// Return the file footer.
virtual bool GetFooter(CowFooter* footer) = 0;
// Return the last valid label
virtual bool GetLastLabel(uint64_t* label) = 0;
// Return an iterator for retrieving CowOperation entries.
virtual std::unique_ptr<ICowOpIter> GetOpIter() = 0;
@ -89,20 +95,30 @@ class CowReader : public ICowReader {
bool Parse(android::base::borrowed_fd fd);
bool GetHeader(CowHeader* header) override;
bool GetFooter(CowFooter* footer) override;
// Create a CowOpIter object which contains header_.num_ops
bool GetLastLabel(uint64_t* label) override;
// Create a CowOpIter object which contains footer_.num_ops
// CowOperation objects. Get() returns a unique CowOperation object
// whose lifeteime depends on the CowOpIter object
// whose lifetime depends on the CowOpIter object
std::unique_ptr<ICowOpIter> GetOpIter() override;
bool ReadData(const CowOperation& op, IByteSink* sink) override;
bool GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read);
private:
bool ParseOps();
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_;
CowFooter footer_;
uint64_t fd_size_;
bool has_footer_;
uint64_t last_label_;
bool has_last_label_;
std::shared_ptr<std::vector<CowOperation>> ops_;
};
} // namespace snapshot

24
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
View file

@ -16,11 +16,13 @@
#include <stdint.h>
#include <memory>
#include <optional>
#include <string>
#include <android-base/unique_fd.h>
#include <libsnapshot/cow_format.h>
#include <libsnapshot/cow_reader.h>
namespace android {
namespace snapshot {
@ -51,9 +53,12 @@ class ICowWriter {
// Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.
bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks);
// Add a label to the op sequence.
bool AddLabel(uint64_t label);
// Flush all pending writes. This must be called before closing the writer
// to ensure that the correct headers and footers are written.
virtual bool Flush() = 0;
virtual bool Finalize() = 0;
// Return number of bytes the cow image occupies on disk.
virtual uint64_t GetCowSize() = 0;
@ -67,6 +72,7 @@ class ICowWriter {
virtual bool EmitCopy(uint64_t new_block, uint64_t old_block) = 0;
virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;
virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
virtual bool EmitLabel(uint64_t label) = 0;
bool ValidateNewBlock(uint64_t new_block);
@ -81,10 +87,18 @@ class CowWriter : public ICowWriter {
explicit CowWriter(const CowOptions& options);
// Set up the writer.
// If opening for write, the file starts from the beginning.
// If opening for append, if the file has a footer, we start appending to the last op.
// If the footer isn't found, the last label is considered corrupt, and dropped.
bool Initialize(android::base::unique_fd&& fd, OpenMode mode = OpenMode::WRITE);
bool Initialize(android::base::borrowed_fd fd, OpenMode mode = OpenMode::WRITE);
// Set up a writer, assuming that the given label is the last valid label.
// This will result in dropping any labels that occur after the given on, and will fail
// if the given label does not appear.
bool InitializeAppend(android::base::unique_fd&&, uint64_t label);
bool InitializeAppend(android::base::borrowed_fd fd, uint64_t label);
bool Flush() override;
bool Finalize() override;
uint64_t GetCowSize() override;
@ -92,14 +106,18 @@ class CowWriter : public ICowWriter {
virtual bool EmitCopy(uint64_t new_block, uint64_t old_block) override;
virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
virtual bool EmitLabel(uint64_t label) override;
private:
void SetupHeaders();
bool ParseOptions();
bool OpenForWrite();
bool OpenForAppend();
bool OpenForAppend(uint64_t label);
bool ImportOps(std::unique_ptr<ICowOpIter> iter);
bool GetDataPos(uint64_t* pos);
bool WriteRawData(const void* data, size_t size);
bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0);
void AddOperation(const CowOperation& op);
std::basic_string<uint8_t> Compress(const void* data, size_t length);
@ -107,7 +125,9 @@ class CowWriter : public ICowWriter {
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_{};
CowFooter footer_{};
int compression_ = 0;
uint64_t next_op_pos_ = 0;
// :TODO: this is not efficient, but stringstream ubsan aborts because some
// bytes overflow a signed char.

6
fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h
View file

@ -55,7 +55,7 @@ class CompressedSnapshotWriter : public ISnapshotWriter {
// Sets the COW device, if needed.
bool SetCowDevice(android::base::unique_fd&& cow_device);
bool Flush() override;
bool Finalize() override;
uint64_t GetCowSize() override;
std::unique_ptr<FileDescriptor> OpenReader() override;
@ -63,6 +63,7 @@ class CompressedSnapshotWriter : public ISnapshotWriter {
bool EmitCopy(uint64_t new_block, uint64_t old_block) override;
bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
bool EmitLabel(uint64_t label) override;
private:
android::base::unique_fd cow_device_;
@ -78,7 +79,7 @@ class OnlineKernelSnapshotWriter : public ISnapshotWriter {
// Set the device used for all writes.
void SetSnapshotDevice(android::base::unique_fd&& snapshot_fd, uint64_t cow_size);
bool Flush() override;
bool Finalize() override;
uint64_t GetCowSize() override { return cow_size_; }
std::unique_ptr<FileDescriptor> OpenReader() override;
@ -86,6 +87,7 @@ class OnlineKernelSnapshotWriter : public ISnapshotWriter {
bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
bool EmitCopy(uint64_t new_block, uint64_t old_block) override;
bool EmitLabel(uint64_t label) override;
private:
android::base::unique_fd snapshot_fd_;

2
fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp
View file

@ -204,7 +204,7 @@ bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) {
}
}
if (!writer_->Flush()) {
if (!writer_->Finalize()) {
LOG(ERROR) << "Unable to finalize COW for " << partition_name;
return false;
}

15
fs_mgr/libsnapshot/snapshot_writer.cpp
View file

@ -59,8 +59,8 @@ bool CompressedSnapshotWriter::SetCowDevice(android::base::unique_fd&& cow_devic
return cow_->Initialize(cow_device_);
}
bool CompressedSnapshotWriter::Flush() {
return cow_->Flush();
bool CompressedSnapshotWriter::Finalize() {
return cow_->Finalize();
}
uint64_t CompressedSnapshotWriter::GetCowSize() {
@ -84,6 +84,10 @@ bool CompressedSnapshotWriter::EmitZeroBlocks(uint64_t new_block_start, uint64_t
return cow_->AddZeroBlocks(new_block_start, num_blocks);
}
bool CompressedSnapshotWriter::EmitLabel(uint64_t label) {
return cow_->AddLabel(label);
}
OnlineKernelSnapshotWriter::OnlineKernelSnapshotWriter(const CowOptions& options)
: ISnapshotWriter(options) {}
@ -93,7 +97,7 @@ void OnlineKernelSnapshotWriter::SetSnapshotDevice(android::base::unique_fd&& sn
cow_size_ = cow_size;
}
bool OnlineKernelSnapshotWriter::Flush() {
bool OnlineKernelSnapshotWriter::Finalize() {
if (fsync(snapshot_fd_.get()) < 0) {
PLOG(ERROR) << "fsync";
return false;
@ -140,6 +144,11 @@ bool OnlineKernelSnapshotWriter::EmitCopy(uint64_t new_block, uint64_t old_block
return EmitRawBlocks(new_block, buffer.data(), buffer.size());
}
bool OnlineKernelSnapshotWriter::EmitLabel(uint64_t) {
// Not Needed
return true;
}
std::unique_ptr<FileDescriptor> OnlineKernelSnapshotWriter::OpenReader() {
unique_fd fd(dup(snapshot_fd_.get()));
if (fd < 0) {

12
fs_mgr/libsnapshot/snapuserd.cpp
View file

@ -326,6 +326,7 @@ int Snapuserd::ReadDiskExceptions(chunk_t chunk, size_t read_size) {
int Snapuserd::ReadMetadata() {
reader_ = std::make_unique<CowReader>();
CowHeader header;
CowFooter footer;
if (!reader_->Parse(cow_fd_)) {
LOG(ERROR) << "Failed to parse";
@ -337,11 +338,15 @@ int Snapuserd::ReadMetadata() {
return 1;
}
if (!reader_->GetFooter(&footer)) {
LOG(ERROR) << "Failed to get footer";
return 1;
}
CHECK(header.block_size == BLOCK_SIZE);
LOG(DEBUG) << "Num-ops: " << std::hex << header.num_ops;
LOG(DEBUG) << "ops-offset: " << std::hex << header.ops_offset;
LOG(DEBUG) << "ops-size: " << std::hex << header.ops_size;
LOG(DEBUG) << "Num-ops: " << std::hex << footer.op.num_ops;
LOG(DEBUG) << "ops-size: " << std::hex << footer.op.ops_size;
cowop_iter_ = reader_->GetOpIter();
@ -373,6 +378,7 @@ int Snapuserd::ReadMetadata() {
struct disk_exception* de =
reinterpret_cast<struct disk_exception*>((char*)de_ptr.get() + offset);
if (cow_op->type == kCowFooterOp || cow_op->type == kCowLabelOp) continue;
if (!(cow_op->type == kCowReplaceOp || cow_op->type == kCowZeroOp ||
cow_op->type == kCowCopyOp)) {
LOG(ERROR) << "Unknown operation-type found: " << cow_op->type;