ACPR/android_system_core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Initial prototype of COW format and API."

Browse source
This commit is contained in:
David Anderson 2020-08-19 23:13:38 +00:00 • committed by Gerrit Code Review
commit 34dda54411
7 changed files with 1094 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
fs_mgr/libsnapshot/Android.bp
View file

@ -122,6 +122,39 @@ cc_library_static {
],
}
cc_defaults {
name: "libsnapshot_cow_defaults",
defaults: [
"fs_mgr_defaults",
],
cflags: [
"-D_FILE_OFFSET_BITS=64",
"-Wall",
"-Werror",
],
export_include_dirs: ["include"],
srcs: [
"cow_reader.cpp",
"cow_writer.cpp",
],
}
cc_library_static {
name: "libsnapshot_cow",
defaults: [
"libsnapshot_cow_defaults",
],
host_supported: true,
shared_libs: [
"libbase",
"libcrypto",
"liblog",
],
static_libs: [
"libz",
],
}
cc_library_static {
name: "libsnapshot_test_helpers",
defaults: ["libsnapshot_defaults"],
@ -343,3 +376,30 @@ cc_binary {
static_executable: true,
system_shared_libs: [],
}
cc_test {
name: "cow_api_test",
defaults: [
"fs_mgr_defaults",
],
srcs: [
"cow_api_test.cpp",
],
cflags: [
"-Wall",
"-Werror",
],
shared_libs: [
"libbase",
"libcrypto",
"liblog",
"libz",
],
static_libs: [
"libgtest",
"libsnapshot_cow",
],
test_min_api_level: 30,
auto_gen_config: true,
require_root: false,
}

244
fs_mgr/libsnapshot/cow_api_test.cpp Normal file
View file

@ -0,0 +1,244 @@
// Copyright (C) 2018 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <iostream>
#include <memory>
#include <string_view>
#include <android-base/file.h>
#include <gtest/gtest.h>
#include <libsnapshot/cow_reader.h>
#include <libsnapshot/cow_writer.h>
namespace android {
namespace snapshot {
class CowTest : public ::testing::Test {
protected:
void SetUp() override {
cow_ = std::make_unique<TemporaryFile>();
ASSERT_GE(cow_->fd, 0) << strerror(errno);
}
void TearDown() override { cow_ = nullptr; }
std::unique_ptr<TemporaryFile> cow_;
};
// Sink that always appends to the end of a string.
class StringSink : public IByteSink {
public:
void* GetBuffer(size_t requested, size_t* actual) override {
size_t old_size = stream_.size();
stream_.resize(old_size + requested, '\0');
*actual = requested;
return stream_.data() + old_size;
}
bool ReturnData(void*, size_t) override { return true; }
void Reset() { stream_.clear(); }
std::string& stream() { return stream_; }
private:
std::string stream_;
};
TEST_F(CowTest, ReadWrite) {
CowOptions options;
CowWriter writer(options);
ASSERT_TRUE(writer.Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddCopy(10, 20));
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
CowHeader header;
ASSERT_TRUE(reader.Parse(cow_->fd));
ASSERT_TRUE(reader.GetHeader(&header));
ASSERT_EQ(header.magic, kCowMagicNumber);
ASSERT_EQ(header.major_version, kCowVersionMajor);
ASSERT_EQ(header.minor_version, kCowVersionMinor);
ASSERT_EQ(header.block_size, options.block_size);
ASSERT_EQ(header.num_ops, 4);
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowCopyOp);
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 0);
ASSERT_EQ(op->new_block, 10);
ASSERT_EQ(op->source, 20);
StringSink sink;
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 104);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
// Note: the zero operation gets split into two blocks.
ASSERT_EQ(op->type, kCowZeroOp);
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 0);
ASSERT_EQ(op->new_block, 51);
ASSERT_EQ(op->source, 0);
iter->Next();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowZeroOp);
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 0);
ASSERT_EQ(op->new_block, 52);
ASSERT_EQ(op->source, 0);
iter->Next();
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, CompressGz) {
CowOptions options;
options.compression = "gz";
CowWriter writer(options);
ASSERT_TRUE(writer.Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
StringSink sink;
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 104);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
iter->Next();
ASSERT_TRUE(iter->Done());
}
TEST_F(CowTest, CompressTwoBlocks) {
CowOptions options;
options.compression = "gz";
CowWriter writer(options);
ASSERT_TRUE(writer.Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
iter->Next();
ASSERT_FALSE(iter->Done());
StringSink sink;
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_EQ(op->new_block, 51);
ASSERT_TRUE(reader.ReadData(*op, &sink));
}
// Only return 1-byte buffers, to stress test the partial read logic in
// CowReader.
class HorribleStringSink : public StringSink {
public:
void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); }
};
TEST_F(CowTest, HorribleSink) {
CowOptions options;
options.compression = "gz";
CowWriter writer(options);
ASSERT_TRUE(writer.Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
HorribleStringSink sink;
auto op = &iter->Get();
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
}
} // namespace snapshot
} // namespace android
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

264
fs_mgr/libsnapshot/cow_reader.cpp Normal file
View file

@ -0,0 +1,264 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <sys/types.h>
#include <unistd.h>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <libsnapshot/cow_reader.h>
#include <openssl/sha.h>
#include <zlib.h>
namespace android {
namespace snapshot {
CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}
static void SHA256(const void* data, size_t length, uint8_t out[32]) {
SHA256_CTX c;
SHA256_Init(&c);
SHA256_Update(&c, data, length);
SHA256_Final(out, &c);
}
bool CowReader::Parse(android::base::unique_fd&& fd) {
owned_fd_ = std::move(fd);
return Parse(android::base::borrowed_fd{owned_fd_});
}
bool CowReader::Parse(android::base::borrowed_fd fd) {
fd_ = fd;
auto pos = lseek(fd_.get(), 0, SEEK_END);
if (pos < 0) {
PLOG(ERROR) << "lseek end failed";
return false;
}
fd_size_ = pos;
if (lseek(fd_.get(), 0, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek header failed";
return false;
}
if (!android::base::ReadFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "read header failed";
return false;
}
// Validity check the ops range.
if (header_.ops_offset >= fd_size_) {
LOG(ERROR) << "ops offset " << header_.ops_offset << " larger than fd size " << fd_size_;
return false;
}
if (fd_size_ - header_.ops_offset < header_.ops_size) {
LOG(ERROR) << "ops size " << header_.ops_size << " is too large";
return false;
}
uint8_t header_csum[32];
{
CowHeader tmp = header_;
memset(&tmp.header_checksum, 0, sizeof(tmp.header_checksum));
SHA256(&tmp, sizeof(tmp), header_csum);
}
if (memcmp(header_csum, header_.header_checksum, sizeof(header_csum)) != 0) {
LOG(ERROR) << "header checksum is invalid";
return false;
}
return true;
}
bool CowReader::GetHeader(CowHeader* header) {
*header = header_;
return true;
}
class CowOpIter final : public ICowOpIter {
public:
CowOpIter(std::unique_ptr<uint8_t[]>&& ops, size_t len);
bool Done() override;
const CowOperation& Get() override;
void Next() override;
private:
bool HasNext();
std::unique_ptr<uint8_t[]> ops_;
const uint8_t* pos_;
const uint8_t* end_;
bool done_;
};
CowOpIter::CowOpIter(std::unique_ptr<uint8_t[]>&& ops, size_t len)
: ops_(std::move(ops)), pos_(ops_.get()), end_(pos_ + len), done_(!HasNext()) {}
bool CowOpIter::Done() {
return done_;
}
bool CowOpIter::HasNext() {
return pos_ < end_ && size_t(end_ - pos_) >= sizeof(CowOperation);
}
void CowOpIter::Next() {
CHECK(!Done());
pos_ += sizeof(CowOperation);
if (!HasNext()) done_ = true;
}
const CowOperation& CowOpIter::Get() {
CHECK(!Done());
CHECK(HasNext());
return *reinterpret_cast<const CowOperation*>(pos_);
}
std::unique_ptr<ICowOpIter> CowReader::GetOpIter() {
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return nullptr;
}
auto ops_buffer = std::make_unique<uint8_t[]>(header_.ops_size);
if (!android::base::ReadFully(fd_, ops_buffer.get(), header_.ops_size)) {
PLOG(ERROR) << "read ops failed";
return nullptr;
}
uint8_t csum[32];
SHA256(ops_buffer.get(), header_.ops_size, csum);
if (memcmp(csum, header_.ops_checksum, sizeof(csum)) != 0) {
LOG(ERROR) << "ops checksum does not match";
return nullptr;
}
return std::make_unique<CowOpIter>(std::move(ops_buffer), header_.ops_size);
}
bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len) {
// Validate the offset, taking care to acknowledge possible overflow of offset+len.
if (offset < sizeof(header_) || offset >= header_.ops_offset || len >= fd_size_ ||
offset + len > header_.ops_offset) {
LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
return false;
}
if (lseek(fd_.get(), offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek to read raw bytes failed";
return false;
}
if (!android::base::ReadFully(fd_, buffer, len)) {
PLOG(ERROR) << "read raw bytes failed";
return false;
}
return true;
}
bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) {
uint64_t offset = op.source;
switch (op.compression) {
case kCowCompressNone: {
size_t remaining = op.data_length;
while (remaining) {
size_t amount = remaining;
void* buffer = sink->GetBuffer(amount, &amount);
if (!buffer) {
LOG(ERROR) << "Could not acquire buffer from sink";
return false;
}
if (!GetRawBytes(offset, buffer, amount)) {
return false;
}
if (!sink->ReturnData(buffer, amount)) {
LOG(ERROR) << "Could not return buffer to sink";
return false;
}
remaining -= amount;
offset += amount;
}
return true;
}
case kCowCompressGz: {
auto input = std::make_unique<Bytef[]>(op.data_length);
if (!GetRawBytes(offset, input.get(), op.data_length)) {
return false;
}
z_stream z = {};
z.next_in = input.get();
z.avail_in = op.data_length;
if (int rv = inflateInit(&z); rv != Z_OK) {
LOG(ERROR) << "inflateInit returned error code " << rv;
return false;
}
while (z.total_out < header_.block_size) {
// If no more output buffer, grab a new buffer.
if (z.avail_out == 0) {
size_t amount = header_.block_size - z.total_out;
z.next_out = reinterpret_cast<Bytef*>(sink->GetBuffer(amount, &amount));
if (!z.next_out) {
LOG(ERROR) << "Could not acquire buffer from sink";
return false;
}
z.avail_out = amount;
}
// Remember the position of the output buffer so we can call ReturnData.
auto buffer = z.next_out;
auto avail_out = z.avail_out;
// Decompress.
int rv = inflate(&z, Z_NO_FLUSH);
if (rv != Z_OK && rv != Z_STREAM_END) {
LOG(ERROR) << "inflate returned error code " << rv;
return false;
}
// Return the section of the buffer that was updated.
if (z.avail_out < avail_out && !sink->ReturnData(buffer, avail_out - z.avail_out)) {
LOG(ERROR) << "Could not return buffer to sink";
return false;
}
if (rv == Z_STREAM_END) {
// Error if the stream has ended, but we didn't fill the entire block.
if (z.total_out != header_.block_size) {
LOG(ERROR) << "Reached gz stream end but did not read a full block of data";
return false;
}
break;
}
CHECK(rv == Z_OK);
// Error if the stream is expecting more data, but we don't have any to read.
if (z.avail_in == 0) {
LOG(ERROR) << "Gz stream ended prematurely";
return false;
}
}
return true;
}
default:
LOG(ERROR) << "Unknown compression type: " << op.compression;
return false;
}
}
} // namespace snapshot
} // namespace android

230
fs_mgr/libsnapshot/cow_writer.cpp Normal file
View file

@ -0,0 +1,230 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <sys/types.h>
#include <unistd.h>
#include <limits>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <libsnapshot/cow_writer.h>
#include <openssl/sha.h>
#include <zlib.h>
namespace android {
namespace snapshot {
static_assert(sizeof(off_t) == sizeof(uint64_t));
CowWriter::CowWriter(const CowOptions& options) : ICowWriter(options), fd_(-1) {
SetupHeaders();
}
void CowWriter::SetupHeaders() {
header_ = {};
header_.magic = kCowMagicNumber;
header_.major_version = kCowVersionMajor;
header_.minor_version = kCowVersionMinor;
header_.block_size = options_.block_size;
}
bool CowWriter::Initialize(android::base::unique_fd&& fd) {
owned_fd_ = std::move(fd);
return Initialize(android::base::borrowed_fd{owned_fd_});
}
bool CowWriter::Initialize(android::base::borrowed_fd fd) {
fd_ = fd;
// This limitation is tied to the data field size in CowOperation.
if (header_.block_size > std::numeric_limits<uint16_t>::max()) {
LOG(ERROR) << "Block size is too large";
return false;
}
if (lseek(fd_.get(), 0, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
if (options_.compression == "gz") {
compression_ = kCowCompressGz;
} else if (!options_.compression.empty()) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
return false;
}
// Headers are not complete, but this ensures the file is at the right
// position.
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write failed";
return false;
}
return true;
}
bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowCopyOp;
op.new_block = new_block;
op.source = old_block;
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
return true;
}
bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
if (size % header_.block_size != 0) {
LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of "
<< header_.block_size;
return false;
}
uint64_t pos;
if (!GetDataPos(&pos)) {
return false;
}
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
for (size_t i = 0; i < size / header_.block_size; i++) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowReplaceOp;
op.new_block = new_block_start + i;
op.source = pos;
if (compression_) {
auto data = Compress(iter, header_.block_size);
if (data.empty()) {
PLOG(ERROR) << "AddRawBlocks: compression failed";
return false;
}
if (data.size() > std::numeric_limits<uint16_t>::max()) {
LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes";
return false;
}
if (!android::base::WriteFully(fd_, data.data(), data.size())) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
op.compression = compression_;
op.data_length = static_cast<uint16_t>(data.size());
pos += data.size();
} else {
op.data_length = static_cast<uint16_t>(header_.block_size);
pos += header_.block_size;
}
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
iter += header_.block_size;
}
if (!compression_ && !android::base::WriteFully(fd_, data, size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
return true;
}
bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
for (uint64_t i = 0; i < num_blocks; i++) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowZeroOp;
op.new_block = new_block_start + i;
op.source = 0;
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
}
return true;
}
std::basic_string<uint8_t> CowWriter::Compress(const void* data, size_t length) {
switch (compression_) {
case kCowCompressGz: {
auto bound = compressBound(length);
auto buffer = std::make_unique<uint8_t[]>(bound);
uLongf dest_len = bound;
auto rv = compress2(buffer.get(), &dest_len, reinterpret_cast<const Bytef*>(data),
length, Z_BEST_COMPRESSION);
if (rv != Z_OK) {
LOG(ERROR) << "compress2 returned: " << rv;
return {};
}
return std::basic_string<uint8_t>(buffer.get(), dest_len);
}
default:
LOG(ERROR) << "unhandled compression type: " << compression_;
break;
}
return {};
}
static void SHA256(const void* data, size_t length, uint8_t out[32]) {
SHA256_CTX c;
SHA256_Init(&c);
SHA256_Update(&c, data, length);
SHA256_Final(out, &c);
}
bool CowWriter::Finalize() {
auto offs = lseek(fd_.get(), 0, SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
header_.ops_offset = offs;
header_.ops_size = ops_.size();
SHA256(ops_.data(), ops_.size(), header_.ops_checksum);
SHA256(&header_, sizeof(header_), header_.header_checksum);
if (lseek(fd_.get(), 0, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek start failed";
return false;
}
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write header failed";
return false;
}
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
if (!android::base::WriteFully(fd_, ops_.data(), ops_.size())) {
PLOG(ERROR) << "write ops failed";
return false;
}
return true;
}
bool CowWriter::GetDataPos(uint64_t* pos) {
off_t offs = lseek(fd_.get(), 0, SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
*pos = offs;
return true;
}
} // namespace snapshot
} // namespace android

103
fs_mgr/libsnapshot/include/libsnapshot/cow_format.h Normal file
View file

@ -0,0 +1,103 @@
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
namespace android {
namespace snapshot {
static constexpr uint64_t kCowMagicNumber = 0x436f77634f572121ULL;
static constexpr uint32_t kCowVersionMajor = 1;
static constexpr uint32_t kCowVersionMinor = 0;
// This header appears as the first sequence of bytes in the COW. All fields
// in the layout are little-endian encoded. The on-disk layout is:
//
// +-----------------------+
// | Header (fixed) |
// +-----------------------+
// | Raw Data (variable) |
// +-----------------------+
// | Operations (variable) |
// +-----------------------+
//
// The "raw data" occurs immediately after the header, and the operation
// sequence occurs after the raw data. This ordering is intentional. While
// streaming an OTA, we can immediately write compressed data, but store the
// metadata in memory. At the end, we can simply append the metadata and flush
// the file. There is no need to create separate files to store the metadata
// and block data.
struct CowHeader {
uint64_t magic;
uint16_t major_version;
uint16_t minor_version;
// Offset to the location of the operation sequence, and size of the
// operation sequence buffer. |ops_offset| is also the end of the
// raw data region.
uint64_t ops_offset;
uint64_t ops_size;
uint64_t num_ops;
// The size of block operations, in bytes.
uint32_t block_size;
// SHA256 checksums of this header, with this field set to 0.
uint8_t header_checksum[32];
// SHA256 of the operation sequence.
uint8_t ops_checksum[32];
} __attribute__((packed));
// Cow operations are currently fixed-size entries, but this may change if
// needed.
struct CowOperation {
// The operation code (see the constants and structures below).
uint8_t type;
// If this operation reads from the data section of the COW, this contains
// the compression type of that data (see constants below).
uint8_t compression;
// If this operation reads from the data section of the COW, this contains
// the length.
uint16_t data_length;
// The block of data in the new image that this operation modifies.
uint64_t new_block;
// The value of |source| depends on the operation code.
//
// For copy operations, this is a block location in the source image.
//
// For replace operations, this is a byte offset within the COW's data
// section (eg, not landing within the header or metadata). It is an
// absolute position within the image.
//
// For zero operations (replace with all zeroes), this is unused and must
// be zero.
uint64_t source;
} __attribute__((packed));
static constexpr uint8_t kCowCopyOp = 1;
static constexpr uint8_t kCowReplaceOp = 2;
static constexpr uint8_t kCowZeroOp = 3;
static constexpr uint8_t kCowCompressNone = 0;
static constexpr uint8_t kCowCompressGz = 1;
} // namespace snapshot
} // namespace android

107
fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h Normal file
View file

@ -0,0 +1,107 @@
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <functional>
#include <memory>
#include <android-base/unique_fd.h>
#include <libsnapshot/cow_format.h>
namespace android {
namespace snapshot {
class ICowOpIter;
// A ByteSink object handles requests for a buffer of a specific size. It
// always owns the underlying buffer. It's designed to minimize potential
// copying as we parse or decompress the COW.
class IByteSink {
public:
virtual ~IByteSink() {}
// Called when the reader has data. The size of the request is given. The
// sink must return a valid pointer (or null on failure), and return the
// maximum number of bytes that can be written to the returned buffer.
//
// The returned buffer is owned by IByteSink, but must remain valid until
// the ready operation has completed (or the entire buffer has been
// covered by calls to ReturnData).
//
// After calling GetBuffer(), all previous buffers returned are no longer
// valid.
virtual void* GetBuffer(size_t requested, size_t* actual) = 0;
// Called when a section returned by |GetBuffer| has been filled with data.
virtual bool ReturnData(void* buffer, size_t length) = 0;
};
// Interface for reading from a snapuserd COW.
class ICowReader {
public:
virtual ~ICowReader() {}
// Return the file header.
virtual bool GetHeader(CowHeader* header) = 0;
// Return an iterator for retrieving CowOperation entries.
virtual std::unique_ptr<ICowOpIter> GetOpIter() = 0;
// Get raw bytes from the data section.
virtual bool GetRawBytes(uint64_t offset, void* buffer, size_t len) = 0;
// Get decoded bytes from the data section, handling any decompression.
// All retrieved data is passed to the sink.
virtual bool ReadData(const CowOperation& op, IByteSink* sink) = 0;
};
// Iterate over a sequence of COW operations.
class ICowOpIter {
public:
virtual ~ICowOpIter() {}
// True if there are more items to read, false otherwise.
virtual bool Done() = 0;
// Read the current operation.
virtual const CowOperation& Get() = 0;
// Advance to the next item.
virtual void Next() = 0;
};
class CowReader : public ICowReader {
public:
CowReader();
bool Parse(android::base::unique_fd&& fd);
bool Parse(android::base::borrowed_fd fd);
bool GetHeader(CowHeader* header) override;
std::unique_ptr<ICowOpIter> GetOpIter() override;
bool GetRawBytes(uint64_t offset, void* buffer, size_t len) override;
bool ReadData(const CowOperation& op, IByteSink* sink) override;
private:
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_;
uint64_t fd_size_;
};
} // namespace snapshot
} // namespace android

86
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h Normal file
View file

@ -0,0 +1,86 @@
// Copyright (C) 2019 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <stdint.h>
#include <string>
#include <android-base/unique_fd.h>
#include <libsnapshot/cow_format.h>
namespace android {
namespace snapshot {
struct CowOptions {
uint32_t block_size = 4096;
std::string compression;
};
// Interface for writing to a snapuserd COW. All operations are ordered; merges
// will occur in the sequence they were added to the COW.
class ICowWriter {
public:
explicit ICowWriter(const CowOptions& options) : options_(options) {}
virtual ~ICowWriter() {}
// Encode an operation that copies the contents of |old_block| to the
// location of |new_block|.
virtual bool AddCopy(uint64_t new_block, uint64_t old_block) = 0;
// Encode a sequence of raw blocks. |size| must be a multiple of the block size.
virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;
// Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.
virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
protected:
CowOptions options_;
};
class CowWriter : public ICowWriter {
public:
explicit CowWriter(const CowOptions& options);
// Set up the writer.
bool Initialize(android::base::unique_fd&& fd);
bool Initialize(android::base::borrowed_fd fd);
bool AddCopy(uint64_t new_block, uint64_t old_block) override;
bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
// Finalize all COW operations and flush pending writes.
bool Finalize();
private:
void SetupHeaders();
bool GetDataPos(uint64_t* pos);
std::basic_string<uint8_t> Compress(const void* data, size_t length);
private:
android::base::unique_fd owned_fd_;
android::base::borrowed_fd fd_;
CowHeader header_;
int compression_ = 0;
// :TODO: this is not efficient, but stringstream ubsan aborts because some
// bytes overflow a signed char.
std::basic_string<uint8_t> ops_;
};
} // namespace snapshot
} // namespace android