ACPR/android_system_core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge changes from topic "vab-brotli"

Browse source 
* changes:
  libsnapshot: Add an append mode to CowWriter.
  libsnapshot: Add support for brotli compression.
This commit is contained in:
David Anderson 2020-09-17 19:32:23 +00:00 • committed by Gerrit Code Review
commit 975ea3217e
11 changed files with 270 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
fs_mgr/libsnapshot/Android.bp
View file

@ -152,6 +152,7 @@ cc_library_static {
"liblog",
],
static_libs: [
"libbrotli",
"libz",
],
ramdisk_available: true,
@ -362,10 +363,11 @@ cc_defaults {
static_libs: [
"libbase",
"libbrotli",
"liblog",
"libdm",
"libz",
"libsnapshot_cow",
"libz",
"libsnapshot_cow",
],
}
@ -403,12 +405,14 @@ cc_test {
"libz",
],
static_libs: [
"libbrotli",
"libgtest",
"libsnapshot_cow",
],
test_min_api_level: 30,
auto_gen_config: true,
require_root: false,
host_supported: true,
}
cc_binary {
@ -494,11 +498,12 @@ cc_test {
shared_libs: [
"libbase",
"liblog",
"libz",
],
static_libs: [
"libbrotli",
"libgtest",
"libsnapshot_cow",
"libz",
],
header_libs: [
"libstorage_literals_headers",

80
fs_mgr/libsnapshot/cow_api_test.cpp
View file

@ -30,12 +30,12 @@ namespace snapshot {
class CowTest : public ::testing::Test {
protected:
void SetUp() override {
virtual void SetUp() override {
cow_ = std::make_unique<TemporaryFile>();
ASSERT_GE(cow_->fd, 0) << strerror(errno);
}
void TearDown() override { cow_ = nullptr; }
virtual void TearDown() override { cow_ = nullptr; }
std::unique_ptr<TemporaryFile> cow_;
};
@ -70,7 +70,7 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_TRUE(writer.AddCopy(10, 20));
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -105,7 +105,7 @@ TEST_F(CowTest, ReadWrite) {
ASSERT_EQ(op->compression, kCowCompressNone);
ASSERT_EQ(op->data_length, 4096);
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 104);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -145,7 +145,7 @@ TEST_F(CowTest, CompressGz) {
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -163,7 +163,7 @@ TEST_F(CowTest, CompressGz) {
ASSERT_EQ(op->compression, kCowCompressGz);
ASSERT_EQ(op->data_length, 56); // compressed!
ASSERT_EQ(op->new_block, 50);
ASSERT_EQ(op->source, 104);
ASSERT_EQ(op->source, 106);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) {
data.resize(options.block_size * 2, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -211,9 +211,11 @@ class HorribleStringSink : public StringSink {
void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); }
};
TEST_F(CowTest, HorribleSink) {
class CompressionTest : public CowTest, public testing::WithParamInterface<const char*> {};
TEST_P(CompressionTest, HorribleSink) {
CowOptions options;
options.compression = "gz";
options.compression = GetParam();
CowWriter writer(options);
ASSERT_TRUE(writer.Initialize(cow_->fd));
@ -222,7 +224,7 @@ TEST_F(CowTest, HorribleSink) {
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@ -239,6 +241,8 @@ TEST_F(CowTest, HorribleSink) {
ASSERT_EQ(sink.stream(), data);
}
INSTANTIATE_TEST_SUITE_P(CowApi, CompressionTest, testing::Values("none", "gz", "brotli"));
TEST_F(CowTest, GetSize) {
CowOptions options;
CowWriter writer(options);
@ -255,7 +259,7 @@ TEST_F(CowTest, GetSize) {
ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
auto size_before = writer.GetCowSize();
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
auto size_after = writer.GetCowSize();
ASSERT_EQ(size_before, size_after);
struct stat buf;
@ -267,6 +271,60 @@ TEST_F(CowTest, GetSize) {
ASSERT_EQ(buf.st_size, writer.GetCowSize());
}
TEST_F(CowTest, Append) {
CowOptions options;
auto writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd));
std::string data = "This is some data, believe it";
data.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));
ASSERT_TRUE(writer->Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
writer = std::make_unique<CowWriter>(options);
ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));
std::string data2 = "More data!";
data2.resize(options.block_size, '\0');
ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));
ASSERT_TRUE(writer->Flush());
ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
struct stat buf;
ASSERT_EQ(fstat(cow_->fd, &buf), 0);
ASSERT_EQ(buf.st_size, writer->GetCowSize());
// Read back both operations.
CowReader reader;
ASSERT_TRUE(reader.Parse(cow_->fd));
StringSink sink;
auto iter = reader.GetOpIter();
ASSERT_NE(iter, nullptr);
ASSERT_FALSE(iter->Done());
auto op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data);
iter->Next();
sink.Reset();
ASSERT_FALSE(iter->Done());
op = &iter->Get();
ASSERT_EQ(op->type, kCowReplaceOp);
ASSERT_TRUE(reader.ReadData(*op, &sink));
ASSERT_EQ(sink.stream(), data2);
iter->Next();
ASSERT_TRUE(iter->Done());
}
} // namespace snapshot
} // namespace android

53
fs_mgr/libsnapshot/cow_decompress.cpp
View file

@ -19,6 +19,7 @@
#include <utility>
#include <android-base/logging.h>
#include <brotli/decode.h>
#include <zlib.h>
namespace android {
@ -207,5 +208,57 @@ std::unique_ptr<IDecompressor> IDecompressor::Gz() {
return std::unique_ptr<IDecompressor>(new GzDecompressor());
}
class BrotliDecompressor final : public StreamDecompressor {
public:
~BrotliDecompressor();
bool Init() override;
bool DecompressInput(const uint8_t* data, size_t length) override;
bool Done() override { return BrotliDecoderIsFinished(decoder_); }
private:
BrotliDecoderState* decoder_ = nullptr;
};
bool BrotliDecompressor::Init() {
decoder_ = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
return true;
}
BrotliDecompressor::~BrotliDecompressor() {
if (decoder_) {
BrotliDecoderDestroyInstance(decoder_);
}
}
bool BrotliDecompressor::DecompressInput(const uint8_t* data, size_t length) {
size_t available_in = length;
const uint8_t* next_in = data;
bool needs_more_output = false;
while (available_in || needs_more_output) {
if (!output_buffer_remaining_ && !GetFreshBuffer()) {
return false;
}
auto output_buffer = output_buffer_;
auto r = BrotliDecoderDecompressStream(decoder_, &available_in, &next_in,
&output_buffer_remaining_, &output_buffer_, nullptr);
if (r == BROTLI_DECODER_RESULT_ERROR) {
LOG(ERROR) << "brotli decode failed";
return false;
}
if (!sink_->ReturnData(output_buffer, output_buffer_ - output_buffer)) {
return false;
}
needs_more_output = (r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
}
return true;
}
std::unique_ptr<IDecompressor> IDecompressor::Brotli() {
return std::unique_ptr<IDecompressor>(new BrotliDecompressor());
}
} // namespace snapshot
} // namespace android

1
fs_mgr/libsnapshot/cow_decompress.h
View file

@ -40,6 +40,7 @@ class IDecompressor {
// Factory methods for decompression methods.
static std::unique_ptr<IDecompressor> Uncompressed();
static std::unique_ptr<IDecompressor> Gz();
static std::unique_ptr<IDecompressor> Brotli();
// |output_bytes| is the expected total number of bytes to sink.
virtual bool Decompress(size_t output_bytes) = 0;

8
fs_mgr/libsnapshot/cow_reader.cpp
View file

@ -78,6 +78,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) {
<< "Expected: " << kCowMagicNumber;
return false;
}
if (header_.header_size != sizeof(CowHeader)) {
LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected "
<< sizeof(CowHeader);
return false;
}
if ((header_.major_version != kCowVersionMajor) ||
(header_.minor_version != kCowVersionMinor)) {
@ -233,6 +238,9 @@ bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) {
case kCowCompressGz:
decompressor = IDecompressor::Gz();
break;
case kCowCompressBrotli:
decompressor = IDecompressor::Brotli();
break;
default:
LOG(ERROR) << "Unknown compression type: " << op.compression;
return false;

2
fs_mgr/libsnapshot/cow_snapuserd_test.cpp
View file

@ -162,7 +162,7 @@ void SnapuserdTest::CreateCowDevice(std::unique_ptr<TemporaryFile>& cow) {
ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_2_.get(), size_));
// Flush operations
ASSERT_TRUE(writer.Finalize());
ASSERT_TRUE(writer.Flush());
ASSERT_EQ(lseek(cow->fd, 0, SEEK_SET), 0);
}

153
fs_mgr/libsnapshot/cow_writer.cpp
View file

@ -22,6 +22,8 @@
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/unique_fd.h>
#include <brotli/encode.h>
#include <libsnapshot/cow_reader.h>
#include <libsnapshot/cow_writer.h>
#include <zlib.h>
@ -39,17 +41,48 @@ void CowWriter::SetupHeaders() {
header_.magic = kCowMagicNumber;
header_.major_version = kCowVersionMajor;
header_.minor_version = kCowVersionMinor;
header_.header_size = sizeof(CowHeader);
header_.block_size = options_.block_size;
}
bool CowWriter::Initialize(android::base::unique_fd&& fd) {
owned_fd_ = std::move(fd);
return Initialize(android::base::borrowed_fd{owned_fd_});
bool CowWriter::ParseOptions() {
if (options_.compression == "gz") {
compression_ = kCowCompressGz;
} else if (options_.compression == "brotli") {
compression_ = kCowCompressBrotli;
} else if (options_.compression == "none") {
compression_ = kCowCompressNone;
} else if (!options_.compression.empty()) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
return false;
}
return true;
}
bool CowWriter::Initialize(android::base::borrowed_fd fd) {
bool CowWriter::Initialize(android::base::unique_fd&& fd, OpenMode mode) {
owned_fd_ = std::move(fd);
return Initialize(android::base::borrowed_fd{owned_fd_}, mode);
}
bool CowWriter::Initialize(android::base::borrowed_fd fd, OpenMode mode) {
fd_ = fd;
if (!ParseOptions()) {
return false;
}
switch (mode) {
case OpenMode::WRITE:
return OpenForWrite();
case OpenMode::APPEND:
return OpenForAppend();
default:
LOG(ERROR) << "Unknown open mode in CowWriter";
return false;
}
}
bool CowWriter::OpenForWrite() {
// This limitation is tied to the data field size in CowOperation.
if (header_.block_size > std::numeric_limits<uint16_t>::max()) {
LOG(ERROR) << "Block size is too large";
@ -61,31 +94,52 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) {
return false;
}
if (options_.compression == "gz") {
compression_ = kCowCompressGz;
} else if (!options_.compression.empty()) {
LOG(ERROR) << "unrecognized compression: " << options_.compression;
// Headers are not complete, but this ensures the file is at the right
// position.
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write failed";
return false;
}
// Headers are not complete, but this ensures the file is at the right
// position.
if (!WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write failed";
header_.ops_offset = header_.header_size;
return true;
}
bool CowWriter::OpenForAppend() {
auto reader = std::make_unique<CowReader>();
if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {
return false;
}
options_.block_size = header_.block_size;
// Reset this, since we're going to reimport all operations.
header_.num_ops = 0;
auto iter = reader->GetOpIter();
while (!iter->Done()) {
auto& op = iter->Get();
AddOperation(op);
iter->Next();
}
// Free reader so we own the descriptor position again.
reader = nullptr;
// Seek to the end of the data section.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
return true;
}
bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowCopyOp;
op.new_block = new_block;
op.source = old_block;
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
AddOperation(op);
return true;
}
@ -103,8 +157,6 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
for (size_t i = 0; i < size / header_.block_size; i++) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowReplaceOp;
op.new_block = new_block_start + i;
@ -120,7 +172,7 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes";
return false;
}
if (!WriteFully(fd_, data.data(), data.size())) {
if (!WriteRawData(data.data(), data.size())) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
@ -132,11 +184,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
pos += header_.block_size;
}
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
AddOperation(op);
iter += header_.block_size;
}
if (!compression_ && !WriteFully(fd_, data, size)) {
if (!compression_ && !WriteRawData(data, size)) {
PLOG(ERROR) << "AddRawBlocks: write failed";
return false;
}
@ -145,13 +197,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
for (uint64_t i = 0; i < num_blocks; i++) {
header_.num_ops++;
CowOperation op = {};
op.type = kCowZeroOp;
op.new_block = new_block_start + i;
op.source = 0;
ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));
AddOperation(op);
}
return true;
}
@ -171,6 +221,24 @@ std::basic_string<uint8_t> CowWriter::Compress(const void* data, size_t length)
}
return std::basic_string<uint8_t>(buffer.get(), dest_len);
}
case kCowCompressBrotli: {
auto bound = BrotliEncoderMaxCompressedSize(length);
if (!bound) {
LOG(ERROR) << "BrotliEncoderMaxCompressedSize returned 0";
return {};
}
auto buffer = std::make_unique<uint8_t[]>(bound);
size_t encoded_size = bound;
auto rv = BrotliEncoderCompress(
BROTLI_DEFAULT_QUALITY, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, length,
reinterpret_cast<const uint8_t*>(data), &encoded_size, buffer.get());
if (!rv) {
LOG(ERROR) << "BrotliEncoderCompress failed";
return {};
}
return std::basic_string<uint8_t>(buffer.get(), encoded_size);
}
default:
LOG(ERROR) << "unhandled compression type: " << compression_;
break;
@ -189,17 +257,7 @@ static void SHA256(const void*, size_t, uint8_t[]) {
#endif
}
bool CowWriter::Finalize() {
// If both fields are set then Finalize is already called.
if (header_.ops_offset > 0 && header_.ops_size > 0) {
return true;
}
auto offs = lseek(fd_.get(), 0, SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
header_.ops_offset = offs;
bool CowWriter::Flush() {
header_.ops_size = ops_.size();
memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32);
@ -212,8 +270,6 @@ bool CowWriter::Finalize() {
PLOG(ERROR) << "lseek failed";
return false;
}
// Header is already written, calling WriteFully will increment
// bytes_written_. So use android::base::WriteFully() here.
if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
PLOG(ERROR) << "write header failed";
return false;
@ -227,13 +283,16 @@ bool CowWriter::Finalize() {
return false;
}
// clear ops_ so that subsequent calls to GetSize() still works.
ops_.clear();
// Re-position for any subsequent writes.
if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek ops failed";
return false;
}
return true;
}
size_t CowWriter::GetCowSize() {
return bytes_written_ + ops_.size() * sizeof(ops_[0]);
return header_.ops_offset + header_.num_ops * sizeof(CowOperation);
}
bool CowWriter::GetDataPos(uint64_t* pos) {
@ -246,9 +305,17 @@ bool CowWriter::GetDataPos(uint64_t* pos) {
return true;
}
bool CowWriter::WriteFully(base::borrowed_fd fd, const void* data, size_t size) {
bytes_written_ += size;
return android::base::WriteFully(fd, data, size);
void CowWriter::AddOperation(const CowOperation& op) {
header_.num_ops++;
ops_.insert(ops_.size(), reinterpret_cast<const uint8_t*>(&op), sizeof(op));
}
bool CowWriter::WriteRawData(const void* data, size_t size) {
if (!android::base::WriteFully(fd_, data, size)) {
return false;
}
header_.ops_offset += size;
return true;
}
} // namespace snapshot

2
fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp
View file

@ -375,7 +375,7 @@ bool NonAbEstimator::AnalyzePartition(const std::string& partition_name) {
}
}
if (!writer->Finalize()) {
if (!writer->Flush()) {
return false;
}

4
fs_mgr/libsnapshot/include/libsnapshot/cow_format.h
View file

@ -45,6 +45,9 @@ struct CowHeader {
uint16_t major_version;
uint16_t minor_version;
// Size of this struct.
uint16_t header_size;
// Offset to the location of the operation sequence, and size of the
// operation sequence buffer. |ops_offset| is also the end of the
// raw data region.
@ -98,6 +101,7 @@ static constexpr uint8_t kCowZeroOp = 3;
static constexpr uint8_t kCowCompressNone = 0;
static constexpr uint8_t kCowCompressGz = 1;
static constexpr uint8_t kCowCompressBrotli = 2;
} // namespace snapshot
} // namespace android

24
fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h
View file

@ -47,12 +47,11 @@ class ICowWriter {
// Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.
virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;
// Finalize all COW operations and flush pending writes.
// Return true if successful.
virtual bool Finalize() = 0;
// Flush all pending writes. This must be called before closing the writer
// to ensure that the correct headers and footers are written.
virtual bool Flush() = 0;
// Return 0 if failed, on success return number of bytes the cow image would be
// after calling Finalize();
// Return number of bytes the cow image occupies on disk.
virtual size_t GetCowSize() = 0;
protected:
@ -61,24 +60,30 @@ class ICowWriter {
class CowWriter : public ICowWriter {
public:
enum class OpenMode { WRITE, APPEND };
explicit CowWriter(const CowOptions& options);
// Set up the writer.
bool Initialize(android::base::unique_fd&& fd);
bool Initialize(android::base::borrowed_fd fd);
bool Initialize(android::base::unique_fd&& fd, OpenMode mode = OpenMode::WRITE);
bool Initialize(android::base::borrowed_fd fd, OpenMode mode = OpenMode::WRITE);
bool AddCopy(uint64_t new_block, uint64_t old_block) override;
bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;
bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;
bool Finalize() override;
bool Flush() override;
size_t GetCowSize() override;
private:
void SetupHeaders();
bool ParseOptions();
bool OpenForWrite();
bool OpenForAppend();
bool GetDataPos(uint64_t* pos);
bool WriteFully(base::borrowed_fd fd, const void* data, size_t size);
bool WriteRawData(const void* data, size_t size);
void AddOperation(const CowOperation& op);
std::basic_string<uint8_t> Compress(const void* data, size_t length);
private:
@ -90,7 +95,6 @@ class CowWriter : public ICowWriter {
// :TODO: this is not efficient, but stringstream ubsan aborts because some
// bytes overflow a signed char.
std::basic_string<uint8_t> ops_;
std::atomic<size_t> bytes_written_ = 0;
};
} // namespace snapshot

2
fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp
View file

@ -204,7 +204,7 @@ bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) {
}
}
if (!writer_->Finalize()) {
if (!writer_->Flush()) {
LOG(ERROR) << "Unable to finalize COW for " << partition_name;
return false;
}