ACPR/android_system_core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "libsnapshot: Remove various unused components." am: 2812fd841e

Browse source 
Original change: https://android-review.googlesource.com/c/platform/system/core/+/2536015

Change-Id: I020cdc0ffd9e77b991b735927521fde4f9264748
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
This commit is contained in:
David Anderson 2023-04-14 23:22:53 +00:00 • committed by Automerger Merge Worker
commit 86813c34a1
7 changed files with 0 additions and 1940 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

90
fs_mgr/libsnapshot/Android.bp
View file

@ -362,24 +362,6 @@ cc_binary {
},
}
cc_test {
name: "snapshot_power_test",
srcs: [
"power_test.cpp",
],
static_libs: [
"libc++fs",
"libsnapshot",
"update_metadata-protos",
],
shared_libs: [
"libbase",
"libfs_mgr_binder",
"liblog",
],
gtest: false,
}
cc_test {
name: "cow_api_test",
defaults: [
@ -416,78 +398,6 @@ cc_test {
host_supported: true,
}
cc_binary {
name: "make_cow_from_ab_ota",
host_supported: true,
device_supported: false,
cflags: [
"-D_FILE_OFFSET_BITS=64",
"-Wall",
"-Werror",
],
static_libs: [
"libbase",
"libbspatch",
"libbrotli",
"libbz",
"libchrome",
"libcrypto",
"libgflags",
"liblog",
"libprotobuf-cpp-lite",
"libpuffpatch",
"libsnapshot_cow",
"libsparse",
"libxz",
"libz",
"liblz4",
"libziparchive",
"update_metadata-protos",
],
srcs: [
"make_cow_from_ab_ota.cpp",
],
target: {
darwin: {
enabled: false,
},
},
}
cc_binary {
name: "estimate_cow_from_nonab_ota",
defaults: [
"libsnapshot_cow_defaults",
],
host_supported: true,
device_supported: false,
cflags: [
"-D_FILE_OFFSET_BITS=64",
"-Wall",
"-Werror",
],
static_libs: [
"libbase",
"libbrotli",
"libbz",
"libcrypto",
"libgflags",
"liblog",
"libsnapshot_cow",
"libsparse",
"libz",
"libziparchive",
],
srcs: [
"estimate_cow_from_nonab_ota.cpp",
],
target: {
darwin: {
enabled: false,
},
},
}
cc_binary {
name: "inspect_cow",
host_supported: true,

40
fs_mgr/libsnapshot/PowerTest.md
View file

@ -1,40 +0,0 @@
snapshot\_power\_test
---------------------
snapshot\_power\_test is a standalone test to simulate power failures during a snapshot-merge operation.
### Test Setup
Start by creating two large files that will be used as the pre-merge and post-merge state. You can take two different partition images (for example, a product.img from two separate builds), or just create random data:
dd if=/dev/urandom of=pre-merge count=1024 bs=1048576
dd if=/dev/urandom of=post-merge count=1024 bs=1048576
Next, push these files to an unencrypted directory on the device:
adb push pre-merge /data/local/unencrypted
adb push post-merge /data/local/unencrypted
Next, run the test setup:
adb sync data
adb shell /data/nativetest64/snapshot_power_test/snapshot_power_test \
/data/local/unencrypted/pre-merge \
/data/local/unencrypted/post-merge
This will create the necessary fiemap-based images.
### Running
The actual test can be run via `run_power_test.sh`. Its syntax is:
run_power_test.sh <POST_MERGE_FILE>
`POST_MERGE_FILE` should be the path on the device of the image to validate the merge against. Example:
run_power_test.sh /data/local/unencrypted/post-merge
The device will begin the merge with a 5% chance of injecting a kernel crash every 10ms. The device should be capable of rebooting normally without user intervention. Once the merge has completed, the test will run a final check command to validate the contents of the snapshot against the post-merge file. It will error if there are any incorrect blocks.
Two environment variables can be passed to `run_power_test.sh`:
1. `FAIL_RATE` - A fraction between 0 and 100 (inclusive) indicating the probability the device should inject a kernel crash every 10ms.
2. `DEVICE_SERIAL` - If multiple devices are attached to adb, this argument is passed as the serial to select (to `adb -s`).

432
fs_mgr/libsnapshot/estimate_cow_from_nonab_ota.cpp
View file

@ -1,432 +0,0 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <gflags/gflags.h>
#include <libsnapshot/cow_writer.h>
#include <openssl/sha.h>
#include <sparse/sparse.h>
#include <ziparchive/zip_archive.h>
DEFINE_string(source_tf, "", "Source target files (dir or zip file)");
DEFINE_string(ota_tf, "", "Target files of the build for an OTA");
DEFINE_string(compression, "gz", "Compression (options: none, gz, brotli)");
namespace android {
namespace snapshot {
using android::base::borrowed_fd;
using android::base::unique_fd;
static constexpr size_t kBlockSize = 4096;
void MyLogger(android::base::LogId, android::base::LogSeverity severity, const char*, const char*,
unsigned int, const char* message) {
if (severity == android::base::ERROR) {
fprintf(stderr, "%s\n", message);
} else {
fprintf(stdout, "%s\n", message);
}
}
class TargetFilesPackage final {
public:
explicit TargetFilesPackage(const std::string& path);
bool Open();
bool HasFile(const std::string& path);
std::unordered_set<std::string> GetDynamicPartitionNames();
unique_fd OpenFile(const std::string& path);
unique_fd OpenImage(const std::string& path);
private:
std::string path_;
unique_fd fd_;
std::unique_ptr<ZipArchive, decltype(&CloseArchive)> zip_;
};
TargetFilesPackage::TargetFilesPackage(const std::string& path)
: path_(path), zip_(nullptr, &CloseArchive) {}
bool TargetFilesPackage::Open() {
fd_.reset(open(path_.c_str(), O_RDONLY));
if (fd_ < 0) {
PLOG(ERROR) << "open failed: " << path_;
return false;
}
struct stat s;
if (fstat(fd_.get(), &s) < 0) {
PLOG(ERROR) << "fstat failed: " << path_;
return false;
}
if (S_ISDIR(s.st_mode)) {
return true;
}
// Otherwise, assume it's a zip file.
ZipArchiveHandle handle;
if (OpenArchiveFd(fd_.get(), path_.c_str(), &handle, false)) {
LOG(ERROR) << "Could not open " << path_ << " as a zip archive.";
return false;
}
zip_.reset(handle);
return true;
}
bool TargetFilesPackage::HasFile(const std::string& path) {
if (zip_) {
ZipEntry64 entry;
return !FindEntry(zip_.get(), path, &entry);
}
auto full_path = path_ + "/" + path;
return access(full_path.c_str(), F_OK) == 0;
}
unique_fd TargetFilesPackage::OpenFile(const std::string& path) {
if (!zip_) {
auto full_path = path_ + "/" + path;
unique_fd fd(open(full_path.c_str(), O_RDONLY));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << full_path;
return {};
}
return fd;
}
ZipEntry64 entry;
if (FindEntry(zip_.get(), path, &entry)) {
LOG(ERROR) << path << " not found in archive: " << path_;
return {};
}
TemporaryFile temp;
if (temp.fd < 0) {
PLOG(ERROR) << "mkstemp failed";
return {};
}
LOG(INFO) << "Extracting " << path << " from " << path_ << " ...";
if (ExtractEntryToFile(zip_.get(), &entry, temp.fd)) {
LOG(ERROR) << "could not extract " << path << " from " << path_;
return {};
}
if (lseek(temp.fd, 0, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return {};
}
return unique_fd{temp.release()};
}
unique_fd TargetFilesPackage::OpenImage(const std::string& path) {
auto fd = OpenFile(path);
if (fd < 0) {
return {};
}
LOG(INFO) << "Unsparsing " << path << " ...";
std::unique_ptr<struct sparse_file, decltype(&sparse_file_destroy)> s(
sparse_file_import(fd.get(), false, false), &sparse_file_destroy);
if (!s) {
return fd;
}
TemporaryFile temp;
if (temp.fd < 0) {
PLOG(ERROR) << "mkstemp failed";
return {};
}
if (sparse_file_write(s.get(), temp.fd, false, false, false) < 0) {
LOG(ERROR) << "sparse_file_write failed";
return {};
}
if (lseek(temp.fd, 0, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return {};
}
fd.reset(temp.release());
return fd;
}
std::unordered_set<std::string> TargetFilesPackage::GetDynamicPartitionNames() {
auto fd = OpenFile("META/misc_info.txt");
if (fd < 0) {
return {};
}
std::string contents;
if (!android::base::ReadFdToString(fd, &contents)) {
PLOG(ERROR) << "read failed";
return {};
}
std::unordered_set<std::string> set;
auto lines = android::base::Split(contents, "\n");
for (const auto& line : lines) {
auto parts = android::base::Split(line, "=");
if (parts.size() == 2 && parts[0] == "dynamic_partition_list") {
auto partitions = android::base::Split(parts[1], " ");
for (const auto& name : partitions) {
if (!name.empty()) {
set.emplace(name);
}
}
break;
}
}
return set;
}
class NonAbEstimator final {
public:
NonAbEstimator(const std::string& ota_tf_path, const std::string& source_tf_path)
: ota_tf_path_(ota_tf_path), source_tf_path_(source_tf_path) {}
bool Run();
private:
bool OpenPackages();
bool AnalyzePartition(const std::string& partition_name);
std::unordered_map<std::string, uint64_t> GetBlockMap(borrowed_fd fd);
std::string ota_tf_path_;
std::string source_tf_path_;
std::unique_ptr<TargetFilesPackage> ota_tf_;
std::unique_ptr<TargetFilesPackage> source_tf_;
uint64_t size_ = 0;
};
bool NonAbEstimator::Run() {
if (!OpenPackages()) {
return false;
}
auto partitions = ota_tf_->GetDynamicPartitionNames();
if (partitions.empty()) {
LOG(ERROR) << "No dynamic partitions found in META/misc_info.txt";
return false;
}
for (const auto& partition : partitions) {
if (!AnalyzePartition(partition)) {
return false;
}
}
int64_t size_in_mb = int64_t(double(size_) / 1024.0 / 1024.0);
std::cout << "Estimated COW size: " << size_ << " (" << size_in_mb << "MiB)\n";
return true;
}
bool NonAbEstimator::OpenPackages() {
ota_tf_ = std::make_unique<TargetFilesPackage>(ota_tf_path_);
if (!ota_tf_->Open()) {
return false;
}
if (!source_tf_path_.empty()) {
source_tf_ = std::make_unique<TargetFilesPackage>(source_tf_path_);
if (!source_tf_->Open()) {
return false;
}
}
return true;
}
static std::string SHA256(const std::string& input) {
std::string hash(32, '\0');
SHA256_CTX c;
SHA256_Init(&c);
SHA256_Update(&c, input.data(), input.size());
SHA256_Final(reinterpret_cast<unsigned char*>(hash.data()), &c);
return hash;
}
bool NonAbEstimator::AnalyzePartition(const std::string& partition_name) {
auto path = "IMAGES/" + partition_name + ".img";
auto fd = ota_tf_->OpenImage(path);
if (fd < 0) {
return false;
}
unique_fd source_fd;
uint64_t source_size = 0;
std::unordered_map<std::string, uint64_t> source_blocks;
if (source_tf_) {
auto dap = source_tf_->GetDynamicPartitionNames();
source_fd = source_tf_->OpenImage(path);
if (source_fd >= 0) {
struct stat s;
if (fstat(source_fd.get(), &s)) {
PLOG(ERROR) << "fstat failed";
return false;
}
source_size = s.st_size;
std::cout << "Hashing blocks for " << partition_name << "...\n";
source_blocks = GetBlockMap(source_fd);
if (source_blocks.empty()) {
LOG(ERROR) << "Could not build a block map for source partition: "
<< partition_name;
return false;
}
} else {
if (dap.count(partition_name)) {
return false;
}
LOG(ERROR) << "Warning: " << partition_name
<< " has no incremental diff since it's not in the source image.";
}
}
TemporaryFile cow;
if (cow.fd < 0) {
PLOG(ERROR) << "mkstemp failed";
return false;
}
CowOptions options;
options.block_size = kBlockSize;
options.compression = FLAGS_compression;
auto writer = std::make_unique<CowWriter>(options);
if (!writer->Initialize(borrowed_fd{cow.fd})) {
LOG(ERROR) << "Could not initialize COW writer";
return false;
}
LOG(INFO) << "Analyzing " << partition_name << " ...";
std::string zeroes(kBlockSize, '\0');
std::string chunk(kBlockSize, '\0');
std::string src_chunk(kBlockSize, '\0');
uint64_t next_block_number = 0;
while (true) {
if (!android::base::ReadFully(fd, chunk.data(), chunk.size())) {
if (errno) {
PLOG(ERROR) << "read failed";
return false;
}
break;
}
uint64_t block_number = next_block_number++;
if (chunk == zeroes) {
if (!writer->AddZeroBlocks(block_number, 1)) {
LOG(ERROR) << "Could not add zero block";
return false;
}
continue;
}
uint64_t source_offset = block_number * kBlockSize;
if (source_fd >= 0 && source_offset <= source_size) {
off64_t offset = block_number * kBlockSize;
if (android::base::ReadFullyAtOffset(source_fd, src_chunk.data(), src_chunk.size(),
offset)) {
if (chunk == src_chunk) {
continue;
}
} else if (errno) {
PLOG(ERROR) << "pread failed";
return false;
}
}
auto hash = SHA256(chunk);
if (auto iter = source_blocks.find(hash); iter != source_blocks.end()) {
if (!writer->AddCopy(block_number, iter->second)) {
return false;
}
continue;
}
if (!writer->AddRawBlocks(block_number, chunk.data(), chunk.size())) {
return false;
}
}
if (!writer->Finalize()) {
return false;
}
struct stat s;
if (fstat(cow.fd, &s) < 0) {
PLOG(ERROR) << "fstat failed";
return false;
}
size_ += s.st_size;
return true;
}
std::unordered_map<std::string, uint64_t> NonAbEstimator::GetBlockMap(borrowed_fd fd) {
std::string chunk(kBlockSize, '\0');
std::unordered_map<std::string, uint64_t> block_map;
uint64_t block_number = 0;
while (true) {
if (!android::base::ReadFully(fd, chunk.data(), chunk.size())) {
if (errno) {
PLOG(ERROR) << "read failed";
return {};
}
break;
}
auto hash = SHA256(chunk);
block_map[hash] = block_number;
block_number++;
}
return block_map;
}
} // namespace snapshot
} // namespace android
using namespace android::snapshot;
int main(int argc, char** argv) {
android::base::InitLogging(argv, android::snapshot::MyLogger);
gflags::SetUsageMessage("Estimate VAB disk usage from Non A/B builds");
gflags::ParseCommandLineFlags(&argc, &argv, false);
if (FLAGS_ota_tf.empty()) {
std::cerr << "Must specify -ota_tf on the command-line." << std::endl;
return 1;
}
NonAbEstimator estimator(FLAGS_ota_tf, FLAGS_source_tf);
if (!estimator.Run()) {
return 1;
}
return 0;
}

692
fs_mgr/libsnapshot/make_cow_from_ab_ota.cpp
View file

@ -1,692 +0,0 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <arpa/inet.h>
#include <errno.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
#include <limits>
#include <string>
#include <unordered_set>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/unique_fd.h>
#include <bsdiff/bspatch.h>
#include <bzlib.h>
#include <gflags/gflags.h>
#include <libsnapshot/cow_writer.h>
#include <puffin/puffpatch.h>
#include <sparse/sparse.h>
#include <update_engine/update_metadata.pb.h>
#include <xz.h>
#include <ziparchive/zip_archive.h>
namespace android {
namespace snapshot {
using android::base::borrowed_fd;
using android::base::unique_fd;
using chromeos_update_engine::DeltaArchiveManifest;
using chromeos_update_engine::Extent;
using chromeos_update_engine::InstallOperation;
using chromeos_update_engine::PartitionUpdate;
static constexpr uint64_t kBlockSize = 4096;
DEFINE_string(source_tf, "", "Source target files (dir or zip file) for incremental payloads");
DEFINE_string(compression, "gz", "Compression type to use (none or gz)");
DEFINE_uint32(cluster_ops, 0, "Number of Cow Ops per cluster (0 or >1)");
void MyLogger(android::base::LogId, android::base::LogSeverity severity, const char*, const char*,
unsigned int, const char* message) {
if (severity == android::base::ERROR) {
fprintf(stderr, "%s\n", message);
} else {
fprintf(stdout, "%s\n", message);
}
}
uint64_t ToLittleEndian(uint64_t value) {
union {
uint64_t u64;
char bytes[8];
} packed;
packed.u64 = value;
std::swap(packed.bytes[0], packed.bytes[7]);
std::swap(packed.bytes[1], packed.bytes[6]);
std::swap(packed.bytes[2], packed.bytes[5]);
std::swap(packed.bytes[3], packed.bytes[4]);
return packed.u64;
}
class PayloadConverter final {
public:
PayloadConverter(const std::string& in_file, const std::string& out_dir)
: in_file_(in_file), out_dir_(out_dir), source_tf_zip_(nullptr, &CloseArchive) {}
bool Run();
private:
bool OpenPayload();
bool OpenSourceTargetFiles();
bool ProcessPartition(const PartitionUpdate& update);
bool ProcessOperation(const InstallOperation& op);
bool ProcessZero(const InstallOperation& op);
bool ProcessCopy(const InstallOperation& op);
bool ProcessReplace(const InstallOperation& op);
bool ProcessDiff(const InstallOperation& op);
borrowed_fd OpenSourceImage();
std::string in_file_;
std::string out_dir_;
unique_fd in_fd_;
uint64_t payload_offset_ = 0;
DeltaArchiveManifest manifest_;
std::unordered_set<std::string> dap_;
unique_fd source_tf_fd_;
std::unique_ptr<ZipArchive, decltype(&CloseArchive)> source_tf_zip_;
// Updated during ProcessPartition().
std::string partition_name_;
std::unique_ptr<CowWriter> writer_;
unique_fd source_image_;
};
bool PayloadConverter::Run() {
if (!OpenPayload()) {
return false;
}
if (manifest_.has_dynamic_partition_metadata()) {
const auto& dpm = manifest_.dynamic_partition_metadata();
for (const auto& group : dpm.groups()) {
for (const auto& partition : group.partition_names()) {
dap_.emplace(partition);
}
}
}
if (dap_.empty()) {
LOG(ERROR) << "No dynamic partitions found.";
return false;
}
if (!OpenSourceTargetFiles()) {
return false;
}
for (const auto& update : manifest_.partitions()) {
if (!ProcessPartition(update)) {
return false;
}
writer_ = nullptr;
source_image_.reset();
}
return true;
}
bool PayloadConverter::OpenSourceTargetFiles() {
if (FLAGS_source_tf.empty()) {
return true;
}
source_tf_fd_.reset(open(FLAGS_source_tf.c_str(), O_RDONLY));
if (source_tf_fd_ < 0) {
LOG(ERROR) << "open failed: " << FLAGS_source_tf;
return false;
}
struct stat s;
if (fstat(source_tf_fd_.get(), &s) < 0) {
LOG(ERROR) << "fstat failed: " << FLAGS_source_tf;
return false;
}
if (S_ISDIR(s.st_mode)) {
return true;
}
// Otherwise, assume it's a zip file.
ZipArchiveHandle handle;
if (OpenArchiveFd(source_tf_fd_.get(), FLAGS_source_tf.c_str(), &handle, false)) {
LOG(ERROR) << "Could not open " << FLAGS_source_tf << " as a zip archive.";
return false;
}
source_tf_zip_.reset(handle);
return true;
}
bool PayloadConverter::ProcessPartition(const PartitionUpdate& update) {
auto partition_name = update.partition_name();
if (dap_.find(partition_name) == dap_.end()) {
// Skip non-DAP partitions.
return true;
}
auto path = out_dir_ + "/" + partition_name + ".cow";
unique_fd fd(open(path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0644));
if (fd < 0) {
PLOG(ERROR) << "open failed: " << path;
return false;
}
CowOptions options;
options.block_size = kBlockSize;
options.compression = FLAGS_compression;
options.cluster_ops = FLAGS_cluster_ops;
writer_ = std::make_unique<CowWriter>(options);
if (!writer_->Initialize(std::move(fd))) {
LOG(ERROR) << "Unable to initialize COW writer";
return false;
}
partition_name_ = partition_name;
for (const auto& op : update.operations()) {
if (!ProcessOperation(op)) {
return false;
}
}
if (!writer_->Finalize()) {
LOG(ERROR) << "Unable to finalize COW for " << partition_name;
return false;
}
return true;
}
bool PayloadConverter::ProcessOperation(const InstallOperation& op) {
switch (op.type()) {
case InstallOperation::SOURCE_COPY:
return ProcessCopy(op);
case InstallOperation::BROTLI_BSDIFF:
case InstallOperation::PUFFDIFF:
return ProcessDiff(op);
case InstallOperation::REPLACE:
case InstallOperation::REPLACE_XZ:
case InstallOperation::REPLACE_BZ:
return ProcessReplace(op);
case InstallOperation::ZERO:
return ProcessZero(op);
default:
LOG(ERROR) << "Unsupported op: " << (int)op.type();
return false;
}
return true;
}
bool PayloadConverter::ProcessZero(const InstallOperation& op) {
for (const auto& extent : op.dst_extents()) {
if (!writer_->AddZeroBlocks(extent.start_block(), extent.num_blocks())) {
LOG(ERROR) << "Could not add zero operation";
return false;
}
}
return true;
}
template <typename T>
static uint64_t SizeOfAllExtents(const T& extents) {
uint64_t total = 0;
for (const auto& extent : extents) {
total += extent.num_blocks() * kBlockSize;
}
return total;
}
class PuffInputStream final : public puffin::StreamInterface {
public:
PuffInputStream(uint8_t* buffer, size_t length) : buffer_(buffer), length_(length), pos_(0) {}
bool GetSize(uint64_t* size) const override {
*size = length_;
return true;
}
bool GetOffset(uint64_t* offset) const override {
*offset = pos_;
return true;
}
bool Seek(uint64_t offset) override {
if (offset > length_) return false;
pos_ = offset;
return true;
}
bool Read(void* buffer, size_t length) override {
if (length_ - pos_ < length) return false;
memcpy(buffer, buffer_ + pos_, length);
pos_ += length;
return true;
}
bool Write(const void*, size_t) override { return false; }
bool Close() override { return true; }
private:
uint8_t* buffer_;
size_t length_;
size_t pos_;
};
class PuffOutputStream final : public puffin::StreamInterface {
public:
PuffOutputStream(std::vector<uint8_t>& stream) : stream_(stream), pos_(0) {}
bool GetSize(uint64_t* size) const override {
*size = stream_.size();
return true;
}
bool GetOffset(uint64_t* offset) const override {
*offset = pos_;
return true;
}
bool Seek(uint64_t offset) override {
if (offset > stream_.size()) {
return false;
}
pos_ = offset;
return true;
}
bool Read(void* buffer, size_t length) override {
if (stream_.size() - pos_ < length) {
return false;
}
memcpy(buffer, &stream_[0] + pos_, length);
pos_ += length;
return true;
}
bool Write(const void* buffer, size_t length) override {
auto remaining = stream_.size() - pos_;
if (remaining < length) {
stream_.resize(stream_.size() + (length - remaining));
}
memcpy(&stream_[0] + pos_, buffer, length);
pos_ += length;
return true;
}
bool Close() override { return true; }
private:
std::vector<uint8_t>& stream_;
size_t pos_;
};
bool PayloadConverter::ProcessDiff(const InstallOperation& op) {
auto source_image = OpenSourceImage();
if (source_image < 0) {
return false;
}
uint64_t src_length = SizeOfAllExtents(op.src_extents());
auto src = std::make_unique<uint8_t[]>(src_length);
size_t src_pos = 0;
// Read source bytes.
for (const auto& extent : op.src_extents()) {
uint64_t offset = extent.start_block() * kBlockSize;
if (lseek(source_image.get(), offset, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek source image failed";
return false;
}
uint64_t size = extent.num_blocks() * kBlockSize;
CHECK(src_length - src_pos >= size);
if (!android::base::ReadFully(source_image, src.get() + src_pos, size)) {
PLOG(ERROR) << "read source image failed";
return false;
}
src_pos += size;
}
CHECK(src_pos == src_length);
// Read patch bytes.
auto patch = std::make_unique<uint8_t[]>(op.data_length());
if (lseek(in_fd_.get(), payload_offset_ + op.data_offset(), SEEK_SET) < 0) {
PLOG(ERROR) << "lseek payload failed";
return false;
}
if (!android::base::ReadFully(in_fd_, patch.get(), op.data_length())) {
PLOG(ERROR) << "read payload failed";
return false;
}
std::vector<uint8_t> dest(SizeOfAllExtents(op.dst_extents()));
// Apply the diff.
if (op.type() == InstallOperation::BROTLI_BSDIFF) {
size_t dest_pos = 0;
auto sink = [&](const uint8_t* data, size_t length) -> size_t {
CHECK(dest.size() - dest_pos >= length);
memcpy(&dest[dest_pos], data, length);
dest_pos += length;
return length;
};
if (int rv = bsdiff::bspatch(src.get(), src_pos, patch.get(), op.data_length(), sink)) {
LOG(ERROR) << "bspatch failed, error code " << rv;
return false;
}
} else if (op.type() == InstallOperation::PUFFDIFF) {
auto src_stream = std::make_unique<PuffInputStream>(src.get(), src_length);
auto dest_stream = std::make_unique<PuffOutputStream>(dest);
bool ok = PuffPatch(std::move(src_stream), std::move(dest_stream), patch.get(),
op.data_length());
if (!ok) {
LOG(ERROR) << "puffdiff operation failed to apply";
return false;
}
} else {
LOG(ERROR) << "unsupported diff operation: " << op.type();
return false;
}
// Write the final blocks to the COW.
size_t dest_pos = 0;
for (const auto& extent : op.dst_extents()) {
uint64_t size = extent.num_blocks() * kBlockSize;
CHECK(dest.size() - dest_pos >= size);
if (!writer_->AddRawBlocks(extent.start_block(), &dest[dest_pos], size)) {
return false;
}
dest_pos += size;
}
return true;
}
borrowed_fd PayloadConverter::OpenSourceImage() {
if (source_image_ >= 0) {
return source_image_;
}
unique_fd unzip_fd;
auto local_path = "IMAGES/" + partition_name_ + ".img";
if (source_tf_zip_) {
{
TemporaryFile tmp;
if (tmp.fd < 0) {
PLOG(ERROR) << "mkstemp failed";
return -1;
}
unzip_fd.reset(tmp.release());
}
ZipEntry64 entry;
if (FindEntry(source_tf_zip_.get(), local_path, &entry)) {
LOG(ERROR) << "not found in archive: " << local_path;
return -1;
}
if (ExtractEntryToFile(source_tf_zip_.get(), &entry, unzip_fd.get())) {
LOG(ERROR) << "could not extract " << local_path;
return -1;
}
if (lseek(unzip_fd.get(), 0, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek failed";
return -1;
}
} else if (source_tf_fd_ >= 0) {
unzip_fd.reset(openat(source_tf_fd_.get(), local_path.c_str(), O_RDONLY));
if (unzip_fd < 0) {
PLOG(ERROR) << "open failed: " << FLAGS_source_tf << "/" << local_path;
return -1;
}
} else {
LOG(ERROR) << "No source target files package was specified; need -source_tf";
return -1;
}
std::unique_ptr<struct sparse_file, decltype(&sparse_file_destroy)> s(
sparse_file_import(unzip_fd.get(), false, false), &sparse_file_destroy);
if (s) {
TemporaryFile tmp;
if (tmp.fd < 0) {
PLOG(ERROR) << "mkstemp failed";
return -1;
}
if (sparse_file_write(s.get(), tmp.fd, false, false, false) < 0) {
LOG(ERROR) << "sparse_file_write failed";
return -1;
}
source_image_.reset(tmp.release());
} else {
source_image_ = std::move(unzip_fd);
}
return source_image_;
}
template <typename ContainerType>
class ExtentIter final {
public:
ExtentIter(const ContainerType& container)
: iter_(container.cbegin()), end_(container.cend()), dst_index_(0) {}
bool GetNext(uint64_t* block) {
while (iter_ != end_) {
if (dst_index_ < iter_->num_blocks()) {
break;
}
iter_++;
dst_index_ = 0;
}
if (iter_ == end_) {
return false;
}
*block = iter_->start_block() + dst_index_;
dst_index_++;
return true;
}
private:
typename ContainerType::const_iterator iter_;
typename ContainerType::const_iterator end_;
uint64_t dst_index_;
};
bool PayloadConverter::ProcessCopy(const InstallOperation& op) {
ExtentIter dst_blocks(op.dst_extents());
for (const auto& extent : op.src_extents()) {
for (uint64_t i = 0; i < extent.num_blocks(); i++) {
uint64_t src_block = extent.start_block() + i;
uint64_t dst_block;
if (!dst_blocks.GetNext(&dst_block)) {
LOG(ERROR) << "SOURCE_COPY contained mismatching extents";
return false;
}
if (src_block == dst_block) continue;
if (!writer_->AddCopy(dst_block, src_block)) {
LOG(ERROR) << "Could not add copy operation";
return false;
}
}
}
return true;
}
bool PayloadConverter::ProcessReplace(const InstallOperation& op) {
auto buffer_size = op.data_length();
auto buffer = std::make_unique<char[]>(buffer_size);
uint64_t offs = payload_offset_ + op.data_offset();
if (lseek(in_fd_.get(), offs, SEEK_SET) < 0) {
PLOG(ERROR) << "lseek " << offs << " failed";
return false;
}
if (!android::base::ReadFully(in_fd_, buffer.get(), buffer_size)) {
PLOG(ERROR) << "read " << buffer_size << " bytes from offset " << offs << "failed";
return false;
}
uint64_t dst_size = 0;
for (const auto& extent : op.dst_extents()) {
dst_size += extent.num_blocks() * kBlockSize;
}
if (op.type() == InstallOperation::REPLACE_BZ) {
auto tmp = std::make_unique<char[]>(dst_size);
uint32_t actual_size;
if (dst_size > std::numeric_limits<typeof(actual_size)>::max()) {
LOG(ERROR) << "too many bytes to decompress: " << dst_size;
return false;
}
actual_size = static_cast<uint32_t>(dst_size);
auto rv = BZ2_bzBuffToBuffDecompress(tmp.get(), &actual_size, buffer.get(), buffer_size, 0,
0);
if (rv) {
LOG(ERROR) << "bz2 decompress failed: " << rv;
return false;
}
if (actual_size != dst_size) {
LOG(ERROR) << "bz2 returned " << actual_size << " bytes, expected " << dst_size;
return false;
}
buffer = std::move(tmp);
buffer_size = dst_size;
} else if (op.type() == InstallOperation::REPLACE_XZ) {
constexpr uint32_t kXzMaxDictSize = 64 * 1024 * 1024;
if (dst_size > std::numeric_limits<size_t>::max()) {
LOG(ERROR) << "too many bytes to decompress: " << dst_size;
return false;
}
std::unique_ptr<struct xz_dec, decltype(&xz_dec_end)> s(
xz_dec_init(XZ_DYNALLOC, kXzMaxDictSize), xz_dec_end);
if (!s) {
LOG(ERROR) << "xz_dec_init failed";
return false;
}
auto tmp = std::make_unique<char[]>(dst_size);
struct xz_buf args;
args.in = reinterpret_cast<const uint8_t*>(buffer.get());
args.in_pos = 0;
args.in_size = buffer_size;
args.out = reinterpret_cast<uint8_t*>(tmp.get());
args.out_pos = 0;
args.out_size = dst_size;
auto rv = xz_dec_run(s.get(), &args);
if (rv != XZ_STREAM_END) {
LOG(ERROR) << "xz decompress failed: " << (int)rv;
return false;
}
buffer = std::move(tmp);
buffer_size = dst_size;
}
uint64_t buffer_pos = 0;
for (const auto& extent : op.dst_extents()) {
uint64_t extent_size = extent.num_blocks() * kBlockSize;
if (buffer_size - buffer_pos < extent_size) {
LOG(ERROR) << "replace op ran out of input buffer";
return false;
}
if (!writer_->AddRawBlocks(extent.start_block(), buffer.get() + buffer_pos, extent_size)) {
LOG(ERROR) << "failed to add raw blocks from replace op";
return false;
}
buffer_pos += extent_size;
}
return true;
}
bool PayloadConverter::OpenPayload() {
in_fd_.reset(open(in_file_.c_str(), O_RDONLY));
if (in_fd_ < 0) {
PLOG(ERROR) << "open " << in_file_;
return false;
}
char magic[4];
if (!android::base::ReadFully(in_fd_, magic, sizeof(magic))) {
PLOG(ERROR) << "read magic";
return false;
}
if (std::string(magic, sizeof(magic)) != "CrAU") {
LOG(ERROR) << "Invalid magic in " << in_file_;
return false;
}
uint64_t version;
uint64_t manifest_size;
uint32_t manifest_signature_size = 0;
if (!android::base::ReadFully(in_fd_, &version, sizeof(version))) {
PLOG(ERROR) << "read version";
return false;
}
version = ToLittleEndian(version);
if (version < 2) {
LOG(ERROR) << "Only payload version 2 or higher is supported.";
return false;
}
if (!android::base::ReadFully(in_fd_, &manifest_size, sizeof(manifest_size))) {
PLOG(ERROR) << "read manifest_size";
return false;
}
manifest_size = ToLittleEndian(manifest_size);
if (!android::base::ReadFully(in_fd_, &manifest_signature_size,
sizeof(manifest_signature_size))) {
PLOG(ERROR) << "read manifest_signature_size";
return false;
}
manifest_signature_size = ntohl(manifest_signature_size);
auto manifest = std::make_unique<uint8_t[]>(manifest_size);
if (!android::base::ReadFully(in_fd_, manifest.get(), manifest_size)) {
PLOG(ERROR) << "read manifest";
return false;
}
// Skip past manifest signature.
auto offs = lseek(in_fd_, manifest_signature_size, SEEK_CUR);
if (offs < 0) {
PLOG(ERROR) << "lseek failed";
return false;
}
payload_offset_ = offs;
if (!manifest_.ParseFromArray(manifest.get(), manifest_size)) {
LOG(ERROR) << "could not parse manifest";
return false;
}
return true;
}
} // namespace snapshot
} // namespace android
int main(int argc, char** argv) {
android::base::InitLogging(argv, android::snapshot::MyLogger);
gflags::SetUsageMessage("Convert OTA payload to a Virtual A/B COW");
int arg_start = gflags::ParseCommandLineFlags(&argc, &argv, false);
xz_crc32_init();
if (argc - arg_start != 2) {
std::cerr << "Usage: [options] <payload.bin> <out-dir>\n";
return 1;
}
android::snapshot::PayloadConverter pc(argv[arg_start], argv[arg_start + 1]);
return pc.Run() ? 0 : 1;
}

559
fs_mgr/libsnapshot/power_test.cpp
View file

@ -1,559 +0,0 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <chrono>
#include <iostream>
#include <random>
#include <string>
#include <thread>
#include <vector>
#include <android-base/file.h>
#include <android-base/parsedouble.h>
#include <android-base/stringprintf.h>
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <ext4_utils/ext4_utils.h>
#include <fstab/fstab.h>
#include <libdm/dm.h>
#include <libfiemap/image_manager.h>
using namespace std::chrono_literals;
using namespace std::string_literals;
using android::base::borrowed_fd;
using android::base::unique_fd;
using android::dm::DeviceMapper;
using android::dm::DmDeviceState;
using android::dm::DmTable;
using android::dm::DmTargetSnapshot;
using android::dm::SnapshotStorageMode;
using android::fiemap::ImageManager;
using android::fs_mgr::Fstab;
namespace android {
namespace snapshot {
static void usage() {
std::cerr << "Usage:\n";
std::cerr << " create <orig-payload> <new-payload>\n";
std::cerr << "\n";
std::cerr << " Create a snapshot device containing the contents of\n";
std::cerr << " orig-payload, and then write the contents of new-payload.\n";
std::cerr << " The original files are not modified.\n";
std::cerr << "\n";
std::cerr << " merge <fail-rate>\n";
std::cerr << "\n";
std::cerr << " Merge the snapshot previously started by create, and wait\n";
std::cerr << " for it to complete. Once done, it is compared to the\n";
std::cerr << " new-payload for consistency. The original files are not \n";
std::cerr << " modified. If a fail-rate is passed (as a fraction between 0\n";
std::cerr << " and 100), every 10ms the device has that percent change of\n";
std::cerr << " injecting a kernel crash.\n";
std::cerr << "\n";
std::cerr << " check <new-payload>\n";
std::cerr << " Verify that all artifacts are correct after a merge\n";
std::cerr << " completes.\n";
std::cerr << "\n";
std::cerr << " cleanup\n";
std::cerr << " Remove all ImageManager artifacts from create/merge.\n";
}
class PowerTest final {
public:
PowerTest();
bool Run(int argc, char** argv);
private:
bool OpenImageManager();
bool Create(int argc, char** argv);
bool Merge(int argc, char** argv);
bool Check(int argc, char** argv);
bool Cleanup();
bool CleanupImage(const std::string& name);
bool SetupImages(const std::string& first_file, borrowed_fd second_fd);
bool MapImages();
bool MapSnapshot(SnapshotStorageMode mode);
bool GetMergeStatus(DmTargetSnapshot::Status* status);
static constexpr char kSnapshotName[] = "snapshot-power-test";
static constexpr char kSnapshotImageName[] = "snapshot-power-test-image";
static constexpr char kSnapshotCowName[] = "snapshot-power-test-cow";
DeviceMapper& dm_;
std::unique_ptr<ImageManager> images_;
std::string image_path_;
std::string cow_path_;
std::string snapshot_path_;
};
PowerTest::PowerTest() : dm_(DeviceMapper::Instance()) {}
bool PowerTest::Run([[maybe_unused]] int argc, [[maybe_unused]] char** argv) {
if (!OpenImageManager()) {
return false;
}
if (argc < 2) {
usage();
return false;
}
if (argv[1] == "create"s) {
return Create(argc, argv);
} else if (argv[1] == "merge"s) {
return Merge(argc, argv);
} else if (argv[1] == "check"s) {
return Check(argc, argv);
} else if (argv[1] == "cleanup"s) {
return Cleanup();
} else {
usage();
return false;
}
}
bool PowerTest::OpenImageManager() {
std::vector<std::string> dirs = {
"/data/gsi/test",
"/metadata/gsi/test",
};
for (const auto& dir : dirs) {
if (mkdir(dir.c_str(), 0700) && errno != EEXIST) {
std::cerr << "mkdir " << dir << ": " << strerror(errno) << "\n";
return false;
}
}
images_ = ImageManager::Open("/metadata/gsi/test", "/data/gsi/test");
if (!images_) {
std::cerr << "Could not open ImageManager\n";
return false;
}
return true;
}
bool PowerTest::Create(int argc, char** argv) {
if (argc < 4) {
usage();
return false;
}
std::string first = argv[2];
std::string second = argv[3];
unique_fd second_fd(open(second.c_str(), O_RDONLY));
if (second_fd < 0) {
std::cerr << "open " << second << ": " << strerror(errno) << "\n";
return false;
}
if (!Cleanup()) {
return false;
}
if (!SetupImages(first, second_fd)) {
return false;
}
if (!MapSnapshot(SnapshotStorageMode::Persistent)) {
return false;
}
struct stat s;
if (fstat(second_fd, &s)) {
std::cerr << "fstat " << second << ": " << strerror(errno) << "\n";
return false;
}
unique_fd snap_fd(open(snapshot_path_.c_str(), O_WRONLY));
if (snap_fd < 0) {
std::cerr << "open " << snapshot_path_ << ": " << strerror(errno) << "\n";
return false;
}
uint8_t chunk[4096];
uint64_t written = 0;
while (written < s.st_size) {
uint64_t remaining = s.st_size - written;
size_t bytes = (size_t)std::min((uint64_t)sizeof(chunk), remaining);
if (!android::base::ReadFully(second_fd, chunk, bytes)) {
std::cerr << "read " << second << ": " << strerror(errno) << "\n";
return false;
}
if (!android::base::WriteFully(snap_fd, chunk, bytes)) {
std::cerr << "write " << snapshot_path_ << ": " << strerror(errno) << "\n";
return false;
}
written += bytes;
}
if (fsync(snap_fd)) {
std::cerr << "fsync: " << strerror(errno) << "\n";
return false;
}
sync();
snap_fd = {};
if (!dm_.DeleteDeviceIfExists(kSnapshotName)) {
std::cerr << "could not delete dm device " << kSnapshotName << "\n";
return false;
}
if (!images_->UnmapImageIfExists(kSnapshotImageName)) {
std::cerr << "failed to unmap " << kSnapshotImageName << "\n";
return false;
}
if (!images_->UnmapImageIfExists(kSnapshotCowName)) {
std::cerr << "failed to unmap " << kSnapshotImageName << "\n";
return false;
}
return true;
}
bool PowerTest::Cleanup() {
if (!dm_.DeleteDeviceIfExists(kSnapshotName)) {
std::cerr << "could not delete dm device " << kSnapshotName << "\n";
return false;
}
if (!CleanupImage(kSnapshotImageName) || !CleanupImage(kSnapshotCowName)) {
return false;
}
return true;
}
bool PowerTest::CleanupImage(const std::string& name) {
if (!images_->UnmapImageIfExists(name)) {
std::cerr << "failed to unmap " << name << "\n";
return false;
}
if (images_->BackingImageExists(name) && !images_->DeleteBackingImage(name)) {
std::cerr << "failed to delete " << name << "\n";
return false;
}
return true;
}
bool PowerTest::SetupImages(const std::string& first, borrowed_fd second_fd) {
unique_fd first_fd(open(first.c_str(), O_RDONLY));
if (first_fd < 0) {
std::cerr << "open " << first << ": " << strerror(errno) << "\n";
return false;
}
struct stat s1, s2;
if (fstat(first_fd.get(), &s1)) {
std::cerr << "first stat: " << strerror(errno) << "\n";
return false;
}
if (fstat(second_fd.get(), &s2)) {
std::cerr << "second stat: " << strerror(errno) << "\n";
return false;
}
// Pick the bigger size of both images, rounding up to the nearest block.
uint64_t s1_size = (s1.st_size + 4095) & ~uint64_t(4095);
uint64_t s2_size = (s2.st_size + 4095) & ~uint64_t(4095);
uint64_t image_size = std::max(s1_size, s2_size) + (1024 * 1024 * 128);
if (!images_->CreateBackingImage(kSnapshotImageName, image_size, 0, nullptr)) {
std::cerr << "failed to create " << kSnapshotImageName << "\n";
return false;
}
// Use the same size for the cow.
if (!images_->CreateBackingImage(kSnapshotCowName, image_size, 0, nullptr)) {
std::cerr << "failed to create " << kSnapshotCowName << "\n";
return false;
}
if (!MapImages()) {
return false;
}
unique_fd image_fd(open(image_path_.c_str(), O_WRONLY));
if (image_fd < 0) {
std::cerr << "open: " << image_path_ << ": " << strerror(errno) << "\n";
return false;
}
uint8_t chunk[4096];
uint64_t written = 0;
while (written < s1.st_size) {
uint64_t remaining = s1.st_size - written;
size_t bytes = (size_t)std::min((uint64_t)sizeof(chunk), remaining);
if (!android::base::ReadFully(first_fd, chunk, bytes)) {
std::cerr << "read: " << strerror(errno) << "\n";
return false;
}
if (!android::base::WriteFully(image_fd, chunk, bytes)) {
std::cerr << "write: " << strerror(errno) << "\n";
return false;
}
written += bytes;
}
if (fsync(image_fd)) {
std::cerr << "fsync: " << strerror(errno) << "\n";
return false;
}
// Zero the first block of the COW.
unique_fd cow_fd(open(cow_path_.c_str(), O_WRONLY));
if (cow_fd < 0) {
std::cerr << "open: " << cow_path_ << ": " << strerror(errno) << "\n";
return false;
}
memset(chunk, 0, sizeof(chunk));
if (!android::base::WriteFully(cow_fd, chunk, sizeof(chunk))) {
std::cerr << "read: " << strerror(errno) << "\n";
return false;
}
if (fsync(cow_fd)) {
std::cerr << "fsync: " << strerror(errno) << "\n";
return false;
}
return true;
}
bool PowerTest::MapImages() {
if (!images_->MapImageDevice(kSnapshotImageName, 10s, &image_path_)) {
std::cerr << "failed to map " << kSnapshotImageName << "\n";
return false;
}
if (!images_->MapImageDevice(kSnapshotCowName, 10s, &cow_path_)) {
std::cerr << "failed to map " << kSnapshotCowName << "\n";
return false;
}
return true;
}
bool PowerTest::MapSnapshot(SnapshotStorageMode mode) {
uint64_t sectors;
{
unique_fd fd(open(image_path_.c_str(), O_RDONLY));
if (fd < 0) {
std::cerr << "open: " << image_path_ << ": " << strerror(errno) << "\n";
return false;
}
sectors = get_block_device_size(fd) / 512;
}
DmTable table;
table.Emplace<DmTargetSnapshot>(0, sectors, image_path_, cow_path_, mode, 8);
if (!dm_.CreateDevice(kSnapshotName, table, &snapshot_path_, 10s)) {
std::cerr << "failed to create snapshot device\n";
return false;
}
return true;
}
bool PowerTest::GetMergeStatus(DmTargetSnapshot::Status* status) {
std::vector<DeviceMapper::TargetInfo> targets;
if (!dm_.GetTableStatus(kSnapshotName, &targets)) {
std::cerr << "failed to get merge status\n";
return false;
}
if (targets.size() != 1) {
std::cerr << "merge device has wrong number of targets\n";
return false;
}
if (!DmTargetSnapshot::ParseStatusText(targets[0].data, status)) {
std::cerr << "could not parse merge target status text\n";
return false;
}
return true;
}
static std::string GetUserdataBlockDeviceName() {
Fstab fstab;
if (!ReadFstabFromFile("/proc/mounts", &fstab)) {
return {};
}
auto entry = android::fs_mgr::GetEntryForMountPoint(&fstab, "/data");
if (!entry) {
return {};
}
auto prefix = "/dev/block/"s;
if (!android::base::StartsWith(entry->blk_device, prefix)) {
return {};
}
return entry->blk_device.substr(prefix.size());
}
bool PowerTest::Merge(int argc, char** argv) {
// Start an f2fs GC to really stress things. :TODO: figure out data device
auto userdata_dev = GetUserdataBlockDeviceName();
if (userdata_dev.empty()) {
std::cerr << "could not locate userdata block device\n";
return false;
}
auto cmd =
android::base::StringPrintf("echo 1 > /sys/fs/f2fs/%s/gc_urgent", userdata_dev.c_str());
system(cmd.c_str());
if (dm_.GetState(kSnapshotName) == DmDeviceState::INVALID) {
if (!MapImages()) {
return false;
}
if (!MapSnapshot(SnapshotStorageMode::Merge)) {
return false;
}
}
std::random_device r;
std::default_random_engine re(r());
std::uniform_real_distribution<double> dist(0.0, 100.0);
std::optional<double> failure_rate;
if (argc >= 3) {
double d;
if (!android::base::ParseDouble(argv[2], &d)) {
std::cerr << "Could not parse failure rate as double: " << argv[2] << "\n";
return false;
}
failure_rate = d;
}
while (true) {
DmTargetSnapshot::Status status;
if (!GetMergeStatus(&status)) {
return false;
}
if (!status.error.empty()) {
std::cerr << "merge reported error: " << status.error << "\n";
return false;
}
if (status.sectors_allocated == status.metadata_sectors) {
break;
}
std::cerr << status.sectors_allocated << " / " << status.metadata_sectors << "\n";
if (failure_rate && *failure_rate >= dist(re)) {
system("echo 1 > /proc/sys/kernel/sysrq");
system("echo c > /proc/sysrq-trigger");
}
std::this_thread::sleep_for(10ms);
}
std::cout << "Merge completed.\n";
return true;
}
bool PowerTest::Check([[maybe_unused]] int argc, [[maybe_unused]] char** argv) {
if (argc < 3) {
std::cerr << "Expected argument: <new-image-path>\n";
return false;
}
std::string md_path, image_path;
std::string canonical_path = argv[2];
if (!dm_.GetDmDevicePathByName(kSnapshotName, &md_path)) {
std::cerr << "could not get dm-path for merge device\n";
return false;
}
if (!images_->GetMappedImageDevice(kSnapshotImageName, &image_path)) {
std::cerr << "could not get image path\n";
return false;
}
unique_fd md_fd(open(md_path.c_str(), O_RDONLY));
if (md_fd < 0) {
std::cerr << "open: " << md_path << ": " << strerror(errno) << "\n";
return false;
}
unique_fd image_fd(open(image_path.c_str(), O_RDONLY));
if (image_fd < 0) {
std::cerr << "open: " << image_path << ": " << strerror(errno) << "\n";
return false;
}
unique_fd canonical_fd(open(canonical_path.c_str(), O_RDONLY));
if (canonical_fd < 0) {
std::cerr << "open: " << canonical_path << ": " << strerror(errno) << "\n";
return false;
}
struct stat s;
if (fstat(canonical_fd, &s)) {
std::cerr << "fstat: " << canonical_path << ": " << strerror(errno) << "\n";
return false;
}
uint64_t canonical_size = s.st_size;
uint64_t md_size = get_block_device_size(md_fd);
uint64_t image_size = get_block_device_size(image_fd);
if (image_size != md_size) {
std::cerr << "image size does not match merge device size\n";
return false;
}
if (canonical_size > image_size) {
std::cerr << "canonical size " << canonical_size << " is greater than image size "
<< image_size << "\n";
return false;
}
constexpr size_t kBlockSize = 4096;
uint8_t canonical_buffer[kBlockSize];
uint8_t image_buffer[kBlockSize];
uint8_t md_buffer[kBlockSize];
uint64_t remaining = canonical_size;
uint64_t blockno = 0;
while (remaining) {
size_t bytes = (size_t)std::min((uint64_t)kBlockSize, remaining);
if (!android::base::ReadFully(canonical_fd, canonical_buffer, bytes)) {
std::cerr << "read: " << canonical_buffer << ": " << strerror(errno) << "\n";
return false;
}
if (!android::base::ReadFully(image_fd, image_buffer, bytes)) {
std::cerr << "read: " << image_buffer << ": " << strerror(errno) << "\n";
return false;
}
if (!android::base::ReadFully(md_fd, md_buffer, bytes)) {
std::cerr << "read: " << md_buffer << ": " << strerror(errno) << "\n";
return false;
}
if (memcmp(canonical_buffer, image_buffer, bytes)) {
std::cerr << "canonical and image differ at block " << blockno << "\n";
return false;
}
if (memcmp(canonical_buffer, md_buffer, bytes)) {
std::cerr << "canonical and image differ at block " << blockno << "\n";
return false;
}
remaining -= bytes;
blockno++;
}
std::cout << "Images all match.\n";
return true;
}
} // namespace snapshot
} // namespace android
int main(int argc, char** argv) {
android::snapshot::PowerTest test;
if (!test.Run(argc, argv)) {
std::cerr << "Unexpected error running test." << std::endl;
return 1;
}
fflush(stdout);
return 0;
}

35
fs_mgr/libsnapshot/run_power_test.sh
View file

@ -1,35 +0,0 @@
#!/bin/bash
set -e
if [ -z "$FAIL_RATE" ]; then
FAIL_RATE=5.0
fi
if [ ! -z "$ANDROID_SERIAL" ]; then
DEVICE_ARGS=-s $ANDROID_SERIAL
else
DEVICE_ARGS=
fi
TEST_BIN=/data/nativetest64/snapshot_power_test/snapshot_power_test
while :
do
adb $DEVICE_ARGS wait-for-device
adb $DEVICE_ARGS root
adb $DEVICE_ARGS shell rm $TEST_BIN
adb $DEVICE_ARGS sync data
set +e
output=$(adb $DEVICE_ARGS shell $TEST_BIN merge $FAIL_RATE 2>&1)
set -e
if [[ "$output" == *"Merge completed"* ]]; then
echo "Merge completed."
break
fi
if [[ "$output" == *"Unexpected error"* ]]; then
echo "Unexpected error."
exit 1
fi
done
adb $DEVICE_ARGS shell $TEST_BIN check $1

92
fs_mgr/libsnapshot/update_engine/update_metadata.proto
View file

@ -1,92 +0,0 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A subset of system/update_engine/update_metadata.proto. A separate file is
// used here because:
// - The original file is optimized for LITE_RUNTIME, but fuzzing needs
// reflection.
// - The definition here has less fields. libsnapshot only uses fields declared
// here, and all fields declared here are fuzzed by libsnapshot_fuzzer. If
// libsnapshot uses more fields in system/update_engine/update_metadata.proto
// in the future, they must be added here too, otherwise it will fail to
// compile.
//
// It is okay that this file is older than
// system/update_engine/update_metadata.proto as long as the messages defined
// here can also be parsed by protobuf defined there. However, it is not
// okay to add fields here without adding them to
// system/update_engine/update_metadata.proto. Doing so will cause a compiler
// error when libsnapshot code starts to use these dangling fields.
syntax = "proto2";
package chromeos_update_engine;
message Extent {
optional uint64 start_block = 1;
optional uint64 num_blocks = 2;
}
message PartitionInfo {
optional uint64 size = 1;
}
message InstallOperation {
enum Type {
SOURCE_COPY = 4;
// Not used by libsnapshot. Declared here so that the fuzzer has an
// alternative value to use for |type|.
ZERO = 6;
}
required Type type = 1;
repeated Extent src_extents = 4;
repeated Extent dst_extents = 6;
}
message PartitionUpdate {
required string partition_name = 1;
optional PartitionInfo new_partition_info = 7;
repeated InstallOperation operations = 8;
optional Extent hash_tree_extent = 11;
optional Extent fec_extent = 15;
optional uint64 estimate_cow_size = 19;
}
message DynamicPartitionGroup {
required string name = 1;
optional uint64 size = 2;
repeated string partition_names = 3;
}
message VABCFeatureSet {
optional bool threaded = 1;
optional bool batch_writes = 2;
}
message DynamicPartitionMetadata {
repeated DynamicPartitionGroup groups = 1;
optional bool vabc_enabled = 3;
optional string vabc_compression_param = 4;
optional uint32 cow_version = 5;
// A collection of knobs to tune Virtual AB Compression
optional VABCFeatureSet vabc_feature_set = 6;
}
message DeltaArchiveManifest {
repeated PartitionUpdate partitions = 13;
optional DynamicPartitionMetadata dynamic_partition_metadata = 15;
optional bool partial_update = 16;
}