From 8a28163d33d85b9742900f173ed765145a449a56 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Tue, 3 Oct 2023 15:55:09 -0700 Subject: [PATCH 1/2] inspect_cow: Add an --extract-to argument. This adds an --extract-to argument to inspect_cow to verify that full OTA snapshots contain correct data. It does not yet work for ordered ops. Test: inspect_cow Bug: N/A Change-Id: I9014da3e83fd4fb5ea54ac1d36e527b3e3e6c9d5 Change-Id: I7e256e8ddec626980cdcf8680bbeac3c2e9d8de1 --- .../libsnapshot_cow/inspect_cow.cpp | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp index a6dee4f43..83b5a1295 100644 --- a/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp +++ b/fs_mgr/libsnapshot/libsnapshot_cow/inspect_cow.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -38,11 +39,13 @@ DEFINE_bool(show_merged, false, DEFINE_bool(verify_merge_sequence, false, "Verify merge order sequencing"); DEFINE_bool(show_merge_sequence, false, "Show merge order sequence"); DEFINE_bool(show_raw_ops, false, "Show raw ops directly from the underlying parser"); +DEFINE_string(extract_to, "", "Extract the COW contents to the given file"); namespace android { namespace snapshot { using android::base::borrowed_fd; +using android::base::unique_fd; void MyLogger(android::base::LogId, android::base::LogSeverity severity, const char*, const char*, unsigned int, const char* message) { @@ -53,7 +56,7 @@ void MyLogger(android::base::LogId, android::base::LogSeverity severity, const c } } -static void ShowBad(CowReader& reader, const struct CowOperation* op) { +static void ShowBad(CowReader& reader, const CowOperation* op) { size_t count; auto buffer = std::make_unique(op->data_length); @@ -104,12 +107,21 @@ static bool ShowRawOpStream(borrowed_fd fd) { } static bool Inspect(const std::string& path) { - android::base::unique_fd fd(open(path.c_str(), O_RDONLY)); + unique_fd fd(open(path.c_str(), O_RDONLY)); if (fd < 0) { PLOG(ERROR) << "open failed: " << path; return false; } + unique_fd extract_to; + if (!FLAGS_extract_to.empty()) { + extract_to.reset(open(FLAGS_extract_to.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0664)); + if (extract_to < 0) { + PLOG(ERROR) << "could not open " << FLAGS_extract_to << " for writing"; + return false; + } + } + CowReader reader; auto start_time = std::chrono::steady_clock::now(); @@ -186,12 +198,23 @@ static bool Inspect(const std::string& path) { if (!FLAGS_silent && FLAGS_show_ops) std::cout << *op << "\n"; - if (FLAGS_decompress && op->type == kCowReplaceOp && op->compression != kCowCompressNone) { + if ((FLAGS_decompress || extract_to >= 0) && op->type == kCowReplaceOp) { if (reader.ReadData(op, buffer.data(), buffer.size()) < 0) { std::cerr << "Failed to decompress for :" << *op << "\n"; success = false; if (FLAGS_show_bad_data) ShowBad(reader, op); } + if (extract_to >= 0) { + off_t offset = uint64_t(op->new_block) * header.block_size; + if (!android::base::WriteFullyAtOffset(extract_to, buffer.data(), buffer.size(), + offset)) { + PLOG(ERROR) << "failed to write block " << op->new_block; + return false; + } + } + } else if (extract_to >= 0 && !IsMetadataOp(*op) && op->type != kCowZeroOp) { + PLOG(ERROR) << "Cannot extract op yet: " << *op; + return false; } if (op->type == kCowSequenceOp && FLAGS_show_merge_sequence) { From c942daf179983d1f9ccf76eb387ec4b809ff6445 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Tue, 3 Oct 2023 19:50:53 -0700 Subject: [PATCH 2/2] snapuserd: Add an extractor tool. This is similar to inspect_cow --extract-to, except it uses snapuserd. It is a diagnostic host tool and uses the tooling added for host testing. Usage: snapuserd_extractor -cow COW_FILE -base BASE_FILE -out OUT_FILE -num_sectors NUM_SECTORS Unlike inspect_cow, this supports xor/copy operations. The extractor code is separated into a utility file so we can use it for additional tests later on. Bug: N/A Test: manual test Change-Id: Ib7509508cba45e6c3a0db8c75454e33c2a503e03 --- fs_mgr/libsnapshot/snapuserd/Android.bp | 45 ++++++++++ .../snapuserd/snapuserd_extractor.cpp | 68 ++++++++++++++ .../snapuserd/user-space-merge/extractor.cpp | 90 +++++++++++++++++++ .../snapuserd/user-space-merge/extractor.h | 51 +++++++++++ 4 files changed, 254 insertions(+) create mode 100644 fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp create mode 100644 fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp create mode 100644 fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h diff --git a/fs_mgr/libsnapshot/snapuserd/Android.bp b/fs_mgr/libsnapshot/snapuserd/Android.bp index 47a86858a..1b0c5639e 100644 --- a/fs_mgr/libsnapshot/snapuserd/Android.bp +++ b/fs_mgr/libsnapshot/snapuserd/Android.bp @@ -293,3 +293,48 @@ cc_test { "vts", ], } + +cc_binary_host { + name: "snapuserd_extractor", + defaults: [ + "fs_mgr_defaults", + "libsnapshot_cow_defaults", + ], + srcs: [ + "testing/dm_user_harness.cpp", + "testing/harness.cpp", + "testing/host_harness.cpp", + "user-space-merge/extractor.cpp", + "snapuserd_extractor.cpp", + ], + cflags: [ + "-D_FILE_OFFSET_BITS=64", + "-Wall", + "-Werror", + ], + shared_libs: [ + "libbase", + "liblog", + ], + static_libs: [ + "libbrotli", + "libcutils_sockets", + "libdm", + "libext2_uuid", + "libext4_utils", + "libfs_mgr_file_wait", + "libgflags", + "libsnapshot_cow", + "libsnapuserd", + "liburing", + "libz", + ], + include_dirs: [ + "bionic/libc/kernel", + ".", + ], + header_libs: [ + "libstorage_literals_headers", + "libfiemap_headers", + ], +} diff --git a/fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp b/fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp new file mode 100644 index 000000000..f46cd5bc5 --- /dev/null +++ b/fs_mgr/libsnapshot/snapuserd/snapuserd_extractor.cpp @@ -0,0 +1,68 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include "user-space-merge/extractor.h" + +using namespace std::string_literals; + +DEFINE_string(base, "", "Base device/image"); +DEFINE_string(cow, "", "COW device/image"); +DEFINE_string(out, "", "Output path"); +DEFINE_int32(num_sectors, 0, "Number of sectors to read"); + +int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) { + android::base::InitLogging(argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + + if (FLAGS_out.empty()) { + LOG(ERROR) << "Missing -out argument."; + return 1; + } + if (FLAGS_base.empty()) { + LOG(ERROR) << "Missing -base argument."; + return 1; + } + if (FLAGS_cow.empty()) { + LOG(ERROR) << "missing -out argument."; + return 1; + } + if (!FLAGS_num_sectors) { + LOG(ERROR) << "missing -num_sectors argument."; + return 1; + } + + android::snapshot::Extractor extractor(FLAGS_base, FLAGS_cow); + if (!extractor.Init()) { + return 1; + } + if (!extractor.Extract(FLAGS_num_sectors, FLAGS_out)) { + return 1; + } + return 0; +} diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp new file mode 100644 index 000000000..c5718d5b7 --- /dev/null +++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.cpp @@ -0,0 +1,90 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "extractor.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +using android::base::unique_fd; +using namespace std::string_literals; + +namespace android { +namespace snapshot { + +Extractor::Extractor(const std::string& base_path, const std::string& cow_path) + : base_path_(base_path), cow_path_(cow_path), control_name_("test") {} + +bool Extractor::Init() { + auto opener = factory_.CreateTestOpener(control_name_); + handler_ = std::make_shared(control_name_, cow_path_, base_path_, base_path_, + opener, 1, false, false); + if (!handler_->InitCowDevice()) { + return false; + } + if (!handler_->InitializeWorkers()) { + return false; + } + + read_worker_ = std::make_unique(cow_path_, base_path_, control_name_, base_path_, + handler_->GetSharedPtr(), opener); + if (!read_worker_->Init()) { + return false; + } + block_server_ = static_cast(read_worker_->block_server()); + + handler_thread_ = std::async(std::launch::async, &SnapshotHandler::Start, handler_.get()); + return true; +} + +Extractor::~Extractor() { + factory_.DeleteQueue(control_name_); +} + +bool Extractor::Extract(off_t num_sectors, const std::string& out_path) { + unique_fd out_fd(open(out_path.c_str(), O_RDWR | O_CLOEXEC | O_TRUNC | O_CREAT, 0664)); + if (out_fd < 0) { + PLOG(ERROR) << "Could not open for writing: " << out_path; + return false; + } + + for (off_t i = 0; i < num_sectors; i++) { + if (!read_worker_->RequestSectors(i, 512)) { + LOG(ERROR) << "Read sector " << i << " failed."; + return false; + } + std::string result = std::move(block_server_->sent_io()); + off_t offset = i * 512; + if (!android::base::WriteFullyAtOffset(out_fd, result.data(), result.size(), offset)) { + PLOG(ERROR) << "write failed"; + return false; + } + } + return true; +} + +} // namespace snapshot +} // namespace android diff --git a/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h new file mode 100644 index 000000000..65285b1fa --- /dev/null +++ b/fs_mgr/libsnapshot/snapuserd/user-space-merge/extractor.h @@ -0,0 +1,51 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include +#include "merge_worker.h" +#include "read_worker.h" +#include "snapuserd_core.h" +#include "testing/host_harness.h" + +namespace android { +namespace snapshot { + +class Extractor final { + public: + Extractor(const std::string& base_path, const std::string& cow_path); + ~Extractor(); + + bool Init(); + bool Extract(off_t num_sectors, const std::string& out_path); + + private: + std::string base_path_; + std::string cow_path_; + + TestBlockServerFactory factory_; + HostTestHarness harness_; + std::string control_name_; + std::shared_ptr handler_; + std::unique_ptr read_worker_; + std::future handler_thread_; + TestBlockServer* block_server_ = nullptr; +}; + +} // namespace snapshot +} // namespace android