From e6cef616f1f9882119e1a7caf84debb247186c6b Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 26 Jan 2023 20:49:11 -0800 Subject: [PATCH 1/4] liblp: Add a helper class for building sparse-compatible super image layouts. This class helps export a list of regions comprising a super partition, and what the contents of those regions should be. It is very similar to ImageBuilder, except that it does not require actual partition image files, nor does it actually write an image file to disk. The goal is to support building an in-memory super image that uses as little memory and backing storage as possible. For example, fastboot can use this to upload a super image without having to read and write gigabytes of unnecessary data. Since the goal is to optimize fastboot, we've taken some shortcuts here. Retrofit devices and other edge-casey behavior are safely rejected. We also don't rely on libsparse behavior here, and attempt to make the translation to sparse records as trivial as possible, by explicitly declaring where fill/dontcare gaps are, and only exporting 4KB aligned regions. Hopefully this will allow the code to be portable to non-fastboot consumers. Bug: 266982466 Test: liblp_test Change-Id: I1b41d233bc4512c4b62e19603e8e77bc5867cfab --- fs_mgr/liblp/Android.bp | 35 ++- .../include/liblp/super_layout_builder.h | 104 ++++++++ fs_mgr/liblp/liblp_test.xml | 26 -- fs_mgr/liblp/super_layout_builder.cpp | 241 ++++++++++++++++++ fs_mgr/liblp/super_layout_builder_test.cpp | 141 ++++++++++ 5 files changed, 507 insertions(+), 40 deletions(-) create mode 100644 fs_mgr/liblp/include/liblp/super_layout_builder.h delete mode 100644 fs_mgr/liblp/liblp_test.xml create mode 100644 fs_mgr/liblp/super_layout_builder.cpp create mode 100644 fs_mgr/liblp/super_layout_builder_test.cpp diff --git a/fs_mgr/liblp/Android.bp b/fs_mgr/liblp/Android.bp index 4b81c2c07..996ffd759 100644 --- a/fs_mgr/liblp/Android.bp +++ b/fs_mgr/liblp/Android.bp @@ -39,6 +39,7 @@ cc_library { ], srcs: [ "builder.cpp", + "super_layout_builder.cpp", "images.cpp", "partition_opener.cpp", "property_fetcher.cpp", @@ -62,17 +63,6 @@ cc_library { export_include_dirs: ["include"], } -filegroup { - name: "liblp_test_srcs", - srcs: [ - "builder_test.cpp", - "device_test.cpp", - "io_test.cpp", - "test_partition_opener.cpp", - "utility_test.cpp", - ], -} - cc_defaults { name: "liblp_test_defaults", defaults: ["fs_mgr_defaults"], @@ -82,22 +72,39 @@ cc_defaults { static_libs: [ "libcutils", "libgmock", - "libfs_mgr", "liblp", "libcrypto_static", ] + liblp_lib_deps, header_libs: [ "libstorage_literals_headers", ], + target: { + android: { + srcs: [ + "device_test.cpp", + "io_test.cpp", + ], + static_libs: [ + "libfs_mgr", + ], + } + }, stl: "libc++_static", - srcs: [":liblp_test_srcs"], + srcs: [ + "builder_test.cpp", + "super_layout_builder_test.cpp", + "test_partition_opener.cpp", + "utility_test.cpp", + ], } cc_test { name: "liblp_test", defaults: ["liblp_test_defaults"], - test_config: "liblp_test.xml", test_suites: ["device-tests"], + auto_gen_config: true, + require_root: true, + host_supported: true } cc_test { diff --git a/fs_mgr/liblp/include/liblp/super_layout_builder.h b/fs_mgr/liblp/include/liblp/super_layout_builder.h new file mode 100644 index 000000000..d92085537 --- /dev/null +++ b/fs_mgr/liblp/include/liblp/super_layout_builder.h @@ -0,0 +1,104 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace android { +namespace fs_mgr { + +struct SuperImageExtent { + enum class Type { INVALID, DATA, PARTITION, ZERO, DONTCARE }; + + SuperImageExtent(const SuperImageExtent& other) = default; + SuperImageExtent(SuperImageExtent&& other) = default; + SuperImageExtent(uint64_t offset, uint64_t size, Type type) + : offset(offset), size(size), type(type) {} + + SuperImageExtent(uint64_t offset, std::shared_ptr blob) + : SuperImageExtent(offset, blob->size(), Type::DATA) { + this->blob = blob; + } + + SuperImageExtent(uint64_t offset, uint64_t size, const std::string& image_name, + uint64_t image_offset) + : SuperImageExtent(offset, size, Type::PARTITION) { + this->image_name = image_name; + this->image_offset = image_offset; + } + + SuperImageExtent& operator=(const SuperImageExtent& other) = default; + SuperImageExtent& operator=(SuperImageExtent&& other) = default; + + bool operator<(const SuperImageExtent& other) const { return offset < other.offset; } + bool operator==(const SuperImageExtent& other) const; + + // Location, size, and type of the extent. + uint64_t offset = 0; + uint64_t size = 0; + Type type = Type::INVALID; + + // If type == DATA, this contains the bytes to write. + std::shared_ptr blob; + // If type == PARTITION, this contains the partition image name and + // offset within that file. + std::string image_name; + uint64_t image_offset = 0; +}; + +// The SuperLayoutBuilder allows building a sparse view of a super image. This +// is useful for efficient flashing, eg to bypass fastbootd and directly flash +// super without physically building and storing the image. +class SuperLayoutBuilder final { + public: + // Open a super_empty.img, return false on failure. This must be called to + // initialize the tool. If it returns false, either the image failed to + // parse, or the tool is not compatible with how the device is configured + // (in which case fastbootd should be preferred). + [[nodiscard]] bool Open(android::base::borrowed_fd fd); + [[nodiscard]] bool Open(const void* data, size_t bytes); + [[nodiscard]] bool Open(const LpMetadata& metadata); + + // Add a partition's image and size to the work list. If false is returned, + // there was either a duplicate partition or not enough space in super. + bool AddPartition(const std::string& partition_name, const std::string& image_name, + uint64_t partition_size); + + // Return the list of extents describing the super image. If this list is + // empty, then there was an unrecoverable error in building the list. + std::vector GetImageLayout(); + + // Return the current metadata. + std::unique_ptr Export() const { return builder_->Export(); } + + private: + std::unique_ptr builder_; + std::unordered_map image_map_; +}; + +std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent); + +} // namespace fs_mgr +} // namespace android diff --git a/fs_mgr/liblp/liblp_test.xml b/fs_mgr/liblp/liblp_test.xml deleted file mode 100644 index 98414b109..000000000 --- a/fs_mgr/liblp/liblp_test.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - diff --git a/fs_mgr/liblp/super_layout_builder.cpp b/fs_mgr/liblp/super_layout_builder.cpp new file mode 100644 index 000000000..37f28e124 --- /dev/null +++ b/fs_mgr/liblp/super_layout_builder.cpp @@ -0,0 +1,241 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include + +#include + +#include "images.h" +#include "utility.h" +#include "writer.h" + +using android::base::borrowed_fd; +using android::base::unique_fd; + +namespace android { +namespace fs_mgr { + +bool SuperLayoutBuilder::Open(borrowed_fd fd) { + auto metadata = ReadFromImageFile(fd.get()); + if (!metadata) { + return false; + } + return Open(*metadata.get()); +} + +bool SuperLayoutBuilder::Open(const void* data, size_t size) { + auto metadata = ReadFromImageBlob(data, size); + if (!metadata) { + return false; + } + return Open(*metadata.get()); +} + +bool SuperLayoutBuilder::Open(const LpMetadata& metadata) { + for (const auto& partition : metadata.partitions) { + if (partition.attributes & LP_PARTITION_ATTR_SLOT_SUFFIXED) { + // Retrofit devices are not supported. + return false; + } + if (!(partition.attributes & LP_PARTITION_ATTR_READONLY)) { + // Writable partitions are not supported. + return false; + } + } + if (!metadata.extents.empty()) { + // Partitions that already have extents are not supported (should + // never be true of super_empty.img). + return false; + } + if (metadata.block_devices.size() != 1) { + // Only one "super" is supported. + return false; + } + + builder_ = MetadataBuilder::New(metadata); + return !!builder_; +} + +bool SuperLayoutBuilder::AddPartition(const std::string& partition_name, + const std::string& image_name, uint64_t partition_size) { + auto p = builder_->FindPartition(partition_name); + if (!p) { + return false; + } + if (!builder_->ResizePartition(p, partition_size)) { + return false; + } + image_map_.emplace(partition_name, image_name); + return true; +} + +// Fill the space between each extent, if any, with either a fill or dontcare +// extent. The caller constructs a sample extent to re-use. +static bool AddGapExtents(std::vector* extents, SuperImageExtent::Type gap_type) { + std::vector old = std::move(*extents); + std::sort(old.begin(), old.end()); + + *extents = {}; + + uint64_t current_offset = 0; + for (const auto& extent : old) { + // Check for overlapping extents - this would be a serious error. + if (current_offset > extent.offset) { + LOG(INFO) << "Overlapping extents detected; cannot layout temporary super image"; + return false; + } + + if (extent.offset != current_offset) { + uint64_t gap_size = extent.offset - current_offset; + extents->emplace_back(current_offset, gap_size, gap_type); + current_offset = extent.offset; + } + + extents->emplace_back(extent); + current_offset += extent.size; + } + return true; +} + +std::vector SuperLayoutBuilder::GetImageLayout() { + auto metadata = builder_->Export(); + if (!metadata) { + return {}; + } + + std::vector extents; + + // Write the primary and backup copies of geometry. + std::string geometry_bytes = SerializeGeometry(metadata->geometry); + auto blob = std::make_shared(std::move(geometry_bytes)); + + extents.emplace_back(0, GetPrimaryGeometryOffset(), SuperImageExtent::Type::ZERO); + extents.emplace_back(GetPrimaryGeometryOffset(), blob); + extents.emplace_back(GetBackupGeometryOffset(), blob); + + // Write the primary and backup copies of each metadata slot. When flashing, + // all metadata copies are the same, even for different slots. + std::string metadata_bytes = SerializeMetadata(*metadata.get()); + + // Align metadata size to 4KB. This makes the layout easily compatible with + // libsparse. + static constexpr size_t kSparseAlignment = 4096; + size_t metadata_aligned_bytes; + if (!AlignTo(metadata_bytes.size(), kSparseAlignment, &metadata_aligned_bytes)) { + LOG(ERROR) << "Unable to align metadata size " << metadata_bytes.size() << " to " + << kSparseAlignment; + return {}; + } + metadata_bytes.resize(metadata_aligned_bytes, '\0'); + + // However, alignment can cause larger-than-supported metadata blocks. Fall + // back to fastbootd/update-super. + if (metadata_bytes.size() > metadata->geometry.metadata_max_size) { + LOG(VERBOSE) << "Aligned metadata size " << metadata_bytes.size() + << " is larger than maximum metadata size " + << metadata->geometry.metadata_max_size; + return {}; + } + + blob = std::make_shared(std::move(metadata_bytes)); + for (uint32_t i = 0; i < metadata->geometry.metadata_slot_count; i++) { + int64_t metadata_primary = GetPrimaryMetadataOffset(metadata->geometry, i); + int64_t metadata_backup = GetBackupMetadataOffset(metadata->geometry, i); + extents.emplace_back(metadata_primary, blob); + extents.emplace_back(metadata_backup, blob); + } + + // Add extents for each partition. + for (const auto& partition : metadata->partitions) { + auto partition_name = GetPartitionName(partition); + auto image_name_iter = image_map_.find(partition_name); + if (image_name_iter == image_map_.end()) { + if (partition.num_extents != 0) { + LOG(ERROR) << "Partition " << partition_name + << " has extents but no image filename"; + return {}; + } + continue; + } + const auto& image_name = image_name_iter->second; + + uint64_t image_offset = 0; + for (uint32_t i = 0; i < partition.num_extents; i++) { + const auto& e = metadata->extents[partition.first_extent_index + i]; + + if (e.target_type != LP_TARGET_TYPE_LINEAR) { + // Any type other than LINEAR isn't understood here. We don't even + // bother with ZERO, which is never generated. + LOG(INFO) << "Unknown extent type from liblp: " << e.target_type; + return {}; + } + + size_t size = e.num_sectors * LP_SECTOR_SIZE; + uint64_t super_offset = e.target_data * LP_SECTOR_SIZE; + extents.emplace_back(super_offset, size, image_name, image_offset); + + image_offset += size; + } + } + + if (!AddGapExtents(&extents, SuperImageExtent::Type::DONTCARE)) { + return {}; + } + return extents; +} + +bool SuperImageExtent::operator==(const SuperImageExtent& other) const { + if (offset != other.offset) { + return false; + } + if (size != other.size) { + return false; + } + if (type != other.type) { + return false; + } + switch (type) { + case Type::DATA: + return *blob == *other.blob; + case Type::PARTITION: + return image_name == other.image_name && image_offset == other.image_offset; + default: + return true; + } +} + +std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent) { + stream << "extent:" << extent.offset << ":" << extent.size << ":"; + switch (extent.type) { + case SuperImageExtent::Type::DATA: + stream << "data"; + break; + case SuperImageExtent::Type::PARTITION: + stream << "partition:" << extent.image_name << ":" << extent.image_offset; + break; + case SuperImageExtent::Type::ZERO: + stream << "zero"; + break; + case SuperImageExtent::Type::DONTCARE: + stream << "dontcare"; + break; + default: + stream << "invalid"; + } + return stream; +} + +} // namespace fs_mgr +} // namespace android diff --git a/fs_mgr/liblp/super_layout_builder_test.cpp b/fs_mgr/liblp/super_layout_builder_test.cpp new file mode 100644 index 000000000..714b6b439 --- /dev/null +++ b/fs_mgr/liblp/super_layout_builder_test.cpp @@ -0,0 +1,141 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include +#include +#include +#include + +#include "images.h" +#include "writer.h" + +using namespace android::fs_mgr; +using namespace android::storage_literals; + +TEST(SuperImageTool, Layout) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_TRUE(tool.Open(*metadata.get())); + ASSERT_TRUE(tool.AddPartition("system_a", "system.img", 16_KiB)); + + // Get a copy of the metadata we'd expect if flashing. + ASSERT_TRUE(builder->ResizePartition(p, 16_KiB)); + metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + auto geometry_blob = std::make_shared(SerializeGeometry(metadata->geometry)); + auto metadata_blob = std::make_shared(SerializeMetadata(*metadata.get())); + metadata_blob->resize(4_KiB, '\0'); + + auto extents = tool.GetImageLayout(); + ASSERT_EQ(extents.size(), 12); + EXPECT_EQ(extents[0], SuperImageExtent(0, 4096, SuperImageExtent::Type::ZERO)); + EXPECT_EQ(extents[1], SuperImageExtent(4096, geometry_blob)); + EXPECT_EQ(extents[2], SuperImageExtent(8192, geometry_blob)); + EXPECT_EQ(extents[3], SuperImageExtent(12288, metadata_blob)); + EXPECT_EQ(extents[4], SuperImageExtent(16384, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[5], SuperImageExtent(20480, metadata_blob)); + EXPECT_EQ(extents[6], SuperImageExtent(24576, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[7], SuperImageExtent(28672, metadata_blob)); + EXPECT_EQ(extents[8], SuperImageExtent(32768, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[9], SuperImageExtent(36864, metadata_blob)); + EXPECT_EQ(extents[10], SuperImageExtent(40960, 4096, SuperImageExtent::Type::DONTCARE)); + EXPECT_EQ(extents[11], SuperImageExtent(45056, 16384, "system.img", 0)); +} + +TEST(SuperImageTool, NoWritablePartitions) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", 0); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, NoRetrofit) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + // Add an extra block device. + metadata->block_devices.emplace_back(metadata->block_devices[0]); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, NoRetrofit2) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition( + "system_a", LP_PARTITION_ATTR_READONLY | LP_PARTITION_ATTR_SLOT_SUFFIXED); + ASSERT_NE(p, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, NoFixedPartitions) { + auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2); + ASSERT_NE(builder, nullptr); + + Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY); + ASSERT_NE(p, nullptr); + ASSERT_TRUE(builder->ResizePartition(p, 4_KiB)); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_FALSE(tool.Open(*metadata.get())); +} + +TEST(SuperImageTool, LargeAlignedMetadata) { + auto builder = MetadataBuilder::New(4_MiB, 512, 2); + ASSERT_NE(builder, nullptr); + + auto metadata = builder->Export(); + ASSERT_NE(metadata, nullptr); + + SuperLayoutBuilder tool; + ASSERT_TRUE(tool.Open(*metadata.get())); + + auto extents = tool.GetImageLayout(); + ASSERT_TRUE(extents.empty()); +} From a1c983e800c02c7b91e0609ed6381db07ffd2f4e Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 27 Jan 2023 11:57:27 -0800 Subject: [PATCH 2/4] fastboot: Use RAII for sparse_file objects. Bug: 266982466 Test: fastboot flash Change-Id: I3d240d6ecc8a37d968ffdef9d50e349e787e8d3e --- fastboot/fastboot.cpp | 46 ++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index 9676f8779..0aa139059 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -106,6 +106,8 @@ static bool g_disable_verification = false; fastboot::FastBootDriver* fb = nullptr; +using SparsePtr = std::unique_ptr; + enum fb_buffer_type { FB_BUFFER_FD, FB_BUFFER_SPARSE, @@ -113,7 +115,7 @@ enum fb_buffer_type { struct fastboot_buffer { enum fb_buffer_type type; - void* data; + std::vector files; int64_t sz; unique_fd fd; int64_t image_size; @@ -824,24 +826,25 @@ static void DumpInfo() { fprintf(stderr, "--------------------------------------------\n"); } -static struct sparse_file** load_sparse_files(int fd, int64_t max_size) { - struct sparse_file* s = sparse_file_import_auto(fd, false, true); +static std::vector load_sparse_files(int fd, int64_t max_size) { + SparsePtr s(sparse_file_import_auto(fd, false, true), sparse_file_destroy); if (!s) die("cannot sparse read file"); if (max_size <= 0 || max_size > std::numeric_limits::max()) { die("invalid max size %" PRId64, max_size); } - int files = sparse_file_resparse(s, max_size, nullptr, 0); + const int files = sparse_file_resparse(s.get(), max_size, nullptr, 0); if (files < 0) die("Failed to resparse"); - sparse_file** out_s = - reinterpret_cast(calloc(sizeof(struct sparse_file*), files + 1)); - if (!out_s) die("Failed to allocate sparse file array"); - - files = sparse_file_resparse(s, max_size, out_s, files); - if (files < 0) die("Failed to resparse"); + auto temp = std::make_unique(files); + const int rv = sparse_file_resparse(s.get(), max_size, temp.get(), files); + if (rv < 0) die("Failed to resparse"); + std::vector out_s; + for (int i = 0; i < files; i++) { + out_s.emplace_back(temp[i], sparse_file_destroy); + } return out_s; } @@ -903,15 +906,13 @@ static bool load_buf_fd(unique_fd fd, struct fastboot_buffer* buf) { int64_t limit = get_sparse_limit(sz); buf->fd = std::move(fd); if (limit) { - sparse_file** s = load_sparse_files(buf->fd.get(), limit); - if (s == nullptr) { + buf->files = load_sparse_files(buf->fd.get(), limit); + if (buf->files.empty()) { return false; } buf->type = FB_BUFFER_SPARSE; - buf->data = s; } else { buf->type = FB_BUFFER_FD; - buf->data = nullptr; buf->sz = sz; } @@ -1079,8 +1080,6 @@ static void copy_avb_footer(const std::string& partition, struct fastboot_buffer } static void flash_buf(const std::string& partition, struct fastboot_buffer* buf) { - sparse_file** s; - if (partition == "boot" || partition == "boot_a" || partition == "boot_b" || partition == "init_boot" || partition == "init_boot_a" || partition == "init_boot_b" || partition == "recovery" || partition == "recovery_a" || partition == "recovery_b") { @@ -1103,17 +1102,10 @@ static void flash_buf(const std::string& partition, struct fastboot_buffer* buf) switch (buf->type) { case FB_BUFFER_SPARSE: { - std::vector> sparse_files; - s = reinterpret_cast(buf->data); - while (*s) { - int64_t sz = sparse_file_len(*s, true, false); - sparse_files.emplace_back(*s, sz); - ++s; - } - - for (size_t i = 0; i < sparse_files.size(); ++i) { - const auto& pair = sparse_files[i]; - fb->FlashPartition(partition, pair.first, pair.second, i + 1, sparse_files.size()); + for (size_t i = 0; i < buf->files.size(); i++) { + sparse_file* s = buf->files[i].get(); + int64_t sz = sparse_file_len(s, true, false); + fb->FlashPartition(partition, s, sz, i + 1, buf->files.size()); } break; } From a67fc32a8afd225e85709301ed249c8e976d8a42 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 27 Jan 2023 16:15:52 -0800 Subject: [PATCH 3/4] fastboot: Allow using LOG(). The die() and verbose() macros are not really standard for AOSP. To allow a gradual transition off them, call InitLogging on startup. This will also allow seeing liblp logs when -v is passed. Bug: 266982466 Test: builds Change-Id: I74278bb5f698edb0dc81477a575b130b0bd11cdf --- fastboot/fastboot.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index 0aa139059..716fc4f67 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -53,6 +53,7 @@ #include #include +#include #include #include #include @@ -1860,7 +1861,19 @@ static void do_wipe_super(const std::string& image, const std::string& slot_over } } +static void FastbootLogger(android::base::LogId /* id */, android::base::LogSeverity /* severity */, + const char* /* tag */, const char* /* file */, unsigned int /* line */, + const char* message) { + verbose("%s", message); +} + +static void FastbootAborter(const char* message) { + die("%s", message); +} + int FastBootTool::Main(int argc, char* argv[]) { + android::base::InitLogging(argv, FastbootLogger, FastbootAborter); + bool wants_wipe = false; bool wants_reboot = false; bool wants_reboot_bootloader = false; From aa87dc5a0d10630c313ec9aebf8a5d51ce3ef8d9 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 27 Jan 2023 20:39:06 -0800 Subject: [PATCH 4/4] fastboot: Move some helpers into util.h/.cpp. Bug: 266982466 Test: builds Change-Id: Ib744d763e11d8a7f7e3f417b331defff61fe4559 --- fastboot/fastboot.cpp | 65 +++++++++++++++---------------------------- fastboot/util.cpp | 35 ++++++++++++++++++++++- fastboot/util.h | 18 ++++++++++++ 3 files changed, 75 insertions(+), 43 deletions(-) diff --git a/fastboot/fastboot.cpp b/fastboot/fastboot.cpp index 716fc4f67..7a5a78203 100644 --- a/fastboot/fastboot.cpp +++ b/fastboot/fastboot.cpp @@ -107,8 +107,6 @@ static bool g_disable_verification = false; fastboot::FastBootDriver* fb = nullptr; -using SparsePtr = std::unique_ptr; - enum fb_buffer_type { FB_BUFFER_FD, FB_BUFFER_SPARSE, @@ -249,14 +247,6 @@ static void InfoMessage(const std::string& info) { fprintf(stderr, "(bootloader) %s\n", info.c_str()); } -static int64_t get_file_size(borrowed_fd fd) { - struct stat sb; - if (fstat(fd.get(), &sb) == -1) { - die("could not get file size"); - } - return sb.st_size; -} - bool ReadFileToVector(const std::string& file, std::vector* out) { out->clear(); @@ -827,19 +817,16 @@ static void DumpInfo() { fprintf(stderr, "--------------------------------------------\n"); } -static std::vector load_sparse_files(int fd, int64_t max_size) { - SparsePtr s(sparse_file_import_auto(fd, false, true), sparse_file_destroy); - if (!s) die("cannot sparse read file"); - +static std::vector resparse_file(sparse_file* s, int64_t max_size) { if (max_size <= 0 || max_size > std::numeric_limits::max()) { die("invalid max size %" PRId64, max_size); } - const int files = sparse_file_resparse(s.get(), max_size, nullptr, 0); + const int files = sparse_file_resparse(s, max_size, nullptr, 0); if (files < 0) die("Failed to resparse"); auto temp = std::make_unique(files); - const int rv = sparse_file_resparse(s.get(), max_size, temp.get(), files); + const int rv = sparse_file_resparse(s, max_size, temp.get(), files); if (rv < 0) die("Failed to resparse"); std::vector out_s; @@ -849,6 +836,13 @@ static std::vector load_sparse_files(int fd, int64_t max_size) { return out_s; } +static std::vector load_sparse_files(int fd, int64_t max_size) { + SparsePtr s(sparse_file_import_auto(fd, false, true), sparse_file_destroy); + if (!s) die("cannot sparse read file"); + + return resparse_file(s.get(), max_size); +} + static uint64_t get_uint_var(const char* var_name) { std::string value_str; if (fb->GetVar(var_name, &value_str) != fastboot::SUCCESS || value_str.empty()) { @@ -998,6 +992,8 @@ static bool has_vbmeta_partition() { fb->GetVar("partition-type:vbmeta_b", &partition_type) == fastboot::SUCCESS; } +// Note: this only works in userspace fastboot. In the bootloader, use +// should_flash_in_userspace(). static bool is_logical(const std::string& partition) { std::string value; return fb->GetVar("is-logical:" + partition, &value) == fastboot::SUCCESS && value == "yes"; @@ -1080,6 +1076,15 @@ static void copy_avb_footer(const std::string& partition, struct fastboot_buffer lseek(buf->fd.get(), 0, SEEK_SET); } +static void flash_partition_files(const std::string& partition, + const std::vector& files) { + for (size_t i = 0; i < files.size(); i++) { + sparse_file* s = files[i].get(); + int64_t sz = sparse_file_len(s, true, false); + fb->FlashPartition(partition, s, sz, i + 1, files.size()); + } +} + static void flash_buf(const std::string& partition, struct fastboot_buffer* buf) { if (partition == "boot" || partition == "boot_a" || partition == "boot_b" || partition == "init_boot" || partition == "init_boot_a" || partition == "init_boot_b" || @@ -1103,11 +1108,7 @@ static void flash_buf(const std::string& partition, struct fastboot_buffer* buf) switch (buf->type) { case FB_BUFFER_SPARSE: { - for (size_t i = 0; i < buf->files.size(); i++) { - sparse_file* s = buf->files[i].get(); - int64_t sz = sparse_file_len(s, true, false); - fb->FlashPartition(partition, s, sz, i + 1, buf->files.size()); - } + flash_partition_files(partition, buf->files); break; } case FB_BUFFER_FD: @@ -1395,13 +1396,6 @@ static void CancelSnapshotIfNeeded() { } } -class ImageSource { - public: - virtual ~ImageSource(){}; - virtual bool ReadFile(const std::string& name, std::vector* out) const = 0; - virtual unique_fd OpenFile(const std::string& name) const = 0; -}; - class FlashAllTool { public: FlashAllTool(const ImageSource& source, const std::string& slot_override, bool skip_secondary, @@ -1775,20 +1769,7 @@ static bool should_flash_in_userspace(const std::string& partition_name) { if (!metadata) { return false; } - for (const auto& partition : metadata->partitions) { - auto candidate = android::fs_mgr::GetPartitionName(partition); - if (partition.attributes & LP_PARTITION_ATTR_SLOT_SUFFIXED) { - // On retrofit devices, we don't know if, or whether, the A or B - // slot has been flashed for dynamic partitions. Instead we add - // both names to the list as a conservative guess. - if (candidate + "_a" == partition_name || candidate + "_b" == partition_name) { - return true; - } - } else if (candidate == partition_name) { - return true; - } - } - return false; + return should_flash_in_userspace(*metadata.get(), partition_name); } static bool wipe_super(const android::fs_mgr::LpMetadata& metadata, const std::string& slot, diff --git a/fastboot/util.cpp b/fastboot/util.cpp index 900d6ea87..ded54a5b9 100644 --- a/fastboot/util.cpp +++ b/fastboot/util.cpp @@ -30,11 +30,13 @@ #include #include #include - +#include #include #include "util.h" +using android::base::borrowed_fd; + static bool g_verbose = false; double now() { @@ -73,3 +75,34 @@ void verbose(const char* fmt, ...) { } fprintf(stderr, "\n"); } + +bool should_flash_in_userspace(const android::fs_mgr::LpMetadata& metadata, + const std::string& partition_name) { + for (const auto& partition : metadata.partitions) { + auto candidate = android::fs_mgr::GetPartitionName(partition); + if (partition.attributes & LP_PARTITION_ATTR_SLOT_SUFFIXED) { + // On retrofit devices, we don't know if, or whether, the A or B + // slot has been flashed for dynamic partitions. Instead we add + // both names to the list as a conservative guess. + if (candidate + "_a" == partition_name || candidate + "_b" == partition_name) { + return true; + } + } else if (candidate == partition_name) { + return true; + } + } + return false; +} + +bool is_sparse_file(borrowed_fd fd) { + SparsePtr s(sparse_file_import(fd.get(), false, false), sparse_file_destroy); + return !!s; +} + +int64_t get_file_size(borrowed_fd fd) { + struct stat sb; + if (fstat(fd.get(), &sb) == -1) { + die("could not get file size"); + } + return sb.st_size; +} diff --git a/fastboot/util.h b/fastboot/util.h index c719df2b5..290d0d5fe 100644 --- a/fastboot/util.h +++ b/fastboot/util.h @@ -4,8 +4,14 @@ #include #include +#include +#include #include +#include +#include + +using SparsePtr = std::unique_ptr; /* util stuff */ double now(); @@ -19,3 +25,15 @@ __attribute__((__format__(__printf__, 1, 2))); void verbose(const char* fmt, ...) __attribute__((__format__(__printf__, 1, 2))); void die(const std::string& str) __attribute__((__noreturn__)); + +bool should_flash_in_userspace(const android::fs_mgr::LpMetadata& metadata, + const std::string& partition_name); +bool is_sparse_file(android::base::borrowed_fd fd); +int64_t get_file_size(android::base::borrowed_fd fd); + +class ImageSource { + public: + virtual ~ImageSource(){}; + virtual bool ReadFile(const std::string& name, std::vector* out) const = 0; + virtual android::base::unique_fd OpenFile(const std::string& name) const = 0; +};