diff --git a/fs_mgr/libsnapshot/Android.bp b/fs_mgr/libsnapshot/Android.bp index f154138ad..aa9bf88c5 100644 --- a/fs_mgr/libsnapshot/Android.bp +++ b/fs_mgr/libsnapshot/Android.bp @@ -35,9 +35,11 @@ cc_defaults { "update_metadata-protos", ], whole_static_libs: [ + "libcutils", "libext2_uuid", "libext4_utils", "libfstab", + "libsnapshot_snapuserd", ], header_libs: [ "libchrome", diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h index 5b50ca97b..d81bbd075 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapshot.h @@ -37,6 +37,7 @@ #include #include #include +#include #ifndef FRIEND_TEST #define FRIEND_TEST(test_set_name, individual_test) \ @@ -358,6 +359,9 @@ class SnapshotManager final : public ISnapshotManager { // This is created lazily since it can connect via binder. bool EnsureImageManager(); + // Ensure we're connected to snapuserd. + bool EnsureSnapuserdConnected(); + // Helper for first-stage init. bool ForceLocalImageManager(); @@ -414,6 +418,11 @@ class SnapshotManager final : public ISnapshotManager { const std::string& cow_device, const std::chrono::milliseconds& timeout_ms, std::string* dev_path); + // Create a dm-user device for a given snapshot. + bool MapDmUserCow(LockedFile* lock, const std::string& name, const std::string& cow_file, + const std::string& base_device, const std::chrono::milliseconds& timeout_ms, + std::string* path); + // Map a COW image that was previous created with CreateCowImage. std::optional MapCowImage(const std::string& name, const std::chrono::milliseconds& timeout_ms); @@ -642,6 +651,7 @@ class SnapshotManager final : public ISnapshotManager { std::unique_ptr images_; bool has_local_image_manager_ = false; bool in_factory_data_reset_ = false; + std::unique_ptr snapuserd_client_; }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h index 2f727d6ee..80f87d968 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd.h @@ -77,7 +77,7 @@ class Snapuserd final { int ReadDiskExceptions(chunk_t chunk, size_t size); int ReadData(chunk_t chunk, size_t size); - std::string GetControlDevicePath() { return control_device_; } + const std::string& GetControlDevicePath() { return control_device_; } private: int ProcessReplaceOp(const CowOperation* cow_op); diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h index dffd4813e..0bbdaa583 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_client.h @@ -53,6 +53,10 @@ class SnapuserdClient { int RestartSnapuserd(std::vector>& vec); bool InitializeSnapuserd(const std::string& cow_device, const std::string& backing_device, const std::string& control_device); + + // Wait for snapuserd to disassociate with a dm-user control device. This + // must ONLY be called if the control device has already been deleted. + bool WaitForDeviceDelete(const std::string& control_device); }; } // namespace snapshot diff --git a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h index 357acac9a..181ee33fe 100644 --- a/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h +++ b/fs_mgr/libsnapshot/include/libsnapshot/snapuserd_server.h @@ -21,12 +21,14 @@ #include #include #include +#include #include #include #include #include #include +#include namespace android { namespace snapshot { @@ -37,19 +39,23 @@ enum class DaemonOperations { START, QUERY, STOP, + DELETE, INVALID, }; class DmUserHandler { private: - std::unique_ptr threadHandler_; + std::thread thread_; + std::unique_ptr snapuserd_; public: - void SetThreadHandler(std::function func) { - threadHandler_ = std::make_unique(func); - } + explicit DmUserHandler(std::unique_ptr&& snapuserd) + : snapuserd_(std::move(snapuserd)) {} - std::unique_ptr& GetThreadHandler() { return threadHandler_; } + const std::unique_ptr& snapuserd() const { return snapuserd_; } + std::thread& thread() { return thread_; } + + const std::string& GetControlDevice() const; }; class Stoppable { @@ -61,9 +67,6 @@ class Stoppable { virtual ~Stoppable() {} - virtual void ThreadStart(std::string cow_device, std::string backing_device, - std::string control_device) = 0; - bool StopRequested() { // checks if value in future object is available if (futureObj_.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout) @@ -78,27 +81,33 @@ class SnapuserdServer : public Stoppable { private: android::base::unique_fd sockfd_; bool terminating_; - std::vector> dm_users_; std::vector watched_fds_; + std::mutex lock_; + std::vector> dm_users_; + void AddWatchedFd(android::base::borrowed_fd fd); void AcceptClient(); bool HandleClient(android::base::borrowed_fd fd, int revents); bool Recv(android::base::borrowed_fd fd, std::string* data); bool Sendmsg(android::base::borrowed_fd fd, const std::string& msg); - bool Receivemsg(android::base::borrowed_fd fd, const std::string& msg); + bool Receivemsg(android::base::borrowed_fd fd, const std::string& str); - void ThreadStart(std::string cow_device, std::string backing_device, - std::string control_device) override; void ShutdownThreads(); + bool WaitForDelete(const std::string& control_device); DaemonOperations Resolveop(std::string& input); std::string GetDaemonStatus(); void Parsemsg(std::string const& msg, const char delim, std::vector& out); void SetTerminating() { terminating_ = true; } - bool IsTerminating() { return terminating_; } + void RunThread(DmUserHandler* handler); + + // Remove a DmUserHandler from dm_users_, searching by its control device. + // If none is found, return nullptr. + std::unique_ptr RemoveHandler(const std::string& control_device); + public: SnapuserdServer() { terminating_ = false; } ~SnapuserdServer(); diff --git a/fs_mgr/libsnapshot/snapshot.cpp b/fs_mgr/libsnapshot/snapshot.cpp index 6574457db..c88c01a50 100644 --- a/fs_mgr/libsnapshot/snapshot.cpp +++ b/fs_mgr/libsnapshot/snapshot.cpp @@ -56,6 +56,7 @@ using android::dm::DmDeviceState; using android::dm::DmTable; using android::dm::DmTargetLinear; using android::dm::DmTargetSnapshot; +using android::dm::DmTargetUser; using android::dm::kSectorSize; using android::dm::SnapshotStorageMode; using android::fiemap::FiemapStatus; @@ -115,6 +116,10 @@ static std::string GetCowName(const std::string& snapshot_name) { return snapshot_name + "-cow"; } +static std::string GetDmUserCowName(const std::string& snapshot_name) { + return snapshot_name + "-user-cow"; +} + static std::string GetCowImageDeviceName(const std::string& snapshot_name) { return snapshot_name + "-cow-img"; } @@ -370,6 +375,45 @@ Return SnapshotManager::CreateCowImage(LockedFile* lock, const std::string& name return Return(images_->CreateBackingImage(cow_image_name, status.cow_file_size(), cow_flags)); } +bool SnapshotManager::MapDmUserCow(LockedFile* lock, const std::string& name, + const std::string& cow_file, const std::string& base_device, + const std::chrono::milliseconds& timeout_ms, std::string* path) { + CHECK(lock); + + auto& dm = DeviceMapper::Instance(); + + // Use the size of the base device for the COW device. It doesn't really + // matter, it just needs to look similar enough so the kernel doesn't complain + // about alignment or being too small. + uint64_t base_sectors = 0; + { + unique_fd fd(open(base_device.c_str(), O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + PLOG(ERROR) << "open failed: " << base_device; + return false; + } + auto dev_size = get_block_device_size(fd); + if (!dev_size) { + PLOG(ERROR) << "Could not determine block device size: " << base_device; + return false; + } + base_sectors = dev_size / kSectorSize; + } + + DmTable table; + table.Emplace(0, base_sectors, name); + if (!dm.CreateDevice(name, table, path, timeout_ms)) { + return false; + } + + if (!EnsureSnapuserdConnected()) { + return false; + } + + auto control_device = "/dev/dm-user/" + name; + return snapuserd_client_->InitializeSnapuserd(cow_file, base_device, control_device); +} + bool SnapshotManager::MapSnapshot(LockedFile* lock, const std::string& name, const std::string& base_device, const std::string& cow_device, const std::chrono::milliseconds& timeout_ms, @@ -1728,6 +1772,30 @@ bool SnapshotManager::MapPartitionWithSnapshot(LockedFile* lock, return true; } + if (IsCompressionEnabled()) { + auto name = GetDmUserCowName(params.GetPartitionName()); + + // :TODO: need to force init to process uevents for these in first-stage. + std::string cow_path; + if (!GetMappedImageDevicePath(cow_name, &cow_path)) { + LOG(ERROR) << "Could not determine path for: " << cow_name; + return false; + } + + std::string new_cow_device; + if (!MapDmUserCow(lock, name, cow_path, base_path, remaining_time, &new_cow_device)) { + LOG(ERROR) << "Could not map dm-user device for partition " + << params.GetPartitionName(); + return false; + } + created_devices.EmplaceBack(&dm, name); + + remaining_time = GetRemainingTime(params.timeout_ms, begin); + if (remaining_time.count() < 0) return false; + + cow_device = new_cow_device; + } + std::string path; if (!MapSnapshot(lock, params.GetPartitionName(), base_device, cow_device, remaining_time, &path)) { @@ -1847,6 +1915,22 @@ bool SnapshotManager::UnmapCowDevices(LockedFile* lock, const std::string& name) if (!EnsureImageManager()) return false; auto& dm = DeviceMapper::Instance(); + + auto dm_user_name = GetDmUserCowName(name); + if (IsCompressionEnabled() && dm.GetState(dm_user_name) != DmDeviceState::INVALID) { + if (!EnsureSnapuserdConnected()) { + return false; + } + if (!dm.DeleteDevice(dm_user_name)) { + LOG(ERROR) << "Cannot unmap " << dm_user_name; + return false; + } + if (!snapuserd_client_->WaitForDeviceDelete("/dev/dm-user/" + dm_user_name)) { + LOG(ERROR) << "Failed to wait for " << dm_user_name << " control device to delete"; + return false; + } + } + auto cow_name = GetCowName(name); if (!dm.DeleteDeviceIfExists(cow_name)) { LOG(ERROR) << "Cannot unmap " << cow_name; @@ -2117,6 +2201,20 @@ bool SnapshotManager::EnsureImageManager() { return true; } +bool SnapshotManager::EnsureSnapuserdConnected() { + if (!snapuserd_client_) { + if (!EnsureSnapuserdStarted()) { + return false; + } + snapuserd_client_ = SnapuserdClient::Connect(kSnapuserdSocket, 10s); + if (!snapuserd_client_) { + LOG(ERROR) << "Unable to connect to snapuserd"; + return false; + } + } + return true; +} + bool SnapshotManager::ForceLocalImageManager() { images_ = android::fiemap::ImageManager::Open(gsid_dir_); if (!images_) { diff --git a/fs_mgr/libsnapshot/snapuserd_client.cpp b/fs_mgr/libsnapshot/snapuserd_client.cpp index 532e5854c..35bb29b8b 100644 --- a/fs_mgr/libsnapshot/snapuserd_client.cpp +++ b/fs_mgr/libsnapshot/snapuserd_client.cpp @@ -123,34 +123,32 @@ bool SnapuserdClient::Sendmsg(const std::string& msg) { return true; } -std::string SnapuserdClient::Receivemsg() { - int ret; - struct timeval tv; - fd_set set; - char msg[PACKET_SIZE]; - std::string msgStr("fail"); - - tv.tv_sec = 2; - tv.tv_usec = 0; - FD_ZERO(&set); - FD_SET(sockfd_, &set); - ret = select(sockfd_ + 1, &set, NULL, NULL, &tv); - if (ret == -1) { // select failed - LOG(ERROR) << "Snapuserd:client: Select call failed"; - } else if (ret == 0) { // timeout - LOG(ERROR) << "Snapuserd:client: Select call timeout"; - } else { - ret = TEMP_FAILURE_RETRY(recv(sockfd_, msg, PACKET_SIZE, 0)); - if (ret < 0) { - PLOG(ERROR) << "Snapuserd:client: recv failed"; - } else if (ret == 0) { - LOG(DEBUG) << "Snapuserd:client disconnected"; - } else { - msgStr.clear(); - msgStr = msg; - } +bool SnapuserdClient::WaitForDeviceDelete(const std::string& control_device) { + std::string msg = "delete," + control_device; + if (!Sendmsg(msg)) { + LOG(ERROR) << "Failed to send message " << msg << " to snapuserd"; + return false; } - return msgStr; + std::string response = Receivemsg(); + if (response != "success") { + LOG(ERROR) << "Failed waiting to delete device " << control_device; + return false; + } + return true; +} + +std::string SnapuserdClient::Receivemsg() { + char msg[PACKET_SIZE]; + ssize_t ret = TEMP_FAILURE_RETRY(recv(sockfd_, msg, sizeof(msg), 0)); + if (ret < 0) { + PLOG(ERROR) << "Snapuserd:client: recv failed"; + return {}; + } + if (ret == 0) { + LOG(DEBUG) << "Snapuserd:client disconnected"; + return {}; + } + return std::string(msg, ret); } bool SnapuserdClient::StopSnapuserd() { diff --git a/fs_mgr/libsnapshot/snapuserd_server.cpp b/fs_mgr/libsnapshot/snapuserd_server.cpp index 48a3b2a79..6b8cdd9e0 100644 --- a/fs_mgr/libsnapshot/snapuserd_server.cpp +++ b/fs_mgr/libsnapshot/snapuserd_server.cpp @@ -36,6 +36,7 @@ DaemonOperations SnapuserdServer::Resolveop(std::string& input) { if (input == "start") return DaemonOperations::START; if (input == "stop") return DaemonOperations::STOP; if (input == "query") return DaemonOperations::QUERY; + if (input == "delete") return DaemonOperations::DELETE; return DaemonOperations::INVALID; } @@ -68,33 +69,25 @@ void SnapuserdServer::Parsemsg(std::string const& msg, const char delim, } } -// new thread -void SnapuserdServer::ThreadStart(std::string cow_device, std::string backing_device, - std::string control_device) { - Snapuserd snapd(cow_device, backing_device, control_device); - if (!snapd.Init()) { - LOG(ERROR) << "Snapuserd: Init failed"; - return; - } - - while (StopRequested() == false) { - int ret = snapd.Run(); - - if (ret < 0) { - LOG(ERROR) << "Snapuserd: Thread terminating as control device is de-registered"; - break; - } - } -} - void SnapuserdServer::ShutdownThreads() { StopThreads(); - for (auto& client : dm_users_) { - auto& th = client->GetThreadHandler(); - - if (th->joinable()) th->join(); + // Acquire the thread list within the lock. + std::vector> dm_users; + { + std::lock_guard guard(lock_); + dm_users = std::move(dm_users_); } + + for (auto& client : dm_users) { + auto& th = client->thread(); + + if (th.joinable()) th.join(); + } +} + +const std::string& DmUserHandler::GetControlDevice() const { + return snapuserd_->GetControlDevicePath(); } bool SnapuserdServer::Sendmsg(android::base::borrowed_fd fd, const std::string& msg) { @@ -135,10 +128,25 @@ bool SnapuserdServer::Receivemsg(android::base::borrowed_fd fd, const std::strin // start,,, // // Start the new thread which binds to dm-user misc device - auto handler = std::make_unique(); - handler->SetThreadHandler( - std::bind(&SnapuserdServer::ThreadStart, this, out[1], out[2], out[3])); - dm_users_.push_back(std::move(handler)); + if (out.size() != 4) { + LOG(ERROR) << "Malformed start message, " << out.size() << " parts"; + return Sendmsg(fd, "fail"); + } + + auto snapuserd = std::make_unique(out[1], out[2], out[3]); + if (!snapuserd->Init()) { + LOG(ERROR) << "Failed to initialize Snapuserd"; + return Sendmsg(fd, "fail"); + } + + auto handler = std::make_unique(std::move(snapuserd)); + { + std::lock_guard lock(lock_); + + handler->thread() = + std::thread(std::bind(&SnapuserdServer::RunThread, this, handler.get())); + dm_users_.push_back(std::move(handler)); + } return Sendmsg(fd, "success"); } case DaemonOperations::STOP: { @@ -162,6 +170,18 @@ bool SnapuserdServer::Receivemsg(android::base::borrowed_fd fd, const std::strin // be ready to receive control message. return Sendmsg(fd, GetDaemonStatus()); } + case DaemonOperations::DELETE: { + // Message format: + // delete, + if (out.size() != 2) { + LOG(ERROR) << "Malformed delete message, " << out.size() << " parts"; + return Sendmsg(fd, "fail"); + } + if (!WaitForDelete(out[1])) { + return Sendmsg(fd, "fail"); + } + return Sendmsg(fd, "success"); + } default: { LOG(ERROR) << "Received unknown message type from client"; Sendmsg(fd, "fail"); @@ -170,6 +190,23 @@ bool SnapuserdServer::Receivemsg(android::base::borrowed_fd fd, const std::strin } } +void SnapuserdServer::RunThread(DmUserHandler* handler) { + while (!StopRequested()) { + if (handler->snapuserd()->Run() < 0) { + LOG(INFO) << "Snapuserd: Thread terminating as control device is de-registered"; + break; + } + } + + if (auto client = RemoveHandler(handler->GetControlDevice())) { + // The main thread did not receive a WaitForDelete request for this + // control device. Since we transferred ownership within the lock, + // we know join() was never called, and will never be called. We can + // safely detach here. + client->thread().detach(); + } +} + bool SnapuserdServer::Start(const std::string& socketname) { sockfd_.reset(android_get_control_socket(socketname.c_str())); if (sockfd_ >= 0) { @@ -260,5 +297,37 @@ void SnapuserdServer::Interrupt() { SetTerminating(); } +std::unique_ptr SnapuserdServer::RemoveHandler(const std::string& control_device) { + std::unique_ptr client; + { + std::lock_guard lock(lock_); + auto iter = dm_users_.begin(); + while (iter != dm_users_.end()) { + if ((*iter)->GetControlDevice() == control_device) { + client = std::move(*iter); + iter = dm_users_.erase(iter); + break; + } + iter++; + } + } + return client; +} + +bool SnapuserdServer::WaitForDelete(const std::string& control_device) { + auto client = RemoveHandler(control_device); + + // Client already deleted. + if (!client) { + return true; + } + + auto& th = client->thread(); + if (th.joinable()) { + th.join(); + } + return true; +} + } // namespace snapshot } // namespace android