From c4a9d16ef1e8d4c277ff5071a6e250acd5e4f5e8 Mon Sep 17 00:00:00 2001 From: Songchun Fan Date: Mon, 9 Mar 2020 11:33:44 -0700 Subject: [PATCH 1/2] [adb data server] wait for installation results before terminates Currently the server often quits before installation finishes. As a result, there is no difference in the commandline output between a successful installation and a failed one. Let adb client wait till installation fails or succeeds by parsing the output from the inc-server process. Test: $ adb install --incremental ~/Downloads/base.apk Test: Performing Incremental Install Test: Serving... Test: All files should be loaded. Notifying the device. Test: Failure [INSTALL_PARSE_FAILED_NOT_APK: Failed to parse /data/app/vmdl749343150.tmp/base.apk: Failed to load asset path /data/app/vmdl749343150.tmp/base.apk] Test: Install command complete (ms: 91 total, 0 apk prep, 91 install) BUG: b/150865433 Change-Id: Ie33505f9cc08fc6d60ad4a5d709526e7aa9a0ad1 --- adb/client/commandline.cpp | 49 ++++++++++++--- adb/client/incremental.cpp | 60 ++++++++++++++++-- adb/client/incremental.h | 3 + adb/client/incremental_server.cpp | 100 ++++++++++++++++++------------ adb/client/incremental_server.h | 2 +- 5 files changed, 160 insertions(+), 54 deletions(-) diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp index 081bac4df..14ccfbe78 100644 --- a/adb/client/commandline.cpp +++ b/adb/client/commandline.cpp @@ -1423,6 +1423,26 @@ static bool _is_valid_ack_reply_fd(const int ack_reply_fd) { #endif } +static bool _is_valid_fd(int fd) { + // Disallow invalid FDs and stdin/out/err as well. + if (fd < 3) { + return false; + } +#ifdef _WIN32 + HANDLE handle = adb_get_os_handle(fd); + DWORD info = 0; + if (GetHandleInformation(handle, &info) == 0) { + return false; + } +#else + int flags = fcntl(fd, F_GETFD); + if (flags == -1) { + return false; + } +#endif + return true; +} + int adb_commandline(int argc, const char** argv) { bool no_daemon = false; bool is_daemon = false; @@ -1977,17 +1997,28 @@ int adb_commandline(int argc, const char** argv) { } } } else if (!strcmp(argv[0], "inc-server")) { - if (argc < 3) { - error_exit("usage: adb inc-server FD FILE1 FILE2 ..."); + if (argc < 4) { +#ifdef _WIN32 + error_exit("usage: adb inc-server CONNECTION_HANDLE OUTPUT_HANDLE FILE1 FILE2 ..."); +#else + error_exit("usage: adb inc-server CONNECTION_FD OUTPUT_FD FILE1 FILE2 ..."); +#endif } - int fd = atoi(argv[1]); - if (fd < 3) { - // Disallow invalid FDs and stdin/out/err as well. - error_exit("Invalid fd number given: %d", fd); + int connection_fd = atoi(argv[1]); + if (!_is_valid_fd(connection_fd)) { + error_exit("Invalid connection_fd number given: %d", connection_fd); } - fd = adb_register_socket(fd); - close_on_exec(fd); - return incremental::serve(fd, argc - 2, argv + 2); + + connection_fd = adb_register_socket(connection_fd); + close_on_exec(connection_fd); + + int output_fd = atoi(argv[2]); + if (!_is_valid_fd(output_fd)) { + error_exit("Invalid output_fd number given: %d", output_fd); + } + output_fd = adb_register_socket(output_fd); + close_on_exec(output_fd); + return incremental::serve(connection_fd, output_fd, argc - 3, argv + 3); } error_exit("unknown command %s", argv[0]); diff --git a/adb/client/incremental.cpp b/adb/client/incremental.cpp index 3ceb3748a..fd608cced 100644 --- a/adb/client/incremental.cpp +++ b/adb/client/incremental.cpp @@ -193,20 +193,72 @@ std::optional install(std::vector files) { auto fd_param = std::to_string(osh); #endif + // pipe for child process to write output + int print_fds[2]; + if (adb_socketpair(print_fds) != 0) { + fprintf(stderr, "Failed to create socket pair for child to print to parent\n"); + return {}; + } + auto [pipe_read_fd, pipe_write_fd] = print_fds; + auto pipe_write_fd_param = std::to_string(pipe_write_fd); + close_on_exec(pipe_read_fd); + std::vector args(std::move(files)); - args.insert(args.begin(), {"inc-server", fd_param}); - auto child = adb_launch_process(adb_path, std::move(args), {connection_fd.get()}); + args.insert(args.begin(), {"inc-server", fd_param, pipe_write_fd_param}); + auto child = + adb_launch_process(adb_path, std::move(args), {connection_fd.get(), pipe_write_fd}); if (!child) { fprintf(stderr, "adb: failed to fork: %s\n", strerror(errno)); return {}; } + adb_close(pipe_write_fd); + auto killOnExit = [](Process* p) { p->kill(); }; std::unique_ptr serverKiller(&child, killOnExit); - // TODO: Terminate server process if installation fails. - serverKiller.release(); + Result result = wait_for_installation(pipe_read_fd); + adb_close(pipe_read_fd); + + if (result == Result::Success) { + // adb client exits now but inc-server can continue + serverKiller.release(); + } return child; } +Result wait_for_installation(int read_fd) { + static constexpr int maxMessageSize = 256; + std::vector child_stdout(CHUNK_SIZE); + int bytes_read; + int buf_size = 0; + // TODO(b/150865433): optimize child's output parsing + while ((bytes_read = adb_read(read_fd, child_stdout.data() + buf_size, + child_stdout.size() - buf_size)) > 0) { + // print to parent's stdout + fprintf(stdout, "%.*s", bytes_read, child_stdout.data() + buf_size); + + buf_size += bytes_read; + const std::string_view stdout_str(child_stdout.data(), buf_size); + // wait till installation either succeeds or fails + if (stdout_str.find("Success") != std::string::npos) { + return Result::Success; + } + // on failure, wait for full message + static constexpr auto failure_msg_head = "Failure ["sv; + if (const auto begin_itr = stdout_str.find(failure_msg_head); + begin_itr != std::string::npos) { + if (buf_size >= maxMessageSize) { + return Result::Failure; + } + const auto end_itr = stdout_str.rfind("]"); + if (end_itr != std::string::npos && end_itr >= begin_itr + failure_msg_head.size()) { + return Result::Failure; + } + } + child_stdout.resize(buf_size + CHUNK_SIZE); + } + return Result::None; +} + } // namespace incremental diff --git a/adb/client/incremental.h b/adb/client/incremental.h index 4b9f6bde0..731e6fb4d 100644 --- a/adb/client/incremental.h +++ b/adb/client/incremental.h @@ -27,4 +27,7 @@ namespace incremental { std::optional install(std::vector files); +enum class Result { Success, Failure, None }; +Result wait_for_installation(int read_fd); + } // namespace incremental diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp index 2512d0562..6c36a0f64 100644 --- a/adb/client/incremental_server.cpp +++ b/adb/client/incremental_server.cpp @@ -18,13 +18,6 @@ #include "incremental_server.h" -#include "adb.h" -#include "adb_io.h" -#include "adb_trace.h" -#include "adb_unique_fd.h" -#include "adb_utils.h" -#include "sysdeps.h" - #include #include #include @@ -41,6 +34,13 @@ #include #include +#include "adb.h" +#include "adb_io.h" +#include "adb_trace.h" +#include "adb_unique_fd.h" +#include "adb_utils.h" +#include "sysdeps.h" + namespace incremental { static constexpr int kBlockSize = 4096; @@ -49,6 +49,7 @@ static constexpr short kCompressionNone = 0; static constexpr short kCompressionLZ4 = 1; static constexpr int kCompressBound = std::max(kBlockSize, LZ4_COMPRESSBOUND(kBlockSize)); static constexpr auto kReadBufferSize = 128 * 1024; +static constexpr int kPollTimeoutMillis = 300000; // 5 minutes using BlockSize = int16_t; using FileId = int16_t; @@ -61,9 +62,10 @@ using MagicType = uint32_t; static constexpr MagicType INCR = 0x494e4352; // LE INCR -static constexpr RequestType EXIT = 0; +static constexpr RequestType SERVING_COMPLETE = 0; static constexpr RequestType BLOCK_MISSING = 1; static constexpr RequestType PREFETCH = 2; +static constexpr RequestType DESTROY = 3; static constexpr inline int64_t roundDownToBlockOffset(int64_t val) { return val & ~(kBlockSize - 1); @@ -162,8 +164,8 @@ class File { class IncrementalServer { public: - IncrementalServer(unique_fd fd, std::vector files) - : adb_fd_(std::move(fd)), files_(std::move(files)) { + IncrementalServer(unique_fd adb_fd, unique_fd output_fd, std::vector files) + : adb_fd_(std::move(adb_fd)), output_fd_(std::move(output_fd)), files_(std::move(files)) { buffer_.reserve(kReadBufferSize); } @@ -197,9 +199,10 @@ class IncrementalServer { void Send(const void* data, size_t size, bool flush); void Flush(); using TimePoint = decltype(std::chrono::high_resolution_clock::now()); - bool Exit(std::optional startTime, int missesCount, int missesSent); + bool ServingComplete(std::optional startTime, int missesCount, int missesSent); unique_fd const adb_fd_; + unique_fd const output_fd_; std::vector files_; // Incoming data buffer. @@ -210,6 +213,9 @@ class IncrementalServer { long long sentSize_ = 0; std::vector pendingBlocks_; + + // True when client notifies that all the data has been received + bool servingComplete_; }; bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) { @@ -217,7 +223,8 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) // Looking for INCR magic. bool magic_found = false; int bcur = 0; - for (int bsize = buffer_.size(); bcur + 4 < bsize; ++bcur) { + int bsize = buffer_.size(); + for (bcur = 0; bcur + 4 < bsize; ++bcur) { uint32_t magic = be32toh(*(uint32_t*)(buffer_.data() + bcur)); if (magic == INCR) { magic_found = true; @@ -226,8 +233,8 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) } if (bcur > 0) { - // Stream the rest to stderr. - fprintf(stderr, "%.*s", bcur, buffer_.data()); + // output the rest. + WriteFdExactly(output_fd_, buffer_.data(), bcur); erase_buffer_head(bcur); } @@ -239,17 +246,26 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) } adb_pollfd pfd = {adb_fd_.get(), POLLIN, 0}; - auto res = adb_poll(&pfd, 1, blocking ? -1 : 0); + auto res = adb_poll(&pfd, 1, blocking ? kPollTimeoutMillis : 0); + if (res != 1) { + WriteFdExactly(output_fd_, buffer_.data(), buffer_.size()); if (res < 0) { - fprintf(stderr, "Failed to poll: %s\n", strerror(errno)); + D("Failed to poll: %s\n", strerror(errno)); + return false; + } + if (blocking) { + fprintf(stderr, "Timed out waiting for data from device.\n"); + } + if (blocking && servingComplete_) { + // timeout waiting from client. Serving is complete, so quit. return false; } *size = 0; return true; } - auto bsize = buffer_.size(); + bsize = buffer_.size(); buffer_.resize(kReadBufferSize); int r = adb_read(adb_fd_, buffer_.data() + bsize, kReadBufferSize - bsize); if (r > 0) { @@ -257,21 +273,19 @@ bool IncrementalServer::SkipToRequest(void* buffer, size_t* size, bool blocking) continue; } - if (r == -1) { - fprintf(stderr, "Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno); - return false; - } - - // socket is closed - return false; + D("Failed to read from fd %d: %d. Exit\n", adb_fd_.get(), errno); + break; } + // socket is closed. print remaining messages + WriteFdExactly(output_fd_, buffer_.data(), buffer_.size()); + return false; } std::optional IncrementalServer::ReadRequest(bool blocking) { uint8_t commandBuf[sizeof(RequestCommand)]; auto size = sizeof(commandBuf); if (!SkipToRequest(&commandBuf, &size, blocking)) { - return {{EXIT}}; + return {{DESTROY}}; } if (size < sizeof(RequestCommand)) { return {}; @@ -391,17 +405,17 @@ void IncrementalServer::Flush() { pendingBlocks_.clear(); } -bool IncrementalServer::Exit(std::optional startTime, int missesCount, int missesSent) { +bool IncrementalServer::ServingComplete(std::optional startTime, int missesCount, + int missesSent) { + servingComplete_ = true; using namespace std::chrono; auto endTime = high_resolution_clock::now(); - fprintf(stderr, - "Connection failed or received exit command. Exit.\n" - "Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: " - "%d, mb: %.3f\n" - "Total time taken: %.3fms\n", - missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0, - duration_cast(endTime - (startTime ? *startTime : endTime)).count() / - 1000.0); + D("Streaming completed.\n" + "Misses: %d, of those unique: %d; sent compressed: %d, uncompressed: " + "%d, mb: %.3f\n" + "Total time taken: %.3fms\n", + missesCount, missesSent, compressed_, uncompressed_, sentSize_ / 1024.0 / 1024.0, + duration_cast(endTime - (startTime ? *startTime : endTime)).count() / 1000.0); return true; } @@ -425,7 +439,7 @@ bool IncrementalServer::Serve() { std::all_of(files_.begin(), files_.end(), [](const File& f) { return f.sentBlocksCount == NumBlocks(f.sentBlocks.size()); })) { - fprintf(stdout, "All files should be loaded. Notifying the device.\n"); + fprintf(stderr, "All files should be loaded. Notifying the device.\n"); SendDone(); doneSent = true; } @@ -446,9 +460,14 @@ bool IncrementalServer::Serve() { BlockIdx blockIdx = request->block_idx; switch (request->request_type) { - case EXIT: { + case DESTROY: { // Stop everything. - return Exit(startTime, missesCount, missesSent); + return true; + } + case SERVING_COMPLETE: { + // Not stopping the server here. + ServingComplete(startTime, missesCount, missesSent); + break; } case BLOCK_MISSING: { ++missesCount; @@ -502,8 +521,9 @@ bool IncrementalServer::Serve() { } } -bool serve(int adb_fd, int argc, const char** argv) { - auto connection_fd = unique_fd(adb_fd); +bool serve(int connection_fd, int output_fd, int argc, const char** argv) { + auto connection_ufd = unique_fd(connection_fd); + auto output_ufd = unique_fd(output_fd); if (argc <= 0) { error_exit("inc-server: must specify at least one file."); } @@ -526,7 +546,7 @@ bool serve(int adb_fd, int argc, const char** argv) { files.emplace_back(filepath, i, st.st_size, std::move(fd)); } - IncrementalServer server(std::move(connection_fd), std::move(files)); + IncrementalServer server(std::move(connection_ufd), std::move(output_ufd), std::move(files)); printf("Serving...\n"); fclose(stdin); fclose(stdout); diff --git a/adb/client/incremental_server.h b/adb/client/incremental_server.h index 53f011eb3..55b8215b8 100644 --- a/adb/client/incremental_server.h +++ b/adb/client/incremental_server.h @@ -21,6 +21,6 @@ namespace incremental { // Expecting arguments like: // {FILE1 FILE2 ...} // Where FILE* are files to serve. -bool serve(int adbFd, int argc, const char** argv); +bool serve(int connection_fd, int output_fd, int argc, const char** argv); } // namespace incremental From 8cdefd4f95c173e9d61b28e219754e4e119fbbb0 Mon Sep 17 00:00:00 2001 From: Songchun Fan Date: Fri, 13 Mar 2020 13:11:43 -0700 Subject: [PATCH 2/2] [adb incremental] send priority blocks first Before this change, "Success" is returned after all data is streamed, around 7.5 seconds for Megacity. After this change, "Success" is returned in about 1.5 seconds, before streaming finishes. BUG: 151676293 Test: manual Change-Id: Ifda7de48da8e82623c99ae0194f70cb162fd72fa --- adb/Android.bp | 1 + adb/client/incremental_server.cpp | 30 ++++- adb/client/incremental_utils.cpp | 216 ++++++++++++++++++++++++++++++ adb/client/incremental_utils.h | 26 ++++ 4 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 adb/client/incremental_utils.cpp create mode 100644 adb/client/incremental_utils.h diff --git a/adb/Android.bp b/adb/Android.bp index 6fd0767fa..2f719456d 100644 --- a/adb/Android.bp +++ b/adb/Android.bp @@ -298,6 +298,7 @@ cc_binary_host { "client/fastdeploycallbacks.cpp", "client/incremental.cpp", "client/incremental_server.cpp", + "client/incremental_utils.cpp", "shell_service_protocol.cpp", ], diff --git a/adb/client/incremental_server.cpp b/adb/client/incremental_server.cpp index 6c36a0f64..1f47de872 100644 --- a/adb/client/incremental_server.cpp +++ b/adb/client/incremental_server.cpp @@ -39,6 +39,7 @@ #include "adb_trace.h" #include "adb_unique_fd.h" #include "adb_utils.h" +#include "incremental_utils.h" #include "sysdeps.h" namespace incremental { @@ -136,6 +137,7 @@ class File { // Plain file File(const char* filepath, FileId id, int64_t size, unique_fd fd) : File(filepath, id, size) { this->fd_ = std::move(fd); + priority_blocks_ = PriorityBlocksForFile(filepath, fd_.get(), size); } int64_t ReadBlock(BlockIdx block_idx, void* buf, bool* is_zip_compressed, std::string* error) const { @@ -147,6 +149,7 @@ class File { } const unique_fd& RawFd() const { return fd_; } + const std::vector& PriorityBlocks() const { return priority_blocks_; } std::vector sentBlocks; NumBlocks sentBlocksCount = 0; @@ -160,6 +163,7 @@ class File { sentBlocks.resize(numBytesToNumBlocks(size)); } unique_fd fd_; + std::vector priority_blocks_; }; class IncrementalServer { @@ -176,14 +180,23 @@ class IncrementalServer { const File* file; BlockIdx overallIndex = 0; BlockIdx overallEnd = 0; + BlockIdx priorityIndex = 0; - PrefetchState(const File& f) : file(&f), overallEnd((BlockIdx)f.sentBlocks.size()) {} - PrefetchState(const File& f, BlockIdx start, int count) + explicit PrefetchState(const File& f, BlockIdx start, int count) : file(&f), overallIndex(start), overallEnd(std::min(start + count, f.sentBlocks.size())) {} - bool done() const { return overallIndex >= overallEnd; } + explicit PrefetchState(const File& f) + : PrefetchState(f, 0, (BlockIdx)f.sentBlocks.size()) {} + + bool done() const { + const bool overallSent = (overallIndex >= overallEnd); + if (file->PriorityBlocks().empty()) { + return overallSent; + } + return overallSent && (priorityIndex >= (BlockIdx)file->PriorityBlocks().size()); + } }; bool SkipToRequest(void* buffer, size_t* size, bool blocking); @@ -365,6 +378,17 @@ void IncrementalServer::RunPrefetching() { while (!prefetches_.empty() && blocksToSend > 0) { auto& prefetch = prefetches_.front(); const auto& file = *prefetch.file; + const auto& priority_blocks = file.PriorityBlocks(); + if (!priority_blocks.empty()) { + for (auto& i = prefetch.priorityIndex; + blocksToSend > 0 && i < (BlockIdx)priority_blocks.size(); ++i) { + if (auto res = SendBlock(file.id, priority_blocks[i]); res == SendResult::Sent) { + --blocksToSend; + } else if (res == SendResult::Error) { + fprintf(stderr, "Failed to send priority block %" PRId32 "\n", i); + } + } + } for (auto& i = prefetch.overallIndex; blocksToSend > 0 && i < prefetch.overallEnd; ++i) { if (auto res = SendBlock(file.id, i); res == SendResult::Sent) { --blocksToSend; diff --git a/adb/client/incremental_utils.cpp b/adb/client/incremental_utils.cpp new file mode 100644 index 000000000..316480c57 --- /dev/null +++ b/adb/client/incremental_utils.cpp @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define TRACE_TAG INCREMENTAL + +#include "incremental_utils.h" + +#include +#include +#include + +#include +#include +#include + +#include "sysdeps.h" + +static constexpr int kBlockSize = 4096; + +static constexpr inline int32_t offsetToBlockIndex(int64_t offset) { + return (offset & ~(kBlockSize - 1)) >> 12; +} + +template +T valueAt(int fd, off64_t offset) { + T t; + memset(&t, 0, sizeof(T)); + if (adb_pread(fd, &t, sizeof(T), offset) != sizeof(T)) { + memset(&t, -1, sizeof(T)); + } + + return t; +} + +static void appendBlocks(int32_t start, int count, std::vector* blocks) { + if (count == 1) { + blocks->push_back(start); + } else { + auto oldSize = blocks->size(); + blocks->resize(oldSize + count); + std::iota(blocks->begin() + oldSize, blocks->end(), start); + } +} + +template +static void unduplicate(std::vector& v) { + std::unordered_set uniques(v.size()); + v.erase(std::remove_if(v.begin(), v.end(), + [&uniques](T t) { return !uniques.insert(t).second; }), + v.end()); +} + +static off64_t CentralDirOffset(int fd, int64_t fileSize) { + static constexpr int kZipEocdRecMinSize = 22; + static constexpr int32_t kZipEocdRecSig = 0x06054b50; + static constexpr int kZipEocdCentralDirSizeFieldOffset = 12; + static constexpr int kZipEocdCommentLengthFieldOffset = 20; + + int32_t sigBuf = 0; + off64_t eocdOffset = -1; + off64_t maxEocdOffset = fileSize - kZipEocdRecMinSize; + int16_t commentLenBuf = 0; + + // Search from the end of zip, backward to find beginning of EOCD + for (int16_t commentLen = 0; commentLen < fileSize; ++commentLen) { + sigBuf = valueAt(fd, maxEocdOffset - commentLen); + if (sigBuf == kZipEocdRecSig) { + commentLenBuf = valueAt( + fd, maxEocdOffset - commentLen + kZipEocdCommentLengthFieldOffset); + if (commentLenBuf == commentLen) { + eocdOffset = maxEocdOffset - commentLen; + break; + } + } + } + + if (eocdOffset < 0) { + return -1; + } + + off64_t cdLen = static_cast( + valueAt(fd, eocdOffset + kZipEocdCentralDirSizeFieldOffset)); + + return eocdOffset - cdLen; +} + +// Does not support APKs larger than 4GB +static off64_t SignerBlockOffset(int fd, int64_t fileSize) { + static constexpr int kApkSigBlockMinSize = 32; + static constexpr int kApkSigBlockFooterSize = 24; + static constexpr int64_t APK_SIG_BLOCK_MAGIC_HI = 0x3234206b636f6c42l; + static constexpr int64_t APK_SIG_BLOCK_MAGIC_LO = 0x20676953204b5041l; + + off64_t cdOffset = CentralDirOffset(fd, fileSize); + if (cdOffset < 0) { + return -1; + } + // CD offset is where original signer block ends. Search backwards for magic and footer. + if (cdOffset < kApkSigBlockMinSize || + valueAt(fd, cdOffset - 2 * sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_LO || + valueAt(fd, cdOffset - sizeof(int64_t)) != APK_SIG_BLOCK_MAGIC_HI) { + return -1; + } + int32_t signerSizeInFooter = valueAt(fd, cdOffset - kApkSigBlockFooterSize); + off64_t signerBlockOffset = cdOffset - signerSizeInFooter - sizeof(int64_t); + if (signerBlockOffset < 0) { + return -1; + } + int32_t signerSizeInHeader = valueAt(fd, signerBlockOffset); + if (signerSizeInFooter != signerSizeInHeader) { + return -1; + } + + return signerBlockOffset; +} + +static std::vector ZipPriorityBlocks(off64_t signerBlockOffset, int64_t fileSize) { + int32_t signerBlockIndex = offsetToBlockIndex(signerBlockOffset); + int32_t lastBlockIndex = offsetToBlockIndex(fileSize); + const auto numPriorityBlocks = lastBlockIndex - signerBlockIndex + 1; + + std::vector zipPriorityBlocks; + + // Some magic here: most of zip libraries perform a scan for EOCD record starting at the offset + // of a maximum comment size from the end of the file. This means the last 65-ish KBs will be + // accessed first, followed by the rest of the central directory blocks. Make sure we + // send the data in the proper order, as central directory can be quite big by itself. + static constexpr auto kMaxZipCommentSize = 64 * 1024; + static constexpr auto kNumBlocksInEocdSearch = kMaxZipCommentSize / kBlockSize + 1; + if (numPriorityBlocks > kNumBlocksInEocdSearch) { + appendBlocks(lastBlockIndex - kNumBlocksInEocdSearch + 1, kNumBlocksInEocdSearch, + &zipPriorityBlocks); + appendBlocks(signerBlockIndex, numPriorityBlocks - kNumBlocksInEocdSearch, + &zipPriorityBlocks); + } else { + appendBlocks(signerBlockIndex, numPriorityBlocks, &zipPriorityBlocks); + } + + // Somehow someone keeps accessing the start of the archive, even if there's nothing really + // interesting there... + appendBlocks(0, 1, &zipPriorityBlocks); + return zipPriorityBlocks; +} + +// TODO(b/151676293): avoid using OpenArchiveFd that reads local file headers +// which causes additional performance cost. Instead, only read from central directory. +static std::vector InstallationPriorityBlocks(int fd, int64_t fileSize) { + std::vector installationPriorityBlocks; + ZipArchiveHandle zip; + if (OpenArchiveFd(fd, "", &zip, false) != 0) { + return {}; + } + void* cookie = nullptr; + if (StartIteration(zip, &cookie) != 0) { + return {}; + } + ZipEntry entry; + std::string_view entryName; + while (Next(cookie, &entry, &entryName) == 0) { + if (entryName == "resources.arsc" || entryName == "AndroidManifest.xml" || + entryName.starts_with("lib/")) { + // Full entries are needed for installation + off64_t entryStartOffset = entry.offset; + off64_t entryEndOffset = + entryStartOffset + + (entry.method == kCompressStored ? entry.uncompressed_length + : entry.compressed_length) + + (entry.has_data_descriptor ? 16 /* sizeof(DataDescriptor) */ : 0); + int32_t startBlockIndex = offsetToBlockIndex(entryStartOffset); + int32_t endBlockIndex = offsetToBlockIndex(entryEndOffset); + int32_t numNewBlocks = endBlockIndex - startBlockIndex + 1; + appendBlocks(startBlockIndex, numNewBlocks, &installationPriorityBlocks); + } else if (entryName == "classes.dex") { + // Only the head is needed for installation + int32_t startBlockIndex = offsetToBlockIndex(entry.offset); + appendBlocks(startBlockIndex, 1, &installationPriorityBlocks); + } + } + + EndIteration(cookie); + CloseArchive(zip); + return installationPriorityBlocks; +} + +namespace incremental { +std::vector PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize) { + if (!android::base::EndsWithIgnoreCase(filepath, ".apk")) { + return {}; + } + off64_t signerOffset = SignerBlockOffset(fd, fileSize); + if (signerOffset < 0) { + // No signer block? not a valid APK + return {}; + } + std::vector priorityBlocks = ZipPriorityBlocks(signerOffset, fileSize); + std::vector installationPriorityBlocks = InstallationPriorityBlocks(fd, fileSize); + + priorityBlocks.insert(priorityBlocks.end(), installationPriorityBlocks.begin(), + installationPriorityBlocks.end()); + unduplicate(priorityBlocks); + return priorityBlocks; +} +} // namespace incremental \ No newline at end of file diff --git a/adb/client/incremental_utils.h b/adb/client/incremental_utils.h new file mode 100644 index 000000000..8bcf6c081 --- /dev/null +++ b/adb/client/incremental_utils.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace incremental { +std::vector PriorityBlocksForFile(const std::string& filepath, int fd, int64_t fileSize); +} // namespace incremental \ No newline at end of file