Merge changes from topic '36810152'

* changes:
  fastboot: Support larger transfers during flash
  fastboot: Cap max size sent to libsparse
  fastboot: add AdbWinUsbApi as a required module

fastboot/Android.mk

@@ -51,7 +51,7 @@ LOCAL_CFLAGS_darwin := -Wno-unused-parameter
 
 LOCAL_SRC_FILES_windows := usb_windows.cpp
 LOCAL_STATIC_LIBRARIES_windows := AdbWinApi
-LOCAL_REQUIRED_MODULES_windows := AdbWinApi
+LOCAL_REQUIRED_MODULES_windows := AdbWinApi AdbWinUsbApi
 LOCAL_LDLIBS_windows := -lws2_32
 LOCAL_C_INCLUDES_windows := development/host/windows/usb/api
 

fastboot/engine.cpp

@@ -44,6 +44,7 @@
 #define OP_NOTICE     4
 #define OP_DOWNLOAD_SPARSE 5
 #define OP_WAIT_FOR_DISCONNECT 6
+#define OP_DOWNLOAD_FD 7
 
 typedef struct Action Action;
 
@@ -56,6 +57,7 @@ struct Action {
     char cmd[CMD_SIZE];
     const char* prod;
     void* data;
+    int fd;
 
     // The protocol only supports 32-bit sizes, so you'll have to break
     // anything larger into chunks.
@@ -142,7 +144,20 @@ void fb_queue_erase(const char *ptn)
     a->msg = mkmsg("erasing '%s'", ptn);
 }
 
-void fb_queue_flash(const char *ptn, void *data, unsigned sz)
+void fb_queue_flash_fd(const char *ptn, int fd, uint32_t sz)
+{
+    Action *a;
+
+    a = queue_action(OP_DOWNLOAD_FD, "");
+    a->fd = fd;
+    a->size = sz;
+    a->msg = mkmsg("sending '%s' (%d KB)", ptn, sz / 1024);
+
+    a = queue_action(OP_COMMAND, "flash:%s", ptn);
+    a->msg = mkmsg("writing '%s'", ptn);
+}
+
+void fb_queue_flash(const char *ptn, void *data, uint32_t sz)
 {
     Action *a;
 
@@ -155,7 +170,7 @@ void fb_queue_flash(const char *ptn, void *data, unsigned sz)
     a->msg = mkmsg("writing '%s'", ptn);
 }
 
-void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, unsigned sz, size_t current,
+void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, uint32_t sz, size_t current,
                            size_t total) {
     Action *a;
 
@@ -282,7 +297,7 @@ static int cb_save(Action* a, int status, const char* resp) {
     return 0;
 }
 
-void fb_queue_query_save(const char *var, char *dest, unsigned dest_size)
+void fb_queue_query_save(const char *var, char *dest, uint32_t dest_size)
 {
     Action *a;
     a = queue_action(OP_QUERY, "getvar:%s", var);
@@ -309,7 +324,7 @@ void fb_queue_command(const char *cmd, const char *msg)
     a->msg = msg;
 }
 
-void fb_queue_download(const char *name, void *data, unsigned size)
+void fb_queue_download(const char *name, void *data, uint32_t size)
 {
     Action *a = queue_action(OP_DOWNLOAD, "");
     a->data = data;
@@ -328,11 +343,11 @@ void fb_queue_wait_for_disconnect(void)
     queue_action(OP_WAIT_FOR_DISCONNECT, "");
 }
 
-int fb_execute_queue(Transport* transport)
+int64_t fb_execute_queue(Transport* transport)
 {
     Action *a;
     char resp[FB_RESPONSE_SZ+1];
-    int status = 0;
+    int64_t status = 0;
 
     a = action_list;
     if (!a)
@@ -351,6 +366,10 @@ int fb_execute_queue(Transport* transport)
             status = fb_download_data(transport, a->data, a->size);
             status = a->func(a, status, status ? fb_get_error().c_str() : "");
             if (status) break;
+        } else if (a->op == OP_DOWNLOAD_FD) {
+            status = fb_download_data_fd(transport, a->fd, a->size);
+            status = a->func(a, status, status ? fb_get_error().c_str() : "");
+            if (status) break;
         } else if (a->op == OP_COMMAND) {
             status = fb_command(transport, a->cmd);
             status = a->func(a, status, status ? fb_get_error().c_str() : "");

fastboot/fastboot.cpp

@@ -55,6 +55,7 @@
 #include <android-base/parsenetaddress.h>
 #include <android-base/stringprintf.h>
 #include <android-base/strings.h>
+#include <android-base/unique_fd.h>
 #include <sparse/sparse.h>
 #include <ziparchive/zip_archive.h>
 
@@ -67,6 +68,8 @@
 #include "udp.h"
 #include "usb.h"
 
+using android::base::unique_fd;
+
 #ifndef O_BINARY
 #define O_BINARY 0
 #endif
@@ -78,6 +81,10 @@ static const char* product = nullptr;
 static const char* cmdline = nullptr;
 static unsigned short vendor_id = 0;
 static int long_listing = 0;
+// Don't resparse files in too-big chunks.
+// libsparse will support INT_MAX, but this results in large allocations, so
+// let's keep it at 1GB to avoid memory pressure on the host.
+static constexpr int64_t RESPARSE_LIMIT = 1 * 1024 * 1024 * 1024;
 static int64_t sparse_limit = -1;
 static int64_t target_sparse_limit = -1;
 
@@ -91,7 +98,7 @@ static unsigned tags_offset    = 0x00000100;
 static const std::string convert_fbe_marker_filename("convert_fbe");
 
 enum fb_buffer_type {
-    FB_BUFFER,
+    FB_BUFFER_FD,
     FB_BUFFER_SPARSE,
 };
 
@@ -99,6 +106,7 @@ struct fastboot_buffer {
     enum fb_buffer_type type;
     void* data;
     int64_t sz;
+    int fd;
 };
 
 static struct {
@@ -789,7 +797,7 @@ static int64_t get_sparse_limit(Transport* transport, int64_t size) {
     }
 
     if (size > limit) {
-        return limit;
+        return std::min(limit, RESPARSE_LIMIT);
     }
 
     return 0;
@@ -822,10 +830,9 @@ static bool load_buf_fd(Transport* transport, int fd, struct fastboot_buffer* bu
         buf->type = FB_BUFFER_SPARSE;
         buf->data = s;
     } else {
-        void* data = load_fd(fd, &sz);
+        buf->type = FB_BUFFER_FD;
-        if (data == nullptr) return -1;
+        buf->data = nullptr;
-        buf->type = FB_BUFFER;
+        buf->fd = fd;
-        buf->data = data;
         buf->sz = sz;
     }
 
@@ -833,11 +840,22 @@ static bool load_buf_fd(Transport* transport, int fd, struct fastboot_buffer* bu
 }
 
 static bool load_buf(Transport* transport, const char* fname, struct fastboot_buffer* buf) {
-    int fd = open(fname, O_RDONLY | O_BINARY);
+    unique_fd fd(TEMP_FAILURE_RETRY(open(fname, O_RDONLY | O_BINARY)));
+
     if (fd == -1) {
         return false;
     }
-    return load_buf_fd(transport, fd, buf);
+
+    struct stat s;
+    if (fstat(fd, &s)) {
+        return false;
+    }
+    if (!S_ISREG(s.st_mode)) {
+        errno = S_ISDIR(s.st_mode) ? EISDIR : EINVAL;
+        return false;
+    }
+
+    return load_buf_fd(transport, fd.release(), buf);
 }
 
 static void flash_buf(const char *pname, struct fastboot_buffer *buf)
@@ -860,9 +878,8 @@ static void flash_buf(const char *pname, struct fastboot_buffer *buf)
             }
             break;
         }
-
+        case FB_BUFFER_FD:
-        case FB_BUFFER:
+            fb_queue_flash_fd(pname, buf->fd, buf->sz);
-            fb_queue_flash(pname, buf->data, buf->sz);
             break;
         default:
             die("unknown buffer type: %d", buf->type);

fastboot/fastboot.h

@@ -41,7 +41,8 @@ struct sparse_file;
 /* protocol.c - fastboot protocol */
 int fb_command(Transport* transport, const char* cmd);
 int fb_command_response(Transport* transport, const char* cmd, char* response);
-int fb_download_data(Transport* transport, const void* data, uint32_t size);
+int64_t fb_download_data(Transport* transport, const void* data, uint32_t size);
+int64_t fb_download_data_fd(Transport* transport, int fd, uint32_t size);
 int fb_download_data_sparse(Transport* transport, struct sparse_file* s);
 const std::string fb_get_error();
 
@@ -51,6 +52,7 @@ const std::string fb_get_error();
 /* engine.c - high level command queue engine */
 bool fb_getvar(Transport* transport, const std::string& key, std::string* value);
 void fb_queue_flash(const char *ptn, void *data, uint32_t sz);
+void fb_queue_flash_fd(const char *ptn, int fd, uint32_t sz);
 void fb_queue_flash_sparse(const char* ptn, struct sparse_file* s, uint32_t sz, size_t current,
                            size_t total);
 void fb_queue_erase(const char *ptn);
@@ -64,7 +66,7 @@ void fb_queue_command(const char *cmd, const char *msg);
 void fb_queue_download(const char *name, void *data, uint32_t size);
 void fb_queue_notice(const char *notice);
 void fb_queue_wait_for_disconnect(void);
-int fb_execute_queue(Transport* transport);
+int64_t fb_execute_queue(Transport* transport);
 void fb_set_active(const char *slot);
 
 /* util stuff */

fastboot/protocol.cpp

@@ -38,6 +38,7 @@
 
 #include <android-base/stringprintf.h>
 #include <sparse/sparse.h>
+#include <utils/FileMap.h>
 
 #include "fastboot.h"
 #include "transport.h"
@@ -48,7 +49,7 @@ const std::string fb_get_error() {
     return g_error;
 }
 
-static int check_response(Transport* transport, uint32_t size, char* response) {
+static int64_t check_response(Transport* transport, uint32_t size, char* response) {
     char status[65];
 
     while (true) {
@@ -105,7 +106,7 @@ static int check_response(Transport* transport, uint32_t size, char* response) {
     return -1;
 }
 
-static int _command_start(Transport* transport, const char* cmd, uint32_t size, char* response) {
+static int64_t _command_start(Transport* transport, const char* cmd, uint32_t size, char* response) {
     size_t cmdsize = strlen(cmd);
     if (cmdsize > 64) {
         g_error = android::base::StringPrintf("command too large (%zu)", cmdsize);
@@ -125,14 +126,14 @@ static int _command_start(Transport* transport, const char* cmd, uint32_t size,
     return check_response(transport, size, response);
 }
 
-static int _command_data(Transport* transport, const void* data, uint32_t size) {
+static int64_t _command_data(Transport* transport, const void* data, uint32_t size) {
-    int r = transport->Write(data, size);
+    int64_t r = transport->Write(data, size);
     if (r < 0) {
         g_error = android::base::StringPrintf("data transfer failure (%s)", strerror(errno));
         transport->Close();
         return -1;
     }
-    if (r != ((int) size)) {
+    if (r != static_cast<int64_t>(size)) {
         g_error = "data transfer failure (short transfer)";
         transport->Close();
         return -1;
@@ -140,17 +141,17 @@ static int _command_data(Transport* transport, const void* data, uint32_t size)
     return r;
 }
 
-static int _command_end(Transport* transport) {
+static int64_t _command_end(Transport* transport) {
     return check_response(transport, 0, 0) < 0 ? -1 : 0;
 }
 
-static int _command_send(Transport* transport, const char* cmd, const void* data, uint32_t size,
+static int64_t _command_send(Transport* transport, const char* cmd, const void* data, uint32_t size,
                          char* response) {
     if (size == 0) {
         return -1;
     }
 
-    int r = _command_start(transport, cmd, size, response);
+    int64_t r = _command_start(transport, cmd, size, response);
     if (r < 0) {
         return -1;
     }
@@ -168,6 +169,39 @@ static int _command_send(Transport* transport, const char* cmd, const void* data
     return size;
 }
 
+static int64_t _command_send_fd(Transport* transport, const char* cmd, int fd, uint32_t size,
+                                char* response) {
+    static constexpr uint32_t MAX_MAP_SIZE = 512 * 1024 * 1024;
+    off64_t offset = 0;
+    uint32_t remaining = size;
+
+    if (_command_start(transport, cmd, size, response) < 0) {
+        return -1;
+    }
+
+    while (remaining) {
+        android::FileMap filemap;
+        size_t len = std::min(remaining, MAX_MAP_SIZE);
+
+        if (!filemap.create(NULL, fd, offset, len, true)) {
+            return -1;
+        }
+
+        if (_command_data(transport, filemap.getDataPtr(), len) < 0) {
+            return -1;
+        }
+
+        remaining -= len;
+        offset += len;
+    }
+
+    if (_command_end(transport) < 0) {
+        return -1;
+    }
+
+    return size;
+}
+
 static int _command_send_no_data(Transport* transport, const char* cmd, char* response) {
     return _command_start(transport, cmd, 0, response);
 }
@@ -180,10 +214,14 @@ int fb_command_response(Transport* transport, const char* cmd, char* response) {
     return _command_send_no_data(transport, cmd, response);
 }
 
-int fb_download_data(Transport* transport, const void* data, uint32_t size) {
+int64_t fb_download_data(Transport* transport, const void* data, uint32_t size) {
-    char cmd[64];
+    std::string cmd(android::base::StringPrintf("download:%08x", size));
-    snprintf(cmd, sizeof(cmd), "download:%08x", size);
+    return _command_send(transport, cmd.c_str(), data, size, 0) < 0 ? -1 : 0;
-    return _command_send(transport, cmd, data, size, 0) < 0 ? -1 : 0;
+}
+
+int64_t fb_download_data_fd(Transport* transport, int fd, uint32_t size) {
+    std::string cmd(android::base::StringPrintf("download:%08x", size));
+    return _command_send_fd(transport, cmd.c_str(), fd, size, 0) < 0 ? -1 : 0;
 }
 
 #define TRANSPORT_BUF_SIZE 1024
@@ -242,7 +280,8 @@ static int fb_download_data_sparse_write(void *priv, const void *data, int len)
 
 static int fb_download_data_sparse_flush(Transport* transport) {
     if (transport_buf_len > 0) {
-        if (_command_data(transport, transport_buf, transport_buf_len) != transport_buf_len) {
+        int64_t r = _command_data(transport, transport_buf, transport_buf_len);
+        if (r != static_cast<int64_t>(transport_buf_len)) {
             return -1;
         }
         transport_buf_len = 0;
@@ -256,9 +295,8 @@ int fb_download_data_sparse(Transport* transport, struct sparse_file* s) {
         return -1;
     }
 
-    char cmd[64];
+    std::string cmd(android::base::StringPrintf("download:%08x", size));
-    snprintf(cmd, sizeof(cmd), "download:%08x", size);
+    int r = _command_start(transport, cmd.c_str(), size, 0);
-    int r = _command_start(transport, cmd, size, 0);
     if (r < 0) {
         return -1;
     }