diff --git a/adb/client/adb_install.cpp b/adb/client/adb_install.cpp index 092a86684..f022b8bcc 100644 --- a/adb/client/adb_install.cpp +++ b/adb/client/adb_install.cpp @@ -290,7 +290,7 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) } } - if (do_sync_push(apk_file, apk_dest.c_str(), false, true)) { + if (do_sync_push(apk_file, apk_dest.c_str(), false, CompressionType::Any)) { result = pm_command(argc, argv); delete_device_file(apk_dest); } diff --git a/adb/client/bugreport.cpp b/adb/client/bugreport.cpp index ab93f7d87..e162aaad1 100644 --- a/adb/client/bugreport.cpp +++ b/adb/client/bugreport.cpp @@ -282,5 +282,5 @@ int Bugreport::SendShellCommand(const std::string& command, bool disable_shell_p bool Bugreport::DoSyncPull(const std::vector& srcs, const char* dst, bool copy_attrs, const char* name) { - return do_sync_pull(srcs, dst, copy_attrs, false, name); + return do_sync_pull(srcs, dst, copy_attrs, CompressionType::None, name); } diff --git a/adb/client/commandline.cpp b/adb/client/commandline.cpp index 04b250df2..9078ae97c 100644 --- a/adb/client/commandline.cpp +++ b/adb/client/commandline.cpp @@ -129,20 +129,20 @@ static void help() { " reverse --remove-all remove all reverse socket connections from device\n" "\n" "file transfer:\n" - " push [--sync] [-zZ] LOCAL... REMOTE\n" + " push [--sync] [-z ALGORITHM] [-Z] LOCAL... REMOTE\n" " copy local files/directories to device\n" " --sync: only push files that are newer on the host than the device\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" - " pull [-azZ] REMOTE... LOCAL\n" + " pull [-a] [-z ALGORITHM] [-Z] REMOTE... LOCAL\n" " copy files/dirs from device\n" " -a: preserve file timestamp and mode\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" - " sync [-lzZ] [all|data|odm|oem|product|system|system_ext|vendor]\n" + " sync [-l] [-z ALGORITHM] [-Z] [all|data|odm|oem|product|system|system_ext|vendor]\n" " sync a local build from $ANDROID_PRODUCT_OUT to the device (default all)\n" " -l: list files that would be copied, but don't copy them\n" - " -z: enable compression\n" + " -z: enable compression with a specified algorithm (any, none, brotli)\n" " -Z: disable compression\n" "\n" "shell:\n" @@ -1314,12 +1314,34 @@ static int restore(int argc, const char** argv) { return 0; } +static CompressionType parse_compression_type(const std::string& str, bool allow_numbers) { + if (allow_numbers) { + if (str == "0") { + return CompressionType::None; + } else if (str == "1") { + return CompressionType::Any; + } + } + + if (str == "any") { + return CompressionType::Any; + } else if (str == "none") { + return CompressionType::None; + } + + if (str == "brotli") { + return CompressionType::Brotli; + } + + error_exit("unexpected compression type %s", str.c_str()); +} + static void parse_push_pull_args(const char** arg, int narg, std::vector* srcs, - const char** dst, bool* copy_attrs, bool* sync, bool* compressed) { + const char** dst, bool* copy_attrs, bool* sync, + CompressionType* compression) { *copy_attrs = false; - const char* adb_compression = getenv("ADB_COMPRESSION"); - if (adb_compression && strcmp(adb_compression, "0") == 0) { - *compressed = false; + if (const char* adb_compression = getenv("ADB_COMPRESSION")) { + *compression = parse_compression_type(adb_compression, true); } srcs->clear(); @@ -1333,13 +1355,13 @@ static void parse_push_pull_args(const char** arg, int narg, std::vector srcs; const char* dst = nullptr; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compressed); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, &sync, &compression); if (srcs.empty() || !dst) error_exit("push requires an argument"); - return do_sync_push(srcs, dst, sync, compressed) ? 0 : 1; + return do_sync_push(srcs, dst, sync, compression) ? 0 : 1; } else if (!strcmp(argv[0], "pull")) { bool copy_attrs = false; - bool compressed = true; + CompressionType compression = CompressionType::Any; std::vector srcs; const char* dst = "."; - parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compressed); + parse_push_pull_args(&argv[1], argc - 1, &srcs, &dst, ©_attrs, nullptr, &compression); if (srcs.empty()) error_exit("pull requires an argument"); - return do_sync_pull(srcs, dst, copy_attrs, compressed) ? 0 : 1; + return do_sync_pull(srcs, dst, copy_attrs, compression) ? 0 : 1; } else if (!strcmp(argv[0], "install")) { if (argc < 2) error_exit("install requires an argument"); return install_app(argc, argv); @@ -1925,27 +1947,26 @@ int adb_commandline(int argc, const char** argv) { } else if (!strcmp(argv[0], "sync")) { std::string src; bool list_only = false; - bool compressed = true; + CompressionType compression = CompressionType::Any; - const char* adb_compression = getenv("ADB_COMPRESSION"); - if (adb_compression && strcmp(adb_compression, "0") == 0) { - compressed = false; + if (const char* adb_compression = getenv("ADB_COMPRESSION"); adb_compression) { + compression = parse_compression_type(adb_compression, true); } int opt; - while ((opt = getopt(argc, const_cast(argv), "lzZ")) != -1) { + while ((opt = getopt(argc, const_cast(argv), "lz:Z")) != -1) { switch (opt) { case 'l': list_only = true; break; case 'z': - compressed = true; + compression = parse_compression_type(optarg, false); break; case 'Z': - compressed = false; + compression = CompressionType::None; break; default: - error_exit("usage: adb sync [-lzZ] [PARTITION]"); + error_exit("usage: adb sync [-l] [-z ALGORITHM] [-Z] [PARTITION]"); } } @@ -1954,7 +1975,7 @@ int adb_commandline(int argc, const char** argv) { } else if (optind + 1 == argc) { src = argv[optind]; } else { - error_exit("usage: adb sync [-lzZ] [PARTITION]"); + error_exit("usage: adb sync [-l] [-z ALGORITHM] [-Z] [PARTITION]"); } std::vector partitions{"data", "odm", "oem", "product", @@ -1965,7 +1986,7 @@ int adb_commandline(int argc, const char** argv) { std::string src_dir{product_file(partition)}; if (!directory_exists(src_dir)) continue; found = true; - if (!do_sync_sync(src_dir, "/" + partition, list_only, compressed)) return 1; + if (!do_sync_sync(src_dir, "/" + partition, list_only, compression)) return 1; } } if (!found) error_exit("don't know how to sync %s partition", src.c_str()); diff --git a/adb/client/fastdeploy.cpp b/adb/client/fastdeploy.cpp index de82e14e5..37f1a90e5 100644 --- a/adb/client/fastdeploy.cpp +++ b/adb/client/fastdeploy.cpp @@ -112,7 +112,7 @@ static void push_to_device(const void* data, size_t byte_count, const char* dst, // but can't be removed until after the push. unix_close(tf.release()); - if (!do_sync_push(srcs, dst, sync, true)) { + if (!do_sync_push(srcs, dst, sync, CompressionType::Any)) { error_exit("Failed to push fastdeploy agent to device."); } } diff --git a/adb/client/file_sync_client.cpp b/adb/client/file_sync_client.cpp index 2ed58b2a3..c71880c2e 100644 --- a/adb/client/file_sync_client.cpp +++ b/adb/client/file_sync_client.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "sysdeps.h" @@ -262,6 +263,18 @@ class SyncConnection { bool HaveSendRecv2() const { return have_sendrecv_v2_; } bool HaveSendRecv2Brotli() const { return have_sendrecv_v2_brotli_; } + // Resolve a compression type which might be CompressionType::Any to a specific compression + // algorithm. + CompressionType ResolveCompressionType(CompressionType compression) const { + if (compression == CompressionType::Any) { + if (HaveSendRecv2Brotli()) { + return CompressionType::Brotli; + } + return CompressionType::None; + } + return compression; + } + const FeatureSet& Features() const { return features_; } bool IsValid() { return fd >= 0; } @@ -323,7 +336,7 @@ class SyncConnection { return WriteFdExactly(fd, buf.data(), buf.size()); } - bool SendSend2(std::string_view path, mode_t mode, bool compressed) { + bool SendSend2(std::string_view path, mode_t mode, CompressionType compression) { if (path.length() > 1024) { Error("SendRequest failed: path too long: %zu", path.length()); errno = ENAMETOOLONG; @@ -339,7 +352,18 @@ class SyncConnection { syncmsg msg; msg.send_v2_setup.id = ID_SEND_V2; msg.send_v2_setup.mode = mode; - msg.send_v2_setup.flags = compressed ? kSyncFlagBrotli : kSyncFlagNone; + msg.send_v2_setup.flags = 0; + switch (compression) { + case CompressionType::None: + break; + + case CompressionType::Brotli: + msg.send_v2_setup.flags = kSyncFlagBrotli; + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.send_v2_setup)); @@ -352,7 +376,7 @@ class SyncConnection { return WriteFdExactly(fd, buf.data(), buf.size()); } - bool SendRecv2(const std::string& path) { + bool SendRecv2(const std::string& path, CompressionType compression) { if (path.length() > 1024) { Error("SendRequest failed: path too long: %zu", path.length()); errno = ENAMETOOLONG; @@ -367,7 +391,18 @@ class SyncConnection { syncmsg msg; msg.recv_v2_setup.id = ID_RECV_V2; - msg.recv_v2_setup.flags = kSyncFlagBrotli; + msg.recv_v2_setup.flags = 0; + switch (compression) { + case CompressionType::None: + break; + + case CompressionType::Brotli: + msg.recv_v2_setup.flags |= kSyncFlagBrotli; + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } buf.resize(sizeof(SyncRequest) + path.length() + sizeof(msg.recv_v2_setup)); @@ -533,9 +568,15 @@ class SyncConnection { return true; } - bool SendLargeFileCompressed(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime) { - if (!SendSend2(path, mode, true)) { + bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime, CompressionType compression) { + if (!HaveSendRecv2()) { + return SendLargeFileLegacy(path, mode, lpath, rpath, mtime); + } + + compression = ResolveCompressionType(compression); + + if (!SendSend2(path, mode, compression)) { Error("failed to send ID_SEND_V2 message '%s': %s", path.c_str(), strerror(errno)); return false; } @@ -558,7 +599,21 @@ class SyncConnection { syncsendbuf sbuf; sbuf.id = ID_DATA; - BrotliEncoder encoder; + std::variant encoder_storage; + Encoder* encoder = nullptr; + switch (compression) { + case CompressionType::None: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Brotli: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + bool sending = true; while (sending) { Block input(SYNC_DATA_MAX); @@ -569,10 +624,10 @@ class SyncConnection { } if (r == 0) { - encoder.Finish(); + encoder->Finish(); } else { input.resize(r); - encoder.Append(std::move(input)); + encoder->Append(std::move(input)); RecordBytesTransferred(r); bytes_copied += r; ReportProgress(rpath, bytes_copied, total_size); @@ -580,7 +635,7 @@ class SyncConnection { while (true) { Block output; - EncodeResult result = encoder.Encode(&output); + EncodeResult result = encoder->Encode(&output); if (result == EncodeResult::Error) { Error("compressing '%s' locally failed", lpath.c_str()); return false; @@ -610,12 +665,8 @@ class SyncConnection { return WriteOrDie(lpath, rpath, &msg.data, sizeof(msg.data)); } - bool SendLargeFile(const std::string& path, mode_t mode, const std::string& lpath, - const std::string& rpath, unsigned mtime, bool compressed) { - if (compressed && HaveSendRecv2Brotli()) { - return SendLargeFileCompressed(path, mode, lpath, rpath, mtime); - } - + bool SendLargeFileLegacy(const std::string& path, mode_t mode, const std::string& lpath, + const std::string& rpath, unsigned mtime) { std::string path_and_mode = android::base::StringPrintf("%s,%d", path.c_str(), mode); if (!SendRequest(ID_SEND_V1, path_and_mode)) { Error("failed to send ID_SEND_V1 message '%s': %s", path_and_mode.c_str(), @@ -921,7 +972,7 @@ static bool sync_stat_fallback(SyncConnection& sc, const std::string& path, stru } static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::string& rpath, - unsigned mtime, mode_t mode, bool sync, bool compressed) { + unsigned mtime, mode_t mode, bool sync, CompressionType compression) { if (sync) { struct stat st; if (sync_lstat(sc, rpath, &st)) { @@ -964,7 +1015,7 @@ static bool sync_send(SyncConnection& sc, const std::string& lpath, const std::s return false; } } else { - if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compressed)) { + if (!sc.SendLargeFile(rpath, mode, lpath, rpath, mtime, compression)) { return false; } } @@ -1027,8 +1078,10 @@ static bool sync_recv_v1(SyncConnection& sc, const char* rpath, const char* lpat } static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, - uint64_t expected_size) { - if (!sc.SendRecv2(rpath)) return false; + uint64_t expected_size, CompressionType compression) { + compression = sc.ResolveCompressionType(compression); + + if (!sc.SendRecv2(rpath, compression)) return false; adb_unlink(lpath); unique_fd lfd(adb_creat(lpath, 0644)); @@ -1040,9 +1093,24 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat uint64_t bytes_copied = 0; Block buffer(SYNC_DATA_MAX); - BrotliDecoder decoder(std::span(buffer.data(), buffer.size())); - bool reading = true; - while (reading) { + std::variant decoder_storage; + Decoder* decoder = nullptr; + + std::span buffer_span(buffer.data(), buffer.size()); + switch (compression) { + case CompressionType::None: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Brotli: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + + while (true) { syncmsg msg; if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { adb_unlink(lpath); @@ -1050,33 +1118,32 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } if (msg.data.id == ID_DONE) { - adb_unlink(lpath); - sc.Error("unexpected ID_DONE"); - return false; - } - - if (msg.data.id != ID_DATA) { + if (!decoder->Finish()) { + sc.Error("unexpected ID_DONE"); + return false; + } + } else if (msg.data.id != ID_DATA) { adb_unlink(lpath); sc.ReportCopyFailure(rpath, lpath, msg); return false; - } + } else { + if (msg.data.size > sc.max) { + sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); + adb_unlink(lpath); + return false; + } - if (msg.data.size > sc.max) { - sc.Error("msg.data.size too large: %u (max %zu)", msg.data.size, sc.max); - adb_unlink(lpath); - return false; + Block block(msg.data.size); + if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { + adb_unlink(lpath); + return false; + } + decoder->Append(std::move(block)); } - Block block(msg.data.size); - if (!ReadFdExactly(sc.fd, block.data(), msg.data.size)) { - adb_unlink(lpath); - return false; - } - decoder.Append(std::move(block)); - while (true) { std::span output; - DecodeResult result = decoder.Decode(&output); + DecodeResult result = decoder->Decode(&output); if (result == DecodeResult::Error) { sc.Error("decompress failed"); @@ -1102,33 +1169,19 @@ static bool sync_recv_v2(SyncConnection& sc, const char* rpath, const char* lpat } else if (result == DecodeResult::MoreOutput) { continue; } else if (result == DecodeResult::Done) { - reading = false; - break; + sc.RecordFilesTransferred(1); + return true; } else { LOG(FATAL) << "invalid DecodeResult: " << static_cast(result); } } } - - syncmsg msg; - if (!ReadFdExactly(sc.fd, &msg.data, sizeof(msg.data))) { - sc.Error("failed to read ID_DONE"); - return false; - } - - if (msg.data.id != ID_DONE) { - sc.Error("unexpected message after transfer: id = %d (expected ID_DONE)", msg.data.id); - return false; - } - - sc.RecordFilesTransferred(1); - return true; } static bool sync_recv(SyncConnection& sc, const char* rpath, const char* lpath, const char* name, - uint64_t expected_size, bool compressed) { - if (sc.HaveSendRecv2() && compressed) { - return sync_recv_v2(sc, rpath, lpath, name, expected_size); + uint64_t expected_size, CompressionType compression) { + if (sc.HaveSendRecv2()) { + return sync_recv_v2(sc, rpath, lpath, name, expected_size, compression); } else { return sync_recv_v1(sc, rpath, lpath, name, expected_size); } @@ -1210,7 +1263,8 @@ static bool is_root_dir(std::string_view path) { } static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::string rpath, - bool check_timestamps, bool list_only, bool compressed) { + bool check_timestamps, bool list_only, + CompressionType compression) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1292,7 +1346,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st if (list_only) { sc.Println("would push: %s -> %s", ci.lpath.c_str(), ci.rpath.c_str()); } else { - if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compressed)) { + if (!sync_send(sc, ci.lpath, ci.rpath, ci.time, ci.mode, false, compression)) { return false; } } @@ -1308,7 +1362,7 @@ static bool copy_local_dir_remote(SyncConnection& sc, std::string lpath, std::st } bool do_sync_push(const std::vector& srcs, const char* dst, bool sync, - bool compressed) { + CompressionType compression) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1373,7 +1427,7 @@ bool do_sync_push(const std::vector& srcs, const char* dst, bool sy dst_dir.append(android::base::Basename(src_path)); } - success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compressed); + success &= copy_local_dir_remote(sc, src_path, dst_dir, sync, false, compression); continue; } else if (!should_push_file(st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, st.st_mode); @@ -1394,7 +1448,7 @@ bool do_sync_push(const std::vector& srcs, const char* dst, bool sy sc.NewTransfer(); sc.SetExpectedTotalBytes(st.st_size); - success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compressed); + success &= sync_send(sc, src_path, dst_path, st.st_mtime, st.st_mode, sync, compression); sc.ReportTransferRate(src_path, TransferDirection::push); } @@ -1480,7 +1534,7 @@ static int set_time_and_mode(const std::string& lpath, time_t time, } static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::string lpath, - bool copy_attrs, bool compressed) { + bool copy_attrs, CompressionType compression) { sc.NewTransfer(); // Make sure that both directory paths end in a slash. @@ -1510,7 +1564,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st continue; } - if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compressed)) { + if (!sync_recv(sc, ci.rpath.c_str(), ci.lpath.c_str(), nullptr, ci.size, compression)) { return false; } @@ -1528,7 +1582,7 @@ static bool copy_remote_dir_local(SyncConnection& sc, std::string rpath, std::st } bool do_sync_pull(const std::vector& srcs, const char* dst, bool copy_attrs, - bool compressed, const char* name) { + CompressionType compression, const char* name) { SyncConnection sc; if (!sc.IsValid()) return false; @@ -1602,7 +1656,7 @@ bool do_sync_pull(const std::vector& srcs, const char* dst, bool co dst_dir.append(android::base::Basename(src_path)); } - success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compressed); + success &= copy_remote_dir_local(sc, src_path, dst_dir, copy_attrs, compression); continue; } else if (!should_pull_file(src_st.st_mode)) { sc.Warning("skipping special file '%s' (mode = 0o%o)", src_path, src_st.st_mode); @@ -1621,7 +1675,7 @@ bool do_sync_pull(const std::vector& srcs, const char* dst, bool co sc.NewTransfer(); sc.SetExpectedTotalBytes(src_st.st_size); - if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compressed)) { + if (!sync_recv(sc, src_path, dst_path, name, src_st.st_size, compression)) { success = false; continue; } @@ -1638,11 +1692,11 @@ bool do_sync_pull(const std::vector& srcs, const char* dst, bool co } bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, - bool compressed) { + CompressionType compression) { SyncConnection sc; if (!sc.IsValid()) return false; - bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compressed); + bool success = copy_local_dir_remote(sc, lpath, rpath, true, list_only, compression); if (!list_only) { sc.ReportOverallTransferRate(TransferDirection::push); } diff --git a/adb/client/file_sync_client.h b/adb/client/file_sync_client.h index de3f19245..aab2e3f86 100644 --- a/adb/client/file_sync_client.h +++ b/adb/client/file_sync_client.h @@ -19,11 +19,13 @@ #include #include +#include "file_sync_protocol.h" + bool do_sync_ls(const char* path); bool do_sync_push(const std::vector& srcs, const char* dst, bool sync, - bool compressed); + CompressionType compression); bool do_sync_pull(const std::vector& srcs, const char* dst, bool copy_attrs, - bool compressed, const char* name = nullptr); + CompressionType compression, const char* name = nullptr); bool do_sync_sync(const std::string& lpath, const std::string& rpath, bool list_only, - bool compressed); + CompressionType compression); diff --git a/adb/compression_utils.h b/adb/compression_utils.h index c445095ca..f349697dd 100644 --- a/adb/compression_utils.h +++ b/adb/compression_utils.h @@ -16,8 +16,12 @@ #pragma once +#include +#include #include +#include + #include #include @@ -37,15 +41,103 @@ enum class EncodeResult { MoreOutput, }; -struct BrotliDecoder { +struct Decoder { + void Append(Block&& block) { input_buffer_.append(std::move(block)); } + bool Finish() { + bool old = std::exchange(finished_, true); + if (old) { + LOG(FATAL) << "Decoder::Finish called while already finished?"; + return false; + } + return true; + } + + virtual DecodeResult Decode(std::span* output) = 0; + + protected: + Decoder(std::span output_buffer) : output_buffer_(output_buffer) {} + ~Decoder() = default; + + bool finished_ = false; + IOVector input_buffer_; + std::span output_buffer_; +}; + +struct Encoder { + void Append(Block input) { input_buffer_.append(std::move(input)); } + bool Finish() { + bool old = std::exchange(finished_, true); + if (old) { + LOG(FATAL) << "Decoder::Finish called while already finished?"; + return false; + } + return true; + } + + virtual EncodeResult Encode(Block* output) = 0; + + protected: + explicit Encoder(size_t output_block_size) : output_block_size_(output_block_size) {} + ~Encoder() = default; + + const size_t output_block_size_; + bool finished_ = false; + IOVector input_buffer_; +}; + +struct NullDecoder final : public Decoder { + explicit NullDecoder(std::span output_buffer) : Decoder(output_buffer) {} + + DecodeResult Decode(std::span* output) final { + size_t available_out = output_buffer_.size(); + void* p = output_buffer_.data(); + while (available_out > 0 && !input_buffer_.empty()) { + size_t len = std::min(available_out, input_buffer_.front_size()); + p = mempcpy(p, input_buffer_.front_data(), len); + available_out -= len; + input_buffer_.drop_front(len); + } + *output = std::span(output_buffer_.data(), static_cast(p)); + if (input_buffer_.empty()) { + return finished_ ? DecodeResult::Done : DecodeResult::NeedInput; + } + return DecodeResult::MoreOutput; + } +}; + +struct NullEncoder final : public Encoder { + explicit NullEncoder(size_t output_block_size) : Encoder(output_block_size) {} + + EncodeResult Encode(Block* output) final { + output->clear(); + output->resize(output_block_size_); + + size_t available_out = output->size(); + void* p = output->data(); + + while (available_out > 0 && !input_buffer_.empty()) { + size_t len = std::min(available_out, input_buffer_.front_size()); + p = mempcpy(p, input_buffer_.front_data(), len); + available_out -= len; + input_buffer_.drop_front(len); + } + + output->resize(output->size() - available_out); + + if (input_buffer_.empty()) { + return finished_ ? EncodeResult::Done : EncodeResult::NeedInput; + } + return EncodeResult::MoreOutput; + } +}; + +struct BrotliDecoder final : public Decoder { explicit BrotliDecoder(std::span output_buffer) - : output_buffer_(output_buffer), + : Decoder(output_buffer), decoder_(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), BrotliDecoderDestroyInstance) {} - void Append(Block&& block) { input_buffer_.append(std::move(block)); } - - DecodeResult Decode(std::span* output) { + DecodeResult Decode(std::span* output) final { size_t available_in = input_buffer_.front_size(); const uint8_t* next_in = reinterpret_cast(input_buffer_.front_data()); @@ -63,7 +155,8 @@ struct BrotliDecoder { switch (r) { case BROTLI_DECODER_RESULT_SUCCESS: - return DecodeResult::Done; + // We need to wait for ID_DONE from the other end. + return finished_ ? DecodeResult::Done : DecodeResult::NeedInput; case BROTLI_DECODER_RESULT_ERROR: return DecodeResult::Error; case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: @@ -77,33 +170,29 @@ struct BrotliDecoder { } private: - IOVector input_buffer_; - std::span output_buffer_; std::unique_ptr decoder_; }; -template -struct BrotliEncoder { - explicit BrotliEncoder() - : output_block_(OutputBlockSize), - output_bytes_left_(OutputBlockSize), +struct BrotliEncoder final : public Encoder { + explicit BrotliEncoder(size_t output_block_size) + : Encoder(output_block_size), + output_block_(output_block_size_), + output_bytes_left_(output_block_size_), encoder_(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), BrotliEncoderDestroyInstance) { BrotliEncoderSetParameter(encoder_.get(), BROTLI_PARAM_QUALITY, 1); } - void Append(Block input) { input_buffer_.append(std::move(input)); } - void Finish() { finished_ = true; } - - EncodeResult Encode(Block* output) { + EncodeResult Encode(Block* output) final { output->clear(); + while (true) { size_t available_in = input_buffer_.front_size(); const uint8_t* next_in = reinterpret_cast(input_buffer_.front_data()); size_t available_out = output_bytes_left_; - uint8_t* next_out = reinterpret_cast(output_block_.data() + - (OutputBlockSize - output_bytes_left_)); + uint8_t* next_out = reinterpret_cast( + output_block_.data() + (output_block_size_ - output_bytes_left_)); BrotliEncoderOperation op = BROTLI_OPERATION_PROCESS; if (finished_) { @@ -121,13 +210,13 @@ struct BrotliEncoder { output_bytes_left_ = available_out; if (BrotliEncoderIsFinished(encoder_.get())) { - output_block_.resize(OutputBlockSize - output_bytes_left_); + output_block_.resize(output_block_size_ - output_bytes_left_); *output = std::move(output_block_); return EncodeResult::Done; } else if (output_bytes_left_ == 0) { *output = std::move(output_block_); - output_block_.resize(OutputBlockSize); - output_bytes_left_ = OutputBlockSize; + output_block_.resize(output_block_size_); + output_bytes_left_ = output_block_size_; return EncodeResult::MoreOutput; } else if (input_buffer_.empty()) { return EncodeResult::NeedInput; @@ -136,8 +225,6 @@ struct BrotliEncoder { } private: - bool finished_ = false; - IOVector input_buffer_; Block output_block_; size_t output_bytes_left_; std::unique_ptr encoder_; diff --git a/adb/daemon/file_sync_service.cpp b/adb/daemon/file_sync_service.cpp index 5ccddeabf..dcd640b57 100644 --- a/adb/daemon/file_sync_service.cpp +++ b/adb/daemon/file_sync_service.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -266,29 +267,45 @@ static bool SendSyncFailErrno(borrowed_fd fd, const std::string& reason) { return SendSyncFail(fd, StringPrintf("%s: %s", reason.c_str(), strerror(errno))); } -static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp) { +static bool handle_send_file_data(borrowed_fd s, unique_fd fd, uint32_t* timestamp, + CompressionType compression) { syncmsg msg; - Block decode_buffer(SYNC_DATA_MAX); - BrotliDecoder decoder(std::span(decode_buffer.data(), decode_buffer.size())); + Block buffer(SYNC_DATA_MAX); + std::span buffer_span(buffer.data(), buffer.size()); + std::variant decoder_storage; + Decoder* decoder = nullptr; + + switch (compression) { + case CompressionType::None: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Brotli: + decoder = &decoder_storage.emplace(buffer_span); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } + while (true) { if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - return true; - } + if (msg.data.id == ID_DONE) { + *timestamp = msg.data.size; + decoder->Finish(); + } else if (msg.data.id == ID_DATA) { + Block block(msg.data.size); + if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; + decoder->Append(std::move(block)); + } else { SendSyncFail(s, "invalid data message"); return false; } - Block block(msg.data.size); - if (!ReadFdExactly(s, block.data(), msg.data.size)) return false; - decoder.Append(std::move(block)); - while (true) { std::span output; - DecodeResult result = decoder.Decode(&output); + DecodeResult result = decoder->Decode(&output); if (result == DecodeResult::Error) { SendSyncFailErrno(s, "decompress failed"); return false; @@ -304,7 +321,7 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t } else if (result == DecodeResult::MoreOutput) { continue; } else if (result == DecodeResult::Done) { - break; + return true; } else { LOG(FATAL) << "invalid DecodeResult: " << static_cast(result); } @@ -314,37 +331,10 @@ static bool handle_send_file_compressed(borrowed_fd s, unique_fd fd, uint32_t* t __builtin_unreachable(); } -static bool handle_send_file_uncompressed(borrowed_fd s, unique_fd fd, uint32_t* timestamp, - std::vector& buffer) { - syncmsg msg; - - while (true) { - if (!ReadFdExactly(s, &msg.data, sizeof(msg.data))) return false; - - if (msg.data.id != ID_DATA) { - if (msg.data.id == ID_DONE) { - *timestamp = msg.data.size; - return true; - } - SendSyncFail(s, "invalid data message"); - return false; - } - - if (msg.data.size > buffer.size()) { // TODO: resize buffer? - SendSyncFail(s, "oversize data message"); - return false; - } - if (!ReadFdExactly(s, &buffer[0], msg.data.size)) return false; - if (!WriteFdExactly(fd, &buffer[0], msg.data.size)) { - SendSyncFailErrno(s, "write failed"); - return false; - } - } -} - static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestamp, uid_t uid, - gid_t gid, uint64_t capabilities, mode_t mode, bool compressed, - std::vector& buffer, bool do_unlink) { + gid_t gid, uint64_t capabilities, mode_t mode, + CompressionType compression, std::vector& buffer, + bool do_unlink) { int rc; syncmsg msg; @@ -389,14 +379,7 @@ static bool handle_send_file(borrowed_fd s, const char* path, uint32_t* timestam D("[ Failed to fadvise: %s ]", strerror(rc)); } - bool result; - if (compressed) { - result = handle_send_file_compressed(s, std::move(fd), timestamp); - } else { - result = handle_send_file_uncompressed(s, std::move(fd), timestamp, buffer); - } - - if (!result) { + if (!handle_send_file_data(s, std::move(fd), timestamp, compression)) { goto fail; } @@ -499,7 +482,7 @@ static bool handle_send_link(int s, const std::string& path, uint32_t* timestamp } #endif -static bool send_impl(int s, const std::string& path, mode_t mode, bool compressed, +static bool send_impl(int s, const std::string& path, mode_t mode, CompressionType compression, std::vector& buffer) { // Don't delete files before copying if they are not "regular" or symlinks. struct stat st; @@ -527,7 +510,7 @@ static bool send_impl(int s, const std::string& path, mode_t mode, bool compress } result = handle_send_file(s, path.c_str(), ×tamp, uid, gid, capabilities, mode, - compressed, buffer, do_unlink); + compression, buffer, do_unlink); } if (!result) { @@ -560,7 +543,7 @@ static bool do_send_v1(int s, const std::string& spec, std::vector& buffer return false; } - return send_impl(s, path, mode, false, buffer); + return send_impl(s, path, mode, CompressionType::None, buffer); } static bool do_send_v2(int s, const std::string& path, std::vector& buffer) { @@ -574,45 +557,60 @@ static bool do_send_v2(int s, const std::string& path, std::vector& buffer PLOG(ERROR) << "failed to read send_v2 setup packet"; } - bool compressed = false; + std::optional compression; if (msg.send_v2_setup.flags & kSyncFlagBrotli) { msg.send_v2_setup.flags &= ~kSyncFlagBrotli; - compressed = true; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + msg.recv_v2_setup.flags)); + return false; + } + compression = CompressionType::Brotli; } + if (msg.send_v2_setup.flags) { SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.send_v2_setup.flags)); return false; } errno = 0; - return send_impl(s, path, msg.send_v2_setup.mode, compressed, buffer); + return send_impl(s, path, msg.send_v2_setup.mode, compression.value_or(CompressionType::None), + buffer); } -static bool recv_uncompressed(borrowed_fd s, unique_fd fd, std::vector& buffer) { - syncmsg msg; - msg.data.id = ID_DATA; - while (true) { - int r = adb_read(fd.get(), &buffer[0], buffer.size() - sizeof(msg.data)); - if (r <= 0) { - if (r == 0) break; - SendSyncFailErrno(s, "read failed"); - return false; - } - msg.data.size = r; +static bool recv_impl(borrowed_fd s, const char* path, CompressionType compression, + std::vector& buffer) { + __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); - if (!WriteFdExactly(s, &msg.data, sizeof(msg.data)) || !WriteFdExactly(s, &buffer[0], r)) { - return false; - } + unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + SendSyncFailErrno(s, "open failed"); + return false; } - return true; -} + int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); + if (rc != 0) { + D("[ Failed to fadvise: %s ]", strerror(rc)); + } -static bool recv_compressed(borrowed_fd s, unique_fd fd) { syncmsg msg; msg.data.id = ID_DATA; - BrotliEncoder encoder; + std::variant encoder_storage; + Encoder* encoder; + + switch (compression) { + case CompressionType::None: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Brotli: + encoder = &encoder_storage.emplace(SYNC_DATA_MAX); + break; + + case CompressionType::Any: + LOG(FATAL) << "unexpected CompressionType::Any"; + } bool sending = true; while (sending) { @@ -624,15 +622,15 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) { } if (r == 0) { - encoder.Finish(); + encoder->Finish(); } else { input.resize(r); - encoder.Append(std::move(input)); + encoder->Append(std::move(input)); } while (true) { Block output; - EncodeResult result = encoder.Encode(&output); + EncodeResult result = encoder->Encode(&output); if (result == EncodeResult::Error) { SendSyncFailErrno(s, "compress failed"); return false; @@ -657,42 +655,13 @@ static bool recv_compressed(borrowed_fd s, unique_fd fd) { } } - return true; -} - -static bool recv_impl(borrowed_fd s, const char* path, bool compressed, std::vector& buffer) { - __android_log_security_bswrite(SEC_TAG_ADB_RECV_FILE, path); - - unique_fd fd(adb_open(path, O_RDONLY | O_CLOEXEC)); - if (fd < 0) { - SendSyncFailErrno(s, "open failed"); - return false; - } - - int rc = posix_fadvise(fd.get(), 0, 0, POSIX_FADV_SEQUENTIAL | POSIX_FADV_NOREUSE); - if (rc != 0) { - D("[ Failed to fadvise: %s ]", strerror(rc)); - } - - bool result; - if (compressed) { - result = recv_compressed(s, std::move(fd)); - } else { - result = recv_uncompressed(s, std::move(fd), buffer); - } - - if (!result) { - return false; - } - - syncmsg msg; msg.data.id = ID_DONE; msg.data.size = 0; return WriteFdExactly(s, &msg.data, sizeof(msg.data)); } static bool do_recv_v1(borrowed_fd s, const char* path, std::vector& buffer) { - return recv_impl(s, path, false, buffer); + return recv_impl(s, path, CompressionType::None, buffer); } static bool do_recv_v2(borrowed_fd s, const char* path, std::vector& buffer) { @@ -706,17 +675,23 @@ static bool do_recv_v2(borrowed_fd s, const char* path, std::vector& buffe PLOG(ERROR) << "failed to read recv_v2 setup packet"; } - bool compressed = false; + std::optional compression; if (msg.recv_v2_setup.flags & kSyncFlagBrotli) { msg.recv_v2_setup.flags &= ~kSyncFlagBrotli; - compressed = true; + if (compression) { + SendSyncFail(s, android::base::StringPrintf("multiple compression flags received: %d", + msg.recv_v2_setup.flags)); + return false; + } + compression = CompressionType::Brotli; } + if (msg.recv_v2_setup.flags) { SendSyncFail(s, android::base::StringPrintf("unknown flags: %d", msg.recv_v2_setup.flags)); return false; } - return recv_impl(s, path, compressed, buffer); + return recv_impl(s, path, compression.value_or(CompressionType::None), buffer); } static const char* sync_id_to_name(uint32_t id) { diff --git a/adb/file_sync_protocol.h b/adb/file_sync_protocol.h index fd9a5169e..70425f7aa 100644 --- a/adb/file_sync_protocol.h +++ b/adb/file_sync_protocol.h @@ -94,6 +94,12 @@ enum SyncFlag : uint32_t { kSyncFlagBrotli = 1, }; +enum class CompressionType { + None, + Any, + Brotli, +}; + // send_v1 sent the path in a buffer, followed by a comma and the mode as a string. // send_v2 sends just the path in the first request, and then sends another syncmsg (with the // same ID!) with details. diff --git a/adb/test_device.py b/adb/test_device.py index 6a9ff89ef..496a0ffff 100755 --- a/adb/test_device.py +++ b/adb/test_device.py @@ -77,8 +77,7 @@ def requires_non_root(func): class DeviceTest(unittest.TestCase): - def setUp(self): - self.device = adb.get_device() + device = adb.get_device() class AbbTest(DeviceTest): @@ -753,535 +752,554 @@ def make_random_device_files(device, in_dir, num_files, prefix='device_tmpfile') return files -class FileOperationsTest(DeviceTest): - SCRATCH_DIR = '/data/local/tmp' - DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file' - DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir' +class FileOperationsTest: + class Base(DeviceTest): + SCRATCH_DIR = '/data/local/tmp' + DEVICE_TEMP_FILE = SCRATCH_DIR + '/adb_test_file' + DEVICE_TEMP_DIR = SCRATCH_DIR + '/adb_test_dir' - def _verify_remote(self, checksum, remote_path): - dev_md5, _ = self.device.shell([get_md5_prog(self.device), - remote_path])[0].split() - self.assertEqual(checksum, dev_md5) + def setUp(self): + self.previous_env = os.environ.get("ADB_COMPRESSION") + os.environ["ADB_COMPRESSION"] = self.compression - def _verify_local(self, checksum, local_path): - with open(local_path, 'rb') as host_file: - host_md5 = compute_md5(host_file.read()) - self.assertEqual(host_md5, checksum) + def tearDown(self): + if self.previous_env is None: + del os.environ["ADB_COMPRESSION"] + else: + os.environ["ADB_COMPRESSION"] = self.previous_env - def test_push(self): - """Push a randomly generated file to specified device.""" - kbytes = 512 - tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False) - rand_str = os.urandom(1024 * kbytes) - tmp.write(rand_str) - tmp.close() + def _verify_remote(self, checksum, remote_path): + dev_md5, _ = self.device.shell([get_md5_prog(self.device), + remote_path])[0].split() + self.assertEqual(checksum, dev_md5) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) - self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE) + def _verify_local(self, checksum, local_path): + with open(local_path, 'rb') as host_file: + host_md5 = compute_md5(host_file.read()) + self.assertEqual(host_md5, checksum) - self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE) - self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) + def test_push(self): + """Push a randomly generated file to specified device.""" + kbytes = 512 + tmp = tempfile.NamedTemporaryFile(mode='wb', delete=False) + rand_str = os.urandom(1024 * kbytes) + tmp.write(rand_str) + tmp.close() - os.remove(tmp.name) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) + self.device.push(local=tmp.name, remote=self.DEVICE_TEMP_FILE) - def test_push_dir(self): - """Push a randomly generated directory of files to the device.""" - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + self._verify_remote(compute_md5(rand_str), self.DEVICE_TEMP_FILE) + self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) - try: - host_dir = tempfile.mkdtemp() + os.remove(tmp.name) - # Make sure the temp directory isn't setuid, or else adb will complain. - os.chmod(host_dir, 0o700) - - # Create 32 random files. - temp_files = make_random_host_files(in_dir=host_dir, num_files=32) - self.device.push(host_dir, self.DEVICE_TEMP_DIR) - - for temp_file in temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - os.path.basename(host_dir), - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) + def test_push_dir(self): + """Push a randomly generated directory of files to the device.""" self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - def disabled_test_push_empty(self): - """Push an empty directory to the device.""" - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + try: + host_dir = tempfile.mkdtemp() - try: - host_dir = tempfile.mkdtemp() + # Make sure the temp directory isn't setuid, or else adb will complain. + os.chmod(host_dir, 0o700) - # Make sure the temp directory isn't setuid, or else adb will complain. - os.chmod(host_dir, 0o700) + # Create 32 random files. + temp_files = make_random_host_files(in_dir=host_dir, num_files=32) + self.device.push(host_dir, self.DEVICE_TEMP_DIR) - # Create an empty directory. - empty_dir_path = os.path.join(host_dir, 'empty') - os.mkdir(empty_dir_path); + for temp_file in temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + os.path.basename(host_dir), + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR) - - remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty") - test_empty_cmd = ["[", "-d", remote_path, "]"] - rc, _, _ = self.device.shell_nocheck(test_empty_cmd) - - self.assertEqual(rc, 0) + def disabled_test_push_empty(self): + """Push an empty directory to the device.""" self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows") - def test_push_symlink(self): - """Push a symlink. + try: + host_dir = tempfile.mkdtemp() - Bug: http://b/31491920 - """ - try: - host_dir = tempfile.mkdtemp() + # Make sure the temp directory isn't setuid, or else adb will complain. + os.chmod(host_dir, 0o700) - # Make sure the temp directory isn't setuid, or else adb will - # complain. - os.chmod(host_dir, 0o700) + # Create an empty directory. + empty_dir_path = os.path.join(host_dir, 'empty') + os.mkdir(empty_dir_path); - with open(os.path.join(host_dir, 'foo'), 'w') as f: - f.write('foo') + self.device.push(empty_dir_path, self.DEVICE_TEMP_DIR) - symlink_path = os.path.join(host_dir, 'symlink') - os.symlink('foo', symlink_path) + remote_path = os.path.join(self.DEVICE_TEMP_DIR, "empty") + test_empty_cmd = ["[", "-d", remote_path, "]"] + rc, _, _ = self.device.shell_nocheck(test_empty_cmd) + + self.assertEqual(rc, 0) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + @unittest.skipIf(sys.platform == "win32", "symlinks require elevated privileges on windows") + def test_push_symlink(self): + """Push a symlink. + + Bug: http://b/31491920 + """ + try: + host_dir = tempfile.mkdtemp() + + # Make sure the temp directory isn't setuid, or else adb will + # complain. + os.chmod(host_dir, 0o700) + + with open(os.path.join(host_dir, 'foo'), 'w') as f: + f.write('foo') + + symlink_path = os.path.join(host_dir, 'symlink') + os.symlink('foo', symlink_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) + self.device.push(symlink_path, self.DEVICE_TEMP_DIR) + rc, out, _ = self.device.shell_nocheck( + ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')]) + self.assertEqual(0, rc) + self.assertEqual(out.strip(), 'foo') + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_multiple_push(self): + """Push multiple files to the device in one adb push command. + + Bug: http://b/25324823 + """ self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - self.device.push(symlink_path, self.DEVICE_TEMP_DIR) - rc, out, _ = self.device.shell_nocheck( - ['cat', posixpath.join(self.DEVICE_TEMP_DIR, 'symlink')]) - self.assertEqual(0, rc) - self.assertEqual(out.strip(), 'foo') - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - def test_multiple_push(self): - """Push multiple files to the device in one adb push command. - - Bug: http://b/25324823 - """ - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', self.DEVICE_TEMP_DIR]) - - try: - host_dir = tempfile.mkdtemp() - - # Create some random files and a subdirectory containing more files. - temp_files = make_random_host_files(in_dir=host_dir, num_files=4) - - subdir = os.path.join(host_dir, 'subdir') - os.mkdir(subdir) - subdir_temp_files = make_random_host_files(in_dir=subdir, - num_files=4) - - paths = [x.full_path for x in temp_files] - paths.append(subdir) - self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR]) - - for temp_file in temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) - - for subdir_temp_file in subdir_temp_files: - remote_path = posixpath.join(self.DEVICE_TEMP_DIR, - # BROKEN: http://b/25394682 - # 'subdir'; - temp_file.base_name) - self._verify_remote(temp_file.checksum, remote_path) - - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - @requires_non_root - def test_push_error_reporting(self): - """Make sure that errors that occur while pushing a file get reported - - Bug: http://b/26816782 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(b'\0' * 1024 * 1024) - tmp_file.flush() try: - self.device.push(local=tmp_file.name, remote='/system/') - self.fail('push should not have succeeded') + host_dir = tempfile.mkdtemp() + + # Create some random files and a subdirectory containing more files. + temp_files = make_random_host_files(in_dir=host_dir, num_files=4) + + subdir = os.path.join(host_dir, 'subdir') + os.mkdir(subdir) + subdir_temp_files = make_random_host_files(in_dir=subdir, + num_files=4) + + paths = [x.full_path for x in temp_files] + paths.append(subdir) + self.device._simple_call(['push'] + paths + [self.DEVICE_TEMP_DIR]) + + for temp_file in temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + + for subdir_temp_file in subdir_temp_files: + remote_path = posixpath.join(self.DEVICE_TEMP_DIR, + # BROKEN: http://b/25394682 + # 'subdir'; + temp_file.base_name) + self._verify_remote(temp_file.checksum, remote_path) + + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + @requires_non_root + def test_push_error_reporting(self): + """Make sure that errors that occur while pushing a file get reported + + Bug: http://b/26816782 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(b'\0' * 1024 * 1024) + tmp_file.flush() + try: + self.device.push(local=tmp_file.name, remote='/system/') + self.fail('push should not have succeeded') + except subprocess.CalledProcessError as e: + output = e.output + + self.assertTrue(b'Permission denied' in output or + b'Read-only file system' in output) + + @requires_non_root + def test_push_directory_creation(self): + """Regression test for directory creation. + + Bug: http://b/110953234 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write(b'\0' * 1024 * 1024) + tmp_file.flush() + remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation' + self.device.shell(['rm', '-rf', remote_path]) + + remote_path += '/filename' + self.device.push(local=tmp_file.name, remote=remote_path) + + def disabled_test_push_multiple_slash_root(self): + """Regression test for pushing to //data/local/tmp. + + Bug: http://b/141311284 + + Disabled because this broken on the adbd side as well: b/141943968 + """ + with tempfile.NamedTemporaryFile() as tmp_file: + tmp_file.write('\0' * 1024 * 1024) + tmp_file.flush() + remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root' + self.device.shell(['rm', '-rf', remote_path]) + self.device.push(local=tmp_file.name, remote=remote_path) + + def _test_pull(self, remote_file, checksum): + tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False) + tmp_write.close() + self.device.pull(remote=remote_file, local=tmp_write.name) + with open(tmp_write.name, 'rb') as tmp_read: + host_contents = tmp_read.read() + host_md5 = compute_md5(host_contents) + self.assertEqual(checksum, host_md5) + os.remove(tmp_write.name) + + @requires_non_root + def test_pull_error_reporting(self): + self.device.shell(['touch', self.DEVICE_TEMP_FILE]) + self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE]) + + try: + output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x') except subprocess.CalledProcessError as e: output = e.output - self.assertTrue(b'Permission denied' in output or - b'Read-only file system' in output) + self.assertIn(b'Permission denied', output) - @requires_non_root - def test_push_directory_creation(self): - """Regression test for directory creation. + self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) - Bug: http://b/110953234 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write(b'\0' * 1024 * 1024) - tmp_file.flush() - remote_path = self.DEVICE_TEMP_DIR + '/test_push_directory_creation' - self.device.shell(['rm', '-rf', remote_path]) + def test_pull(self): + """Pull a randomly generated file from specified device.""" + kbytes = 512 + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) + cmd = ['dd', 'if=/dev/urandom', + 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024', + 'count={}'.format(kbytes)] + self.device.shell(cmd) + dev_md5, _ = self.device.shell( + [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split() + self._test_pull(self.DEVICE_TEMP_FILE, dev_md5) + self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE]) - remote_path += '/filename' - self.device.push(local=tmp_file.name, remote=remote_path) + def test_pull_dir(self): + """Pull a randomly generated directory of files from the device.""" + try: + host_dir = tempfile.mkdtemp() - def disabled_test_push_multiple_slash_root(self): - """Regression test for pushing to //data/local/tmp. + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - Bug: http://b/141311284 + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - Disabled because this broken on the adbd side as well: b/141943968 - """ - with tempfile.NamedTemporaryFile() as tmp_file: - tmp_file.write('\0' * 1024 * 1024) - tmp_file.flush() - remote_path = '/' + self.DEVICE_TEMP_DIR + '/test_push_multiple_slash_root' - self.device.shell(['rm', '-rf', remote_path]) - self.device.push(local=tmp_file.name, remote=remote_path) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) - def _test_pull(self, remote_file, checksum): - tmp_write = tempfile.NamedTemporaryFile(mode='wb', delete=False) - tmp_write.close() - self.device.pull(remote=remote_file, local=tmp_write.name) - with open(tmp_write.name, 'rb') as tmp_read: - host_contents = tmp_read.read() - host_md5 = compute_md5(host_contents) - self.assertEqual(checksum, host_md5) - os.remove(tmp_write.name) + for temp_file in temp_files: + host_path = os.path.join( + host_dir, posixpath.basename(self.DEVICE_TEMP_DIR), + temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - @requires_non_root - def test_pull_error_reporting(self): - self.device.shell(['touch', self.DEVICE_TEMP_FILE]) - self.device.shell(['chmod', 'a-rwx', self.DEVICE_TEMP_FILE]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) - try: - output = self.device.pull(remote=self.DEVICE_TEMP_FILE, local='x') - except subprocess.CalledProcessError as e: - output = e.output + def test_pull_dir_symlink(self): + """Pull a directory into a symlink to a directory. - self.assertIn(b'Permission denied', output) + Bug: http://b/27362811 + """ + if os.name != 'posix': + raise unittest.SkipTest('requires POSIX') - self.device.shell(['rm', '-f', self.DEVICE_TEMP_FILE]) + try: + host_dir = tempfile.mkdtemp() + real_dir = os.path.join(host_dir, 'dir') + symlink = os.path.join(host_dir, 'symlink') + os.mkdir(real_dir) + os.symlink(real_dir, symlink) - def test_pull(self): - """Pull a randomly generated file from specified device.""" - kbytes = 512 - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_FILE]) - cmd = ['dd', 'if=/dev/urandom', - 'of={}'.format(self.DEVICE_TEMP_FILE), 'bs=1024', - 'count={}'.format(kbytes)] - self.device.shell(cmd) - dev_md5, _ = self.device.shell( - [get_md5_prog(self.device), self.DEVICE_TEMP_FILE])[0].split() - self._test_pull(self.DEVICE_TEMP_FILE, dev_md5) - self.device.shell_nocheck(['rm', self.DEVICE_TEMP_FILE]) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) - def test_pull_dir(self): - """Pull a randomly generated directory of files from the device.""" - try: - host_dir = tempfile.mkdtemp() + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + for temp_file in temp_files: + host_path = os.path.join( + real_dir, posixpath.basename(self.DEVICE_TEMP_DIR), + temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + def test_pull_dir_symlink_collision(self): + """Pull a directory into a colliding symlink to directory.""" + if os.name != 'posix': + raise unittest.SkipTest('requires POSIX') + + try: + host_dir = tempfile.mkdtemp() + real_dir = os.path.join(host_dir, 'real') + tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR) + symlink = os.path.join(host_dir, tmp_dirname) + os.mkdir(real_dir) + os.symlink(real_dir, symlink) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + + for temp_file in temp_files: + host_path = os.path.join(real_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_pull_dir_nonexistent(self): + """Pull a directory of files from the device to a nonexistent path.""" + try: + host_dir = tempfile.mkdtemp() + dest_dir = os.path.join(host_dir, 'dest') + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + + self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir) + + for temp_file in temp_files: + host_path = os.path.join(dest_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + # selinux prevents adbd from accessing symlinks on /data/local/tmp. + def disabled_test_pull_symlink_dir(self): + """Pull a symlink to a directory of symlinks to files.""" + try: + host_dir = tempfile.mkdtemp() + + remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents') + remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links') + remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink') + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', remote_dir, remote_links]) + self.device.shell(['ln', '-s', remote_links, remote_symlink]) + + # Populate device directory with random files. + temp_files = make_random_device_files( + self.device, in_dir=remote_dir, num_files=32) + + for temp_file in temp_files: + self.device.shell( + ['ln', '-s', '../contents/{}'.format(temp_file.base_name), + posixpath.join(remote_links, temp_file.base_name)]) + + self.device.pull(remote=remote_symlink, local=host_dir) + + for temp_file in temp_files: + host_path = os.path.join( + host_dir, 'symlink', temp_file.base_name) + self._verify_local(temp_file.checksum, host_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_pull_empty(self): + """Pull a directory containing an empty directory from the device.""" + try: + host_dir = tempfile.mkdtemp() + + remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty') + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', remote_empty_path]) + + self.device.pull(remote=remote_empty_path, local=host_dir) + self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty'))) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def test_multiple_pull(self): + """Pull a randomly generated directory of files from the device.""" + + try: + host_dir = tempfile.mkdtemp() + + subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir') + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + self.device.shell(['mkdir', '-p', subdir]) + + # Create some random files and a subdirectory containing more files. + temp_files = make_random_device_files( + self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4) + + subdir_temp_files = make_random_device_files( + self.device, in_dir=subdir, num_files=4, prefix='subdir_') + + paths = [x.full_path for x in temp_files] + paths.append(subdir) + self.device._simple_call(['pull'] + paths + [host_dir]) + + for temp_file in temp_files: + local_path = os.path.join(host_dir, temp_file.base_name) + self._verify_local(temp_file.checksum, local_path) + + for subdir_temp_file in subdir_temp_files: + local_path = os.path.join(host_dir, + 'subdir', + subdir_temp_file.base_name) + self._verify_local(subdir_temp_file.checksum, local_path) + + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if host_dir is not None: + shutil.rmtree(host_dir) + + def verify_sync(self, device, temp_files, device_dir): + """Verifies that a list of temp files was synced to the device.""" + # Confirm that every file on the device mirrors that on the host. for temp_file in temp_files: - host_path = os.path.join( - host_dir, posixpath.basename(self.DEVICE_TEMP_DIR), - temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + device_full_path = posixpath.join( + device_dir, temp_file.base_name) + dev_md5, _ = device.shell( + [get_md5_prog(self.device), device_full_path])[0].split() + self.assertEqual(temp_file.checksum, dev_md5) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_sync(self): + """Sync a host directory to the data partition.""" - def test_pull_dir_symlink(self): - """Pull a directory into a symlink to a directory. + try: + base_dir = tempfile.mkdtemp() - Bug: http://b/27362811 - """ - if os.name != 'posix': - raise unittest.SkipTest('requires POSIX') + # Create mirror device directory hierarchy within base_dir. + full_dir_path = base_dir + self.DEVICE_TEMP_DIR + os.makedirs(full_dir_path) - try: - host_dir = tempfile.mkdtemp() - real_dir = os.path.join(host_dir, 'dir') - symlink = os.path.join(host_dir, 'symlink') - os.mkdir(real_dir) - os.symlink(real_dir, symlink) + # Create 32 random files within the host mirror. + temp_files = make_random_host_files( + in_dir=full_dir_path, num_files=32) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + # Clean up any stale files on the device. + device = adb.get_device() # pylint: disable=no-member + device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + old_product_out = os.environ.get('ANDROID_PRODUCT_OUT') + os.environ['ANDROID_PRODUCT_OUT'] = base_dir + device.sync('data') + if old_product_out is None: + del os.environ['ANDROID_PRODUCT_OUT'] + else: + os.environ['ANDROID_PRODUCT_OUT'] = old_product_out - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=symlink) + self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR) - for temp_file in temp_files: - host_path = os.path.join( - real_dir, posixpath.basename(self.DEVICE_TEMP_DIR), - temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if base_dir is not None: + shutil.rmtree(base_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_push_sync(self): + """Sync a host directory to a specific path.""" - def test_pull_dir_symlink_collision(self): - """Pull a directory into a colliding symlink to directory.""" - if os.name != 'posix': - raise unittest.SkipTest('requires POSIX') + try: + temp_dir = tempfile.mkdtemp() + temp_files = make_random_host_files(in_dir=temp_dir, num_files=32) - try: - host_dir = tempfile.mkdtemp() - real_dir = os.path.join(host_dir, 'real') - tmp_dirname = os.path.basename(self.DEVICE_TEMP_DIR) - symlink = os.path.join(host_dir, tmp_dirname) - os.mkdir(real_dir) - os.symlink(real_dir, symlink) + device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + # Clean up any stale files on the device. + device = adb.get_device() # pylint: disable=no-member + device.shell(['rm', '-rf', device_dir]) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + device.push(temp_dir, device_dir, sync=True) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=host_dir) + self.verify_sync(device, temp_files, device_dir) - for temp_file in temp_files: - host_path = os.path.join(real_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) + self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) + def test_unicode_paths(self): + """Ensure that we can support non-ASCII paths, even on Windows.""" + name = u'로보카 폴리' - def test_pull_dir_nonexistent(self): - """Pull a directory of files from the device to a nonexistent path.""" - try: - host_dir = tempfile.mkdtemp() - dest_dir = os.path.join(host_dir, 'dest') + self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) + remote_path = u'/data/local/tmp/adb-test-{}'.format(name) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', self.DEVICE_TEMP_DIR]) + ## push. + tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False) + tf.close() + self.device.push(tf.name, remote_path) + os.remove(tf.name) + self.assertFalse(os.path.exists(tf.name)) - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=32) + # Verify that the device ended up with the expected UTF-8 path + output = self.device.shell( + ['ls', '/data/local/tmp/adb-test-*'])[0].strip() + self.assertEqual(remote_path, output) - self.device.pull(remote=self.DEVICE_TEMP_DIR, local=dest_dir) + # pull. + self.device.pull(remote_path, tf.name) + self.assertTrue(os.path.exists(tf.name)) + os.remove(tf.name) + self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) - for temp_file in temp_files: - host_path = os.path.join(dest_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) +class FileOperationsTestUncompressed(FileOperationsTest.Base): + compression = "none" - # selinux prevents adbd from accessing symlinks on /data/local/tmp. - def disabled_test_pull_symlink_dir(self): - """Pull a symlink to a directory of symlinks to files.""" - try: - host_dir = tempfile.mkdtemp() - remote_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'contents') - remote_links = posixpath.join(self.DEVICE_TEMP_DIR, 'links') - remote_symlink = posixpath.join(self.DEVICE_TEMP_DIR, 'symlink') - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', remote_dir, remote_links]) - self.device.shell(['ln', '-s', remote_links, remote_symlink]) - - # Populate device directory with random files. - temp_files = make_random_device_files( - self.device, in_dir=remote_dir, num_files=32) - - for temp_file in temp_files: - self.device.shell( - ['ln', '-s', '../contents/{}'.format(temp_file.base_name), - posixpath.join(remote_links, temp_file.base_name)]) - - self.device.pull(remote=remote_symlink, local=host_dir) - - for temp_file in temp_files: - host_path = os.path.join( - host_dir, 'symlink', temp_file.base_name) - self._verify_local(temp_file.checksum, host_path) - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - def test_pull_empty(self): - """Pull a directory containing an empty directory from the device.""" - try: - host_dir = tempfile.mkdtemp() - - remote_empty_path = posixpath.join(self.DEVICE_TEMP_DIR, 'empty') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', remote_empty_path]) - - self.device.pull(remote=remote_empty_path, local=host_dir) - self.assertTrue(os.path.isdir(os.path.join(host_dir, 'empty'))) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - def test_multiple_pull(self): - """Pull a randomly generated directory of files from the device.""" - - try: - host_dir = tempfile.mkdtemp() - - subdir = posixpath.join(self.DEVICE_TEMP_DIR, 'subdir') - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - self.device.shell(['mkdir', '-p', subdir]) - - # Create some random files and a subdirectory containing more files. - temp_files = make_random_device_files( - self.device, in_dir=self.DEVICE_TEMP_DIR, num_files=4) - - subdir_temp_files = make_random_device_files( - self.device, in_dir=subdir, num_files=4, prefix='subdir_') - - paths = [x.full_path for x in temp_files] - paths.append(subdir) - self.device._simple_call(['pull'] + paths + [host_dir]) - - for temp_file in temp_files: - local_path = os.path.join(host_dir, temp_file.base_name) - self._verify_local(temp_file.checksum, local_path) - - for subdir_temp_file in subdir_temp_files: - local_path = os.path.join(host_dir, - 'subdir', - subdir_temp_file.base_name) - self._verify_local(subdir_temp_file.checksum, local_path) - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if host_dir is not None: - shutil.rmtree(host_dir) - - def verify_sync(self, device, temp_files, device_dir): - """Verifies that a list of temp files was synced to the device.""" - # Confirm that every file on the device mirrors that on the host. - for temp_file in temp_files: - device_full_path = posixpath.join( - device_dir, temp_file.base_name) - dev_md5, _ = device.shell( - [get_md5_prog(self.device), device_full_path])[0].split() - self.assertEqual(temp_file.checksum, dev_md5) - - def test_sync(self): - """Sync a host directory to the data partition.""" - - try: - base_dir = tempfile.mkdtemp() - - # Create mirror device directory hierarchy within base_dir. - full_dir_path = base_dir + self.DEVICE_TEMP_DIR - os.makedirs(full_dir_path) - - # Create 32 random files within the host mirror. - temp_files = make_random_host_files( - in_dir=full_dir_path, num_files=32) - - # Clean up any stale files on the device. - device = adb.get_device() # pylint: disable=no-member - device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - - old_product_out = os.environ.get('ANDROID_PRODUCT_OUT') - os.environ['ANDROID_PRODUCT_OUT'] = base_dir - device.sync('data') - if old_product_out is None: - del os.environ['ANDROID_PRODUCT_OUT'] - else: - os.environ['ANDROID_PRODUCT_OUT'] = old_product_out - - self.verify_sync(device, temp_files, self.DEVICE_TEMP_DIR) - - #self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if base_dir is not None: - shutil.rmtree(base_dir) - - def test_push_sync(self): - """Sync a host directory to a specific path.""" - - try: - temp_dir = tempfile.mkdtemp() - temp_files = make_random_host_files(in_dir=temp_dir, num_files=32) - - device_dir = posixpath.join(self.DEVICE_TEMP_DIR, 'sync_src_dst') - - # Clean up any stale files on the device. - device = adb.get_device() # pylint: disable=no-member - device.shell(['rm', '-rf', device_dir]) - - device.push(temp_dir, device_dir, sync=True) - - self.verify_sync(device, temp_files, device_dir) - - self.device.shell(['rm', '-rf', self.DEVICE_TEMP_DIR]) - finally: - if temp_dir is not None: - shutil.rmtree(temp_dir) - - def test_unicode_paths(self): - """Ensure that we can support non-ASCII paths, even on Windows.""" - name = u'로보카 폴리' - - self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) - remote_path = u'/data/local/tmp/adb-test-{}'.format(name) - - ## push. - tf = tempfile.NamedTemporaryFile('wb', suffix=name, delete=False) - tf.close() - self.device.push(tf.name, remote_path) - os.remove(tf.name) - self.assertFalse(os.path.exists(tf.name)) - - # Verify that the device ended up with the expected UTF-8 path - output = self.device.shell( - ['ls', '/data/local/tmp/adb-test-*'])[0].strip() - self.assertEqual(remote_path, output) - - # pull. - self.device.pull(remote_path, tf.name) - self.assertTrue(os.path.exists(tf.name)) - os.remove(tf.name) - self.device.shell(['rm', '-f', '/data/local/tmp/adb-test-*']) +class FileOperationsTestBrotli(FileOperationsTest.Base): + compression = "brotli" class DeviceOfflineTest(DeviceTest):