ACPR/android_system_core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

FastDeploy refactor: 2+GB APK support, optimizations, tests.

Browse source 
- removed 2GB apk size cap,
- removed zip archive parsing on device (1.1M->236K agent size reduction),
- optimized matching entries search,
- added more robust matching entries search based on hash of CDr entry,
- reduced patch size by reusing Local File Header of matched entries,
- removed extra manifest parsing and extra agent calls,
- added device-side tests for agent,
- fix for Windows patch creation.

Test: atest adb_test fastdeploy_test FastDeployTests

Total time for 0-size patch reduction for 1.7G apk: 1m1.778s->0m36.234s.

Change-Id: I66d2cef1adf5b2be3325e355a7e72e9c99992369
This commit is contained in:
Alex Buynytskyy 2019-09-16 12:10:54 -07:00
parent c63ef7fc36
commit 665f3ff5a7
34 changed files with 1867 additions and 749 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
adb/Android.bp
View file

@ -27,7 +27,6 @@ cc_defaults {
"-DADB_HOST=1", // overridden by adbd_defaults
"-DALLOW_ADBD_ROOT=0", // overridden by adbd_defaults
"-DANDROID_BASE_UNIQUE_FD_DISABLE_IMPLICIT_CONVERSION=1",
"-DENABLE_FASTDEPLOY=1", // enable fast deploy
],
cpp_std: "experimental",
@ -691,6 +690,7 @@ cc_library_host_static {
name: "libfastdeploy_host",
defaults: ["adb_defaults"],
srcs: [
"fastdeploy/deploypatchgenerator/apk_archive.cpp",
"fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp",
"fastdeploy/deploypatchgenerator/patch_utils.cpp",
"fastdeploy/proto/ApkEntry.proto",
@ -727,6 +727,7 @@ cc_test_host {
name: "fastdeploy_test",
defaults: ["adb_defaults"],
srcs: [
"fastdeploy/deploypatchgenerator/apk_archive_test.cpp",
"fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp",
"fastdeploy/deploypatchgenerator/patch_utils_test.cpp",
],
@ -754,6 +755,9 @@ cc_test_host {
},
},
data: [
"fastdeploy/testdata/rotating_cube-metadata-release.data",
"fastdeploy/testdata/rotating_cube-release.apk",
"fastdeploy/testdata/sample.apk",
"fastdeploy/testdata/sample.cd",
],
}

118
adb/client/adb_install.cpp
View file

@ -37,9 +37,7 @@
#include "commandline.h"
#include "fastdeploy.h"
#if defined(ENABLE_FASTDEPLOY)
static constexpr int kFastDeployMinApi = 24;
#endif
namespace {
@ -146,15 +144,7 @@ static void read_status_line(int fd, char* buf, size_t count) {
*buf = '\0';
}
#if defined(ENABLE_FASTDEPLOY)
static int delete_device_patch_file(const char* apkPath) {
std::string patchDevicePath = get_patch_path(apkPath);
return delete_device_file(patchDevicePath);
}
#endif
static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy,
bool use_localagent) {
static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy) {
printf("Performing Streamed Install\n");
// The last argument must be the APK file
@ -176,31 +166,15 @@ static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy
error_exit("--fastdeploy doesn't support .apex files");
}
if (use_fastdeploy == true) {
#if defined(ENABLE_FASTDEPLOY)
TemporaryFile metadataTmpFile;
std::string patchTmpFilePath;
{
TemporaryFile patchTmpFile;
patchTmpFile.DoNotRemove();
patchTmpFilePath = patchTmpFile.path;
if (use_fastdeploy) {
auto metadata = extract_metadata(file);
if (metadata.has_value()) {
// pass all but 1st (command) and last (apk path) parameters through to pm for
// session creation
std::vector<const char*> pm_args{argv + 1, argv + argc - 1};
auto patchFd = install_patch(pm_args.size(), pm_args.data());
return stream_patch(file, std::move(metadata.value()), std::move(patchFd));
}
FILE* metadataFile = fopen(metadataTmpFile.path, "wb");
extract_metadata(file, metadataFile);
fclose(metadataFile);
create_patch(file, metadataTmpFile.path, patchTmpFilePath.c_str());
// pass all but 1st (command) and last (apk path) parameters through to pm for
// session creation
std::vector<const char*> pm_args{argv + 1, argv + argc - 1};
install_patch(file, patchTmpFilePath.c_str(), pm_args.size(), pm_args.data());
adb_unlink(patchTmpFilePath.c_str());
delete_device_patch_file(file);
return 0;
#else
error_exit("fastdeploy is disabled");
#endif
}
struct stat sb;
@ -265,8 +239,7 @@ static int install_app_streamed(int argc, const char** argv, bool use_fastdeploy
return 1;
}
static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy,
bool use_localagent) {
static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy) {
printf("Performing Push Install\n");
// Find last APK argument.
@ -287,35 +260,26 @@ static int install_app_legacy(int argc, const char** argv, bool use_fastdeploy,
int result = -1;
std::vector<const char*> apk_file = {argv[last_apk]};
std::string apk_dest = "/data/local/tmp/" + android::base::Basename(argv[last_apk]);
if (use_fastdeploy == true) {
#if defined(ENABLE_FASTDEPLOY)
TemporaryFile metadataTmpFile;
TemporaryFile patchTmpFile;
FILE* metadataFile = fopen(metadataTmpFile.path, "wb");
extract_metadata(apk_file[0], metadataFile);
fclose(metadataFile);
create_patch(apk_file[0], metadataTmpFile.path, patchTmpFile.path);
apply_patch_on_device(apk_file[0], patchTmpFile.path, apk_dest.c_str());
#else
error_exit("fastdeploy is disabled");
#endif
} else {
if (!do_sync_push(apk_file, apk_dest.c_str(), false)) goto cleanup_apk;
}
argv[last_apk] = apk_dest.c_str(); /* destination name, not source location */
result = pm_command(argc, argv);
cleanup_apk:
if (use_fastdeploy == true) {
#if defined(ENABLE_FASTDEPLOY)
delete_device_patch_file(apk_file[0]);
#endif
if (use_fastdeploy) {
auto metadata = extract_metadata(apk_file[0]);
if (metadata.has_value()) {
auto patchFd = apply_patch_on_device(apk_dest.c_str());
int status = stream_patch(apk_file[0], std::move(metadata.value()), std::move(patchFd));
result = pm_command(argc, argv);
delete_device_file(apk_dest);
return status;
}
}
delete_device_file(apk_dest);
if (do_sync_push(apk_file, apk_dest.c_str(), false)) {
result = pm_command(argc, argv);
delete_device_file(apk_dest);
}
return result;
}
@ -324,7 +288,6 @@ int install_app(int argc, const char** argv) {
InstallMode installMode = INSTALL_DEFAULT;
bool use_fastdeploy = false;
bool is_reinstall = false;
bool use_localagent = false;
FastDeploy_AgentUpdateStrategy agent_update_strategy = FastDeploy_AgentUpdateDifferentVersion;
for (int i = 1; i < argc; i++) {
@ -353,11 +316,6 @@ int install_app(int argc, const char** argv) {
} else if (!strcmp(argv[i], "--version-check-agent")) {
processedArgIndicies.push_back(i);
agent_update_strategy = FastDeploy_AgentUpdateDifferentVersion;
#ifndef _WIN32
} else if (!strcmp(argv[i], "--local-agent")) {
processedArgIndicies.push_back(i);
use_localagent = true;
#endif
}
}
@ -369,14 +327,13 @@ int install_app(int argc, const char** argv) {
error_exit("Attempting to use streaming install on unsupported device");
}
#if defined(ENABLE_FASTDEPLOY)
if (use_fastdeploy == true && get_device_api_level() < kFastDeployMinApi) {
if (use_fastdeploy && get_device_api_level() < kFastDeployMinApi) {
printf("Fast Deploy is only compatible with devices of API version %d or higher, "
"ignoring.\n",
kFastDeployMinApi);
use_fastdeploy = false;
}
#endif
fastdeploy_set_agent_update_strategy(agent_update_strategy);
std::vector<const char*> passthrough_argv;
for (int i = 0; i < argc; i++) {
@ -389,26 +346,13 @@ int install_app(int argc, const char** argv) {
error_exit("install requires an apk argument");
}
if (use_fastdeploy == true) {
#if defined(ENABLE_FASTDEPLOY)
fastdeploy_set_local_agent(use_localagent);
update_agent(agent_update_strategy);
// The last argument must be the APK file
const char* file = passthrough_argv.back();
use_fastdeploy = find_package(file);
#else
error_exit("fastdeploy is disabled");
#endif
}
switch (installMode) {
case INSTALL_PUSH:
return install_app_legacy(passthrough_argv.size(), passthrough_argv.data(),
use_fastdeploy, use_localagent);
use_fastdeploy);
case INSTALL_STREAM:
return install_app_streamed(passthrough_argv.size(), passthrough_argv.data(),
use_fastdeploy, use_localagent);
use_fastdeploy);
case INSTALL_DEFAULT:
default:
return 1;

9
adb/client/commandline.cpp
View file

@ -255,13 +255,8 @@ static void stdin_raw_restore() {
}
#endif
// Reads from |fd| and prints received data. If |use_shell_protocol| is true
// this expects that incoming data will use the shell protocol, in which case
// stdout/stderr are routed independently and the remote exit code will be
// returned.
// if |callback| is non-null, stdout/stderr output will be handled by it.
int read_and_dump(borrowed_fd fd, bool use_shell_protocol = false,
StandardStreamsCallbackInterface* callback = &DEFAULT_STANDARD_STREAMS_CALLBACK) {
int read_and_dump(borrowed_fd fd, bool use_shell_protocol,
StandardStreamsCallbackInterface* callback) {
int exit_code = 0;
if (fd < 0) return exit_code;

8
adb/client/commandline.h
View file

@ -109,6 +109,14 @@ int send_shell_command(
const std::string& command, bool disable_shell_protocol = false,
StandardStreamsCallbackInterface* callback = &DEFAULT_STANDARD_STREAMS_CALLBACK);
// Reads from |fd| and prints received data. If |use_shell_protocol| is true
// this expects that incoming data will use the shell protocol, in which case
// stdout/stderr are routed independently and the remote exit code will be
// returned.
// if |callback| is non-null, stdout/stderr output will be handled by it.
int read_and_dump(borrowed_fd fd, bool use_shell_protocol = false,
StandardStreamsCallbackInterface* callback = &DEFAULT_STANDARD_STREAMS_CALLBACK);
// Connects to the device "abb" service with |command| and returns the fd.
template <typename ContainerT>
unique_fd send_abb_exec_command(const ContainerT& command_args, std::string* error) {

352
adb/client/fastdeploy.cpp
View file

@ -30,112 +30,114 @@
#include "deployagent.inc" // Generated include via build rule.
#include "deployagentscript.inc" // Generated include via build rule.
#include "fastdeploy/deploypatchgenerator/deploy_patch_generator.h"
#include "fastdeploy/deploypatchgenerator/patch_utils.h"
#include "fastdeploy/proto/ApkEntry.pb.h"
#include "fastdeploycallbacks.h"
#include "sysdeps.h"
#include "adb_utils.h"
static constexpr long kRequiredAgentVersion = 0x00000002;
static constexpr long kRequiredAgentVersion = 0x00000003;
static constexpr int kPackageMissing = 3;
static constexpr int kInvalidAgentVersion = 4;
static constexpr const char* kDeviceAgentPath = "/data/local/tmp/";
static constexpr const char* kDeviceAgentFile = "/data/local/tmp/deployagent.jar";
static constexpr const char* kDeviceAgentScript = "/data/local/tmp/deployagent";
static bool g_use_localagent = false;
static constexpr bool g_verbose_timings = false;
static FastDeploy_AgentUpdateStrategy g_agent_update_strategy =
FastDeploy_AgentUpdateDifferentVersion;
long get_agent_version() {
std::vector<char> versionOutputBuffer;
std::vector<char> versionErrorBuffer;
using APKMetaData = com::android::fastdeploy::APKMetaData;
int statusCode = capture_shell_command("/data/local/tmp/deployagent version",
&versionOutputBuffer, &versionErrorBuffer);
long version = -1;
namespace {
if (statusCode == 0 && versionOutputBuffer.size() > 0) {
version = strtol((char*)versionOutputBuffer.data(), NULL, 16);
struct TimeReporter {
TimeReporter(const char* label) : label_(label) {}
~TimeReporter() {
if (g_verbose_timings) {
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - start_);
fprintf(stderr, "%s finished in %lldms\n", label_,
static_cast<long long>(duration.count()));
}
}
return version;
}
private:
const char* label_;
std::chrono::steady_clock::time_point start_ = std::chrono::steady_clock::now();
};
#define REPORT_FUNC_TIME() TimeReporter reporter(__func__)
struct FileDeleter {
FileDeleter(const char* path) : path_(path) {}
~FileDeleter() { adb_unlink(path_); }
private:
const char* const path_;
};
} // namespace
int get_device_api_level() {
std::vector<char> sdkVersionOutputBuffer;
std::vector<char> sdkVersionErrorBuffer;
REPORT_FUNC_TIME();
std::vector<char> sdk_version_output_buffer;
std::vector<char> sdk_version_error_buffer;
int api_level = -1;
int statusCode = capture_shell_command("getprop ro.build.version.sdk", &sdkVersionOutputBuffer,
&sdkVersionErrorBuffer);
if (statusCode == 0 && sdkVersionOutputBuffer.size() > 0) {
api_level = strtol((char*)sdkVersionOutputBuffer.data(), NULL, 10);
int statusCode = capture_shell_command("getprop ro.build.version.sdk",
&sdk_version_output_buffer, &sdk_version_error_buffer);
if (statusCode == 0 && sdk_version_output_buffer.size() > 0) {
api_level = strtol((char*)sdk_version_output_buffer.data(), NULL, 10);
}
return api_level;
}
void fastdeploy_set_local_agent(bool use_localagent) {
g_use_localagent = use_localagent;
void fastdeploy_set_agent_update_strategy(FastDeploy_AgentUpdateStrategy agent_update_strategy) {
g_agent_update_strategy = agent_update_strategy;
}
static bool deploy_agent(bool checkTimeStamps) {
static void push_to_device(const void* data, size_t byte_count, const char* dst, bool sync) {
std::vector<const char*> srcs;
// TODO: Deploy agent from bin2c directly instead of writing to disk first.
TemporaryFile tempAgent;
android::base::WriteFully(tempAgent.fd, kDeployAgent, sizeof(kDeployAgent));
srcs.push_back(tempAgent.path);
if (!do_sync_push(srcs, kDeviceAgentFile, checkTimeStamps)) {
{
TemporaryFile temp;
android::base::WriteFully(temp.fd, data, byte_count);
srcs.push_back(temp.path);
// On Windows, the file needs to be flushed before pushing to device.
// closing the file flushes its content, but we still need to remove it after push.
// FileDeleter does exactly that.
temp.DoNotRemove();
}
FileDeleter temp_deleter(srcs.back());
if (!do_sync_push(srcs, dst, sync)) {
error_exit("Failed to push fastdeploy agent to device.");
}
srcs.clear();
// TODO: Deploy agent from bin2c directly instead of writing to disk first.
TemporaryFile tempAgentScript;
android::base::WriteFully(tempAgentScript.fd, kDeployAgentScript, sizeof(kDeployAgentScript));
srcs.push_back(tempAgentScript.path);
if (!do_sync_push(srcs, kDeviceAgentScript, checkTimeStamps)) {
error_exit("Failed to push fastdeploy agent script to device.");
}
srcs.clear();
}
static bool deploy_agent(bool check_time_stamps) {
REPORT_FUNC_TIME();
push_to_device(kDeployAgent, sizeof(kDeployAgent), kDeviceAgentFile, check_time_stamps);
push_to_device(kDeployAgentScript, sizeof(kDeployAgentScript), kDeviceAgentScript,
check_time_stamps);
// on windows the shell script might have lost execute permission
// so need to set this explicitly
const char* kChmodCommandPattern = "chmod 777 %s";
std::string chmodCommand =
std::string chmod_command =
android::base::StringPrintf(kChmodCommandPattern, kDeviceAgentScript);
int ret = send_shell_command(chmodCommand);
int ret = send_shell_command(chmod_command);
if (ret != 0) {
error_exit("Error executing %s returncode: %d", chmodCommand.c_str(), ret);
error_exit("Error executing %s returncode: %d", chmod_command.c_str(), ret);
}
return true;
}
void update_agent(FastDeploy_AgentUpdateStrategy agentUpdateStrategy) {
long agent_version = get_agent_version();
switch (agentUpdateStrategy) {
case FastDeploy_AgentUpdateAlways:
deploy_agent(false);
break;
case FastDeploy_AgentUpdateNewerTimeStamp:
deploy_agent(true);
break;
case FastDeploy_AgentUpdateDifferentVersion:
if (agent_version != kRequiredAgentVersion) {
if (agent_version < 0) {
printf("Could not detect agent on device, deploying\n");
} else {
printf("Device agent version is (%ld), (%ld) is required, re-deploying\n",
agent_version, kRequiredAgentVersion);
}
deploy_agent(false);
}
break;
}
agent_version = get_agent_version();
if (agent_version != kRequiredAgentVersion) {
error_exit("After update agent version remains incorrect! Expected %ld but version is %ld",
kRequiredAgentVersion, agent_version);
}
}
static std::string get_string_from_utf16(const char16_t* input, int input_len) {
ssize_t utf8_length = utf16_to_utf8_length(input, input_len);
if (utf8_length <= 0) {
@ -147,29 +149,29 @@ static std::string get_string_from_utf16(const char16_t* input, int input_len) {
return utf8;
}
static std::string get_packagename_from_apk(const char* apkPath) {
static std::string get_package_name_from_apk(const char* apk_path) {
#undef open
std::unique_ptr<android::ZipFileRO> zipFile(android::ZipFileRO::open(apkPath));
std::unique_ptr<android::ZipFileRO> zip_file((android::ZipFileRO::open)(apk_path));
#define open ___xxx_unix_open
if (zipFile == nullptr) {
perror_exit("Could not open %s", apkPath);
if (zip_file == nullptr) {
perror_exit("Could not open %s", apk_path);
}
android::ZipEntryRO entry = zipFile->findEntryByName("AndroidManifest.xml");
android::ZipEntryRO entry = zip_file->findEntryByName("AndroidManifest.xml");
if (entry == nullptr) {
error_exit("Could not find AndroidManifest.xml inside %s", apkPath);
error_exit("Could not find AndroidManifest.xml inside %s", apk_path);
}
uint32_t manifest_len = 0;
if (!zipFile->getEntryInfo(entry, NULL, &manifest_len, NULL, NULL, NULL, NULL)) {
error_exit("Could not read AndroidManifest.xml inside %s", apkPath);
if (!zip_file->getEntryInfo(entry, NULL, &manifest_len, NULL, NULL, NULL, NULL)) {
error_exit("Could not read AndroidManifest.xml inside %s", apk_path);
}
std::vector<char> manifest_data(manifest_len);
if (!zipFile->uncompressEntry(entry, manifest_data.data(), manifest_len)) {
error_exit("Could not uncompress AndroidManifest.xml inside %s", apkPath);
if (!zip_file->uncompressEntry(entry, manifest_data.data(), manifest_len)) {
error_exit("Could not uncompress AndroidManifest.xml inside %s", apk_path);
}
android::ResXMLTree tree;
android::status_t setto_status = tree.setTo(manifest_data.data(), manifest_len, true);
if (setto_status != android::OK) {
error_exit("Could not parse AndroidManifest.xml inside %s", apkPath);
error_exit("Could not parse AndroidManifest.xml inside %s", apk_path);
}
android::ResXMLParser::event_code_t code;
while ((code = tree.next()) != android::ResXMLParser::BAD_DOCUMENT &&
@ -210,80 +212,97 @@ static std::string get_packagename_from_apk(const char* apkPath) {
break;
}
}
error_exit("Could not find package name tag in AndroidManifest.xml inside %s", apkPath);
error_exit("Could not find package name tag in AndroidManifest.xml inside %s", apk_path);
}
void extract_metadata(const char* apkPath, FILE* outputFp) {
std::string packageName = get_packagename_from_apk(apkPath);
const char* kAgentExtractCommandPattern = "/data/local/tmp/deployagent extract %s";
std::string extractCommand =
android::base::StringPrintf(kAgentExtractCommandPattern, packageName.c_str());
static long parse_agent_version(const std::vector<char>& version_buffer) {
long version = -1;
if (!version_buffer.empty()) {
version = strtol((char*)version_buffer.data(), NULL, 16);
}
return version;
}
std::vector<char> extractErrorBuffer;
DeployAgentFileCallback cb(outputFp, &extractErrorBuffer);
int returnCode = send_shell_command(extractCommand, false, &cb);
static void update_agent_if_necessary() {
switch (g_agent_update_strategy) {
case FastDeploy_AgentUpdateAlways:
deploy_agent(/*check_time_stamps=*/false);
break;
case FastDeploy_AgentUpdateNewerTimeStamp:
deploy_agent(/*check_time_stamps=*/true);
break;
default:
break;
}
}
std::optional<APKMetaData> extract_metadata(const char* apk_path) {
// Update agent if there is a command line argument forcing to do so.
update_agent_if_necessary();
REPORT_FUNC_TIME();
std::string package_name = get_package_name_from_apk(apk_path);
// Dump apk command checks the required vs current agent version and if they match then returns
// the APK dump for package. Doing this in a single call saves round-trip and agent launch time.
constexpr const char* kAgentDumpCommandPattern = "/data/local/tmp/deployagent dump %ld %s";
std::string dump_command = android::base::StringPrintf(
kAgentDumpCommandPattern, kRequiredAgentVersion, package_name.c_str());
std::vector<char> dump_out_buffer;
std::vector<char> dump_error_buffer;
int returnCode =
capture_shell_command(dump_command.c_str(), &dump_out_buffer, &dump_error_buffer);
if (returnCode >= kInvalidAgentVersion) {
// Agent has wrong version or missing.
long agent_version = parse_agent_version(dump_out_buffer);
if (agent_version < 0) {
printf("Could not detect agent on device, deploying\n");
} else {
printf("Device agent version is (%ld), (%ld) is required, re-deploying\n",
agent_version, kRequiredAgentVersion);
}
deploy_agent(/*check_time_stamps=*/false);
// Retry with new agent.
dump_out_buffer.clear();
dump_error_buffer.clear();
returnCode =
capture_shell_command(dump_command.c_str(), &dump_out_buffer, &dump_error_buffer);
}
if (returnCode != 0) {
fprintf(stderr, "Executing %s returned %d\n", extractCommand.c_str(), returnCode);
fprintf(stderr, "%*s\n", int(extractErrorBuffer.size()), extractErrorBuffer.data());
if (returnCode == kInvalidAgentVersion) {
long agent_version = parse_agent_version(dump_out_buffer);
error_exit(
"After update agent version remains incorrect! Expected %ld but version is %ld",
kRequiredAgentVersion, agent_version);
}
if (returnCode == kPackageMissing) {
fprintf(stderr, "Package %s not found, falling back to install\n",
package_name.c_str());
return {};
}
fprintf(stderr, "Executing %s returned %d\n", dump_command.c_str(), returnCode);
fprintf(stderr, "%*s\n", int(dump_error_buffer.size()), dump_error_buffer.data());
error_exit("Aborting");
}
com::android::fastdeploy::APKDump dump;
if (!dump.ParseFromArray(dump_out_buffer.data(), dump_out_buffer.size())) {
fprintf(stderr, "Can't parse output of %s\n", dump_command.c_str());
error_exit("Aborting");
}
return PatchUtils::GetDeviceAPKMetaData(dump);
}
void create_patch(const char* apkPath, const char* metadataPath, const char* patchPath) {
DeployPatchGenerator generator(false);
unique_fd patchFd(adb_open(patchPath, O_WRONLY | O_CREAT | O_CLOEXEC));
if (patchFd < 0) {
perror_exit("adb: failed to create %s", patchPath);
}
bool success = generator.CreatePatch(apkPath, metadataPath, patchFd);
if (!success) {
error_exit("Failed to create patch for %s", apkPath);
}
}
unique_fd install_patch(int argc, const char** argv) {
REPORT_FUNC_TIME();
constexpr char kAgentApplyServicePattern[] = "shell:/data/local/tmp/deployagent apply - -pm %s";
std::string get_patch_path(const char* apkPath) {
std::string packageName = get_packagename_from_apk(apkPath);
std::string patchDevicePath =
android::base::StringPrintf("%s%s.patch", kDeviceAgentPath, packageName.c_str());
return patchDevicePath;
}
void apply_patch_on_device(const char* apkPath, const char* patchPath, const char* outputPath) {
const std::string kAgentApplyCommandPattern = "/data/local/tmp/deployagent apply %s %s -o %s";
std::string packageName = get_packagename_from_apk(apkPath);
std::string patchDevicePath = get_patch_path(apkPath);
std::vector<const char*> srcs = {patchPath};
bool push_ok = do_sync_push(srcs, patchDevicePath.c_str(), false);
if (!push_ok) {
error_exit("Error pushing %s to %s returned", patchPath, patchDevicePath.c_str());
}
std::string applyPatchCommand =
android::base::StringPrintf(kAgentApplyCommandPattern.c_str(), packageName.c_str(),
patchDevicePath.c_str(), outputPath);
int returnCode = send_shell_command(applyPatchCommand);
if (returnCode != 0) {
error_exit("Executing %s returned %d", applyPatchCommand.c_str(), returnCode);
}
}
void install_patch(const char* apkPath, const char* patchPath, int argc, const char** argv) {
const std::string kAgentApplyCommandPattern = "/data/local/tmp/deployagent apply %s %s -pm %s";
std::string packageName = get_packagename_from_apk(apkPath);
std::string patchDevicePath =
android::base::StringPrintf("%s%s.patch", kDeviceAgentPath, packageName.c_str());
std::vector<const char*> srcs{patchPath};
bool push_ok = do_sync_push(srcs, patchDevicePath.c_str(), false);
if (!push_ok) {
error_exit("Error pushing %s to %s returned", patchPath, patchDevicePath.c_str());
}
std::vector<unsigned char> applyOutputBuffer;
std::vector<unsigned char> applyErrorBuffer;
std::vector<unsigned char> apply_output_buffer;
std::vector<unsigned char> apply_error_buffer;
std::string argsString;
bool rSwitchPresent = false;
@ -298,17 +317,42 @@ void install_patch(const char* apkPath, const char* patchPath, int argc, const c
argsString.append("-r");
}
std::string applyPatchCommand =
android::base::StringPrintf(kAgentApplyCommandPattern.c_str(), packageName.c_str(),
patchDevicePath.c_str(), argsString.c_str());
int returnCode = send_shell_command(applyPatchCommand);
if (returnCode != 0) {
error_exit("Executing %s returned %d", applyPatchCommand.c_str(), returnCode);
std::string error;
std::string apply_patch_service_string =
android::base::StringPrintf(kAgentApplyServicePattern, argsString.c_str());
unique_fd fd{adb_connect(apply_patch_service_string, &error)};
if (fd < 0) {
error_exit("Executing %s returned %s", apply_patch_service_string.c_str(), error.c_str());
}
return fd;
}
unique_fd apply_patch_on_device(const char* output_path) {
REPORT_FUNC_TIME();
constexpr char kAgentApplyServicePattern[] = "shell:/data/local/tmp/deployagent apply - -o %s";
std::string error;
std::string apply_patch_service_string =
android::base::StringPrintf(kAgentApplyServicePattern, output_path);
unique_fd fd{adb_connect(apply_patch_service_string, &error)};
if (fd < 0) {
error_exit("Executing %s returned %s", apply_patch_service_string.c_str(), error.c_str());
}
return fd;
}
static void create_patch(const char* apk_path, APKMetaData metadata, borrowed_fd patch_fd) {
REPORT_FUNC_TIME();
DeployPatchGenerator generator(/*is_verbose=*/false);
bool success = generator.CreatePatch(apk_path, std::move(metadata), patch_fd);
if (!success) {
error_exit("Failed to create patch for %s", apk_path);
}
}
bool find_package(const char* apkPath) {
const std::string findCommand =
"/data/local/tmp/deployagent find " + get_packagename_from_apk(apkPath);
return !send_shell_command(findCommand);
int stream_patch(const char* apk_path, APKMetaData metadata, unique_fd patch_fd) {
create_patch(apk_path, std::move(metadata), patch_fd);
REPORT_FUNC_TIME();
return read_and_dump(patch_fd.get());
}

20
adb/client/fastdeploy.h
View file

@ -16,6 +16,11 @@
#pragma once
#include "adb_unique_fd.h"
#include "fastdeploy/proto/ApkEntry.pb.h"
#include <optional>
#include <string>
enum FastDeploy_AgentUpdateStrategy {
@ -24,12 +29,11 @@ enum FastDeploy_AgentUpdateStrategy {
FastDeploy_AgentUpdateDifferentVersion
};
void fastdeploy_set_local_agent(bool use_localagent);
void fastdeploy_set_agent_update_strategy(FastDeploy_AgentUpdateStrategy agent_update_strategy);
int get_device_api_level();
void update_agent(FastDeploy_AgentUpdateStrategy agentUpdateStrategy);
void extract_metadata(const char* apkPath, FILE* outputFp);
void create_patch(const char* apkPath, const char* metadataPath, const char* patchPath);
void apply_patch_on_device(const char* apkPath, const char* patchPath, const char* outputPath);
void install_patch(const char* apkPath, const char* patchPath, int argc, const char** argv);
std::string get_patch_path(const char* apkPath);
bool find_package(const char* apkPath);
std::optional<com::android::fastdeploy::APKMetaData> extract_metadata(const char* apk_path);
unique_fd install_patch(int argc, const char** argv);
unique_fd apply_patch_on_device(const char* output_path);
int stream_patch(const char* apk_path, com::android::fastdeploy::APKMetaData metadata,
unique_fd patch_fd);

30
adb/client/fastdeploycallbacks.cpp
View file

@ -49,35 +49,7 @@ class DeployAgentBufferCallback : public StandardStreamsCallbackInterface {
int capture_shell_command(const char* command, std::vector<char>* outBuffer,
std::vector<char>* errBuffer) {
DeployAgentBufferCallback cb(outBuffer, errBuffer);
return send_shell_command(command, false, &cb);
}
DeployAgentFileCallback::DeployAgentFileCallback(FILE* outputFile, std::vector<char>* errBuffer) {
mpOutFile = outputFile;
mpErrBuffer = errBuffer;
mBytesWritten = 0;
}
void DeployAgentFileCallback::OnStdout(const char* buffer, int length) {
if (mpOutFile != NULL) {
int bytes_written = fwrite(buffer, 1, length, mpOutFile);
if (bytes_written != length) {
printf("Write error %d\n", bytes_written);
}
mBytesWritten += bytes_written;
}
}
void DeployAgentFileCallback::OnStderr(const char* buffer, int length) {
appendBuffer(mpErrBuffer, buffer, length);
}
int DeployAgentFileCallback::Done(int status) {
return status;
}
int DeployAgentFileCallback::getBytesWritten() {
return mBytesWritten;
return send_shell_command(command, /*disable_shell_protocol=*/false, &cb);
}
DeployAgentBufferCallback::DeployAgentBufferCallback(std::vector<char>* outBuffer,

17
adb/client/fastdeploycallbacks.h
View file

@ -17,23 +17,6 @@
#pragma once
#include <vector>
#include "commandline.h"
class DeployAgentFileCallback : public StandardStreamsCallbackInterface {
public:
DeployAgentFileCallback(FILE* outputFile, std::vector<char>* errBuffer);
virtual void OnStdout(const char* buffer, int length);
virtual void OnStderr(const char* buffer, int length);
virtual int Done(int status);
int getBytesWritten();
private:
FILE* mpOutFile;
std::vector<char>* mpErrBuffer;
int mBytesWritten;
};
int capture_shell_command(const char* command, std::vector<char>* outBuffer,
std::vector<char>* errBuffer);

71
adb/fastdeploy/Android.bp
View file

@ -13,15 +13,76 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//
java_binary {
name: "deployagent",
java_library {
name: "deployagent_lib",
sdk_version: "24",
srcs: ["deployagent/src/**/*.java", "deploylib/src/**/*.java", "proto/**/*.proto"],
static_libs: ["apkzlib_zip"],
srcs: [
"deployagent/src/**/*.java",
"proto/**/*.proto",
],
proto: {
type: "lite",
},
}
java_binary {
name: "deployagent",
static_libs: [
"deployagent_lib",
],
dex_preopt: {
enabled: false,
}
}
}
android_test {
name: "FastDeployTests",
manifest: "AndroidManifest.xml",
srcs: [
"deployagent/test/com/android/fastdeploy/ApkArchiveTest.java",
],
static_libs: [
"androidx.test.core",
"androidx.test.runner",
"androidx.test.rules",
"deployagent_lib",
"mockito-target-inline-minus-junit4",
],
libs: [
"android.test.runner",
"android.test.base",
"android.test.mock",
],
data: [
"testdata/sample.apk",
"testdata/sample.cd",
],
optimize: {
enabled: false,
},
}
java_test_host {
name: "FastDeployHostTests",
srcs: [
"deployagent/test/com/android/fastdeploy/FastDeployTest.java",
],
data: [
"testdata/helloworld5.apk",
"testdata/helloworld7.apk",
],
libs: [
"compatibility-host-util",
"cts-tradefed",
"tradefed",
],
test_suites: [
"general-tests",
],
}

29
adb/fastdeploy/AndroidManifest.xml Normal file
View file

@ -0,0 +1,29 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Copyright (C) 2019 The Android Open Source Project
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.android.fastdeploytests">
<application android:testOnly="true"
android:debuggable="true">
<uses-library android:name="android.test.runner" />
</application>
<instrumentation
android:name="androidx.test.runner.AndroidJUnitRunner"
android:targetPackage="com.android.fastdeploytests"
android:label="FastDeploy Tests" />
</manifest>

40
adb/fastdeploy/AndroidTest.xml Normal file
View file

@ -0,0 +1,40 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright (C) 2019 The Android Open Source Project
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License
-->
<configuration description="Runs Device Tests for FastDeploy.">
<option name="test-suite-tag" value="FastDeployTests"/>
<target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
<option name="cleanup-apks" value="true"/>
<option name="install-arg" value="-t"/>
<option name="test-file-name" value="FastDeployTests.apk"/>
</target_preparer>
<target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
<option name="cleanup" value="false" />
<option name="push-file" key="sample.apk" value="/data/local/tmp/FastDeployTests/sample.apk" />
<option name="push-file" key="sample.cd" value="/data/local/tmp/FastDeployTests/sample.cd" />
</target_preparer>
<test class="com.android.tradefed.testtype.AndroidJUnitTest">
<option name="package" value="com.android.fastdeploytests"/>
<option name="runner" value="androidx.test.runner.AndroidJUnitRunner"/>
</test>
<test class="com.android.compatibility.common.tradefed.testtype.JarHostTest" >
<option name="jar" value="FastDeployHostTests.jar" />
</test>
</configuration>

193
adb/fastdeploy/deployagent/src/com/android/fastdeploy/ApkArchive.java Normal file
View file

@ -0,0 +1,193 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.fastdeploy;
import android.util.Log;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
/**
* Extremely light-weight APK parser class.
* Aware of Central Directory, Local File Headers and Signature.
* No Zip64 support yet.
*/
public final class ApkArchive {
private static final String TAG = "ApkArchive";
// Central Directory constants.
private static final int EOCD_SIGNATURE = 0x06054b50;
private static final int EOCD_MIN_SIZE = 22;
private static final long EOCD_MAX_SIZE = 65_535L + EOCD_MIN_SIZE;
private static final int CD_ENTRY_HEADER_SIZE_BYTES = 22;
private static final int CD_LOCAL_FILE_HEADER_SIZE_OFFSET = 12;
// Signature constants.
private static final int EOSIGNATURE_SIZE = 24;
public final static class Dump {
final byte[] cd;
final byte[] signature;
Dump(byte[] cd, byte[] signature) {
this.cd = cd;
this.signature = signature;
}
}
final static class Location {
final long offset;
final long size;
public Location(long offset, long size) {
this.offset = offset;
this.size = size;
}
}
private final RandomAccessFile mFile;
private final FileChannel mChannel;
public ApkArchive(File apk) throws IOException {
mFile = new RandomAccessFile(apk, "r");
mChannel = mFile.getChannel();
}
/**
* Extract the APK metadata: content of Central Directory and Signature.
*
* @return raw content from APK representing CD and Signature data.
*/
public Dump extractMetadata() throws IOException {
Location cdLoc = getCDLocation();
byte[] cd = readMetadata(cdLoc);
byte[] signature = null;
Location sigLoc = getSignatureLocation(cdLoc.offset);
if (sigLoc != null) {
signature = readMetadata(sigLoc);
long size = ByteBuffer.wrap(signature).order(ByteOrder.LITTLE_ENDIAN).getLong();
if (sigLoc.size != size) {
Log.e(TAG, "Mismatching signature sizes: " + sigLoc.size + " != " + size);
signature = null;
}
}
return new Dump(cd, signature);
}
private long findEndOfCDRecord() throws IOException {
final long fileSize = mChannel.size();
int sizeToRead = Math.toIntExact(Math.min(fileSize, EOCD_MAX_SIZE));
final long readOffset = fileSize - sizeToRead;
ByteBuffer buffer = mChannel.map(FileChannel.MapMode.READ_ONLY, readOffset,
sizeToRead).order(ByteOrder.LITTLE_ENDIAN);
buffer.position(sizeToRead - EOCD_MIN_SIZE);
while (true) {
int signature = buffer.getInt(); // Read 4 bytes.
if (signature == EOCD_SIGNATURE) {
return readOffset + buffer.position() - 4;
}
if (buffer.position() == 4) {
break;
}
buffer.position(buffer.position() - Integer.BYTES - 1); // Backtrack 5 bytes.
}
return -1L;
}
private Location findCDRecord(ByteBuffer buf) {
if (buf.order() != ByteOrder.LITTLE_ENDIAN) {
throw new IllegalArgumentException("ByteBuffer byte order must be little endian");
}
if (buf.remaining() < CD_ENTRY_HEADER_SIZE_BYTES) {
throw new IllegalArgumentException(
"Input too short. Need at least " + CD_ENTRY_HEADER_SIZE_BYTES
+ " bytes, available: " + buf.remaining() + "bytes.");
}
int originalPosition = buf.position();
int recordSignature = buf.getInt();
if (recordSignature != EOCD_SIGNATURE) {
throw new IllegalArgumentException(
"Not a Central Directory record. Signature: 0x"
+ Long.toHexString(recordSignature & 0xffffffffL));
}
buf.position(originalPosition + CD_LOCAL_FILE_HEADER_SIZE_OFFSET);
long size = buf.getInt() & 0xffffffffL;
long offset = buf.getInt() & 0xffffffffL;
return new Location(offset, size);
}
// Retrieve the location of the Central Directory Record.
Location getCDLocation() throws IOException {
long eocdRecord = findEndOfCDRecord();
if (eocdRecord < 0) {
throw new IllegalArgumentException("Unable to find End of Central Directory record.");
}
Location location = findCDRecord(mChannel.map(FileChannel.MapMode.READ_ONLY, eocdRecord,
CD_ENTRY_HEADER_SIZE_BYTES).order(ByteOrder.LITTLE_ENDIAN));
if (location == null) {
throw new IllegalArgumentException("Unable to find Central Directory File Header.");
}
return location;
}
// Retrieve the location of the signature block starting from Central
// Directory Record or null if signature is not found.
Location getSignatureLocation(long cdRecordOffset) throws IOException {
long signatureOffset = cdRecordOffset - EOSIGNATURE_SIZE;
if (signatureOffset < 0) {
Log.e(TAG, "Unable to find Signature.");
return null;
}
ByteBuffer signature = mChannel.map(FileChannel.MapMode.READ_ONLY, signatureOffset,
EOSIGNATURE_SIZE).order(ByteOrder.LITTLE_ENDIAN);
long size = signature.getLong();
byte[] sign = new byte[16];
signature.get(sign);
String signAsString = new String(sign);
if (!"APK Sig Block 42".equals(signAsString)) {
Log.e(TAG, "Signature magic does not match: " + signAsString);
return null;
}
long offset = cdRecordOffset - size - 8;
return new Location(offset, size);
}
private byte[] readMetadata(Location loc) throws IOException {
byte[] payload = new byte[(int) loc.size];
ByteBuffer buffer = mChannel.map(FileChannel.MapMode.READ_ONLY, loc.offset, loc.size);
buffer.get(payload);
return payload;
}
}

260
adb/fastdeploy/deployagent/src/com/android/fastdeploy/DeployAgent.java
View file

@ -24,18 +24,22 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.util.Set;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import com.android.fastdeploy.PatchFormatException;
import com.android.fastdeploy.ApkArchive;
import com.android.fastdeploy.APKDump;
import com.android.fastdeploy.APKMetaData;
import com.android.fastdeploy.PatchUtils;
import com.google.protobuf.ByteString;
public final class DeployAgent {
private static final int BUFFER_SIZE = 128 * 1024;
private static final int AGENT_VERSION = 0x00000002;
private static final int AGENT_VERSION = 0x00000003;
public static void main(String[] args) {
int exitCode = 0;
@ -45,68 +49,70 @@ public final class DeployAgent {
}
String commandString = args[0];
switch (commandString) {
case "dump": {
if (args.length != 3) {
showUsage(1);
}
if (commandString.equals("extract")) {
if (args.length != 2) {
showUsage(1);
}
String packageName = args[1];
extractMetaData(packageName);
} else if (commandString.equals("find")) {
if (args.length != 2) {
showUsage(1);
}
String packageName = args[1];
if (getFilenameFromPackageName(packageName) == null) {
exitCode = 3;
}
} else if (commandString.equals("apply")) {
if (args.length < 4) {
showUsage(1);
}
String packageName = args[1];
String patchPath = args[2];
String outputParam = args[3];
InputStream deltaInputStream = null;
if (patchPath.compareTo("-") == 0) {
deltaInputStream = System.in;
} else {
deltaInputStream = new FileInputStream(patchPath);
}
if (outputParam.equals("-o")) {
OutputStream outputStream = null;
if (args.length > 4) {
String outputPath = args[4];
if (!outputPath.equals("-")) {
outputStream = new FileOutputStream(outputPath);
String requiredVersion = args[1];
if (AGENT_VERSION == Integer.parseInt(requiredVersion)) {
String packageName = args[2];
String packagePath = getFilenameFromPackageName(packageName);
if (packagePath != null) {
dumpApk(packageName, packagePath);
} else {
exitCode = 3;
}
} else {
System.out.printf("0x%08X\n", AGENT_VERSION);
exitCode = 4;
}
if (outputStream == null) {
outputStream = System.out;
}
File deviceFile = getFileFromPackageName(packageName);
writePatchToStream(
new RandomAccessFile(deviceFile, "r"), deltaInputStream, outputStream);
} else if (outputParam.equals("-pm")) {
String[] sessionArgs = null;
if (args.length > 4) {
int numSessionArgs = args.length-4;
sessionArgs = new String[numSessionArgs];
for (int i=0 ; i<numSessionArgs ; i++) {
sessionArgs[i] = args[i+4];
}
}
exitCode = applyPatch(packageName, deltaInputStream, sessionArgs);
break;
}
} else if (commandString.equals("version")) {
System.out.printf("0x%08X\n", AGENT_VERSION);
} else {
showUsage(1);
case "apply": {
if (args.length < 3) {
showUsage(1);
}
String patchPath = args[1];
String outputParam = args[2];
InputStream deltaInputStream = null;
if (patchPath.compareTo("-") == 0) {
deltaInputStream = System.in;
} else {
deltaInputStream = new FileInputStream(patchPath);
}
if (outputParam.equals("-o")) {
OutputStream outputStream = null;
if (args.length > 3) {
String outputPath = args[3];
if (!outputPath.equals("-")) {
outputStream = new FileOutputStream(outputPath);
}
}
if (outputStream == null) {
outputStream = System.out;
}
writePatchToStream(deltaInputStream, outputStream);
} else if (outputParam.equals("-pm")) {
String[] sessionArgs = null;
if (args.length > 3) {
int numSessionArgs = args.length - 3;
sessionArgs = new String[numSessionArgs];
for (int i = 0; i < numSessionArgs; i++) {
sessionArgs[i] = args[i + 3];
}
}
exitCode = applyPatch(deltaInputStream, sessionArgs);
}
break;
}
default:
showUsage(1);
break;
}
} catch (Exception e) {
System.err.println("Error: " + e);
@ -118,16 +124,16 @@ public final class DeployAgent {
private static void showUsage(int exitCode) {
System.err.println(
"usage: deployagent <command> [<args>]\n\n" +
"commands:\n" +
"version get the version\n" +
"find PKGNAME return zero if package found, else non-zero\n" +
"extract PKGNAME extract an installed package's metadata\n" +
"apply PKGNAME PATCHFILE [-o|-pm] apply a patch from PATCHFILE (- for stdin) to an installed package\n" +
" -o <FILE> directs output to FILE, default or - for stdout\n" +
" -pm <ARGS> directs output to package manager, passes <ARGS> to 'pm install-create'\n"
);
"usage: deployagent <command> [<args>]\n\n" +
"commands:\n" +
"dump VERSION PKGNAME dump info for an installed package given that " +
"VERSION equals current agent's version\n" +
"apply PATCHFILE [-o|-pm] apply a patch from PATCHFILE " +
"(- for stdin) to an installed package\n" +
" -o <FILE> directs output to FILE, default or - for stdout\n" +
" -pm <ARGS> directs output to package manager, passes <ARGS> to " +
"'pm install-create'\n"
);
System.exit(exitCode);
}
@ -162,32 +168,34 @@ public final class DeployAgent {
}
int equalsIndex = line.lastIndexOf(packageSuffix);
String fileName =
line.substring(packageIndex + packagePrefix.length(), equalsIndex);
line.substring(packageIndex + packagePrefix.length(), equalsIndex);
return fileName;
}
}
return null;
}
private static File getFileFromPackageName(String packageName) throws IOException {
String filename = getFilenameFromPackageName(packageName);
if (filename == null) {
// Should not happen (function is only called when we know the package exists)
throw new IOException("package not found");
}
return new File(filename);
}
private static void dumpApk(String packageName, String packagePath) throws IOException {
File apk = new File(packagePath);
ApkArchive.Dump dump = new ApkArchive(apk).extractMetadata();
private static void extractMetaData(String packageName) throws IOException {
File apkFile = getFileFromPackageName(packageName);
APKMetaData apkMetaData = PatchUtils.getAPKMetaData(apkFile);
apkMetaData.writeTo(System.out);
APKDump.Builder apkDumpBuilder = APKDump.newBuilder();
apkDumpBuilder.setName(packageName);
if (dump.cd != null) {
apkDumpBuilder.setCd(ByteString.copyFrom(dump.cd));
}
if (dump.signature != null) {
apkDumpBuilder.setSignature(ByteString.copyFrom(dump.signature));
}
apkDumpBuilder.setAbsolutePath(apk.getAbsolutePath());
apkDumpBuilder.build().writeTo(System.out);
}
private static int createInstallSession(String[] args) throws IOException {
StringBuilder commandBuilder = new StringBuilder();
commandBuilder.append("pm install-create ");
for (int i=0 ; args != null && i<args.length ; i++) {
for (int i = 0; args != null && i < args.length; i++) {
commandBuilder.append(args[i] + " ");
}
@ -199,7 +207,8 @@ public final class DeployAgent {
String successLineEnd = "]";
while ((line = reader.readLine()) != null) {
if (line.startsWith(successLineStart) && line.endsWith(successLineEnd)) {
return Integer.parseInt(line.substring(successLineStart.length(), line.lastIndexOf(successLineEnd)));
return Integer.parseInt(line.substring(successLineStart.length(),
line.lastIndexOf(successLineEnd)));
}
}
@ -213,16 +222,15 @@ public final class DeployAgent {
return p.exitValue();
}
private static int applyPatch(String packageName, InputStream deltaStream, String[] sessionArgs)
private static int applyPatch(InputStream deltaStream, String[] sessionArgs)
throws IOException, PatchFormatException {
File deviceFile = getFileFromPackageName(packageName);
int sessionId = createInstallSession(sessionArgs);
if (sessionId < 0) {
System.err.println("PM Create Session Failed");
return -1;
}
int writeExitCode = writePatchedDataToSession(new RandomAccessFile(deviceFile, "r"), deltaStream, sessionId);
int writeExitCode = writePatchedDataToSession(deltaStream, sessionId);
if (writeExitCode == 0) {
return commitInstallSession(sessionId);
} else {
@ -230,84 +238,94 @@ public final class DeployAgent {
}
}
private static long writePatchToStream(RandomAccessFile oldData, InputStream patchData,
OutputStream outputStream) throws IOException, PatchFormatException {
private static long writePatchToStream(InputStream patchData,
OutputStream outputStream) throws IOException, PatchFormatException {
long newSize = readPatchHeader(patchData);
long bytesWritten = writePatchedDataToStream(oldData, newSize, patchData, outputStream);
long bytesWritten = writePatchedDataToStream(newSize, patchData, outputStream);
outputStream.flush();
if (bytesWritten != newSize) {
throw new PatchFormatException(String.format(
"output size mismatch (expected %ld but wrote %ld)", newSize, bytesWritten));
"output size mismatch (expected %ld but wrote %ld)", newSize, bytesWritten));
}
return bytesWritten;
}
private static long readPatchHeader(InputStream patchData)
throws IOException, PatchFormatException {
throws IOException, PatchFormatException {
byte[] signatureBuffer = new byte[PatchUtils.SIGNATURE.length()];
try {
PatchUtils.readFully(patchData, signatureBuffer, 0, signatureBuffer.length);
PatchUtils.readFully(patchData, signatureBuffer);
} catch (IOException e) {
throw new PatchFormatException("truncated signature");
}
String signature = new String(signatureBuffer, 0, signatureBuffer.length, "US-ASCII");
String signature = new String(signatureBuffer);
if (!PatchUtils.SIGNATURE.equals(signature)) {
throw new PatchFormatException("bad signature");
}
long newSize = PatchUtils.readBsdiffLong(patchData);
if (newSize < 0 || newSize > Integer.MAX_VALUE) {
throw new PatchFormatException("bad newSize");
long newSize = PatchUtils.readLELong(patchData);
if (newSize < 0) {
throw new PatchFormatException("bad newSize: " + newSize);
}
return newSize;
}
// Note that this function assumes patchData has been seek'ed to the start of the delta stream
// (i.e. the signature has already been read by readPatchHeader). For a stream that points to the
// start of a patch file call writePatchToStream
private static long writePatchedDataToStream(RandomAccessFile oldData, long newSize,
InputStream patchData, OutputStream outputStream) throws IOException {
// (i.e. the signature has already been read by readPatchHeader). For a stream that points to
// the start of a patch file call writePatchToStream
private static long writePatchedDataToStream(long newSize, InputStream patchData,
OutputStream outputStream) throws IOException {
String deviceFile = PatchUtils.readString(patchData);
RandomAccessFile oldDataFile = new RandomAccessFile(deviceFile, "r");
FileChannel oldData = oldDataFile.getChannel();
WritableByteChannel newData = Channels.newChannel(outputStream);
long newDataBytesWritten = 0;
byte[] buffer = new byte[BUFFER_SIZE];
while (newDataBytesWritten < newSize) {
long copyLen = PatchUtils.readFormattedLong(patchData);
if (copyLen > 0) {
PatchUtils.pipe(patchData, outputStream, buffer, (int) copyLen);
long newDataLen = PatchUtils.readLELong(patchData);
if (newDataLen > 0) {
PatchUtils.pipe(patchData, outputStream, buffer, newDataLen);
}
long oldDataOffset = PatchUtils.readFormattedLong(patchData);
long oldDataLen = PatchUtils.readFormattedLong(patchData);
oldData.seek(oldDataOffset);
if (oldDataLen > 0) {
PatchUtils.pipe(oldData, outputStream, buffer, (int) oldDataLen);
long oldDataOffset = PatchUtils.readLELong(patchData);
long oldDataLen = PatchUtils.readLELong(patchData);
if (oldDataLen >= 0) {
long offset = oldDataOffset;
long len = oldDataLen;
while (len > 0) {
long chunkLen = Math.min(len, 1024*1024*1024);
oldData.transferTo(offset, chunkLen, newData);
offset += chunkLen;
len -= chunkLen;
}
}
newDataBytesWritten += copyLen + oldDataLen;
newDataBytesWritten += newDataLen + oldDataLen;
}
return newDataBytesWritten;
}
private static int writePatchedDataToSession(RandomAccessFile oldData, InputStream patchData, int sessionId)
private static int writePatchedDataToSession(InputStream patchData, int sessionId)
throws IOException, PatchFormatException {
try {
Process p;
long newSize = readPatchHeader(patchData);
StringBuilder commandBuilder = new StringBuilder();
commandBuilder.append(String.format("pm install-write -S %d %d -- -", newSize, sessionId));
String command = commandBuilder.toString();
String command = String.format("pm install-write -S %d %d -- -", newSize, sessionId);
p = Runtime.getRuntime().exec(command);
OutputStream sessionOutputStream = p.getOutputStream();
long bytesWritten = writePatchedDataToStream(oldData, newSize, patchData, sessionOutputStream);
long bytesWritten = writePatchedDataToStream(newSize, patchData, sessionOutputStream);
sessionOutputStream.flush();
p.waitFor();
if (bytesWritten != newSize) {
throw new PatchFormatException(
String.format("output size mismatch (expected %d but wrote %)", newSize, bytesWritten));
String.format("output size mismatch (expected %d but wrote %)", newSize,
bytesWritten));
}
return p.exitValue();
} catch (InterruptedException e) {

0
adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchFormatException.java → adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchFormatException.java
View file

74
adb/fastdeploy/deployagent/src/com/android/fastdeploy/PatchUtils.java Normal file
View file

@ -0,0 +1,74 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.fastdeploy;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
class PatchUtils {
public static final String SIGNATURE = "FASTDEPLOY";
/**
* Reads a 64-bit signed integer in Little Endian format from the specified {@link
* DataInputStream}.
*
* @param in the stream to read from.
*/
static long readLELong(InputStream in) throws IOException {
byte[] buffer = new byte[Long.BYTES];
readFully(in, buffer);
ByteBuffer buf = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN);
return buf.getLong();
}
static String readString(InputStream in) throws IOException {
int size = (int) readLELong(in);
byte[] buffer = new byte[size];
readFully(in, buffer);
return new String(buffer);
}
static void readFully(final InputStream in, final byte[] destination, final int startAt,
final int numBytes) throws IOException {
int numRead = 0;
while (numRead < numBytes) {
int readNow = in.read(destination, startAt + numRead, numBytes - numRead);
if (readNow == -1) {
throw new IOException("truncated input stream");
}
numRead += readNow;
}
}
static void readFully(final InputStream in, final byte[] destination) throws IOException {
readFully(in, destination, 0, destination.length);
}
static void pipe(final InputStream in, final OutputStream out, final byte[] buffer,
long copyLength) throws IOException {
while (copyLength > 0) {
int maxCopy = (int) Math.min(buffer.length, copyLength);
readFully(in, buffer, 0, maxCopy);
out.write(buffer, 0, maxCopy);
copyLength -= maxCopy;
}
}
}

74
adb/fastdeploy/deployagent/test/com/android/fastdeploy/ApkArchiveTest.java Normal file
View file

@ -0,0 +1,74 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.fastdeploy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import androidx.test.filters.SmallTest;
import androidx.test.runner.AndroidJUnit4;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import com.android.fastdeploy.ApkArchive;
import java.io.File;
import java.io.IOException;
@SmallTest
@RunWith(AndroidJUnit4.class)
public class ApkArchiveTest {
private static final File SAMPLE_APK = new File("/data/local/tmp/FastDeployTests/sample.apk");
private static final File WRONG_APK = new File("/data/local/tmp/FastDeployTests/sample.cd");
@Test
public void testApkArchiveSizes() throws IOException {
ApkArchive archive = new ApkArchive(SAMPLE_APK);
ApkArchive.Location cdLoc = archive.getCDLocation();
assertNotEquals(cdLoc, null);
assertEquals(cdLoc.offset, 2044145);
assertEquals(cdLoc.size, 49390);
// Check that block can be retrieved
ApkArchive.Location sigLoc = archive.getSignatureLocation(cdLoc.offset);
assertNotEquals(sigLoc, null);
assertEquals(sigLoc.offset, 2040049);
assertEquals(sigLoc.size, 4088);
}
@Test
public void testApkArchiveDump() throws IOException {
ApkArchive archive = new ApkArchive(SAMPLE_APK);
ApkArchive.Dump dump = archive.extractMetadata();
assertNotEquals(dump, null);
assertNotEquals(dump.cd, null);
assertNotEquals(dump.signature, null);
assertEquals(dump.cd.length, 49390);
assertEquals(dump.signature.length, 4088);
}
@Test(expected = IllegalArgumentException.class)
public void testApkArchiveDumpWrongApk() throws IOException {
ApkArchive archive = new ApkArchive(WRONG_APK);
archive.extractMetadata();
}
}

85
adb/fastdeploy/deployagent/test/com/android/fastdeploy/FastDeployTest.java Normal file
View file

@ -0,0 +1,85 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.fastdeploy;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.io.IOException;
import java.util.Arrays;
@RunWith(DeviceJUnit4ClassRunner.class)
public class FastDeployTest extends BaseHostJUnit4Test {
private static final String TEST_APP_PACKAGE = "com.example.helloworld";
private static final String TEST_APK5_NAME = "helloworld5.apk";
private static final String TEST_APK7_NAME = "helloworld7.apk";
private String mTestApk5Path;
private String mTestApk7Path;
@Before
public void setUp() throws Exception {
CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(getBuild());
getDevice().uninstallPackage(TEST_APP_PACKAGE);
mTestApk5Path = buildHelper.getTestFile(TEST_APK5_NAME).getAbsolutePath();
mTestApk7Path = buildHelper.getTestFile(TEST_APK7_NAME).getAbsolutePath();
}
@Test
public void testAppInstalls() throws Exception {
fastInstallPackage(mTestApk5Path);
assertTrue(isAppInstalled(TEST_APP_PACKAGE));
getDevice().uninstallPackage(TEST_APP_PACKAGE);
assertFalse(isAppInstalled(TEST_APP_PACKAGE));
}
@Test
public void testAppPatch() throws Exception {
fastInstallPackage(mTestApk5Path);
assertTrue(isAppInstalled(TEST_APP_PACKAGE));
fastInstallPackage(mTestApk7Path);
assertTrue(isAppInstalled(TEST_APP_PACKAGE));
getDevice().uninstallPackage(TEST_APP_PACKAGE);
assertFalse(isAppInstalled(TEST_APP_PACKAGE));
}
private boolean isAppInstalled(String packageName) throws DeviceNotAvailableException {
final String commandResult = getDevice().executeShellCommand("pm list packages");
final int prefixLength = "package:".length();
return Arrays.stream(commandResult.split("\\r?\\n"))
.anyMatch(line -> line.substring(prefixLength).equals(packageName));
}
// Mostly copied from PkgInstallSignatureVerificationTest.java.
private String fastInstallPackage(String apkPath)
throws IOException, DeviceNotAvailableException {
return getDevice().executeAdbCommand("install", "-t", "--fastdeploy", "--force-agent",
apkPath);
}
}

186
adb/fastdeploy/deploylib/src/com/android/fastdeploy/PatchUtils.java
View file

@ -1,186 +0,0 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.fastdeploy;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import com.android.tools.build.apkzlib.zip.ZFile;
import com.android.tools.build.apkzlib.zip.ZFileOptions;
import com.android.tools.build.apkzlib.zip.StoredEntry;
import com.android.tools.build.apkzlib.zip.StoredEntryType;
import com.android.tools.build.apkzlib.zip.CentralDirectoryHeaderCompressInfo;
import com.android.tools.build.apkzlib.zip.CentralDirectoryHeader;
import com.android.fastdeploy.APKMetaData;
import com.android.fastdeploy.APKEntry;
class PatchUtils {
private static final long NEGATIVE_MASK = 1L << 63;
private static final long NEGATIVE_LONG_SIGN_MASK = 1L << 63;
public static final String SIGNATURE = "FASTDEPLOY";
private static long getOffsetFromEntry(StoredEntry entry) {
return entry.getCentralDirectoryHeader().getOffset() + entry.getLocalHeaderSize();
}
public static APKMetaData getAPKMetaData(File apkFile) throws IOException {
APKMetaData.Builder apkEntriesBuilder = APKMetaData.newBuilder();
ZFileOptions options = new ZFileOptions();
ZFile zFile = new ZFile(apkFile, options);
ArrayList<StoredEntry> metaDataEntries = new ArrayList<StoredEntry>();
for (StoredEntry entry : zFile.entries()) {
if (entry.getType() != StoredEntryType.FILE) {
continue;
}
metaDataEntries.add(entry);
}
Collections.sort(metaDataEntries, new Comparator<StoredEntry>() {
private long getOffsetFromEntry(StoredEntry entry) {
return PatchUtils.getOffsetFromEntry(entry);
}
@Override
public int compare(StoredEntry lhs, StoredEntry rhs) {
// -1 - less than, 1 - greater than, 0 - equal, all inversed for descending
return Long.compare(getOffsetFromEntry(lhs), getOffsetFromEntry(rhs));
}
});
for (StoredEntry entry : metaDataEntries) {
CentralDirectoryHeader cdh = entry.getCentralDirectoryHeader();
CentralDirectoryHeaderCompressInfo cdhci = cdh.getCompressionInfoWithWait();
APKEntry.Builder entryBuilder = APKEntry.newBuilder();
entryBuilder.setCrc32(cdh.getCrc32());
entryBuilder.setFileName(cdh.getName());
entryBuilder.setCompressedSize(cdhci.getCompressedSize());
entryBuilder.setUncompressedSize(cdh.getUncompressedSize());
entryBuilder.setDataOffset(getOffsetFromEntry(entry));
apkEntriesBuilder.addEntries(entryBuilder);
apkEntriesBuilder.build();
}
return apkEntriesBuilder.build();
}
/**
* Writes a 64-bit signed integer to the specified {@link OutputStream}. The least significant
* byte is written first and the most significant byte is written last.
* @param value the value to write
* @param outputStream the stream to write to
*/
static void writeFormattedLong(final long value, OutputStream outputStream) throws IOException {
long y = value;
if (y < 0) {
y = (-y) | NEGATIVE_MASK;
}
for (int i = 0; i < 8; ++i) {
outputStream.write((byte) (y & 0xff));
y >>>= 8;
}
}
/**
* Reads a 64-bit signed integer written by {@link #writeFormattedLong(long, OutputStream)} from
* the specified {@link InputStream}.
* @param inputStream the stream to read from
*/
static long readFormattedLong(InputStream inputStream) throws IOException {
long result = 0;
for (int bitshift = 0; bitshift < 64; bitshift += 8) {
result |= ((long) inputStream.read()) << bitshift;
}
if ((result - NEGATIVE_MASK) > 0) {
result = (result & ~NEGATIVE_MASK) * -1;
}
return result;
}
static final long readBsdiffLong(InputStream in) throws PatchFormatException, IOException {
long result = 0;
for (int bitshift = 0; bitshift < 64; bitshift += 8) {
result |= ((long) in.read()) << bitshift;
}
if (result == NEGATIVE_LONG_SIGN_MASK) {
// "Negative zero", which is valid in signed-magnitude format.
// NB: No sane patch generator should ever produce such a value.
throw new PatchFormatException("read negative zero");
}
if ((result & NEGATIVE_LONG_SIGN_MASK) != 0) {
result = -(result & ~NEGATIVE_LONG_SIGN_MASK);
}
return result;
}
static void readFully(final InputStream in, final byte[] destination, final int startAt,
final int numBytes) throws IOException {
int numRead = 0;
while (numRead < numBytes) {
int readNow = in.read(destination, startAt + numRead, numBytes - numRead);
if (readNow == -1) {
throw new IOException("truncated input stream");
}
numRead += readNow;
}
}
static void pipe(final InputStream in, final OutputStream out, final byte[] buffer,
long copyLength) throws IOException {
while (copyLength > 0) {
int maxCopy = Math.min(buffer.length, (int) copyLength);
readFully(in, buffer, 0, maxCopy);
out.write(buffer, 0, maxCopy);
copyLength -= maxCopy;
}
}
static void pipe(final RandomAccessFile in, final OutputStream out, final byte[] buffer,
long copyLength) throws IOException {
while (copyLength > 0) {
int maxCopy = Math.min(buffer.length, (int) copyLength);
in.readFully(buffer, 0, maxCopy);
out.write(buffer, 0, maxCopy);
copyLength -= maxCopy;
}
}
static void fill(byte value, final OutputStream out, final byte[] buffer, long fillLength)
throws IOException {
while (fillLength > 0) {
int maxCopy = Math.min(buffer.length, (int) fillLength);
Arrays.fill(buffer, 0, maxCopy, value);
out.write(buffer, 0, maxCopy);
fillLength -= maxCopy;
}
}
}

415
adb/fastdeploy/deploypatchgenerator/apk_archive.cpp Normal file
View file

@ -0,0 +1,415 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define TRACE_TAG ADB
#include "apk_archive.h"
#include "adb_trace.h"
#include "sysdeps.h"
#include <android-base/endian.h>
#include <android-base/mapped_file.h>
#include <openssl/md5.h>
constexpr uint16_t kCompressStored = 0;
// mask value that signifies that the entry has a DD
static const uint32_t kGPBDDFlagMask = 0x0008;
namespace {
struct FileRegion {
FileRegion(borrowed_fd fd, off64_t offset, size_t length)
: mapped_(android::base::MappedFile::FromOsHandle(adb_get_os_handle(fd), offset, length,
PROT_READ)) {
if (mapped_.data() != nullptr) {
return;
}
// Mapped file failed, falling back to pread.
buffer_.resize(length);
if (auto err = adb_pread(fd.get(), buffer_.data(), length, offset); size_t(err) != length) {
fprintf(stderr, "Unable to read %lld bytes at offset %" PRId64 " \n",
static_cast<long long>(length), offset);
buffer_.clear();
return;
}
}
const char* data() const { return mapped_.data() ? mapped_.data() : buffer_.data(); }
size_t size() const { return mapped_.data() ? mapped_.size() : buffer_.size(); }
private:
FileRegion() = default;
DISALLOW_COPY_AND_ASSIGN(FileRegion);
android::base::MappedFile mapped_;
std::string buffer_;
};
} // namespace
using com::android::fastdeploy::APKDump;
ApkArchive::ApkArchive(const std::string& path) : path_(path), size_(0) {
fd_.reset(adb_open(path_.c_str(), O_RDONLY));
if (fd_ == -1) {
fprintf(stderr, "Unable to open file '%s'\n", path_.c_str());
return;
}
struct stat st;
if (stat(path_.c_str(), &st) == -1) {
fprintf(stderr, "Unable to stat file '%s'\n", path_.c_str());
return;
}
size_ = st.st_size;
}
ApkArchive::~ApkArchive() {}
APKDump ApkArchive::ExtractMetadata() {
D("ExtractMetadata");
if (!ready()) {
return {};
}
Location cdLoc = GetCDLocation();
if (!cdLoc.valid) {
return {};
}
APKDump dump;
dump.set_absolute_path(path_);
dump.set_cd(ReadMetadata(cdLoc));
Location sigLoc = GetSignatureLocation(cdLoc.offset);
if (sigLoc.valid) {
dump.set_signature(ReadMetadata(sigLoc));
}
return dump;
}
off_t ApkArchive::FindEndOfCDRecord() const {
constexpr int endOfCDSignature = 0x06054b50;
constexpr off_t endOfCDMinSize = 22;
constexpr off_t endOfCDMaxSize = 65535 + endOfCDMinSize;
auto sizeToRead = std::min(size_, endOfCDMaxSize);
auto readOffset = size_ - sizeToRead;
FileRegion mapped(fd_, readOffset, sizeToRead);
// Start scanning from the end
auto* start = mapped.data();
auto* cursor = start + mapped.size() - sizeof(endOfCDSignature);
// Search for End of Central Directory record signature.
while (cursor >= start) {
if (*(int32_t*)cursor == endOfCDSignature) {
return readOffset + (cursor - start);
}
cursor--;
}
return -1;
}
ApkArchive::Location ApkArchive::FindCDRecord(const char* cursor) {
struct ecdr_t {
int32_t signature;
uint16_t diskNumber;
uint16_t numDisk;
uint16_t diskEntries;
uint16_t numEntries;
uint32_t crSize;
uint32_t offsetToCdHeader;
uint16_t commentSize;
uint8_t comment[0];
} __attribute__((packed));
ecdr_t* header = (ecdr_t*)cursor;
Location location;
location.offset = header->offsetToCdHeader;
location.size = header->crSize;
location.valid = true;
return location;
}
ApkArchive::Location ApkArchive::GetCDLocation() {
constexpr off_t cdEntryHeaderSizeBytes = 22;
Location location;
// Find End of Central Directory Record
off_t eocdRecord = FindEndOfCDRecord();
if (eocdRecord < 0) {
fprintf(stderr, "Unable to find End of Central Directory record in file '%s'\n",
path_.c_str());
return location;
}
// Find Central Directory Record
FileRegion mapped(fd_, eocdRecord, cdEntryHeaderSizeBytes);
location = FindCDRecord(mapped.data());
if (!location.valid) {
fprintf(stderr, "Unable to find Central Directory File Header in file '%s'\n",
path_.c_str());
return location;
}
return location;
}
ApkArchive::Location ApkArchive::GetSignatureLocation(off_t cdRecordOffset) {
Location location;
// Signature constants.
constexpr off_t endOfSignatureSize = 24;
off_t signatureOffset = cdRecordOffset - endOfSignatureSize;
if (signatureOffset < 0) {
fprintf(stderr, "Unable to find signature in file '%s'\n", path_.c_str());
return location;
}
FileRegion mapped(fd_, signatureOffset, endOfSignatureSize);
uint64_t signatureSize = *(uint64_t*)mapped.data();
auto* signature = mapped.data() + sizeof(signatureSize);
// Check if there is a v2/v3 Signature block here.
if (memcmp(signature, "APK Sig Block 42", 16)) {
return location;
}
// This is likely a signature block.
location.size = signatureSize;
location.offset = cdRecordOffset - location.size - 8;
location.valid = true;
return location;
}
std::string ApkArchive::ReadMetadata(Location loc) const {
FileRegion mapped(fd_, loc.offset, loc.size);
return {mapped.data(), mapped.size()};
}
size_t ApkArchive::ParseCentralDirectoryRecord(const char* input, size_t size, std::string* md5Hash,
int64_t* localFileHeaderOffset, int64_t* dataSize) {
// A structure representing the fixed length fields for a single
// record in the central directory of the archive. In addition to
// the fixed length fields listed here, each central directory
// record contains a variable length "file_name" and "extra_field"
// whose lengths are given by |file_name_length| and |extra_field_length|
// respectively.
static constexpr int kCDFileHeaderMagic = 0x02014b50;
struct CentralDirectoryRecord {
// The start of record signature. Must be |kSignature|.
uint32_t record_signature;
// Source tool version. Top byte gives source OS.
uint16_t version_made_by;
// Tool version. Ignored by this implementation.
uint16_t version_needed;
// The "general purpose bit flags" for this entry. The only
// flag value that we currently check for is the "data descriptor"
// flag.
uint16_t gpb_flags;
// The compression method for this entry, one of |kCompressStored|
// and |kCompressDeflated|.
uint16_t compression_method;
// The file modification time and date for this entry.
uint16_t last_mod_time;
uint16_t last_mod_date;
// The CRC-32 checksum for this entry.
uint32_t crc32;
// The compressed size (in bytes) of this entry.
uint32_t compressed_size;
// The uncompressed size (in bytes) of this entry.
uint32_t uncompressed_size;
// The length of the entry file name in bytes. The file name
// will appear immediately after this record.
uint16_t file_name_length;
// The length of the extra field info (in bytes). This data
// will appear immediately after the entry file name.
uint16_t extra_field_length;
// The length of the entry comment (in bytes). This data will
// appear immediately after the extra field.
uint16_t comment_length;
// The start disk for this entry. Ignored by this implementation).
uint16_t file_start_disk;
// File attributes. Ignored by this implementation.
uint16_t internal_file_attributes;
// File attributes. For archives created on Unix, the top bits are the
// mode.
uint32_t external_file_attributes;
// The offset to the local file header for this entry, from the
// beginning of this archive.
uint32_t local_file_header_offset;
private:
CentralDirectoryRecord() = default;
DISALLOW_COPY_AND_ASSIGN(CentralDirectoryRecord);
} __attribute__((packed));
const CentralDirectoryRecord* cdr;
if (size < sizeof(*cdr)) {
return 0;
}
auto begin = input;
cdr = reinterpret_cast<const CentralDirectoryRecord*>(begin);
if (cdr->record_signature != kCDFileHeaderMagic) {
fprintf(stderr, "Invalid Central Directory Record signature\n");
return 0;
}
auto end = begin + sizeof(*cdr) + cdr->file_name_length + cdr->extra_field_length +
cdr->comment_length;
uint8_t md5Digest[MD5_DIGEST_LENGTH];
MD5((const unsigned char*)begin, end - begin, md5Digest);
md5Hash->assign((const char*)md5Digest, sizeof(md5Digest));
*localFileHeaderOffset = cdr->local_file_header_offset;
*dataSize = (cdr->compression_method == kCompressStored) ? cdr->uncompressed_size
: cdr->compressed_size;
return end - begin;
}
size_t ApkArchive::CalculateLocalFileEntrySize(int64_t localFileHeaderOffset,
int64_t dataSize) const {
// The local file header for a given entry. This duplicates information
// present in the central directory of the archive. It is an error for
// the information here to be different from the central directory
// information for a given entry.
static constexpr int kLocalFileHeaderMagic = 0x04034b50;
struct LocalFileHeader {
// The local file header signature, must be |kSignature|.
uint32_t lfh_signature;
// Tool version. Ignored by this implementation.
uint16_t version_needed;
// The "general purpose bit flags" for this entry. The only
// flag value that we currently check for is the "data descriptor"
// flag.
uint16_t gpb_flags;
// The compression method for this entry, one of |kCompressStored|
// and |kCompressDeflated|.
uint16_t compression_method;
// The file modification time and date for this entry.
uint16_t last_mod_time;
uint16_t last_mod_date;
// The CRC-32 checksum for this entry.
uint32_t crc32;
// The compressed size (in bytes) of this entry.
uint32_t compressed_size;
// The uncompressed size (in bytes) of this entry.
uint32_t uncompressed_size;
// The length of the entry file name in bytes. The file name
// will appear immediately after this record.
uint16_t file_name_length;
// The length of the extra field info (in bytes). This data
// will appear immediately after the entry file name.
uint16_t extra_field_length;
private:
LocalFileHeader() = default;
DISALLOW_COPY_AND_ASSIGN(LocalFileHeader);
} __attribute__((packed));
static constexpr int kLocalFileHeaderSize = sizeof(LocalFileHeader);
CHECK(ready()) << path_;
const LocalFileHeader* lfh;
if (localFileHeaderOffset + kLocalFileHeaderSize > size_) {
fprintf(stderr,
"Invalid Local File Header offset in file '%s' at offset %lld, file size %lld\n",
path_.c_str(), static_cast<long long>(localFileHeaderOffset),
static_cast<long long>(size_));
return 0;
}
FileRegion lfhMapped(fd_, localFileHeaderOffset, sizeof(LocalFileHeader));
lfh = reinterpret_cast<const LocalFileHeader*>(lfhMapped.data());
if (lfh->lfh_signature != kLocalFileHeaderMagic) {
fprintf(stderr, "Invalid Local File Header signature in file '%s' at offset %lld\n",
path_.c_str(), static_cast<long long>(localFileHeaderOffset));
return 0;
}
// The *optional* data descriptor start signature.
static constexpr int kOptionalDataDescriptorMagic = 0x08074b50;
struct DataDescriptor {
// CRC-32 checksum of the entry.
uint32_t crc32;
// Compressed size of the entry.
uint32_t compressed_size;
// Uncompressed size of the entry.
uint32_t uncompressed_size;
private:
DataDescriptor() = default;
DISALLOW_COPY_AND_ASSIGN(DataDescriptor);
} __attribute__((packed));
static constexpr int kDataDescriptorSize = sizeof(DataDescriptor);
off_t ddOffset = localFileHeaderOffset + kLocalFileHeaderSize + lfh->file_name_length +
lfh->extra_field_length + dataSize;
int64_t ddSize = 0;
int64_t localDataSize;
if (lfh->gpb_flags & kGPBDDFlagMask) {
// There is trailing data descriptor.
const DataDescriptor* dd;
if (ddOffset + int(sizeof(uint32_t)) > size_) {
fprintf(stderr,
"Error reading trailing data descriptor signature in file '%s' at offset %lld, "
"file size %lld\n",
path_.c_str(), static_cast<long long>(ddOffset), static_cast<long long>(size_));
return 0;
}
FileRegion ddMapped(fd_, ddOffset, sizeof(uint32_t) + sizeof(DataDescriptor));
off_t localDDOffset = 0;
if (kOptionalDataDescriptorMagic == *(uint32_t*)ddMapped.data()) {
ddOffset += sizeof(uint32_t);
localDDOffset += sizeof(uint32_t);
ddSize += sizeof(uint32_t);
}
if (ddOffset + kDataDescriptorSize > size_) {
fprintf(stderr,
"Error reading trailing data descriptor in file '%s' at offset %lld, file size "
"%lld\n",
path_.c_str(), static_cast<long long>(ddOffset), static_cast<long long>(size_));
return 0;
}
dd = reinterpret_cast<const DataDescriptor*>(ddMapped.data() + localDDOffset);
localDataSize = (lfh->compression_method == kCompressStored) ? dd->uncompressed_size
: dd->compressed_size;
ddSize += sizeof(*dd);
} else {
localDataSize = (lfh->compression_method == kCompressStored) ? lfh->uncompressed_size
: lfh->compressed_size;
}
if (localDataSize != dataSize) {
fprintf(stderr,
"Data sizes mismatch in file '%s' at offset %lld, CDr: %lld vs LHR/DD: %lld\n",
path_.c_str(), static_cast<long long>(localFileHeaderOffset),
static_cast<long long>(dataSize), static_cast<long long>(localDataSize));
return 0;
}
return kLocalFileHeaderSize + lfh->file_name_length + lfh->extra_field_length + dataSize +
ddSize;
}

80
adb/fastdeploy/deploypatchgenerator/apk_archive.h Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <memory>
#include <string>
#include <vector>
#include <adb_unique_fd.h>
#include "fastdeploy/proto/ApkEntry.pb.h"
class ApkArchiveTester;
// Manipulates an APK archive. Process it by mmaping it in order to minimize
// I/Os.
class ApkArchive {
public:
friend ApkArchiveTester;
// A convenience struct to store the result of search operation when
// locating the EoCDr, CDr, and Signature Block.
struct Location {
off_t offset = 0;
off_t size = 0;
bool valid = false;
};
ApkArchive(const std::string& path);
~ApkArchive();
com::android::fastdeploy::APKDump ExtractMetadata();
// Parses the CDr starting from |input| and returns number of bytes consumed.
// Extracts local file header offset, data size and calculates MD5 hash of the record.
// 0 indicates invalid CDr.
static size_t ParseCentralDirectoryRecord(const char* input, size_t size, std::string* md5Hash,
int64_t* localFileHeaderOffset, int64_t* dataSize);
// Calculates Local File Entry size including header using offset and data size from CDr.
// 0 indicates invalid Local File Entry.
size_t CalculateLocalFileEntrySize(int64_t localFileHeaderOffset, int64_t dataSize) const;
private:
std::string ReadMetadata(Location loc) const;
// Retrieve the location of the Central Directory Record.
Location GetCDLocation();
// Retrieve the location of the signature block starting from Central
// Directory Record
Location GetSignatureLocation(off_t cdRecordOffset);
// Find the End of Central Directory Record, starting from the end of the
// file.
off_t FindEndOfCDRecord() const;
// Find Central Directory Record, starting from the end of the file.
Location FindCDRecord(const char* cursor);
// Checks if the archive can be used.
bool ready() const { return fd_ >= 0; }
std::string path_;
off_t size_;
unique_fd fd_;
};

73
adb/fastdeploy/deploypatchgenerator/apk_archive_test.cpp Normal file
View file

@ -0,0 +1,73 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <iostream>
#include <gtest/gtest.h>
#include "apk_archive.h"
// Friend test to get around private scope of ApkArchive private functions.
class ApkArchiveTester {
public:
ApkArchiveTester(const std::string& path) : archive_(path) {}
bool ready() { return archive_.ready(); }
auto ExtractMetadata() { return archive_.ExtractMetadata(); }
ApkArchive::Location GetCDLocation() { return archive_.GetCDLocation(); }
ApkArchive::Location GetSignatureLocation(size_t start) {
return archive_.GetSignatureLocation(start);
}
private:
ApkArchive archive_;
};
TEST(ApkArchiveTest, TestApkArchiveSizes) {
ApkArchiveTester archiveTester("fastdeploy/testdata/sample.apk");
EXPECT_TRUE(archiveTester.ready());
ApkArchive::Location cdLoc = archiveTester.GetCDLocation();
EXPECT_TRUE(cdLoc.valid);
ASSERT_EQ(cdLoc.offset, 2044145u);
ASSERT_EQ(cdLoc.size, 49390u);
// Check that block can be retrieved
ApkArchive::Location sigLoc = archiveTester.GetSignatureLocation(cdLoc.offset);
EXPECT_TRUE(sigLoc.valid);
ASSERT_EQ(sigLoc.offset, 2040049u);
ASSERT_EQ(sigLoc.size, 4088u);
}
TEST(ApkArchiveTest, TestApkArchiveDump) {
ApkArchiveTester archiveTester("fastdeploy/testdata/sample.apk");
EXPECT_TRUE(archiveTester.ready());
auto dump = archiveTester.ExtractMetadata();
ASSERT_EQ(dump.cd().size(), 49390u);
ASSERT_EQ(dump.signature().size(), 4088u);
}
TEST(ApkArchiveTest, WrongApk) {
ApkArchiveTester archiveTester("fastdeploy/testdata/sample.cd");
EXPECT_TRUE(archiveTester.ready());
auto dump = archiveTester.ExtractMetadata();
ASSERT_EQ(dump.cd().size(), 0u);
ASSERT_EQ(dump.signature().size(), 0u);
}

196
adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.cpp
View file

@ -25,8 +25,12 @@
#include <iostream>
#include <sstream>
#include <string>
#include <unordered_map>
#include <openssl/md5.h>
#include "adb_unique_fd.h"
#include "adb_utils.h"
#include "android-base/file.h"
#include "patch_utils.h"
#include "sysdeps.h"
@ -34,9 +38,6 @@
using namespace com::android::fastdeploy;
void DeployPatchGenerator::Log(const char* fmt, ...) {
if (!is_verbose_) {
return;
}
va_list ap;
va_start(ap, fmt);
vprintf(fmt, ap);
@ -44,19 +45,34 @@ void DeployPatchGenerator::Log(const char* fmt, ...) {
va_end(ap);
}
void DeployPatchGenerator::APKEntryToLog(const APKEntry& entry) {
Log("Filename: %s", entry.filename().c_str());
Log("CRC32: 0x%08" PRIX64, entry.crc32());
Log("Data Offset: %" PRId64, entry.dataoffset());
Log("Compressed Size: %" PRId64, entry.compressedsize());
Log("Uncompressed Size: %" PRId64, entry.uncompressedsize());
static std::string HexEncode(const void* in_buffer, unsigned int size) {
static const char kHexChars[] = "0123456789ABCDEF";
// Each input byte creates two output hex characters.
std::string out_buffer(size * 2, '\0');
for (unsigned int i = 0; i < size; ++i) {
char byte = ((const uint8_t*)in_buffer)[i];
out_buffer[(i << 1)] = kHexChars[(byte >> 4) & 0xf];
out_buffer[(i << 1) + 1] = kHexChars[byte & 0xf];
}
return out_buffer;
}
void DeployPatchGenerator::APKMetaDataToLog(const char* file, const APKMetaData& metadata) {
void DeployPatchGenerator::APKEntryToLog(const APKEntry& entry) {
if (!is_verbose_) {
return;
}
Log("APK Metadata: %s", file);
Log("MD5: %s", HexEncode(entry.md5().data(), entry.md5().size()).c_str());
Log("Data Offset: %" PRId64, entry.dataoffset());
Log("Data Size: %" PRId64, entry.datasize());
}
void DeployPatchGenerator::APKMetaDataToLog(const APKMetaData& metadata) {
if (!is_verbose_) {
return;
}
Log("APK Metadata: %s", metadata.absolute_path().c_str());
for (int i = 0; i < metadata.entries_size(); i++) {
const APKEntry& entry = metadata.entries(i);
APKEntryToLog(entry);
@ -65,49 +81,93 @@ void DeployPatchGenerator::APKMetaDataToLog(const char* file, const APKMetaData&
void DeployPatchGenerator::ReportSavings(const std::vector<SimpleEntry>& identicalEntries,
uint64_t totalSize) {
long totalEqualBytes = 0;
int totalEqualFiles = 0;
uint64_t totalEqualBytes = 0;
uint64_t totalEqualFiles = 0;
for (size_t i = 0; i < identicalEntries.size(); i++) {
if (identicalEntries[i].deviceEntry != nullptr) {
totalEqualBytes += identicalEntries[i].localEntry->compressedsize();
totalEqualBytes += identicalEntries[i].localEntry->datasize();
totalEqualFiles++;
}
}
float savingPercent = (totalEqualBytes * 100.0f) / totalSize;
fprintf(stderr, "Detected %d equal APK entries\n", totalEqualFiles);
fprintf(stderr, "%ld bytes are equal out of %" PRIu64 " (%.2f%%)\n", totalEqualBytes, totalSize,
savingPercent);
double savingPercent = (totalEqualBytes * 100.0f) / totalSize;
fprintf(stderr, "Detected %" PRIu64 " equal APK entries\n", totalEqualFiles);
fprintf(stderr, "%" PRIu64 " bytes are equal out of %" PRIu64 " (%.2f%%)\n", totalEqualBytes,
totalSize, savingPercent);
}
struct PatchEntry {
int64_t deltaFromDeviceDataStart = 0;
int64_t deviceDataOffset = 0;
int64_t deviceDataLength = 0;
};
static void WritePatchEntry(const PatchEntry& patchEntry, borrowed_fd input, borrowed_fd output,
size_t* realSizeOut) {
if (!(patchEntry.deltaFromDeviceDataStart | patchEntry.deviceDataOffset |
patchEntry.deviceDataLength)) {
return;
}
PatchUtils::WriteLong(patchEntry.deltaFromDeviceDataStart, output);
if (patchEntry.deltaFromDeviceDataStart > 0) {
PatchUtils::Pipe(input, output, patchEntry.deltaFromDeviceDataStart);
}
auto hostDataLength = patchEntry.deviceDataLength;
adb_lseek(input, hostDataLength, SEEK_CUR);
PatchUtils::WriteLong(patchEntry.deviceDataOffset, output);
PatchUtils::WriteLong(patchEntry.deviceDataLength, output);
*realSizeOut += patchEntry.deltaFromDeviceDataStart + hostDataLength;
}
void DeployPatchGenerator::GeneratePatch(const std::vector<SimpleEntry>& entriesToUseOnDevice,
const char* localApkPath, borrowed_fd output) {
unique_fd input(adb_open(localApkPath, O_RDONLY | O_CLOEXEC));
const std::string& localApkPath,
const std::string& deviceApkPath, borrowed_fd output) {
unique_fd input(adb_open(localApkPath.c_str(), O_RDONLY | O_CLOEXEC));
size_t newApkSize = adb_lseek(input, 0L, SEEK_END);
adb_lseek(input, 0L, SEEK_SET);
// Header.
PatchUtils::WriteSignature(output);
PatchUtils::WriteLong(newApkSize, output);
PatchUtils::WriteString(deviceApkPath, output);
size_t currentSizeOut = 0;
size_t realSizeOut = 0;
// Write data from the host upto the first entry we have that matches a device entry. Then write
// the metadata about the device entry and repeat for all entries that match on device. Finally
// write out any data left. If the device and host APKs are exactly the same this ends up
// writing out zip metadata from the local APK followed by offsets to the data to use from the
// device APK.
for (auto&& entry : entriesToUseOnDevice) {
int64_t deviceDataOffset = entry.deviceEntry->dataoffset();
PatchEntry patchEntry;
for (size_t i = 0, size = entriesToUseOnDevice.size(); i < size; ++i) {
auto&& entry = entriesToUseOnDevice[i];
int64_t hostDataOffset = entry.localEntry->dataoffset();
int64_t deviceDataLength = entry.deviceEntry->compressedsize();
int64_t hostDataLength = entry.localEntry->datasize();
int64_t deviceDataOffset = entry.deviceEntry->dataoffset();
// Both entries are the same, using host data length.
int64_t deviceDataLength = hostDataLength;
int64_t deltaFromDeviceDataStart = hostDataOffset - currentSizeOut;
PatchUtils::WriteLong(deltaFromDeviceDataStart, output);
if (deltaFromDeviceDataStart > 0) {
PatchUtils::Pipe(input, output, deltaFromDeviceDataStart);
WritePatchEntry(patchEntry, input, output, &realSizeOut);
patchEntry.deltaFromDeviceDataStart = deltaFromDeviceDataStart;
patchEntry.deviceDataOffset = deviceDataOffset;
patchEntry.deviceDataLength = deviceDataLength;
} else {
patchEntry.deviceDataLength += deviceDataLength;
}
PatchUtils::WriteLong(deviceDataOffset, output);
PatchUtils::WriteLong(deviceDataLength, output);
adb_lseek(input, deviceDataLength, SEEK_CUR);
currentSizeOut += deltaFromDeviceDataStart + deviceDataLength;
currentSizeOut += deltaFromDeviceDataStart + hostDataLength;
}
if (currentSizeOut != newApkSize) {
WritePatchEntry(patchEntry, input, output, &realSizeOut);
if (realSizeOut != currentSizeOut) {
fprintf(stderr, "Size mismatch current %lld vs real %lld\n",
static_cast<long long>(currentSizeOut), static_cast<long long>(realSizeOut));
error_exit("Aborting");
}
if (newApkSize > currentSizeOut) {
PatchUtils::WriteLong(newApkSize - currentSizeOut, output);
PatchUtils::Pipe(input, output, newApkSize - currentSizeOut);
PatchUtils::WriteLong(0, output);
@ -115,44 +175,72 @@ void DeployPatchGenerator::GeneratePatch(const std::vector<SimpleEntry>& entries
}
}
bool DeployPatchGenerator::CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath,
borrowed_fd output) {
std::string content;
APKMetaData deviceApkMetadata;
if (android::base::ReadFileToString(deviceApkMetadataPath, &content)) {
deviceApkMetadata.ParsePartialFromString(content);
} else {
// TODO: What do we want to do if we don't find any metadata.
// The current fallback behavior is to build a patch with the contents of |localApkPath|.
}
bool DeployPatchGenerator::CreatePatch(const char* localApkPath, APKMetaData deviceApkMetadata,
android::base::borrowed_fd output) {
return CreatePatch(PatchUtils::GetHostAPKMetaData(localApkPath), std::move(deviceApkMetadata),
output);
}
APKMetaData localApkMetadata = PatchUtils::GetAPKMetaData(localApkPath);
// Log gathered metadata info.
APKMetaDataToLog(deviceApkMetadataPath, deviceApkMetadata);
APKMetaDataToLog(localApkPath, localApkMetadata);
bool DeployPatchGenerator::CreatePatch(APKMetaData localApkMetadata, APKMetaData deviceApkMetadata,
borrowed_fd output) {
// Log metadata info.
APKMetaDataToLog(deviceApkMetadata);
APKMetaDataToLog(localApkMetadata);
const std::string localApkPath = localApkMetadata.absolute_path();
const std::string deviceApkPath = deviceApkMetadata.absolute_path();
std::vector<SimpleEntry> identicalEntries;
uint64_t totalSize =
BuildIdenticalEntries(identicalEntries, localApkMetadata, deviceApkMetadata);
ReportSavings(identicalEntries, totalSize);
GeneratePatch(identicalEntries, localApkPath, output);
GeneratePatch(identicalEntries, localApkPath, deviceApkPath, output);
return true;
}
uint64_t DeployPatchGenerator::BuildIdenticalEntries(std::vector<SimpleEntry>& outIdenticalEntries,
const APKMetaData& localApkMetadata,
const APKMetaData& deviceApkMetadata) {
outIdenticalEntries.reserve(
std::min(localApkMetadata.entries_size(), deviceApkMetadata.entries_size()));
using md5Digest = std::pair<uint64_t, uint64_t>;
struct md5Hash {
size_t operator()(const md5Digest& digest) const {
std::hash<uint64_t> hasher;
size_t seed = 0;
seed ^= hasher(digest.first) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
seed ^= hasher(digest.second) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
return seed;
}
};
static_assert(sizeof(md5Digest) == MD5_DIGEST_LENGTH);
std::unordered_map<md5Digest, std::vector<const APKEntry*>, md5Hash> deviceEntries;
for (const auto& deviceEntry : deviceApkMetadata.entries()) {
md5Digest md5;
memcpy(&md5, deviceEntry.md5().data(), deviceEntry.md5().size());
deviceEntries[md5].push_back(&deviceEntry);
}
uint64_t totalSize = 0;
for (int i = 0; i < localApkMetadata.entries_size(); i++) {
const APKEntry& localEntry = localApkMetadata.entries(i);
totalSize += localEntry.compressedsize();
for (int j = 0; j < deviceApkMetadata.entries_size(); j++) {
const APKEntry& deviceEntry = deviceApkMetadata.entries(j);
if (deviceEntry.crc32() == localEntry.crc32() &&
deviceEntry.filename().compare(localEntry.filename()) == 0) {
for (const auto& localEntry : localApkMetadata.entries()) {
totalSize += localEntry.datasize();
md5Digest md5;
memcpy(&md5, localEntry.md5().data(), localEntry.md5().size());
auto deviceEntriesIt = deviceEntries.find(md5);
if (deviceEntriesIt == deviceEntries.end()) {
continue;
}
for (const auto* deviceEntry : deviceEntriesIt->second) {
if (deviceEntry->md5() == localEntry.md5()) {
SimpleEntry simpleEntry;
simpleEntry.localEntry = const_cast<APKEntry*>(&localEntry);
simpleEntry.deviceEntry = const_cast<APKEntry*>(&deviceEntry);
simpleEntry.localEntry = &localEntry;
simpleEntry.deviceEntry = deviceEntry;
APKEntryToLog(localEntry);
outIdenticalEntries.push_back(simpleEntry);
break;

37
adb/fastdeploy/deploypatchgenerator/deploy_patch_generator.h
View file

@ -27,12 +27,15 @@
*/
class DeployPatchGenerator {
public:
using APKEntry = com::android::fastdeploy::APKEntry;
using APKMetaData = com::android::fastdeploy::APKMetaData;
/**
* Simple struct to hold mapping between local metadata and device metadata.
*/
struct SimpleEntry {
com::android::fastdeploy::APKEntry* localEntry;
com::android::fastdeploy::APKEntry* deviceEntry;
const APKEntry* localEntry;
const APKEntry* deviceEntry;
};
/**
@ -41,10 +44,10 @@ class DeployPatchGenerator {
*/
explicit DeployPatchGenerator(bool is_verbose) : is_verbose_(is_verbose) {}
/**
* Given a |localApkPath|, and the |deviceApkMetadataPath| from an installed APK this function
* Given a |localApkPath|, and the |deviceApkMetadata| from an installed APK this function
* writes a patch to the given |output|.
*/
bool CreatePatch(const char* localApkPath, const char* deviceApkMetadataPath,
bool CreatePatch(const char* localApkPath, APKMetaData deviceApkMetadata,
android::base::borrowed_fd output);
private:
@ -57,14 +60,20 @@ class DeployPatchGenerator {
/**
* Helper function to log the APKMetaData structure. If |is_verbose_| is false this function
* early outs. |file| is the path to the file represented by |metadata|. This function is used
* for debugging / information.
* early outs. This function is used for debugging / information.
*/
void APKMetaDataToLog(const char* file, const com::android::fastdeploy::APKMetaData& metadata);
void APKMetaDataToLog(const APKMetaData& metadata);
/**
* Helper function to log APKEntry.
*/
void APKEntryToLog(const com::android::fastdeploy::APKEntry& entry);
void APKEntryToLog(const APKEntry& entry);
/**
* Given the |localApkMetadata| metadata, and the |deviceApkMetadata| from an installed APK this
* function writes a patch to the given |output|.
*/
bool CreatePatch(APKMetaData localApkMetadata, APKMetaData deviceApkMetadata,
android::base::borrowed_fd output);
/**
* Helper function to report savings by fastdeploy. This function prints out savings even with
@ -92,11 +101,11 @@ class DeployPatchGenerator {
* highest.
*/
void GeneratePatch(const std::vector<SimpleEntry>& entriesToUseOnDevice,
const char* localApkPath, android::base::borrowed_fd output);
const std::string& localApkPath, const std::string& deviceApkPath,
android::base::borrowed_fd output);
protected:
uint64_t BuildIdenticalEntries(
std::vector<SimpleEntry>& outIdenticalEntries,
const com::android::fastdeploy::APKMetaData& localApkMetadata,
const com::android::fastdeploy::APKMetaData& deviceApkMetadataPath);
};
uint64_t BuildIdenticalEntries(std::vector<SimpleEntry>& outIdenticalEntries,
const APKMetaData& localApkMetadata,
const APKMetaData& deviceApkMetadata);
};

40
adb/fastdeploy/deploypatchgenerator/deploy_patch_generator_test.cpp
View file

@ -15,6 +15,7 @@
*/
#include "deploy_patch_generator.h"
#include "apk_archive.h"
#include "patch_utils.h"
#include <android-base/file.h>
@ -31,21 +32,17 @@ static std::string GetTestFile(const std::string& name) {
return "fastdeploy/testdata/" + name;
}
class TestPatchGenerator : DeployPatchGenerator {
public:
TestPatchGenerator() : DeployPatchGenerator(false) {}
void GatherIdenticalEntries(std::vector<DeployPatchGenerator::SimpleEntry>& outIdenticalEntries,
const APKMetaData& metadataA, const APKMetaData& metadataB) {
BuildIdenticalEntries(outIdenticalEntries, metadataA, metadataB);
}
struct TestPatchGenerator : DeployPatchGenerator {
using DeployPatchGenerator::BuildIdenticalEntries;
using DeployPatchGenerator::DeployPatchGenerator;
};
TEST(DeployPatchGeneratorTest, IdenticalFileEntries) {
std::string apkPath = GetTestFile("rotating_cube-release.apk");
APKMetaData metadataA = PatchUtils::GetAPKMetaData(apkPath.c_str());
TestPatchGenerator generator;
APKMetaData metadataA = PatchUtils::GetHostAPKMetaData(apkPath.c_str());
TestPatchGenerator generator(false);
std::vector<DeployPatchGenerator::SimpleEntry> entries;
generator.GatherIdenticalEntries(entries, metadataA, metadataA);
generator.BuildIdenticalEntries(entries, metadataA, metadataA);
// Expect the entry count to match the number of entries in the metadata.
const uint32_t identicalCount = entries.size();
const uint32_t entriesCount = metadataA.entries_size();
@ -64,9 +61,28 @@ TEST(DeployPatchGeneratorTest, NoDeviceMetadata) {
// Create a patch that is 100% different.
TemporaryFile output;
DeployPatchGenerator generator(true);
generator.CreatePatch(apkPath.c_str(), "", output.fd);
generator.CreatePatch(apkPath.c_str(), {}, output.fd);
// Expect a patch file that has a size at least the size of our initial APK.
long patchSize = adb_lseek(output.fd, 0L, SEEK_END);
EXPECT_GT(patchSize, apkSize);
}
}
TEST(DeployPatchGeneratorTest, ZeroSizePatch) {
std::string apkPath = GetTestFile("rotating_cube-release.apk");
ApkArchive archive(apkPath);
auto dump = archive.ExtractMetadata();
EXPECT_NE(dump.cd().size(), 0u);
APKMetaData metadata = PatchUtils::GetDeviceAPKMetaData(dump);
// Create a patch that is 100% the same.
TemporaryFile output;
output.DoNotRemove();
DeployPatchGenerator generator(true);
generator.CreatePatch(apkPath.c_str(), metadata, output.fd);
// Expect a patch file that is smaller than 0.5K.
int64_t patchSize = adb_lseek(output.fd, 0L, SEEK_END);
EXPECT_LE(patchSize, 512);
}

96
adb/fastdeploy/deploypatchgenerator/patch_utils.cpp
View file

@ -16,72 +16,94 @@
#include "patch_utils.h"
#include <androidfw/ZipFileRO.h>
#include <stdio.h>
#include "adb_io.h"
#include "adb_utils.h"
#include "android-base/endian.h"
#include "sysdeps.h"
#include "apk_archive.h"
using namespace com::android;
using namespace com::android::fastdeploy;
using namespace android::base;
static constexpr char kSignature[] = "FASTDEPLOY";
APKMetaData PatchUtils::GetAPKMetaData(const char* apkPath) {
APKMetaData PatchUtils::GetDeviceAPKMetaData(const APKDump& apk_dump) {
APKMetaData apkMetaData;
#undef open
std::unique_ptr<android::ZipFileRO> zipFile(android::ZipFileRO::open(apkPath));
#define open ___xxx_unix_open
if (zipFile == nullptr) {
printf("Could not open %s", apkPath);
exit(1);
}
void* cookie;
if (zipFile->startIteration(&cookie)) {
android::ZipEntryRO entry;
while ((entry = zipFile->nextEntry(cookie)) != NULL) {
char fileName[256];
// Make sure we have a file name.
// TODO: Handle filenames longer than 256.
if (zipFile->getEntryFileName(entry, fileName, sizeof(fileName))) {
continue;
}
apkMetaData.set_absolute_path(apk_dump.absolute_path());
uint32_t uncompressedSize, compressedSize, crc32;
int64_t dataOffset;
zipFile->getEntryInfo(entry, nullptr, &uncompressedSize, &compressedSize, &dataOffset,
nullptr, &crc32);
APKEntry* apkEntry = apkMetaData.add_entries();
apkEntry->set_crc32(crc32);
apkEntry->set_filename(fileName);
apkEntry->set_compressedsize(compressedSize);
apkEntry->set_uncompressedsize(uncompressedSize);
apkEntry->set_dataoffset(dataOffset);
}
std::string md5Hash;
int64_t localFileHeaderOffset;
int64_t dataSize;
const auto& cd = apk_dump.cd();
auto cur = cd.data();
int64_t size = cd.size();
while (auto consumed = ApkArchive::ParseCentralDirectoryRecord(
cur, size, &md5Hash, &localFileHeaderOffset, &dataSize)) {
cur += consumed;
size -= consumed;
auto apkEntry = apkMetaData.add_entries();
apkEntry->set_md5(md5Hash);
apkEntry->set_dataoffset(localFileHeaderOffset);
apkEntry->set_datasize(dataSize);
}
return apkMetaData;
}
APKMetaData PatchUtils::GetHostAPKMetaData(const char* apkPath) {
ApkArchive archive(apkPath);
auto dump = archive.ExtractMetadata();
if (dump.cd().empty()) {
fprintf(stderr, "adb: Could not extract Central Directory from %s\n", apkPath);
error_exit("Aborting");
}
auto apkMetaData = GetDeviceAPKMetaData(dump);
// Now let's set data sizes.
for (auto& apkEntry : *apkMetaData.mutable_entries()) {
auto dataSize =
archive.CalculateLocalFileEntrySize(apkEntry.dataoffset(), apkEntry.datasize());
if (dataSize == 0) {
error_exit("Aborting");
}
apkEntry.set_datasize(dataSize);
}
return apkMetaData;
}
void PatchUtils::WriteSignature(borrowed_fd output) {
WriteFdExactly(output, kSignature, sizeof(kSignature) - 1);
}
void PatchUtils::WriteLong(int64_t value, borrowed_fd output) {
int64_t toLittleEndian = htole64(value);
WriteFdExactly(output, &toLittleEndian, sizeof(int64_t));
int64_t littleEndian = htole64(value);
WriteFdExactly(output, &littleEndian, sizeof(littleEndian));
}
void PatchUtils::WriteString(const std::string& value, android::base::borrowed_fd output) {
WriteLong(value.size(), output);
WriteFdExactly(output, value);
}
void PatchUtils::Pipe(borrowed_fd input, borrowed_fd output, size_t amount) {
constexpr static int BUFFER_SIZE = 128 * 1024;
constexpr static size_t BUFFER_SIZE = 128 * 1024;
char buffer[BUFFER_SIZE];
size_t transferAmount = 0;
while (transferAmount != amount) {
long chunkAmount =
amount - transferAmount > BUFFER_SIZE ? BUFFER_SIZE : amount - transferAmount;
long readAmount = adb_read(input, buffer, chunkAmount);
auto chunkAmount = std::min(amount - transferAmount, BUFFER_SIZE);
auto readAmount = adb_read(input, buffer, chunkAmount);
if (readAmount < 0) {
fprintf(stderr, "adb: failed to read from input: %s\n", strerror(errno));
error_exit("Aborting");
}
WriteFdExactly(output, buffer, readAmount);
transferAmount += readAmount;
}
}
}

15
adb/fastdeploy/deploypatchgenerator/patch_utils.h
View file

@ -24,12 +24,19 @@
*/
class PatchUtils {
public:
/**
* This function takes the dump of Central Directly and builds the APKMetaData required by the
* patching algorithm. The if this function has an error a string is printed to the terminal and
* exit(1) is called.
*/
static com::android::fastdeploy::APKMetaData GetDeviceAPKMetaData(
const com::android::fastdeploy::APKDump& apk_dump);
/**
* This function takes a local APK file and builds the APKMetaData required by the patching
* algorithm. The if this function has an error a string is printed to the terminal and exit(1)
* is called.
*/
static com::android::fastdeploy::APKMetaData GetAPKMetaData(const char* file);
static com::android::fastdeploy::APKMetaData GetHostAPKMetaData(const char* file);
/**
* Writes a fixed signature string to the header of the patch.
*/
@ -38,9 +45,13 @@ class PatchUtils {
* Writes an int64 to the |output| reversing the bytes.
*/
static void WriteLong(int64_t value, android::base::borrowed_fd output);
/**
* Writes string to the |output|.
*/
static void WriteString(const std::string& value, android::base::borrowed_fd output);
/**
* Copy |amount| of data from |input| to |output|.
*/
static void Pipe(android::base::borrowed_fd input, android::base::borrowed_fd output,
size_t amount);
};
};

54
adb/fastdeploy/deploypatchgenerator/patch_utils_test.cpp
View file

@ -23,10 +23,13 @@
#include <sstream>
#include <string>
#include <google/protobuf/util/message_differencer.h>
#include "adb_io.h"
#include "sysdeps.h"
using namespace com::android::fastdeploy;
using google::protobuf::util::MessageDifferencer;
static std::string GetTestFile(const std::string& name) {
return "fastdeploy/testdata/" + name;
@ -86,11 +89,56 @@ TEST(PatchUtilsTest, SignatureConstMatches) {
TEST(PatchUtilsTest, GatherMetadata) {
std::string apkFile = GetTestFile("rotating_cube-release.apk");
APKMetaData metadata = PatchUtils::GetAPKMetaData(apkFile.c_str());
APKMetaData actual = PatchUtils::GetHostAPKMetaData(apkFile.c_str());
std::string expectedMetadata;
android::base::ReadFileToString(GetTestFile("rotating_cube-metadata-release.data"),
&expectedMetadata);
APKMetaData expected;
EXPECT_TRUE(expected.ParseFromString(expectedMetadata));
// Test paths might vary.
expected.set_absolute_path(actual.absolute_path());
std::string actualMetadata;
metadata.SerializeToString(&actualMetadata);
actual.SerializeToString(&actualMetadata);
expected.SerializeToString(&expectedMetadata);
EXPECT_EQ(expectedMetadata, actualMetadata);
}
}
static inline void sanitize(APKMetaData& metadata) {
metadata.clear_absolute_path();
for (auto&& entry : *metadata.mutable_entries()) {
entry.clear_datasize();
}
}
TEST(PatchUtilsTest, GatherDumpMetadata) {
APKMetaData hostMetadata;
APKMetaData deviceMetadata;
hostMetadata = PatchUtils::GetHostAPKMetaData(GetTestFile("sample.apk").c_str());
{
std::string cd;
android::base::ReadFileToString(GetTestFile("sample.cd"), &cd);
APKDump dump;
dump.set_cd(std::move(cd));
deviceMetadata = PatchUtils::GetDeviceAPKMetaData(dump);
}
sanitize(hostMetadata);
sanitize(deviceMetadata);
std::string expectedMetadata;
hostMetadata.SerializeToString(&expectedMetadata);
std::string actualMetadata;
deviceMetadata.SerializeToString(&actualMetadata);
EXPECT_EQ(expectedMetadata, actualMetadata);
}

22
adb/fastdeploy/proto/ApkEntry.proto
View file

@ -1,18 +1,26 @@
syntax = "proto2";
syntax = "proto3";
package com.android.fastdeploy;
option java_package = "com.android.fastdeploy";
option java_outer_classname = "ApkEntryProto";
option java_multiple_files = true;
option optimize_for = LITE_RUNTIME;
message APKDump {
string name = 1;
bytes cd = 2;
bytes signature = 3;
string absolute_path = 4;
}
message APKEntry {
required int64 crc32 = 1;
required string fileName = 2;
required int64 dataOffset = 3;
required int64 compressedSize = 4;
required int64 uncompressedSize = 5;
bytes md5 = 1;
int64 dataOffset = 2;
int64 dataSize = 3;
}
message APKMetaData {
repeated APKEntry entries = 1;
string absolute_path = 1;
repeated APKEntry entries = 2;
}

BIN
adb/fastdeploy/testdata/helloworld5.apk vendored Normal file
View file

Binary file not shown.

BIN
adb/fastdeploy/testdata/helloworld7.apk vendored Normal file
View file

Binary file not shown.

12
adb/fastdeploy/testdata/rotating_cube-metadata-release.data vendored
View file

@ -1,6 +1,8 @@
#Ç‚Ï« META-INF/MANIFEST.MFÇ Q(W
#ƒ<>•ŽAndroidManifest.xml1 ä(è
6¦µ€>#lib/armeabi-v7a/libvulkan_sample.so<18> ÀÒQ(Œ²ì
—ã—‘resources.arscôàQ ´(´
‹œÂÉ classes.dexÁ ÿ(ô
-fastdeploy/testdata/rotating_cube-release.apk
ìv4@ob<>O#&kýn•
K›• 3Qcp^<Ð̽sF0•ƒ
xB ™a­©–2áÒ_O'˜¨
¶ÅhsêÃDÍY
ª^"cvÀ <18>ÓQ
q<>žp¶Îó{2ÐÄÒÙ«ÂÁàQç

BIN
adb/fastdeploy/testdata/sample.apk vendored Normal file
View file

Binary file not shown.

BIN
adb/fastdeploy/testdata/sample.cd vendored Normal file
View file

Binary file not shown.

4
adb/sysdeps.h
View file

@ -601,6 +601,10 @@ static __inline__ int adb_is_absolute_host_path(const char* path) {
return path[0] == '/';
}
static __inline__ int adb_get_os_handle(borrowed_fd fd) {
return fd.get();
}
#endif /* !_WIN32 */
static inline void disable_tcp_nagle(borrowed_fd fd) {