From cf9f0870e4fa9504cd2e124456cdb8bdd44dd49f Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 7 Mar 2024 17:56:14 -0800 Subject: [PATCH] Add support for tombstone symbolization to pbtombstone. This patch teaches pbtombstone to use llvm-symbolizer to symbolize stack traces and augment the protobuf tombstones with the symbol information, before printing tombstones with the symbolized stack traces included. The main advantage of adding this information to the tombstone as opposed to having developers use the stack tool is that stack does not print all of the information in the original tombstone, which means that both reports may be required to understand a crash. Furthermore, stack traces printed by stack are not correlated with the stack traces in the tombstone, making the report harder to read, especially with GWP-ASan and MTE which may produce multiple stack traces for the crashing thread. Although we could teach stack to print more information, this would continue to be fragile because stack relies on parsing textual tombstones. Switching stack to read proto tombstones would be tantamount to a full rewrite and would require duplicating the C++ proto-to-text logic that we already have in Python. It seems better to reuse the C++ code for the proto-based symbolization tool. llvm-symbolizer will look up the symbol files by build ID using a .build-id directory following the standard here: https://fedoraproject.org/wiki/RolandMcGrath/BuildID It will look for .build-id directories under paths specified with --debug-file-directory, which pbtombstone will pass through to llvm-symbolizer using its own --debug-file-directory flag. The intent is that tools for platform developers will pass the flag --debug-file-directory $ANDROID_PRODUCT_OUT/symbols to pbtombstone. Soong will start creating .build-id under symbols after a corresponding Soong CL lands. Bug: 328531087 Change-Id: Ia4676821cf980c69487cf11aefa2a02dc0c1626f --- debuggerd/Android.bp | 5 +- .../libdebuggerd/tombstone_proto_to_text.h | 4 +- .../test/tombstone_proto_to_text_test.cpp | 16 +- debuggerd/libdebuggerd/tombstone.cpp | 9 +- .../libdebuggerd/tombstone_proto_to_text.cpp | 47 ++--- debuggerd/pbtombstone.cpp | 44 ++++- debuggerd/tombstone_symbolize.cpp | 160 ++++++++++++++++++ debuggerd/tombstone_symbolize.h | 42 +++++ 8 files changed, 292 insertions(+), 35 deletions(-) create mode 100644 debuggerd/tombstone_symbolize.cpp create mode 100644 debuggerd/tombstone_symbolize.h diff --git a/debuggerd/Android.bp b/debuggerd/Android.bp index d4dc9a340..3257a2c50 100644 --- a/debuggerd/Android.bp +++ b/debuggerd/Android.bp @@ -333,7 +333,10 @@ cc_binary { name: "pbtombstone", host_supported: true, defaults: ["debuggerd_defaults"], - srcs: ["pbtombstone.cpp"], + srcs: [ + "pbtombstone.cpp", + "tombstone_symbolize.cpp", + ], static_libs: [ "libbase", "libdebuggerd_tombstone_proto_to_text", diff --git a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h index 515a15f0a..2de972344 100644 --- a/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h +++ b/debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h @@ -19,8 +19,10 @@ #include #include +class BacktraceFrame; class Tombstone; bool tombstone_proto_to_text( const Tombstone& tombstone, - std::function callback); + std::function callback, + std::function symbolize); diff --git a/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp b/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp index 3fdb71d48..aad209a06 100644 --- a/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp +++ b/debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp @@ -61,12 +61,16 @@ class TombstoneProtoToTextTest : public ::testing::Test { void ProtoToString() { text_ = ""; - EXPECT_TRUE( - tombstone_proto_to_text(*tombstone_, [this](const std::string& line, bool should_log) { + EXPECT_TRUE(tombstone_proto_to_text( + *tombstone_, + [this](const std::string& line, bool should_log) { if (should_log) { text_ += "LOG "; } text_ += line + '\n'; + }, + [&](const BacktraceFrame& frame) { + text_ += "SYMBOLIZE " + frame.build_id() + " " + std::to_string(frame.pc()) + "\n"; })); } @@ -163,3 +167,11 @@ TEST_F(TombstoneProtoToTextTest, stack_record) { EXPECT_MATCH(text_, "stack_record fp:0x1 tag:0xb pc:foo\\.so\\+0x567 \\(BuildId: ABC123\\)"); EXPECT_MATCH(text_, "stack_record fp:0x2 tag:0xc pc:bar\\.so\\+0x678"); } + +TEST_F(TombstoneProtoToTextTest, symbolize) { + BacktraceFrame* frame = main_thread_->add_current_backtrace(); + frame->set_pc(12345); + frame->set_build_id("0123456789abcdef"); + ProtoToString(); + EXPECT_MATCH(text_, "\\(BuildId: 0123456789abcdef\\)\\nSYMBOLIZE 0123456789abcdef 12345\\n"); +} diff --git a/debuggerd/libdebuggerd/tombstone.cpp b/debuggerd/libdebuggerd/tombstone.cpp index d483b9889..30c6fe4c5 100644 --- a/debuggerd/libdebuggerd/tombstone.cpp +++ b/debuggerd/libdebuggerd/tombstone.cpp @@ -146,7 +146,10 @@ void engrave_tombstone(unique_fd output_fd, unique_fd proto_fd, log.tfd = output_fd.get(); log.amfd_data = amfd_data; - tombstone_proto_to_text(tombstone, [&log](const std::string& line, bool should_log) { - _LOG(&log, should_log ? logtype::HEADER : logtype::LOGS, "%s\n", line.c_str()); - }); + tombstone_proto_to_text( + tombstone, + [&log](const std::string& line, bool should_log) { + _LOG(&log, should_log ? logtype::HEADER : logtype::LOGS, "%s\n", line.c_str()); + }, + [](const BacktraceFrame&) {}); } diff --git a/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp b/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp index 611e237b4..fedafc0c3 100644 --- a/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp +++ b/debuggerd/libdebuggerd/tombstone_proto_to_text.cpp @@ -41,6 +41,7 @@ using android::base::StringPrintf; #define CBL(...) CB(true, __VA_ARGS__) #define CBS(...) CB(false, __VA_ARGS__) using CallbackType = std::function; +using SymbolizeCallbackType = std::function; #define DESCRIBE_FLAG(flag) \ if (value & flag) { \ @@ -184,7 +185,8 @@ static void print_thread_registers(CallbackType callback, const Tombstone& tombs print_register_row(callback, word_size, special_row, should_log); } -static void print_backtrace(CallbackType callback, const Tombstone& tombstone, +static void print_backtrace(CallbackType callback, SymbolizeCallbackType symbolize, + const Tombstone& tombstone, const google::protobuf::RepeatedPtrField& backtrace, bool should_log) { int index = 0; @@ -209,11 +211,14 @@ static void print_backtrace(CallbackType callback, const Tombstone& tombstone, } line += function + build_id; CB(should_log, "%s", line.c_str()); + + symbolize(frame); } } -static void print_thread_backtrace(CallbackType callback, const Tombstone& tombstone, - const Thread& thread, bool should_log) { +static void print_thread_backtrace(CallbackType callback, SymbolizeCallbackType symbolize, + const Tombstone& tombstone, const Thread& thread, + bool should_log) { CBS(""); CB(should_log, "%d total frames", thread.current_backtrace().size()); CB(should_log, "backtrace:"); @@ -221,7 +226,7 @@ static void print_thread_backtrace(CallbackType callback, const Tombstone& tombs CB(should_log, " NOTE: %s", android::base::Join(thread.backtrace_note(), "\n NOTE: ").c_str()); } - print_backtrace(callback, tombstone, thread.current_backtrace(), should_log); + print_backtrace(callback, symbolize, tombstone, thread.current_backtrace(), should_log); } static void print_thread_memory_dump(CallbackType callback, const Tombstone& tombstone, @@ -274,10 +279,11 @@ static void print_thread_memory_dump(CallbackType callback, const Tombstone& tom } } -static void print_thread(CallbackType callback, const Tombstone& tombstone, const Thread& thread) { +static void print_thread(CallbackType callback, SymbolizeCallbackType symbolize, + const Tombstone& tombstone, const Thread& thread) { print_thread_header(callback, tombstone, thread, false); print_thread_registers(callback, tombstone, thread, false); - print_thread_backtrace(callback, tombstone, thread, false); + print_thread_backtrace(callback, symbolize, tombstone, thread, false); print_thread_memory_dump(callback, tombstone, thread); } @@ -433,8 +439,8 @@ static std::string oct_encode(const std::string& data) { return oct_encoded; } -static void print_main_thread(CallbackType callback, const Tombstone& tombstone, - const Thread& thread) { +static void print_main_thread(CallbackType callback, SymbolizeCallbackType symbolize, + const Tombstone& tombstone, const Thread& thread) { print_thread_header(callback, tombstone, thread, true); const Signal& signal_info = tombstone.signal_info(); @@ -488,7 +494,7 @@ static void print_main_thread(CallbackType callback, const Tombstone& tombstone, CBL(" in this process. The stack trace below is the first system call or context"); CBL(" switch that was executed after the memory corruption happened."); } - print_thread_backtrace(callback, tombstone, thread, true); + print_thread_backtrace(callback, symbolize, tombstone, thread, true); if (tombstone.causes_size() > 1) { CBS(""); @@ -521,13 +527,13 @@ static void print_main_thread(CallbackType callback, const Tombstone& tombstone, if (heap_object.deallocation_backtrace_size() != 0) { CBS(""); CBL("deallocated by thread %" PRIu64 ":", heap_object.deallocation_tid()); - print_backtrace(callback, tombstone, heap_object.deallocation_backtrace(), true); + print_backtrace(callback, symbolize, tombstone, heap_object.deallocation_backtrace(), true); } if (heap_object.allocation_backtrace_size() != 0) { CBS(""); CBL("allocated by thread %" PRIu64 ":", heap_object.allocation_tid()); - print_backtrace(callback, tombstone, heap_object.allocation_backtrace(), true); + print_backtrace(callback, symbolize, tombstone, heap_object.allocation_backtrace(), true); } } } @@ -576,8 +582,9 @@ void print_logs(CallbackType callback, const Tombstone& tombstone, int tail) { } } -static void print_guest_thread(CallbackType callback, const Tombstone& tombstone, - const Thread& guest_thread, pid_t tid, bool should_log) { +static void print_guest_thread(CallbackType callback, SymbolizeCallbackType symbolize, + const Tombstone& tombstone, const Thread& guest_thread, pid_t tid, + bool should_log) { CBS("--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---"); CBS("Guest thread information for tid: %d", tid); print_thread_registers(callback, tombstone, guest_thread, should_log); @@ -585,12 +592,13 @@ static void print_guest_thread(CallbackType callback, const Tombstone& tombstone CBS(""); CB(true, "%d total frames", guest_thread.current_backtrace().size()); CB(true, "backtrace:"); - print_backtrace(callback, tombstone, guest_thread.current_backtrace(), should_log); + print_backtrace(callback, symbolize, tombstone, guest_thread.current_backtrace(), should_log); print_thread_memory_dump(callback, tombstone, guest_thread); } -bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback) { +bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback, + SymbolizeCallbackType symbolize) { CBL("*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***"); CBL("Build fingerprint: '%s'", tombstone.build_fingerprint().c_str()); CBL("Revision: '%s'", tombstone.revision().c_str()); @@ -618,14 +626,15 @@ bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback) const auto& main_thread = main_thread_it->second; - print_main_thread(callback, tombstone, main_thread); + print_main_thread(callback, symbolize, tombstone, main_thread); print_logs(callback, tombstone, 50); const auto& guest_threads = tombstone.guest_threads(); auto main_guest_thread_it = guest_threads.find(tombstone.tid()); if (main_guest_thread_it != threads.end()) { - print_guest_thread(callback, tombstone, main_guest_thread_it->second, tombstone.tid(), true); + print_guest_thread(callback, symbolize, tombstone, main_guest_thread_it->second, + tombstone.tid(), true); } // protobuf's map is unordered, so sort the keys first. @@ -638,10 +647,10 @@ bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback) for (const auto& tid : thread_ids) { CBS("--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---"); - print_thread(callback, tombstone, threads.find(tid)->second); + print_thread(callback, symbolize, tombstone, threads.find(tid)->second); auto guest_thread_it = guest_threads.find(tid); if (guest_thread_it != guest_threads.end()) { - print_guest_thread(callback, tombstone, guest_thread_it->second, tid, false); + print_guest_thread(callback, symbolize, tombstone, guest_thread_it->second, tid, false); } } diff --git a/debuggerd/pbtombstone.cpp b/debuggerd/pbtombstone.cpp index dcb7c6c21..0902b386f 100644 --- a/debuggerd/pbtombstone.cpp +++ b/debuggerd/pbtombstone.cpp @@ -16,32 +16,55 @@ #include #include +#include #include #include +#include +#include + #include #include #include "tombstone.pb.h" +#include "tombstone_symbolize.h" using android::base::unique_fd; [[noreturn]] void usage(bool error) { - fprintf(stderr, "usage: pbtombstone TOMBSTONE.PB\n"); + fprintf(stderr, "usage: pbtombstone [OPTION] TOMBSTONE.PB\n"); fprintf(stderr, "Convert a protobuf tombstone to text.\n"); + fprintf(stderr, "Arguments:\n"); + fprintf(stderr, " -h, --help print this message\n"); + fprintf(stderr, " --debug-file-directory PATH specify the path to a symbols directory\n"); exit(error); } -int main(int argc, const char* argv[]) { - if (argc != 2) { +int main(int argc, char* argv[]) { + std::vector debug_file_directories; + static struct option long_options[] = { + {"debug-file-directory", required_argument, 0, 0}, + {"help", no_argument, 0, 'h'}, + {}, + }; + int c; + while ((c = getopt_long(argc, argv, "h", long_options, 0)) != -1) { + switch (c) { + case 0: + debug_file_directories.push_back(optarg); + break; + + case 'h': + usage(false); + break; + } + } + + if (optind != argc-1) { usage(true); } - if (strcmp("-h", argv[1]) == 0 || strcmp("--help", argv[1]) == 0) { - usage(false); - } - - unique_fd fd(open(argv[1], O_RDONLY | O_CLOEXEC)); + unique_fd fd(open(argv[optind], O_RDONLY | O_CLOEXEC)); if (fd == -1) { err(1, "failed to open tombstone '%s'", argv[1]); } @@ -51,8 +74,11 @@ int main(int argc, const char* argv[]) { err(1, "failed to parse tombstone"); } + Symbolizer sym; + sym.Start(debug_file_directories); bool result = tombstone_proto_to_text( - tombstone, [](const std::string& line, bool) { printf("%s\n", line.c_str()); }); + tombstone, [](const std::string& line, bool) { printf("%s\n", line.c_str()); }, + [&](const BacktraceFrame& frame) { symbolize_backtrace_frame(frame, sym); }); if (!result) { errx(1, "tombstone was malformed"); diff --git a/debuggerd/tombstone_symbolize.cpp b/debuggerd/tombstone_symbolize.cpp new file mode 100644 index 000000000..07735d0ee --- /dev/null +++ b/debuggerd/tombstone_symbolize.cpp @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tombstone_symbolize.h" + +#include +#include +#include + +#include +#include + +#include "android-base/stringprintf.h" +#include "android-base/unique_fd.h" + +#include "tombstone.pb.h" + +using android::base::StringPrintf; +using android::base::unique_fd; + +bool Symbolizer::Start(const std::vector& debug_file_directories) { + unique_fd parent_in, parent_out, child_in, child_out; + if (!Pipe(&parent_in, &child_out) || !Pipe(&child_in, &parent_out)) { + return false; + } + + std::vector args; + args.push_back("llvm-symbolizer"); + for (const std::string &dir : debug_file_directories) { + args.push_back("--debug-file-directory"); + args.push_back(dir.c_str()); + } + args.push_back(0); + + int pid = fork(); + if (pid == -1) { + return false; + } else if (pid == 0) { + parent_in.reset(); + parent_out.reset(); + + dup2(child_in.get(), STDIN_FILENO); + dup2(child_out.get(), STDOUT_FILENO); + + execvp("llvm-symbolizer", const_cast(args.data())); + + fprintf(stderr, "unable to start llvm-symbolizer: %s\n", strerror(errno)); + _exit(1); + } else { + child_in.reset(); + child_out.reset(); + + // TODO: Check that llvm-symbolizer started up successfully. + // There used to be an easy way to do this, but it was removed in: + // https://github.com/llvm/llvm-project/commit/1792852f86dc75efa1f44d46b1a0daf386d64afa + + in_fd = std::move(parent_in); + out_fd = std::move(parent_out); + return true; + } +} + +std::string Symbolizer::read_response() { + std::string resp; + + while (resp.size() < 2 || resp[resp.size() - 2] != '\n' || resp[resp.size() - 1] != '\n') { + char buf[4096]; + ssize_t size = read(in_fd, buf, 4096); + if (size <= 0) { + return ""; + } + resp.append(buf, size); + } + + return resp; +} + +std::vector Symbolizer::SymbolizeCode(std::string path, uint64_t rel_pc) { + std::string request = StringPrintf("CODE %s 0x%" PRIx64 "\n", path.c_str(), rel_pc); + if (write(out_fd, request.c_str(), request.size()) != static_cast(request.size())) { + return {}; + } + + std::string response = read_response(); + if (response.empty()) { + return {}; + } + + std::vector frames; + + size_t frame_start = 0; + while (frame_start < response.size() - 1) { + Symbolizer::Frame frame; + + size_t second_line_start = response.find('\n', frame_start) + 1; + if (second_line_start == std::string::npos + 1) { + return {}; + } + + size_t third_line_start = response.find('\n', second_line_start) + 1; + if (third_line_start == std::string::npos + 1) { + return {}; + } + + frame.function_name = response.substr(frame_start, second_line_start - frame_start - 1); + + size_t column_number_start = response.rfind(':', third_line_start); + if (column_number_start == std::string::npos) { + return {}; + } + + size_t line_number_start = response.rfind(':', column_number_start - 1); + if (line_number_start == std::string::npos) { + return {}; + } + + frame.file = response.substr(second_line_start, line_number_start - second_line_start); + + errno = 0; + frame.line = strtoull(response.c_str() + line_number_start + 1, 0, 10); + frame.column = strtoull(response.c_str() + column_number_start + 1, 0, 10); + if (errno != 0) { + return {}; + } + + frames.push_back(frame); + + frame_start = third_line_start; + } + + if (frames.size() == 1 && frames[0].file == "??") { + return {}; + } + + return frames; +} + +void symbolize_backtrace_frame(const BacktraceFrame& frame, Symbolizer& sym) { + if (frame.build_id().empty()) { + return; + } + + for (Symbolizer::Frame f : sym.SymbolizeCode("BUILDID:" + frame.build_id(), frame.rel_pc())) { + printf(" %s:%" PRId64 ":%" PRId64 " (%s)\n", f.file.c_str(), f.line, f.column, + f.function_name.c_str()); + } +} diff --git a/debuggerd/tombstone_symbolize.h b/debuggerd/tombstone_symbolize.h new file mode 100644 index 000000000..c22d677ee --- /dev/null +++ b/debuggerd/tombstone_symbolize.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "android-base/unique_fd.h" + +class BacktraceFrame; + +class Symbolizer { + android::base::unique_fd in_fd, out_fd; + + std::string read_response(); + + public: + bool Start(const std::vector& debug_file_directories); + + struct Frame { + std::string function_name, file; + uint64_t line, column; + }; + + std::vector SymbolizeCode(std::string path, uint64_t rel_pc); +}; + +void symbolize_backtrace_frame(const BacktraceFrame& frame, Symbolizer& sym);