Add support for tombstone symbolization to pbtombstone.

This patch teaches pbtombstone to use llvm-symbolizer to symbolize
stack traces and augment the protobuf tombstones with the symbol
information, before printing tombstones with the symbolized stack
traces included.

The main advantage of adding this information to the tombstone
as opposed to having developers use the stack tool is that stack
does not print all of the information in the original tombstone,
which means that both reports may be required to understand a crash.
Furthermore, stack traces printed by stack are not correlated with
the stack traces in the tombstone, making the report harder to read,
especially with GWP-ASan and MTE which may produce multiple stack
traces for the crashing thread.

Although we could teach stack to print more information, this would
continue to be fragile because stack relies on parsing textual
tombstones. Switching stack to read proto tombstones would be
tantamount to a full rewrite and would require duplicating the C++
proto-to-text logic that we already have in Python. It seems better
to reuse the C++ code for the proto-based symbolization tool.

llvm-symbolizer will look up the symbol files by build ID using a
.build-id directory following the standard here:
https://fedoraproject.org/wiki/RolandMcGrath/BuildID

It will look for .build-id directories under paths specified with
--debug-file-directory, which pbtombstone will pass through to
llvm-symbolizer using its own --debug-file-directory flag. The
intent is that tools for platform developers will pass the flag
--debug-file-directory $ANDROID_PRODUCT_OUT/symbols to pbtombstone.
Soong will start creating .build-id under symbols after a corresponding
Soong CL lands.

Bug: 328531087
Change-Id: Ia4676821cf980c69487cf11aefa2a02dc0c1626f

debuggerd/Android.bp

@@ -333,7 +333,10 @@ cc_binary {
     name: "pbtombstone",
     host_supported: true,
     defaults: ["debuggerd_defaults"],
-    srcs: ["pbtombstone.cpp"],
+    srcs: [
+        "pbtombstone.cpp",
+        "tombstone_symbolize.cpp",
+    ],
     static_libs: [
         "libbase",
         "libdebuggerd_tombstone_proto_to_text",

debuggerd/libdebuggerd/include/libdebuggerd/tombstone_proto_to_text.h

@@ -19,8 +19,10 @@
 #include <functional>
 #include <string>
 
+class BacktraceFrame;
 class Tombstone;
 
 bool tombstone_proto_to_text(
     const Tombstone& tombstone,
-    std::function<void(const std::string& line, bool should_log)> callback);
+    std::function<void(const std::string& line, bool should_log)> callback,
+    std::function<void(const BacktraceFrame& frame)> symbolize);

debuggerd/libdebuggerd/test/tombstone_proto_to_text_test.cpp

@@ -61,12 +61,16 @@ class TombstoneProtoToTextTest : public ::testing::Test {
 
   void ProtoToString() {
     text_ = "";
-    EXPECT_TRUE(
-        tombstone_proto_to_text(*tombstone_, [this](const std::string& line, bool should_log) {
+    EXPECT_TRUE(tombstone_proto_to_text(
+        *tombstone_,
+        [this](const std::string& line, bool should_log) {
           if (should_log) {
             text_ += "LOG ";
           }
           text_ += line + '\n';
+        },
+        [&](const BacktraceFrame& frame) {
+          text_ += "SYMBOLIZE " + frame.build_id() + " " + std::to_string(frame.pc()) + "\n";
         }));
   }
 
@@ -163,3 +167,11 @@ TEST_F(TombstoneProtoToTextTest, stack_record) {
   EXPECT_MATCH(text_, "stack_record fp:0x1 tag:0xb pc:foo\\.so\\+0x567 \\(BuildId: ABC123\\)");
   EXPECT_MATCH(text_, "stack_record fp:0x2 tag:0xc pc:bar\\.so\\+0x678");
 }
+
+TEST_F(TombstoneProtoToTextTest, symbolize) {
+  BacktraceFrame* frame = main_thread_->add_current_backtrace();
+  frame->set_pc(12345);
+  frame->set_build_id("0123456789abcdef");
+  ProtoToString();
+  EXPECT_MATCH(text_, "\\(BuildId: 0123456789abcdef\\)\\nSYMBOLIZE 0123456789abcdef 12345\\n");
+}

debuggerd/libdebuggerd/tombstone.cpp

@@ -146,7 +146,10 @@ void engrave_tombstone(unique_fd output_fd, unique_fd proto_fd,
   log.tfd = output_fd.get();
   log.amfd_data = amfd_data;
 
-  tombstone_proto_to_text(tombstone, [&log](const std::string& line, bool should_log) {
-    _LOG(&log, should_log ? logtype::HEADER : logtype::LOGS, "%s\n", line.c_str());
-  });
+  tombstone_proto_to_text(
+      tombstone,
+      [&log](const std::string& line, bool should_log) {
+        _LOG(&log, should_log ? logtype::HEADER : logtype::LOGS, "%s\n", line.c_str());
+      },
+      [](const BacktraceFrame&) {});
 }

debuggerd/libdebuggerd/tombstone_proto_to_text.cpp

@@ -41,6 +41,7 @@ using android::base::StringPrintf;
 #define CBL(...) CB(true, __VA_ARGS__)
 #define CBS(...) CB(false, __VA_ARGS__)
 using CallbackType = std::function<void(const std::string& line, bool should_log)>;
+using SymbolizeCallbackType = std::function<void(const BacktraceFrame& frame)>;
 
 #define DESCRIBE_FLAG(flag) \
   if (value & flag) {       \
@@ -184,7 +185,8 @@ static void print_thread_registers(CallbackType callback, const Tombstone& tombs
   print_register_row(callback, word_size, special_row, should_log);
 }
 
-static void print_backtrace(CallbackType callback, const Tombstone& tombstone,
+static void print_backtrace(CallbackType callback, SymbolizeCallbackType symbolize,
+                            const Tombstone& tombstone,
                             const google::protobuf::RepeatedPtrField<BacktraceFrame>& backtrace,
                             bool should_log) {
   int index = 0;
@@ -209,11 +211,14 @@ static void print_backtrace(CallbackType callback, const Tombstone& tombstone,
     }
     line += function + build_id;
     CB(should_log, "%s", line.c_str());
+
+    symbolize(frame);
   }
 }
 
-static void print_thread_backtrace(CallbackType callback, const Tombstone& tombstone,
-                                   const Thread& thread, bool should_log) {
+static void print_thread_backtrace(CallbackType callback, SymbolizeCallbackType symbolize,
+                                   const Tombstone& tombstone, const Thread& thread,
+                                   bool should_log) {
   CBS("");
   CB(should_log, "%d total frames", thread.current_backtrace().size());
   CB(should_log, "backtrace:");
@@ -221,7 +226,7 @@ static void print_thread_backtrace(CallbackType callback, const Tombstone& tombs
     CB(should_log, "  NOTE: %s",
        android::base::Join(thread.backtrace_note(), "\n  NOTE: ").c_str());
   }
-  print_backtrace(callback, tombstone, thread.current_backtrace(), should_log);
+  print_backtrace(callback, symbolize, tombstone, thread.current_backtrace(), should_log);
 }
 
 static void print_thread_memory_dump(CallbackType callback, const Tombstone& tombstone,
@@ -274,10 +279,11 @@ static void print_thread_memory_dump(CallbackType callback, const Tombstone& tom
   }
 }
 
-static void print_thread(CallbackType callback, const Tombstone& tombstone, const Thread& thread) {
+static void print_thread(CallbackType callback, SymbolizeCallbackType symbolize,
+                         const Tombstone& tombstone, const Thread& thread) {
   print_thread_header(callback, tombstone, thread, false);
   print_thread_registers(callback, tombstone, thread, false);
-  print_thread_backtrace(callback, tombstone, thread, false);
+  print_thread_backtrace(callback, symbolize, tombstone, thread, false);
   print_thread_memory_dump(callback, tombstone, thread);
 }
 
@@ -433,8 +439,8 @@ static std::string oct_encode(const std::string& data) {
   return oct_encoded;
 }
 
-static void print_main_thread(CallbackType callback, const Tombstone& tombstone,
-                              const Thread& thread) {
+static void print_main_thread(CallbackType callback, SymbolizeCallbackType symbolize,
+                              const Tombstone& tombstone, const Thread& thread) {
   print_thread_header(callback, tombstone, thread, true);
 
   const Signal& signal_info = tombstone.signal_info();
@@ -488,7 +494,7 @@ static void print_main_thread(CallbackType callback, const Tombstone& tombstone,
     CBL("      in this process. The stack trace below is the first system call or context");
     CBL("      switch that was executed after the memory corruption happened.");
   }
-  print_thread_backtrace(callback, tombstone, thread, true);
+  print_thread_backtrace(callback, symbolize, tombstone, thread, true);
 
   if (tombstone.causes_size() > 1) {
     CBS("");
@@ -521,13 +527,13 @@ static void print_main_thread(CallbackType callback, const Tombstone& tombstone,
       if (heap_object.deallocation_backtrace_size() != 0) {
         CBS("");
         CBL("deallocated by thread %" PRIu64 ":", heap_object.deallocation_tid());
-        print_backtrace(callback, tombstone, heap_object.deallocation_backtrace(), true);
+        print_backtrace(callback, symbolize, tombstone, heap_object.deallocation_backtrace(), true);
       }
 
       if (heap_object.allocation_backtrace_size() != 0) {
         CBS("");
         CBL("allocated by thread %" PRIu64 ":", heap_object.allocation_tid());
-        print_backtrace(callback, tombstone, heap_object.allocation_backtrace(), true);
+        print_backtrace(callback, symbolize, tombstone, heap_object.allocation_backtrace(), true);
       }
     }
   }
@@ -576,8 +582,9 @@ void print_logs(CallbackType callback, const Tombstone& tombstone, int tail) {
   }
 }
 
-static void print_guest_thread(CallbackType callback, const Tombstone& tombstone,
-                               const Thread& guest_thread, pid_t tid, bool should_log) {
+static void print_guest_thread(CallbackType callback, SymbolizeCallbackType symbolize,
+                               const Tombstone& tombstone, const Thread& guest_thread, pid_t tid,
+                               bool should_log) {
   CBS("--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---");
   CBS("Guest thread information for tid: %d", tid);
   print_thread_registers(callback, tombstone, guest_thread, should_log);
@@ -585,12 +592,13 @@ static void print_guest_thread(CallbackType callback, const Tombstone& tombstone
   CBS("");
   CB(true, "%d total frames", guest_thread.current_backtrace().size());
   CB(true, "backtrace:");
-  print_backtrace(callback, tombstone, guest_thread.current_backtrace(), should_log);
+  print_backtrace(callback, symbolize, tombstone, guest_thread.current_backtrace(), should_log);
 
   print_thread_memory_dump(callback, tombstone, guest_thread);
 }
 
-bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback) {
+bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback,
+                             SymbolizeCallbackType symbolize) {
   CBL("*** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***");
   CBL("Build fingerprint: '%s'", tombstone.build_fingerprint().c_str());
   CBL("Revision: '%s'", tombstone.revision().c_str());
@@ -618,14 +626,15 @@ bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback)
 
   const auto& main_thread = main_thread_it->second;
 
-  print_main_thread(callback, tombstone, main_thread);
+  print_main_thread(callback, symbolize, tombstone, main_thread);
 
   print_logs(callback, tombstone, 50);
 
   const auto& guest_threads = tombstone.guest_threads();
   auto main_guest_thread_it = guest_threads.find(tombstone.tid());
   if (main_guest_thread_it != threads.end()) {
-    print_guest_thread(callback, tombstone, main_guest_thread_it->second, tombstone.tid(), true);
+    print_guest_thread(callback, symbolize, tombstone, main_guest_thread_it->second,
+                       tombstone.tid(), true);
   }
 
   // protobuf's map is unordered, so sort the keys first.
@@ -638,10 +647,10 @@ bool tombstone_proto_to_text(const Tombstone& tombstone, CallbackType callback)
 
   for (const auto& tid : thread_ids) {
     CBS("--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---");
-    print_thread(callback, tombstone, threads.find(tid)->second);
+    print_thread(callback, symbolize, tombstone, threads.find(tid)->second);
     auto guest_thread_it = guest_threads.find(tid);
     if (guest_thread_it != guest_threads.end()) {
-      print_guest_thread(callback, tombstone, guest_thread_it->second, tid, false);
+      print_guest_thread(callback, symbolize, tombstone, guest_thread_it->second, tid, false);
     }
   }
 

debuggerd/pbtombstone.cpp

@@ -16,32 +16,55 @@
 
 #include <err.h>
 #include <fcntl.h>
+#include <getopt.h>
 #include <stdio.h>
 #include <unistd.h>
 
+#include <string>
+#include <vector>
+
 #include <android-base/unique_fd.h>
 #include <libdebuggerd/tombstone_proto_to_text.h>
 
 #include "tombstone.pb.h"
+#include "tombstone_symbolize.h"
 
 using android::base::unique_fd;
 
 [[noreturn]] void usage(bool error) {
-  fprintf(stderr, "usage: pbtombstone TOMBSTONE.PB\n");
+  fprintf(stderr, "usage: pbtombstone [OPTION] TOMBSTONE.PB\n");
   fprintf(stderr, "Convert a protobuf tombstone to text.\n");
+  fprintf(stderr, "Arguments:\n");
+  fprintf(stderr, "  -h, --help                   print this message\n");
+  fprintf(stderr, "  --debug-file-directory PATH  specify the path to a symbols directory\n");
   exit(error);
 }
 
-int main(int argc, const char* argv[]) {
-  if (argc != 2) {
+int main(int argc, char* argv[]) {
+  std::vector<std::string> debug_file_directories;
+  static struct option long_options[] = {
+      {"debug-file-directory", required_argument, 0, 0},
+      {"help", no_argument, 0, 'h'},
+      {},
+  };
+  int c;
+  while ((c = getopt_long(argc, argv, "h", long_options, 0)) != -1) {
+    switch (c) {
+      case 0:
+        debug_file_directories.push_back(optarg);
+        break;
+
+      case 'h':
+        usage(false);
+        break;
+    }
+  }
+
+  if (optind != argc-1) {
     usage(true);
   }
 
-  if (strcmp("-h", argv[1]) == 0 || strcmp("--help", argv[1]) == 0) {
-    usage(false);
-  }
-
-  unique_fd fd(open(argv[1], O_RDONLY | O_CLOEXEC));
+  unique_fd fd(open(argv[optind], O_RDONLY | O_CLOEXEC));
   if (fd == -1) {
     err(1, "failed to open tombstone '%s'", argv[1]);
   }
@@ -51,8 +74,11 @@ int main(int argc, const char* argv[]) {
     err(1, "failed to parse tombstone");
   }
 
+  Symbolizer sym;
+  sym.Start(debug_file_directories);
   bool result = tombstone_proto_to_text(
-      tombstone, [](const std::string& line, bool) { printf("%s\n", line.c_str()); });
+      tombstone, [](const std::string& line, bool) { printf("%s\n", line.c_str()); },
+      [&](const BacktraceFrame& frame) { symbolize_backtrace_frame(frame, sym); });
 
   if (!result) {
     errx(1, "tombstone was malformed");

debuggerd/tombstone_symbolize.cpp

@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "tombstone_symbolize.h"
+
+#include <fcntl.h>
+#include <inttypes.h>
+#include <unistd.h>
+
+#include <string>
+#include <vector>
+
+#include "android-base/stringprintf.h"
+#include "android-base/unique_fd.h"
+
+#include "tombstone.pb.h"
+
+using android::base::StringPrintf;
+using android::base::unique_fd;
+
+bool Symbolizer::Start(const std::vector<std::string>& debug_file_directories) {
+  unique_fd parent_in, parent_out, child_in, child_out;
+  if (!Pipe(&parent_in, &child_out) || !Pipe(&child_in, &parent_out)) {
+    return false;
+  }
+
+  std::vector<const char *> args;
+  args.push_back("llvm-symbolizer");
+  for (const std::string &dir : debug_file_directories) {
+    args.push_back("--debug-file-directory");
+    args.push_back(dir.c_str());
+  }
+  args.push_back(0);
+
+  int pid = fork();
+  if (pid == -1) {
+    return false;
+  } else if (pid == 0) {
+    parent_in.reset();
+    parent_out.reset();
+
+    dup2(child_in.get(), STDIN_FILENO);
+    dup2(child_out.get(), STDOUT_FILENO);
+
+    execvp("llvm-symbolizer", const_cast<char *const *>(args.data()));
+
+    fprintf(stderr, "unable to start llvm-symbolizer: %s\n", strerror(errno));
+    _exit(1);
+  } else {
+    child_in.reset();
+    child_out.reset();
+
+    // TODO: Check that llvm-symbolizer started up successfully.
+    // There used to be an easy way to do this, but it was removed in:
+    // https://github.com/llvm/llvm-project/commit/1792852f86dc75efa1f44d46b1a0daf386d64afa
+
+    in_fd = std::move(parent_in);
+    out_fd = std::move(parent_out);
+    return true;
+  }
+}
+
+std::string Symbolizer::read_response() {
+  std::string resp;
+
+  while (resp.size() < 2 || resp[resp.size() - 2] != '\n' || resp[resp.size() - 1] != '\n') {
+    char buf[4096];
+    ssize_t size = read(in_fd, buf, 4096);
+    if (size <= 0) {
+      return "";
+    }
+    resp.append(buf, size);
+  }
+
+  return resp;
+}
+
+std::vector<Symbolizer::Frame> Symbolizer::SymbolizeCode(std::string path, uint64_t rel_pc) {
+  std::string request = StringPrintf("CODE %s 0x%" PRIx64 "\n", path.c_str(), rel_pc);
+  if (write(out_fd, request.c_str(), request.size()) != static_cast<ssize_t>(request.size())) {
+    return {};
+  }
+
+  std::string response = read_response();
+  if (response.empty()) {
+    return {};
+  }
+
+  std::vector<Symbolizer::Frame> frames;
+
+  size_t frame_start = 0;
+  while (frame_start < response.size() - 1) {
+    Symbolizer::Frame frame;
+
+    size_t second_line_start = response.find('\n', frame_start) + 1;
+    if (second_line_start == std::string::npos + 1) {
+      return {};
+    }
+
+    size_t third_line_start = response.find('\n', second_line_start) + 1;
+    if (third_line_start == std::string::npos + 1) {
+      return {};
+    }
+
+    frame.function_name = response.substr(frame_start, second_line_start - frame_start - 1);
+
+    size_t column_number_start = response.rfind(':', third_line_start);
+    if (column_number_start == std::string::npos) {
+      return {};
+    }
+
+    size_t line_number_start = response.rfind(':', column_number_start - 1);
+    if (line_number_start == std::string::npos) {
+      return {};
+    }
+
+    frame.file = response.substr(second_line_start, line_number_start - second_line_start);
+
+    errno = 0;
+    frame.line = strtoull(response.c_str() + line_number_start + 1, 0, 10);
+    frame.column = strtoull(response.c_str() + column_number_start + 1, 0, 10);
+    if (errno != 0) {
+      return {};
+    }
+
+    frames.push_back(frame);
+
+    frame_start = third_line_start;
+  }
+
+  if (frames.size() == 1 && frames[0].file == "??") {
+    return {};
+  }
+
+  return frames;
+}
+
+void symbolize_backtrace_frame(const BacktraceFrame& frame, Symbolizer& sym) {
+  if (frame.build_id().empty()) {
+    return;
+  }
+
+  for (Symbolizer::Frame f : sym.SymbolizeCode("BUILDID:" + frame.build_id(), frame.rel_pc())) {
+    printf("          %s:%" PRId64 ":%" PRId64 " (%s)\n", f.file.c_str(), f.line, f.column,
+           f.function_name.c_str());
+  }
+}

debuggerd/tombstone_symbolize.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "android-base/unique_fd.h"
+
+class BacktraceFrame;
+
+class Symbolizer {
+  android::base::unique_fd in_fd, out_fd;
+
+  std::string read_response();
+
+ public:
+  bool Start(const std::vector<std::string>& debug_file_directories);
+
+  struct Frame {
+    std::string function_name, file;
+    uint64_t line, column;
+  };
+
+  std::vector<Frame> SymbolizeCode(std::string path, uint64_t rel_pc);
+};
+
+void symbolize_backtrace_frame(const BacktraceFrame& frame, Symbolizer& sym);