//===- DwarfTransformer.cpp -----------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include <thread>
#include <unordered_set>
#include "llvm/DebugInfo/DIContext.h"
#include "llvm/DebugInfo/DWARF/DWARFContext.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/DebugInfo/GSYM/DwarfTransformer.h"
#include "llvm/DebugInfo/GSYM/FunctionInfo.h"
#include "llvm/DebugInfo/GSYM/GsymCreator.h"
#include "llvm/DebugInfo/GSYM/GsymReader.h"
#include "llvm/DebugInfo/GSYM/InlineInfo.h"
using namespace llvm;
using namespace gsym;
struct llvm::gsym::CUInfo {
const DWARFDebugLine::LineTable *LineTable;
const char *CompDir;
std::vector<uint32_t> FileCache;
uint64_t Language = 0;
uint8_t AddrSize = 0;
CUInfo(DWARFContext &DICtx, DWARFCompileUnit *CU) {
LineTable = DICtx.getLineTableForUnit(CU);
CompDir = CU->getCompilationDir();
FileCache.clear();
if (LineTable)
FileCache.assign(LineTable->Prologue.FileNames.size() + 1, UINT32_MAX);
DWARFDie Die = CU->getUnitDIE();
Language = dwarf::toUnsigned(Die.find(dwarf::DW_AT_language), 0);
AddrSize = CU->getAddressByteSize();
}
/// Return true if Addr is the highest address for a given compile unit. The
/// highest address is encoded as -1, of all ones in the address. These high
/// addresses are used by some linkers to indicate that a function has been
/// dead stripped or didn't end up in the linked executable.
bool isHighestAddress(uint64_t Addr) const {
if (AddrSize == 4)
return Addr == UINT32_MAX;
else if (AddrSize == 8)
return Addr == UINT64_MAX;
return false;
}
/// Convert a DWARF compile unit file index into a GSYM global file index.
///
/// Each compile unit in DWARF has its own file table in the line table
/// prologue. GSYM has a single large file table that applies to all files
/// from all of the info in a GSYM file. This function converts between the
/// two and caches and DWARF CU file index that has already been converted so
/// the first client that asks for a compile unit file index will end up
/// doing the conversion, and subsequent clients will get the cached GSYM
/// index.
uint32_t DWARFToGSYMFileIndex(GsymCreator &Gsym, uint32_t DwarfFileIdx) {
if (!LineTable)
return 0;
assert(DwarfFileIdx < FileCache.size());
uint32_t &GsymFileIdx = FileCache[DwarfFileIdx];
if (GsymFileIdx != UINT32_MAX)
return GsymFileIdx;
std::string File;
if (LineTable->getFileNameByIndex(
DwarfFileIdx, CompDir,
DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath, File))
GsymFileIdx = Gsym.insertFile(File);
else
GsymFileIdx = 0;
return GsymFileIdx;
}
};
static DWARFDie GetParentDeclContextDIE(DWARFDie &Die) {
if (DWARFDie SpecDie =
Die.getAttributeValueAsReferencedDie(dwarf::DW_AT_specification)) {
if (DWARFDie SpecParent = GetParentDeclContextDIE(SpecDie))
return SpecParent;
}
if (DWARFDie AbstDie =
Die.getAttributeValueAsReferencedDie(dwarf::DW_AT_abstract_origin)) {
if (DWARFDie AbstParent = GetParentDeclContextDIE(AbstDie))
return AbstParent;
}
// We never want to follow parent for inlined subroutine - that would
// give us information about where the function is inlined, not what
// function is inlined
if (Die.getTag() == dwarf::DW_TAG_inlined_subroutine)
return DWARFDie();
DWARFDie ParentDie = Die.getParent();
if (!ParentDie)
return DWARFDie();
switch (ParentDie.getTag()) {
case dwarf::DW_TAG_namespace:
case dwarf::DW_TAG_structure_type:
case dwarf::DW_TAG_union_type:
case dwarf::DW_TAG_class_type:
case dwarf::DW_TAG_subprogram:
return ParentDie; // Found parent decl context DIE
case dwarf::DW_TAG_lexical_block:
return GetParentDeclContextDIE(ParentDie);
default:
break;
}
return DWARFDie();
}
/// Get the GsymCreator string table offset for the qualified name for the
/// DIE passed in. This function will avoid making copies of any strings in
/// the GsymCreator when possible. We don't need to copy a string when the
/// string comes from our .debug_str section or is an inlined string in the
/// .debug_info. If we create a qualified name string in this function by
/// combining multiple strings in the DWARF string table or info, we will make
/// a copy of the string when we add it to the string table.
static Optional<uint32_t> getQualifiedNameIndex(DWARFDie &Die,
uint64_t Language,
GsymCreator &Gsym) {
// If the dwarf has mangled name, use mangled name
if (auto LinkageName =
dwarf::toString(Die.findRecursively({dwarf::DW_AT_MIPS_linkage_name,
dwarf::DW_AT_linkage_name}),
nullptr))
return Gsym.insertString(LinkageName, /* Copy */ false);
StringRef ShortName(Die.getName(DINameKind::ShortName));
if (ShortName.empty())
return llvm::None;
// For C++ and ObjC, prepend names of all parent declaration contexts
if (!(Language == dwarf::DW_LANG_C_plus_plus ||
Language == dwarf::DW_LANG_C_plus_plus_03 ||
Language == dwarf::DW_LANG_C_plus_plus_11 ||
Language == dwarf::DW_LANG_C_plus_plus_14 ||
Language == dwarf::DW_LANG_ObjC_plus_plus ||
// This should not be needed for C, but we see C++ code marked as C
// in some binaries. This should hurt, so let's do it for C as well
Language == dwarf::DW_LANG_C))
return Gsym.insertString(ShortName, /* Copy */ false);
// Some GCC optimizations create functions with names ending with .isra.<num>
// or .part.<num> and those names are just DW_AT_name, not DW_AT_linkage_name
// If it looks like it could be the case, don't add any prefix
if (ShortName.startswith("_Z") &&
(ShortName.contains(".isra.") || ShortName.contains(".part.")))
return Gsym.insertString(ShortName, /* Copy */ false);
DWARFDie ParentDeclCtxDie = GetParentDeclContextDIE(Die);
if (ParentDeclCtxDie) {
std::string Name = ShortName.str();
while (ParentDeclCtxDie) {
StringRef ParentName(ParentDeclCtxDie.getName(DINameKind::ShortName));
if (!ParentName.empty()) {
// "lambda" names are wrapped in < >. Replace with { }
// to be consistent with demangled names and not to confuse with
// templates
if (ParentName.front() == '<' && ParentName.back() == '>')
Name = "{" + ParentName.substr(1, ParentName.size() - 2).str() + "}" +
"::" + Name;
else
Name = ParentName.str() + "::" + Name;
}
ParentDeclCtxDie = GetParentDeclContextDIE(ParentDeclCtxDie);
}
// Copy the name since we created a new name in a std::string.
return Gsym.insertString(Name, /* Copy */ true);
}
// Don't copy the name since it exists in the DWARF object file.
return Gsym.insertString(ShortName, /* Copy */ false);
}
static bool hasInlineInfo(DWARFDie Die, uint32_t Depth) {
bool CheckChildren = true;
switch (Die.getTag()) {
case dwarf::DW_TAG_subprogram:
// Don't look into functions within functions.
CheckChildren = Depth == 0;
break;
case dwarf::DW_TAG_inlined_subroutine:
return true;
default:
break;
}
if (!CheckChildren)
return false;
for (DWARFDie ChildDie : Die.children()) {
if (hasInlineInfo(ChildDie, Depth + 1))
return true;
}
return false;
}
static void parseInlineInfo(GsymCreator &Gsym, CUInfo &CUI, DWARFDie Die,
uint32_t Depth, FunctionInfo &FI,
InlineInfo &parent) {
if (!hasInlineInfo(Die, Depth))
return;
dwarf::Tag Tag = Die.getTag();
if (Tag == dwarf::DW_TAG_inlined_subroutine) {
// create new InlineInfo and append to parent.children
InlineInfo II;
DWARFAddressRange FuncRange =
DWARFAddressRange(FI.startAddress(), FI.endAddress());
Expected<DWARFAddressRangesVector> RangesOrError = Die.getAddressRanges();
if (RangesOrError) {
for (const DWARFAddressRange &Range : RangesOrError.get()) {
// Check that the inlined function is within the range of the function
// info, it might not be in case of split functions
if (FuncRange.LowPC <= Range.LowPC && Range.HighPC <= FuncRange.HighPC)
II.Ranges.insert(AddressRange(Range.LowPC, Range.HighPC));
}
}
if (II.Ranges.empty())
return;
if (auto NameIndex = getQualifiedNameIndex(Die, CUI.Language, Gsym))
II.Name = *NameIndex;
II.CallFile = CUI.DWARFToGSYMFileIndex(
Gsym, dwarf::toUnsigned(Die.find(dwarf::DW_AT_call_file), 0));
II.CallLine = dwarf::toUnsigned(Die.find(dwarf::DW_AT_call_line), 0);
// parse all children and append to parent
for (DWARFDie ChildDie : Die.children())
parseInlineInfo(Gsym, CUI, ChildDie, Depth + 1, FI, II);
parent.Children.emplace_back(std::move(II));
return;
}
if (Tag == dwarf::DW_TAG_subprogram || Tag == dwarf::DW_TAG_lexical_block) {
// skip this Die and just recurse down
for (DWARFDie ChildDie : Die.children())
parseInlineInfo(Gsym, CUI, ChildDie, Depth + 1, FI, parent);
}
}
static void convertFunctionLineTable(raw_ostream &Log, CUInfo &CUI,
DWARFDie Die, GsymCreator &Gsym,
FunctionInfo &FI) {
std::vector<uint32_t> RowVector;
const uint64_t StartAddress = FI.startAddress();
const uint64_t EndAddress = FI.endAddress();
const uint64_t RangeSize = EndAddress - StartAddress;
const object::SectionedAddress SecAddress{
StartAddress, object::SectionedAddress::UndefSection};
if (!CUI.LineTable->lookupAddressRange(SecAddress, RangeSize, RowVector)) {
// If we have a DW_TAG_subprogram but no line entries, fall back to using
// the DW_AT_decl_file an d DW_AT_decl_line if we have both attributes.
if (auto FileIdx =
dwarf::toUnsigned(Die.findRecursively({dwarf::DW_AT_decl_file}))) {
if (auto Line =
dwarf::toUnsigned(Die.findRecursively({dwarf::DW_AT_decl_line}))) {
LineEntry LE(StartAddress, CUI.DWARFToGSYMFileIndex(Gsym, *FileIdx),
*Line);
FI.OptLineTable = LineTable();
FI.OptLineTable->push(LE);
// LE.Addr = EndAddress;
// FI.OptLineTable->push(LE);
}
}
return;
}
FI.OptLineTable = LineTable();
DWARFDebugLine::Row PrevRow;
for (uint32_t RowIndex : RowVector) {
// Take file number and line/column from the row.
const DWARFDebugLine::Row &Row = CUI.LineTable->Rows[RowIndex];
const uint32_t FileIdx = CUI.DWARFToGSYMFileIndex(Gsym, Row.File);
uint64_t RowAddress = Row.Address.Address;
// Watch out for a RowAddress that is in the middle of a line table entry
// in the DWARF. If we pass an address in between two line table entries
// we will get a RowIndex for the previous valid line table row which won't
// be contained in our function. This is usually a bug in the DWARF due to
// linker problems or LTO or other DWARF re-linking so it is worth emitting
// an error, but not worth stopping the creation of the GSYM.
if (!FI.Range.contains(RowAddress)) {
if (RowAddress < FI.Range.Start) {
Log << "error: DIE has a start address whose LowPC is between the "
"line table Row[" << RowIndex << "] with address "
<< HEX64(RowAddress) << " and the next one.\n";
Die.dump(Log, 0, DIDumpOptions::getForSingleDIE());
RowAddress = FI.Range.Start;
} else {
continue;
}
}
LineEntry LE(RowAddress, FileIdx, Row.Line);
if (RowIndex != RowVector[0] && Row.Address < PrevRow.Address) {
// We have seen full duplicate line tables for functions in some
// DWARF files. Watch for those here by checking the the last
// row was the function's end address (HighPC) and that the
// current line table entry's address is the same as the first
// line entry we already have in our "function_info.Lines". If
// so break out after printing a warning.
auto FirstLE = FI.OptLineTable->first();
if (FirstLE && *FirstLE == LE) {
Log << "warning: duplicate line table detected for DIE:\n";
Die.dump(Log, 0, DIDumpOptions::getForSingleDIE());
} else {
// Print out (ignore if os == nulls as this is expensive)
Log << "error: line table has addresses that do not "
<< "monotonically increase:\n";
for (uint32_t RowIndex2 : RowVector) {
CUI.LineTable->Rows[RowIndex2].dump(Log);
}
Die.dump(Log, 0, DIDumpOptions::getForSingleDIE());
}
break;
}
// Skip multiple line entries for the same file and line.
auto LastLE = FI.OptLineTable->last();
if (LastLE && LastLE->File == FileIdx && LastLE->Line == Row.Line)
continue;
// Only push a row if it isn't an end sequence. End sequence markers are
// included for the last address in a function or the last contiguous
// address in a sequence.
if (Row.EndSequence) {
// End sequence means that the next line entry could have a lower address
// that the previous entries. So we clear the previous row so we don't
// trigger the line table error about address that do not monotonically
// increase.
PrevRow = DWARFDebugLine::Row();
} else {
FI.OptLineTable->push(LE);
PrevRow = Row;
}
}
// If not line table rows were added, clear the line table so we don't encode
// on in the GSYM file.
if (FI.OptLineTable->empty())
FI.OptLineTable = llvm::None;
}
void DwarfTransformer::handleDie(raw_ostream &OS, CUInfo &CUI, DWARFDie Die) {
switch (Die.getTag()) {
case dwarf::DW_TAG_subprogram: {
Expected<DWARFAddressRangesVector> RangesOrError = Die.getAddressRanges();
if (!RangesOrError) {
consumeError(RangesOrError.takeError());
break;
}
const DWARFAddressRangesVector &Ranges = RangesOrError.get();
if (Ranges.empty())
break;
auto NameIndex = getQualifiedNameIndex(Die, CUI.Language, Gsym);
if (!NameIndex) {
OS << "error: function at " << HEX64(Die.getOffset())
<< " has no name\n ";
Die.dump(OS, 0, DIDumpOptions::getForSingleDIE());
break;
}
// Create a function_info for each range
for (const DWARFAddressRange &Range : Ranges) {
// The low PC must be less than the high PC. Many linkers don't remove
// DWARF for functions that don't get linked into the final executable.
// If both the high and low pc have relocations, linkers will often set
// the address values for both to the same value to indicate the function
// has been remove. Other linkers have been known to set the one or both
// PC values to a UINT32_MAX for 4 byte addresses and UINT64_MAX for 8
// byte addresses to indicate the function isn't valid. The check below
// tries to watch for these cases and abort if it runs into them.
if (Range.LowPC >= Range.HighPC || CUI.isHighestAddress(Range.LowPC))
break;
// Many linkers can't remove DWARF and might set the LowPC to zero. Since
// high PC can be an offset from the low PC in more recent DWARF versions
// we need to watch for a zero'ed low pc which we do using
// ValidTextRanges below.
if (!Gsym.IsValidTextAddress(Range.LowPC)) {
// We expect zero and -1 to be invalid addresses in DWARF depending
// on the linker of the DWARF. This indicates a function was stripped
// and the debug info wasn't able to be stripped from the DWARF. If
// the LowPC isn't zero or -1, then we should emit an error.
if (Range.LowPC != 0) {
// Unexpected invalid address, emit an error
Log << "warning: DIE has an address range whose start address is "
"not in any executable sections (" <<
*Gsym.GetValidTextRanges() << ") and will not be processed:\n";
Die.dump(Log, 0, DIDumpOptions::getForSingleDIE());
}
break;
}
FunctionInfo FI;
FI.setStartAddress(Range.LowPC);
FI.setEndAddress(Range.HighPC);
FI.Name = *NameIndex;
if (CUI.LineTable) {
convertFunctionLineTable(OS, CUI, Die, Gsym, FI);
}
if (hasInlineInfo(Die, 0)) {
FI.Inline = InlineInfo();
FI.Inline->Name = *NameIndex;
FI.Inline->Ranges.insert(FI.Range);
parseInlineInfo(Gsym, CUI, Die, 0, FI, *FI.Inline);
}
Gsym.addFunctionInfo(std::move(FI));
}
} break;
default:
break;
}
for (DWARFDie ChildDie : Die.children())
handleDie(OS, CUI, ChildDie);
}
Error DwarfTransformer::convert(uint32_t NumThreads) {
size_t NumBefore = Gsym.getNumFunctionInfos();
if (NumThreads == 1) {
// Parse all DWARF data from this thread, use the same string/file table
// for everything
for (const auto &CU : DICtx.compile_units()) {
DWARFDie Die = CU->getUnitDIE(false);
CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
handleDie(Log, CUI, Die);
}
} else {
// LLVM Dwarf parser is not thread-safe and we need to parse all DWARF up
// front before we start accessing any DIEs since there might be
// cross compile unit references in the DWARF. If we don't do this we can
// end up crashing.
// We need to call getAbbreviations sequentially first so that getUnitDIE()
// only works with its local data.
for (const auto &CU : DICtx.compile_units())
CU->getAbbreviations();
// Now parse all DIEs in case we have cross compile unit references in a
// thread pool.
ThreadPool pool(hardware_concurrency(NumThreads));
for (const auto &CU : DICtx.compile_units())
pool.async([&CU]() { CU->getUnitDIE(false /*CUDieOnly*/); });
pool.wait();
// Now convert all DWARF to GSYM in a thread pool.
std::mutex LogMutex;
for (const auto &CU : DICtx.compile_units()) {
DWARFDie Die = CU->getUnitDIE(false /*CUDieOnly*/);
if (Die) {
CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
pool.async([this, CUI, &LogMutex, Die]() mutable {
std::string ThreadLogStorage;
raw_string_ostream ThreadOS(ThreadLogStorage);
handleDie(ThreadOS, CUI, Die);
ThreadOS.flush();
if (!ThreadLogStorage.empty()) {
// Print ThreadLogStorage lines into an actual stream under a lock
std::lock_guard<std::mutex> guard(LogMutex);
Log << ThreadLogStorage;
}
});
}
}
pool.wait();
}
size_t FunctionsAddedCount = Gsym.getNumFunctionInfos() - NumBefore;
Log << "Loaded " << FunctionsAddedCount << " functions from DWARF.\n";
return Error::success();
}
llvm::Error DwarfTransformer::verify(StringRef GsymPath) {
Log << "Verifying GSYM file \"" << GsymPath << "\":\n";
auto Gsym = GsymReader::openFile(GsymPath);
if (!Gsym)
return Gsym.takeError();
auto NumAddrs = Gsym->getNumAddresses();
DILineInfoSpecifier DLIS(
DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath,
DILineInfoSpecifier::FunctionNameKind::LinkageName);
std::string gsymFilename;
for (uint32_t I = 0; I < NumAddrs; ++I) {
auto FuncAddr = Gsym->getAddress(I);
if (!FuncAddr)
return createStringError(std::errc::invalid_argument,
"failed to extract address[%i]", I);
auto FI = Gsym->getFunctionInfo(*FuncAddr);
if (!FI)
return createStringError(std::errc::invalid_argument,
"failed to extract function info for address 0x%"
PRIu64, *FuncAddr);
for (auto Addr = *FuncAddr; Addr < *FuncAddr + FI->size(); ++Addr) {
const object::SectionedAddress SectAddr{
Addr, object::SectionedAddress::UndefSection};
auto LR = Gsym->lookup(Addr);
if (!LR)
return LR.takeError();
auto DwarfInlineInfos =
DICtx.getInliningInfoForAddress(SectAddr, DLIS);
uint32_t NumDwarfInlineInfos = DwarfInlineInfos.getNumberOfFrames();
if (NumDwarfInlineInfos == 0) {
DwarfInlineInfos.addFrame(
DICtx.getLineInfoForAddress(SectAddr, DLIS));
}
// Check for 1 entry that has no file and line info
if (NumDwarfInlineInfos == 1 &&
DwarfInlineInfos.getFrame(0).FileName == "<invalid>") {
DwarfInlineInfos = DIInliningInfo();
NumDwarfInlineInfos = 0;
}
if (NumDwarfInlineInfos > 0 &&
NumDwarfInlineInfos != LR->Locations.size()) {
Log << "error: address " << HEX64(Addr) << " has "
<< NumDwarfInlineInfos << " DWARF inline frames and GSYM has "
<< LR->Locations.size() << "\n";
Log << " " << NumDwarfInlineInfos << " DWARF frames:\n";
for (size_t Idx = 0; Idx < NumDwarfInlineInfos; ++Idx) {
const auto dii = DwarfInlineInfos.getFrame(Idx);
Log << " [" << Idx << "]: " << dii.FunctionName << " @ "
<< dii.FileName << ':' << dii.Line << '\n';
}
Log << " " << LR->Locations.size() << " GSYM frames:\n";
for (size_t Idx = 0, count = LR->Locations.size();
Idx < count; ++Idx) {
const auto &gii = LR->Locations[Idx];
Log << " [" << Idx << "]: " << gii.Name << " @ " << gii.Dir
<< '/' << gii.Base << ':' << gii.Line << '\n';
}
DwarfInlineInfos = DICtx.getInliningInfoForAddress(SectAddr, DLIS);
Gsym->dump(Log, *FI);
continue;
}
for (size_t Idx = 0, count = LR->Locations.size(); Idx < count;
++Idx) {
const auto &gii = LR->Locations[Idx];
if (Idx < NumDwarfInlineInfos) {
const auto dii = DwarfInlineInfos.getFrame(Idx);
gsymFilename = LR->getSourceFile(Idx);
// Verify function name
if (dii.FunctionName.find(gii.Name.str()) != 0)
Log << "error: address " << HEX64(Addr) << " DWARF function \""
<< dii.FunctionName.c_str()
<< "\" doesn't match GSYM function \"" << gii.Name << "\"\n";
// Verify source file path
if (dii.FileName != gsymFilename)
Log << "error: address " << HEX64(Addr) << " DWARF path \""
<< dii.FileName.c_str() << "\" doesn't match GSYM path \""
<< gsymFilename.c_str() << "\"\n";
// Verify source file line
if (dii.Line != gii.Line)
Log << "error: address " << HEX64(Addr) << " DWARF line "
<< dii.Line << " != GSYM line " << gii.Line << "\n";
}
}
}
}
return Error::success();
}