[JITLink][AArch64] Add LD64_GOTPAGE_LO15 rel support

This relocation is used in order to address GOT entries using 15 bit
offset in ldr instruction. The offset is calculated relative to GOT
section page address.

Author: yota9

llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h

@@ -233,6 +233,21 @@ enum EdgeKind_aarch64 : Edge::Kind {
   ///     out-of-range error will be returned.
   PageOffset12,
 
+  /// The 15-bit offset of the GOT entry from the GOT table.
+  ///
+  /// Used for load/store instructions addressing a GOT entry.
+  ///
+  /// Fixup expression:
+  ///
+  ///   Fixup <- ((Target + Addend - Page(GOT))) & 0x7fff) >> 3 : uint12
+  ///
+  /// Errors:
+  ///   - The result of the unshifted part of the fixup expression must be
+  ///     aligned otherwise an alignment error will be returned.
+  ///   - The result of the fixup expression must fit into a uint12 otherwise an
+  ///     out-of-range error will be returned.
+  GotPageOffset15,
+
   /// A GOT entry getter/constructor, transformed to Page21 pointing at the GOT
   /// entry for the original target.
   ///
@@ -273,6 +288,23 @@ enum EdgeKind_aarch64 : Edge::Kind {
   ///
   RequestGOTAndTransformToPageOffset12,
 
+  /// A GOT entry getter/constructor, transformed to Pageoffset15 pointing at
+  /// the GOT entry for the original target.
+  ///
+  /// Indicates that this edge should be transformed into a GotPageOffset15
+  /// targeting the GOT entry for the edge's current target, maintaining the
+  /// same addend. A GOT entry for the target should be created if one does not
+  /// already exist.
+  ///
+  /// Fixup expression:
+  ///   NONE
+  ///
+  /// Errors:
+  ///   - *ASSERTION* Failure to handle edges of this kind prior to the fixup
+  ///     phase will result in an assert/unreachable during the fixup phase.
+  ///
+  RequestGOTAndTransformToPageOffset15,
+
   /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT
   /// entry for the original target.
   ///
@@ -430,7 +462,8 @@ inline unsigned getMoveWide16Shift(uint32_t Instr) {
 }
 
 /// Apply fixup expression for edge to block content.
-inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
+inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
+                        const Symbol *GOTSymbol) {
   using namespace support;
 
   char *BlockWorkingMem = B.getAlreadyMutableContent().data();
@@ -603,6 +636,24 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
     *(ulittle32_t *)FixupPtr = FixedInstr;
     break;
   }
+  case GotPageOffset15: {
+    assert(GOTSymbol && "No GOT section symbol");
+    uint64_t TargetOffset =
+        (E.getTarget().getAddress() + E.getAddend()).getValue() -
+        (GOTSymbol->getAddress().getValue() & ~static_cast<uint64_t>(4096 - 1));
+    if (TargetOffset > 0x7fff)
+      return make_error<JITLinkError>("PAGEOFF15 target is out of range");
+
+    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+    const unsigned ImmShift = 3;
+    if (TargetOffset & ((1 << ImmShift) - 1))
+      return make_error<JITLinkError>("PAGEOFF15 target is not aligned");
+
+    uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
+    uint32_t FixedInstr = RawInstr | EncodedImm;
+    *(ulittle32_t *)FixupPtr = FixedInstr;
+    break;
+  }
   default:
     return make_error<JITLinkError>(
         "In graph " + G.getName() + ", section " + B.getSection().getName() +
@@ -701,6 +752,15 @@ class GOTTableManager : public TableManager<GOTTableManager> {
              "RawInstr isn't a 64-bit LDR immediate");
       break;
     }
+    case aarch64::RequestGOTAndTransformToPageOffset15: {
+      KindToSet = aarch64::GotPageOffset15;
+      uint32_t RawInstr = *(const support::ulittle32_t *)FixupPtr;
+      (void)RawInstr;
+      assert(E.getAddend() == 0 && "GOTPageOffset15 with non-zero addend");
+      assert((RawInstr & 0xfffffc00) == 0xf9400000 &&
+             "RawInstr isn't a 64-bit LDR immediate");
+      break;
+    }
     case aarch64::RequestGOTAndTransformToDelta32: {
       KindToSet = aarch64::Delta32;
       break;

llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp

@@ -29,18 +29,92 @@ using namespace llvm::jitlink;
 
 namespace {
 
+constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
+
 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
   friend class JITLinker<ELFJITLinker_aarch64>;
 
 public:
   ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx,
                        std::unique_ptr<LinkGraph> G,
                        PassConfiguration PassConfig)
-      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
+      : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
+    if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))
+      getPassConfig().PostAllocationPasses.push_back(
+          [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
+  }
 
 private:
+  Symbol *GOTSymbol = nullptr;
+
   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
-    return aarch64::applyFixup(G, B, E);
+    return aarch64::applyFixup(G, B, E, GOTSymbol);
+  }
+
+  Error getOrCreateGOTSymbol(LinkGraph &G) {
+    auto DefineExternalGOTSymbolIfPresent =
+        createDefineExternalSectionStartAndEndSymbolsPass(
+            [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
+              if (Sym.getName() == ELFGOTSymbolName)
+                if (auto *GOTSection = G.findSectionByName(
+                        aarch64::GOTTableManager::getSectionName())) {
+                  GOTSymbol = &Sym;
+                  return {*GOTSection, true};
+                }
+              return {};
+            });
+
+    // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
+    // external.
+    if (auto Err = DefineExternalGOTSymbolIfPresent(G))
+      return Err;
+
+    // If we succeeded then we're done.
+    if (GOTSymbol)
+      return Error::success();
+
+    // Otherwise look for a GOT section: If it already has a start symbol we'll
+    // record it, otherwise we'll create our own.
+    // If there's a GOT section but we didn't find an external GOT symbol...
+    if (auto *GOTSection =
+            G.findSectionByName(aarch64::GOTTableManager::getSectionName())) {
+
+      // Check for an existing defined symbol.
+      for (auto *Sym : GOTSection->symbols())
+        if (Sym->getName() == ELFGOTSymbolName) {
+          GOTSymbol = Sym;
+          return Error::success();
+        }
+
+      // If there's no defined symbol then create one.
+      SectionRange SR(*GOTSection);
+      if (SR.empty())
+        GOTSymbol =
+            &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
+                                 Linkage::Strong, Scope::Local, true);
+      else
+        GOTSymbol =
+            &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
+                                Linkage::Strong, Scope::Local, false, true);
+    }
+
+    // If we still haven't found a GOT symbol then double check the externals.
+    // We may have a GOT-relative reference but no GOT section, in which case
+    // we just need to point the GOT symbol at some address in this graph.
+    if (!GOTSymbol) {
+      for (auto *Sym : G.external_symbols()) {
+        if (Sym->getName() == ELFGOTSymbolName) {
+          auto Blocks = G.blocks();
+          if (!Blocks.empty()) {
+            G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
+            GOTSymbol = Sym;
+            break;
+          }
+        }
+      }
+    }
+
+    return Error::success();
   }
 };
 
@@ -70,6 +144,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
     ELFPrel64,
     ELFAdrGOTPage21,
     ELFLd64GOTLo12,
+    ELFLd64GOTPAGELo15,
     ELFTLSDescAdrPage21,
     ELFTLSDescAddLo12,
     ELFTLSDescLd64Lo12,
@@ -125,6 +200,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       return ELFAdrGOTPage21;
     case ELF::R_AARCH64_LD64_GOT_LO12_NC:
       return ELFLd64GOTLo12;
+    case ELF::R_AARCH64_LD64_GOTPAGE_LO15:
+      return ELFLd64GOTPAGELo15;
     case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
       return ELFTLSDescAdrPage21;
     case ELF::R_AARCH64_TLSDESC_ADD_LO12:
@@ -362,6 +439,10 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       Kind = aarch64::RequestGOTAndTransformToPageOffset12;
       break;
     }
+    case ELFLd64GOTPAGELo15: {
+      Kind = aarch64::RequestGOTAndTransformToPageOffset15;
+      break;
+    }
     case ELFTLSDescAdrPage21: {
       Kind = aarch64::RequestTLSDescEntryAndTransformToPage21;
       break;
@@ -427,6 +508,8 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
       return "ELFAdrGOTPage21";
     case ELFLd64GOTLo12:
       return "ELFLd64GOTLo12";
+    case ELFLd64GOTPAGELo15:
+      return "ELFLd64GOTPAGELo15";
     case ELFTLSDescAdrPage21:
       return "ELFTLSDescAdrPage21";
     case ELFTLSDescAddLo12:

llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp

@@ -552,7 +552,7 @@ class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> {
 
 private:
   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
-    return aarch64::applyFixup(G, B, E);
+    return aarch64::applyFixup(G, B, E, nullptr);
   }
 
   uint64_t NullValue = 0;

llvm/lib/ExecutionEngine/JITLink/aarch64.cpp

@@ -57,10 +57,14 @@ const char *getEdgeKindName(Edge::Kind R) {
     return "Page21";
   case PageOffset12:
     return "PageOffset12";
+  case GotPageOffset15:
+    return "GotPageOffset15";
   case RequestGOTAndTransformToPage21:
     return "RequestGOTAndTransformToPage21";
   case RequestGOTAndTransformToPageOffset12:
     return "RequestGOTAndTransformToPageOffset12";
+  case RequestGOTAndTransformToPageOffset15:
+    return "RequestGOTAndTransformToPageOffset15";
   case RequestGOTAndTransformToDelta32:
     return "RequestGOTAndTransformToDelta32";
   case RequestTLVPAndTransformToPage21:

llvm/test/ExecutionEngine/JITLink/AArch64/ELF_relocations.s

@@ -264,6 +264,20 @@ test_ld64_gotlo12_external:
         ldr   x0, [x0, :got_lo12:external_data]
         .size test_ld64_gotlo12_external, .-test_ld64_gotlo12_external
 
+# Check R_AARCH64_LD64_GOTPAGE_LO15 handling with a reference to an external
+# symbol. Validate the reference to the GOT entry.
+# For the LDR :gotpage_lo15: instruction we have the 15-bit offset of the GOT
+# entry from the page containing the GOT.
+# jitlink-check: decode_operand(test_ld64_gotpagelo15_external, 2) = \
+# jitlink-check:     (got_addr(elf_reloc.o, external_data) - \
+# jitlink-check:     (section_addr(elf_reloc.o, $__GOT) & 0xfffffffffffff000)) \
+# jitlink-check:     [15:3]
+        .globl  test_ld64_gotpagelo15_external
+        .p2align  2
+test_ld64_gotpagelo15_external:
+        ldr   x0, [x0, :gotpage_lo15:external_data]
+        .size test_ld64_gotpagelo15_external, .-test_ld64_gotpagelo15_external
+
 # Check R_AARCH64_TSTBR14 for tbz
 #
 # jitlink-check: decode_operand(test_tstbr14_tbz, 2) = \