ItaniumCXXABI.cpp

Go to the documentation of this file.

//===------- ItaniumCXXABI.cpp - Emit LLVM Code from ASTs for a Module ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This provides C++ code generation targeting the Itanium C++ ABI.  The class
// in this file generates structures that follow the Itanium C++ ABI, which is
// documented at:
//  https://itanium-cxx-abi.github.io/cxx-abi/abi.html
//  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
//
// It also supports the closely-related ARM ABI, documented at:
// https://developer.arm.com/documentation/ihi0041/g/
//
//===----------------------------------------------------------------------===//
 
#include "CGCXXABI.h"
#include "CGCleanup.h"
#include "CGRecordLayout.h"
#include "CGVTables.h"
#include "CodeGenFunction.h"
#include "CodeGenModule.h"
#include "TargetInfo.h"
#include "clang/AST/Attr.h"
#include "clang/AST/Mangle.h"
#include "clang/AST/StmtCXX.h"
#include "clang/AST/Type.h"
#include "clang/CodeGen/ConstantInitBuilder.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/ScopedPrinter.h"
 
#include <optional>
 
using namespace clang;
using namespace CodeGen;
 
namespace {
class ItaniumCXXABI : public CodeGen::CGCXXABI {
  /// VTables - All the vtables which have been defined.
  llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
 
  /// All the thread wrapper functions that have been used.
  llvm::SmallVector<std::pair<const VarDecl *, llvm::Function *>, 8>
      ThreadWrappers;
 
protected:
  bool UseARMMethodPtrABI;
  bool UseARMGuardVarABI;
  bool Use32BitVTableOffsetABI;
 
  ItaniumMangleContext &getMangleContext() {
    return cast<ItaniumMangleContext>(CodeGen::CGCXXABI::getMangleContext());
  }
 
public:
  ItaniumCXXABI(CodeGen::CodeGenModule &CGM,
                bool UseARMMethodPtrABI = false,
                bool UseARMGuardVarABI = false) :
    CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI),
    UseARMGuardVarABI(UseARMGuardVarABI),
    Use32BitVTableOffsetABI(false) { }
 
  bool classifyReturnType(CGFunctionInfo &FI) const override;
 
  RecordArgABI getRecordArgABI(const CXXRecordDecl *RD) const override {
    // If C++ prohibits us from making a copy, pass by address.
    if (!RD->canPassInRegisters())
      return RAA_Indirect;
    return RAA_Default;
  }
 
  bool isThisCompleteObject(GlobalDecl GD) const override {
    // The Itanium ABI has separate complete-object vs.  base-object
    // variants of both constructors and destructors.
    if (isa<CXXDestructorDecl>(GD.getDecl())) {
      switch (GD.getDtorType()) {
      case Dtor_Complete:
      case Dtor_Deleting:
        return true;
 
      case Dtor_Base:
        return false;
 
      case Dtor_Comdat:
        llvm_unreachable("emitting dtor comdat as function?");
      }
      llvm_unreachable("bad dtor kind");
    }
    if (isa<CXXConstructorDecl>(GD.getDecl())) {
      switch (GD.getCtorType()) {
      case Ctor_Complete:
        return true;
 
      case Ctor_Base:
        return false;
 
      case Ctor_CopyingClosure:
      case Ctor_DefaultClosure:
        llvm_unreachable("closure ctors in Itanium ABI?");
 
      case Ctor_Comdat:
        llvm_unreachable("emitting ctor comdat as function?");
      }
      llvm_unreachable("bad dtor kind");
    }
 
    // No other kinds.
    return false;
  }
 
  bool isZeroInitializable(const MemberPointerType *MPT) override;
 
  llvm::Type *ConvertMemberPointerType(const MemberPointerType *MPT) override;
 
  CGCallee
    EmitLoadOfMemberFunctionPointer(CodeGenFunction &CGF,
                                    const Expr *E,
                                    Address This,
                                    llvm::Value *&ThisPtrForCall,
                                    llvm::Value *MemFnPtr,
                                    const MemberPointerType *MPT) override;
 
  llvm::Value *
    EmitMemberDataPointerAddress(CodeGenFunction &CGF, const Expr *E,
                                 Address Base,
                                 llvm::Value *MemPtr,
                                 const MemberPointerType *MPT) override;
 
  llvm::Value *EmitMemberPointerConversion(CodeGenFunction &CGF,
                                           const CastExpr *E,
                                           llvm::Value *Src) override;
  llvm::Constant *EmitMemberPointerConversion(const CastExpr *E,
                                              llvm::Constant *Src) override;
 
  llvm::Constant *EmitNullMemberPointer(const MemberPointerType *MPT) override;
 
  llvm::Constant *EmitMemberFunctionPointer(const CXXMethodDecl *MD) override;
  llvm::Constant *EmitMemberDataPointer(const MemberPointerType *MPT,
                                        CharUnits offset) override;
  llvm::Constant *EmitMemberPointer(const APValue &MP, QualType MPT) override;
  llvm::Constant *BuildMemberPointer(const CXXMethodDecl *MD,
                                     CharUnits ThisAdjustment);
 
  llvm::Value *EmitMemberPointerComparison(CodeGenFunction &CGF,
                                           llvm::Value *L, llvm::Value *R,
                                           const MemberPointerType *MPT,
                                           bool Inequality) override;
 
  llvm::Value *EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
                                         llvm::Value *Addr,
                                         const MemberPointerType *MPT) override;
 
  void emitVirtualObjectDelete(CodeGenFunction &CGF, const CXXDeleteExpr *DE,
                               Address Ptr, QualType ElementType,
                               const CXXDestructorDecl *Dtor) override;
 
  void emitRethrow(CodeGenFunction &CGF, bool isNoReturn) override;
  void emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) override;
 
  void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
 
  llvm::CallInst *
  emitTerminateForUnexpectedException(CodeGenFunction &CGF,
                                      llvm::Value *Exn) override;
 
  void EmitFundamentalRTTIDescriptors(const CXXRecordDecl *RD);
  llvm::Constant *getAddrOfRTTIDescriptor(QualType Ty) override;
  CatchTypeInfo
  getAddrOfCXXCatchHandlerType(QualType Ty,
                               QualType CatchHandlerType) override {
    return CatchTypeInfo{getAddrOfRTTIDescriptor(Ty), 0};
  }
 
  bool shouldTypeidBeNullChecked(QualType SrcRecordTy) override;
  void EmitBadTypeidCall(CodeGenFunction &CGF) override;
  llvm::Value *EmitTypeid(CodeGenFunction &CGF, QualType SrcRecordTy,
                          Address ThisPtr,
                          llvm::Type *StdTypeInfoPtrTy) override;
 
  bool shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
                                          QualType SrcRecordTy) override;
 
  /// Determine whether we know that all instances of type RecordTy will have
  /// the same vtable pointer values, that is distinct from all other vtable
  /// pointers. While this is required by the Itanium ABI, it doesn't happen in
  /// practice in some cases due to language extensions.
  bool hasUniqueVTablePointer(QualType RecordTy) {
    const CXXRecordDecl *RD = RecordTy->getAsCXXRecordDecl();
 
    // Under -fapple-kext, multiple definitions of the same vtable may be
    // emitted.
    if (!CGM.getCodeGenOpts().AssumeUniqueVTables ||
        getContext().getLangOpts().AppleKext)
      return false;
 
    // If the type_info* would be null, the vtable might be merged with that of
    // another type.
    if (!CGM.shouldEmitRTTI())
      return false;
 
    // If there's only one definition of the vtable in the program, it has a
    // unique address.
    if (!llvm::GlobalValue::isWeakForLinker(CGM.getVTableLinkage(RD)))
      return true;
 
    // Even if there are multiple definitions of the vtable, they are required
    // by the ABI to use the same symbol name, so should be merged at load
    // time. However, if the class has hidden visibility, there can be
    // different versions of the class in different modules, and the ABI
    // library might treat them as being the same.
    if (CGM.GetLLVMVisibility(RD->getVisibility()) !=
        llvm::GlobalValue::DefaultVisibility)
      return false;
 
    return true;
  }
 
  bool shouldEmitExactDynamicCast(QualType DestRecordTy) override {
    return hasUniqueVTablePointer(DestRecordTy);
  }
 
  llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
                                   QualType SrcRecordTy, QualType DestTy,
                                   QualType DestRecordTy,
                                   llvm::BasicBlock *CastEnd) override;
 
  llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address ThisAddr,
                                    QualType SrcRecordTy, QualType DestTy,
                                    QualType DestRecordTy,
                                    llvm::BasicBlock *CastSuccess,
                                    llvm::BasicBlock *CastFail) override;
 
  llvm::Value *emitDynamicCastToVoid(CodeGenFunction &CGF, Address Value,
                                     QualType SrcRecordTy) override;
 
  bool EmitBadCastCall(CodeGenFunction &CGF) override;
 
  llvm::Value *
    GetVirtualBaseClassOffset(CodeGenFunction &CGF, Address This,
                              const CXXRecordDecl *ClassDecl,
                              const CXXRecordDecl *BaseClassDecl) override;
 
  void EmitCXXConstructors(const CXXConstructorDecl *D) override;
 
  AddedStructorArgCounts
  buildStructorSignature(GlobalDecl GD,
                         SmallVectorImpl<CanQualType> &ArgTys) override;
 
  bool useThunkForDtorVariant(const CXXDestructorDecl *Dtor,
                              CXXDtorType DT) const override {
    // Itanium does not emit any destructor variant as an inline thunk.
    // Delegating may occur as an optimization, but all variants are either
    // emitted with external linkage or as linkonce if they are inline and used.
    return false;
  }
 
  void EmitCXXDestructors(const CXXDestructorDecl *D) override;
 
  void addImplicitStructorParams(CodeGenFunction &CGF, QualType &ResTy,
                                 FunctionArgList &Params) override;
 
  void EmitInstanceFunctionProlog(CodeGenFunction &CGF) override;
 
  AddedStructorArgs getImplicitConstructorArgs(CodeGenFunction &CGF,
                                               const CXXConstructorDecl *D,
                                               CXXCtorType Type,
                                               bool ForVirtualBase,
                                               bool Delegating) override;
 
  llvm::Value *getCXXDestructorImplicitParam(CodeGenFunction &CGF,
                                             const CXXDestructorDecl *DD,
                                             CXXDtorType Type,
                                             bool ForVirtualBase,
                                             bool Delegating) override;
 
  void EmitDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *DD,
                          CXXDtorType Type, bool ForVirtualBase,
                          bool Delegating, Address This,
                          QualType ThisTy) override;
 
  void emitVTableDefinitions(CodeGenVTables &CGVT,
                             const CXXRecordDecl *RD) override;
 
  bool isVirtualOffsetNeededForVTableField(CodeGenFunction &CGF,
                                           CodeGenFunction::VPtr Vptr) override;
 
  bool doStructorsInitializeVPtrs(const CXXRecordDecl *VTableClass) override {
    return true;
  }
 
  llvm::Constant *
  getVTableAddressPoint(BaseSubobject Base,
                        const CXXRecordDecl *VTableClass) override;
 
  llvm::Value *getVTableAddressPointInStructor(
      CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
      BaseSubobject Base, const CXXRecordDecl *NearestVBase) override;
 
  llvm::Value *getVTableAddressPointInStructorWithVTT(
      CodeGenFunction &CGF, const CXXRecordDecl *VTableClass,
      BaseSubobject Base, const CXXRecordDecl *NearestVBase);
 
  llvm::GlobalVariable *getAddrOfVTable(const CXXRecordDecl *RD,
                                        CharUnits VPtrOffset) override;
 
  CGCallee getVirtualFunctionPointer(CodeGenFunction &CGF, GlobalDecl GD,
                                     Address This, llvm::Type *Ty,
                                     SourceLocation Loc) override;
 
  llvm::Value *
  EmitVirtualDestructorCall(CodeGenFunction &CGF, const CXXDestructorDecl *Dtor,
                            CXXDtorType DtorType, Address This,
                            DeleteOrMemberCallExpr E,
                            llvm::CallBase **CallOrInvoke) override;
 
  void emitVirtualInheritanceTables(const CXXRecordDecl *RD) override;
 
  bool canSpeculativelyEmitVTable(const CXXRecordDecl *RD) const override;
  bool canSpeculativelyEmitVTableAsBaseClass(const CXXRecordDecl *RD) const;
 
  void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD,
                       bool ReturnAdjustment) override {
    // Allow inlining of thunks by emitting them with available_externally
    // linkage together with vtables when needed.
    if (ForVTable && !Thunk->hasLocalLinkage())
      Thunk->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
    CGM.setGVProperties(Thunk, GD);
  }
 
  bool exportThunk() override { return true; }
 
  llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This,
                                     const CXXRecordDecl *UnadjustedThisClass,
                                     const ThunkInfo &TI) override;
 
  llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret,
                                       const CXXRecordDecl *UnadjustedRetClass,
                                       const ReturnAdjustment &RA) override;
 
  size_t getSrcArgforCopyCtor(const CXXConstructorDecl *,
                              FunctionArgList &Args) const override {
    assert(!Args.empty() && "expected the arglist to not be empty!");
    return Args.size() - 1;
  }
 
  StringRef GetPureVirtualCallName() override { return "__cxa_pure_virtual"; }
  StringRef GetDeletedVirtualCallName() override
    { return "__cxa_deleted_virtual"; }
 
  CharUnits getArrayCookieSizeImpl(QualType elementType) override;
  Address InitializeArrayCookie(CodeGenFunction &CGF,
                                Address NewPtr,
                                llvm::Value *NumElements,
                                const CXXNewExpr *expr,
                                QualType ElementType) override;
  llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
                                   Address allocPtr,
                                   CharUnits cookieSize) override;
 
  void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
                       llvm::GlobalVariable *DeclPtr,
                       bool PerformInit) override;
  void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
                          llvm::FunctionCallee dtor,
                          llvm::Constant *addr) override;
 
  llvm::Function *getOrCreateThreadLocalWrapper(const VarDecl *VD,
                                                llvm::Value *Val);
  void EmitThreadLocalInitFuncs(
      CodeGenModule &CGM,
      ArrayRef<const VarDecl *> CXXThreadLocals,
      ArrayRef<llvm::Function *> CXXThreadLocalInits,
      ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
 
  bool usesThreadWrapperFunction(const VarDecl *VD) const override {
    return !isEmittedWithConstantInitializer(VD) ||
           mayNeedDestruction(VD);
  }
  LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
                                      QualType LValType) override;
 
  bool NeedsVTTParameter(GlobalDecl GD) override;
 
  llvm::Constant *
  getOrCreateVirtualFunctionPointerThunk(const CXXMethodDecl *MD);
 
  /**************************** RTTI Uniqueness ******************************/
 
protected:
  /// Returns true if the ABI requires RTTI type_info objects to be unique
  /// across a program.
  virtual bool shouldRTTIBeUnique() const { return true; }
 
public:
  /// What sort of unique-RTTI behavior should we use?
  enum RTTIUniquenessKind {
    /// We are guaranteeing, or need to guarantee, that the RTTI string
    /// is unique.
    RUK_Unique,
 
    /// We are not guaranteeing uniqueness for the RTTI string, so we
    /// can demote to hidden visibility but must use string comparisons.
    RUK_NonUniqueHidden,
 
    /// We are not guaranteeing uniqueness for the RTTI string, so we
    /// have to use string comparisons, but we also have to emit it with
    /// non-hidden visibility.
    RUK_NonUniqueVisible
  };
 
  /// Return the required visibility status for the given type and linkage in
  /// the current ABI.
  RTTIUniquenessKind
  classifyRTTIUniqueness(QualType CanTy,
                         llvm::GlobalValue::LinkageTypes Linkage) const;
  friend class ItaniumRTTIBuilder;
 
  void emitCXXStructor(GlobalDecl GD) override;
 
  std::pair<llvm::Value *, const CXXRecordDecl *>
  LoadVTablePtr(CodeGenFunction &CGF, Address This,
                const CXXRecordDecl *RD) override;
 
 private:
   llvm::Constant *
   getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD);
 
   bool hasAnyUnusedVirtualInlineFunction(const CXXRecordDecl *RD) const {
     const auto &VtableLayout =
         CGM.getItaniumVTableContext().getVTableLayout(RD);
 
     for (const auto &VtableComponent : VtableLayout.vtable_components()) {
       // Skip empty slot.
       if (!VtableComponent.isUsedFunctionPointerKind())
         continue;
 
       const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
       const FunctionDecl *FD = Method->getDefinition();
       const bool IsInlined =
           Method->getCanonicalDecl()->isInlined() || (FD && FD->isInlined());
       if (!IsInlined)
         continue;
 
       StringRef Name = CGM.getMangledName(VtableComponent.getGlobalDecl());
       auto *Entry = CGM.GetGlobalValue(Name);
       // This checks if virtual inline function has already been emitted.
       // Note that it is possible that this inline function would be emitted
       // after trying to emit vtable speculatively. Because of this we do
       // an extra pass after emitting all deferred vtables to find and emit
       // these vtables opportunistically.
       if (!Entry || Entry->isDeclaration())
         return true;
     }
     return false;
  }
 
  bool isVTableHidden(const CXXRecordDecl *RD) const {
    const auto &VtableLayout =
            CGM.getItaniumVTableContext().getVTableLayout(RD);
 
    for (const auto &VtableComponent : VtableLayout.vtable_components()) {
      if (VtableComponent.isRTTIKind()) {
        const CXXRecordDecl *RTTIDecl = VtableComponent.getRTTIDecl();
        if (RTTIDecl->getVisibility() == Visibility::HiddenVisibility)
          return true;
      } else if (VtableComponent.isUsedFunctionPointerKind()) {
        const CXXMethodDecl *Method = VtableComponent.getFunctionDecl();
        if (Method->getVisibility() == Visibility::HiddenVisibility &&
            !Method->isDefined())
          return true;
      }
    }
    return false;
  }
};
 
class ARMCXXABI : public ItaniumCXXABI {
public:
  ARMCXXABI(CodeGen::CodeGenModule &CGM) :
    ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
                  /*UseARMGuardVarABI=*/true) {}
 
  bool constructorsAndDestructorsReturnThis() const override { return true; }
 
  void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV,
                           QualType ResTy) override;
 
  CharUnits getArrayCookieSizeImpl(QualType elementType) override;
  Address InitializeArrayCookie(CodeGenFunction &CGF,
                                Address NewPtr,
                                llvm::Value *NumElements,
                                const CXXNewExpr *expr,
                                QualType ElementType) override;
  llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF, Address allocPtr,
                                   CharUnits cookieSize) override;
};
 
class AppleARM64CXXABI : public ARMCXXABI {
public:
  AppleARM64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {
    Use32BitVTableOffsetABI = true;
  }
 
  // ARM64 libraries are prepared for non-unique RTTI.
  bool shouldRTTIBeUnique() const override { return false; }
};
 
class FuchsiaCXXABI final : public ItaniumCXXABI {
public:
  explicit FuchsiaCXXABI(CodeGen::CodeGenModule &CGM)
      : ItaniumCXXABI(CGM) {}
 
private:
  bool constructorsAndDestructorsReturnThis() const override { return true; }
};
 
class WebAssemblyCXXABI final : public ItaniumCXXABI {
public:
  explicit WebAssemblyCXXABI(CodeGen::CodeGenModule &CGM)
      : ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
                      /*UseARMGuardVarABI=*/true) {}
  void emitBeginCatch(CodeGenFunction &CGF, const CXXCatchStmt *C) override;
  llvm::CallInst *
  emitTerminateForUnexpectedException(CodeGenFunction &CGF,
                                      llvm::Value *Exn) override;
 
private:
  bool constructorsAndDestructorsReturnThis() const override { return true; }
  bool canCallMismatchedFunctionType() const override { return false; }
};
 
class XLCXXABI final : public ItaniumCXXABI {
public:
  explicit XLCXXABI(CodeGen::CodeGenModule &CGM)
      : ItaniumCXXABI(CGM) {}
 
  void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
                          llvm::FunctionCallee dtor,
                          llvm::Constant *addr) override;
 
  bool useSinitAndSterm() const override { return true; }
 
private:
  void emitCXXStermFinalizer(const VarDecl &D, llvm::Function *dtorStub,
                             llvm::Constant *addr);
};
}
 
CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
  switch (CGM.getContext().getCXXABIKind()) {
  // For IR-generation purposes, there's no significant difference
  // between the ARM and iOS ABIs.
  case TargetCXXABI::GenericARM:
  case TargetCXXABI::iOS:
  case TargetCXXABI::WatchOS:
    return new ARMCXXABI(CGM);
 
  case TargetCXXABI::AppleARM64:
    return new AppleARM64CXXABI(CGM);
 
  case TargetCXXABI::Fuchsia:
    return new FuchsiaCXXABI(CGM);
 
  // Note that AArch64 uses the generic ItaniumCXXABI class since it doesn't
  // include the other 32-bit ARM oddities: constructor/destructor return values
  // and array cookies.
  case TargetCXXABI::GenericAArch64:
    return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true,
                             /*UseARMGuardVarABI=*/true);
 
  case TargetCXXABI::GenericMIPS:
    return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true);
 
  case TargetCXXABI::WebAssembly:
    return new WebAssemblyCXXABI(CGM);
 
  case TargetCXXABI::XL:
    return new XLCXXABI(CGM);
 
  case TargetCXXABI::GenericItanium:
    return new ItaniumCXXABI(CGM);
 
  case TargetCXXABI::Microsoft:
    llvm_unreachable("Microsoft ABI is not Itanium-based");
  }
  llvm_unreachable("bad ABI kind");
}
 
llvm::Type *
ItaniumCXXABI::ConvertMemberPointerType(const MemberPointerType *MPT) {
  if (MPT->isMemberDataPointer())
    return CGM.PtrDiffTy;
  return llvm::StructType::get(CGM.PtrDiffTy, CGM.PtrDiffTy);
}
 
/// In the Itanium and ARM ABIs, method pointers have the form:
///   struct { ptrdiff_t ptr; ptrdiff_t adj; } memptr;
///
/// In the Itanium ABI:
///  - method pointers are virtual if (memptr.ptr & 1) is nonzero
///  - the this-adjustment is (memptr.adj)
///  - the virtual offset is (memptr.ptr - 1)
///
/// In the ARM ABI:
///  - method pointers are virtual if (memptr.adj & 1) is nonzero
///  - the this-adjustment is (memptr.adj >> 1)
///  - the virtual offset is (memptr.ptr)
/// ARM uses 'adj' for the virtual flag because Thumb functions
/// may be only single-byte aligned.
///
/// If the member is virtual, the adjusted 'this' pointer points
/// to a vtable pointer from which the virtual offset is applied.
///
/// If the member is non-virtual, memptr.ptr is the address of
/// the function to call.
CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
    CodeGenFunction &CGF, const Expr *E, Address ThisAddr,
    llvm::Value *&ThisPtrForCall,
    llvm::Value *MemFnPtr, const MemberPointerType *MPT) {
  CGBuilderTy &Builder = CGF.Builder;
 
  const FunctionProtoType *FPT =
      MPT->getPointeeType()->castAs<FunctionProtoType>();
  auto *RD =
      cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
 
  llvm::Constant *ptrdiff_1 = llvm::ConstantInt::get(CGM.PtrDiffTy, 1);
 
  llvm::BasicBlock *FnVirtual = CGF.createBasicBlock("memptr.virtual");
  llvm::BasicBlock *FnNonVirtual = CGF.createBasicBlock("memptr.nonvirtual");
  llvm::BasicBlock *FnEnd = CGF.createBasicBlock("memptr.end");
 
  // Extract memptr.adj, which is in the second field.
  llvm::Value *RawAdj = Builder.CreateExtractValue(MemFnPtr, 1, "memptr.adj");
 
  // Compute the true adjustment.
  llvm::Value *Adj = RawAdj;
  if (UseARMMethodPtrABI)
    Adj = Builder.CreateAShr(Adj, ptrdiff_1, "memptr.adj.shifted");
 
  // Apply the adjustment and cast back to the original struct type
  // for consistency.
  llvm::Value *This = ThisAddr.emitRawPointer(CGF);
  This = Builder.CreateInBoundsGEP(Builder.getInt8Ty(), This, Adj);
  ThisPtrForCall = This;
 
  // Load the function pointer.
  llvm::Value *FnAsInt = Builder.CreateExtractValue(MemFnPtr, 0, "memptr.ptr");
 
  // If the LSB in the function pointer is 1, the function pointer points to
  // a virtual function.
  llvm::Value *IsVirtual;
  if (UseARMMethodPtrABI)
    IsVirtual = Builder.CreateAnd(RawAdj, ptrdiff_1);
  else
    IsVirtual = Builder.CreateAnd(FnAsInt, ptrdiff_1);
  IsVirtual = Builder.CreateIsNotNull(IsVirtual, "memptr.isvirtual");
  Builder.CreateCondBr(IsVirtual, FnVirtual, FnNonVirtual);
 
  // In the virtual path, the adjustment left 'This' pointing to the
  // vtable of the correct base subobject.  The "function pointer" is an
  // offset within the vtable (+1 for the virtual flag on non-ARM).
  CGF.EmitBlock(FnVirtual);
 
  // Cast the adjusted this to a pointer to vtable pointer and load.
  llvm::Type *VTableTy = CGF.CGM.GlobalsInt8PtrTy;
  CharUnits VTablePtrAlign =
    CGF.CGM.getDynamicOffsetAlignment(ThisAddr.getAlignment(), RD,
                                      CGF.getPointerAlign());
  llvm::Value *VTable = CGF.GetVTablePtr(
      Address(This, ThisAddr.getElementType(), VTablePtrAlign), VTableTy, RD);
 
  // Apply the offset.
  // On ARM64, to reserve extra space in virtual member function pointers,
  // we only pay attention to the low 32 bits of the offset.
  llvm::Value *VTableOffset = FnAsInt;
  if (!UseARMMethodPtrABI)
    VTableOffset = Builder.CreateSub(VTableOffset, ptrdiff_1);
  if (Use32BitVTableOffsetABI) {
    VTableOffset = Builder.CreateTrunc(VTableOffset, CGF.Int32Ty);
    VTableOffset = Builder.CreateZExt(VTableOffset, CGM.PtrDiffTy);
  }
 
  // Check the address of the function pointer if CFI on member function
  // pointers is enabled.
  llvm::Constant *CheckSourceLocation;
  llvm::Constant *CheckTypeDesc;
  bool ShouldEmitCFICheck = CGF.SanOpts.has(SanitizerKind::CFIMFCall) &&
                            CGM.HasHiddenLTOVisibility(RD);
  bool ShouldEmitVFEInfo = CGM.getCodeGenOpts().VirtualFunctionElimination &&
                           CGM.HasHiddenLTOVisibility(RD);
  bool ShouldEmitWPDInfo =
      CGM.getCodeGenOpts().WholeProgramVTables &&
      // Don't insert type tests if we are forcing public visibility.
      !CGM.AlwaysHasLTOVisibilityPublic(RD);
  llvm::Value *VirtualFn = nullptr;
 
  {
    CodeGenFunction::SanitizerScope SanScope(&CGF);
    llvm::Value *TypeId = nullptr;
    llvm::Value *CheckResult = nullptr;
 
    if (ShouldEmitCFICheck || ShouldEmitVFEInfo || ShouldEmitWPDInfo) {
      // If doing CFI, VFE or WPD, we will need the metadata node to check
      // against.
      llvm::Metadata *MD =
          CGM.CreateMetadataIdentifierForVirtualMemPtrType(QualType(MPT, 0));
      TypeId = llvm::MetadataAsValue::get(CGF.getLLVMContext(), MD);
    }
 
    if (ShouldEmitVFEInfo) {
      llvm::Value *VFPAddr =
          Builder.CreateGEP(CGF.Int8Ty, VTable, VTableOffset);
 
      // If doing VFE, load from the vtable with a type.checked.load intrinsic
      // call. Note that we use the GEP to calculate the address to load from
      // and pass 0 as the offset to the intrinsic. This is because every
      // vtable slot of the correct type is marked with matching metadata, and
      // we know that the load must be from one of these slots.
      llvm::Value *CheckedLoad = Builder.CreateCall(
          CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
          {VFPAddr, llvm::ConstantInt::get(CGM.Int32Ty, 0), TypeId});
      CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
      VirtualFn = Builder.CreateExtractValue(CheckedLoad, 0);
    } else {
      // When not doing VFE, emit a normal load, as it allows more
      // optimisations than type.checked.load.
      if (ShouldEmitCFICheck || ShouldEmitWPDInfo) {
        llvm::Value *VFPAddr =
            Builder.CreateGEP(CGF.Int8Ty, VTable, VTableOffset);
        llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)
                                      ? llvm::Intrinsic::type_test
                                      : llvm::Intrinsic::public_type_test;
 
        CheckResult =
            Builder.CreateCall(CGM.getIntrinsic(IID), {VFPAddr, TypeId});
      }
 
      if (CGM.getItaniumVTableContext().isRelativeLayout()) {
        VirtualFn = CGF.Builder.CreateCall(
            CGM.getIntrinsic(llvm::Intrinsic::load_relative,
                             {VTableOffset->getType()}),
            {VTable, VTableOffset});
      } else {
        llvm::Value *VFPAddr =
            CGF.Builder.CreateGEP(CGF.Int8Ty, VTable, VTableOffset);
        VirtualFn = CGF.Builder.CreateAlignedLoad(CGF.UnqualPtrTy, VFPAddr,
                                                  CGF.getPointerAlign(),
                                                  "memptr.virtualfn");
      }
    }
    assert(VirtualFn && "Virtual fuction pointer not created!");
    assert((!ShouldEmitCFICheck || !ShouldEmitVFEInfo || !ShouldEmitWPDInfo ||
            CheckResult) &&
           "Check result required but not created!");
 
    if (ShouldEmitCFICheck) {
      // If doing CFI, emit the check.
      CheckSourceLocation = CGF.EmitCheckSourceLocation(E->getBeginLoc());
      CheckTypeDesc = CGF.EmitCheckTypeDescriptor(QualType(MPT, 0));
      llvm::Constant *StaticData[] = {
          llvm::ConstantInt::get(CGF.Int8Ty, CodeGenFunction::CFITCK_VMFCall),
          CheckSourceLocation,
          CheckTypeDesc,
      };
 
      if (CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIMFCall)) {
        CGF.EmitTrapCheck(CheckResult, SanitizerHandler::CFICheckFail);
      } else {
        llvm::Value *AllVtables = llvm::MetadataAsValue::get(
            CGM.getLLVMContext(),
            llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
        llvm::Value *ValidVtable = Builder.CreateCall(
            CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});
        CGF.EmitCheck(std::make_pair(CheckResult, SanitizerKind::SO_CFIMFCall),
                      SanitizerHandler::CFICheckFail, StaticData,
                      {VTable, ValidVtable});
      }
 
      FnVirtual = Builder.GetInsertBlock();
    }
  } // End of sanitizer scope
 
  CGF.EmitBranch(FnEnd);
 
  // In the non-virtual path, the function pointer is actually a
  // function pointer.
  CGF.EmitBlock(FnNonVirtual);
  llvm::Value *NonVirtualFn =
      Builder.CreateIntToPtr(FnAsInt, CGF.UnqualPtrTy, "memptr.nonvirtualfn");
 
  // Check the function pointer if CFI on member function pointers is enabled.
  if (ShouldEmitCFICheck) {
    CXXRecordDecl *RD = MPT->getClass()->getAsCXXRecordDecl();
    if (RD->hasDefinition()) {
      CodeGenFunction::SanitizerScope SanScope(&CGF);
 
      llvm::Constant *StaticData[] = {
          llvm::ConstantInt::get(CGF.Int8Ty, CodeGenFunction::CFITCK_NVMFCall),
          CheckSourceLocation,
          CheckTypeDesc,
      };
 
      llvm::Value *Bit = Builder.getFalse();
      for (const CXXRecordDecl *Base : CGM.getMostBaseClasses(RD)) {
        llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(
            getContext().getMemberPointerType(
                MPT->getPointeeType(),
                getContext().getRecordType(Base).getTypePtr()));
        llvm::Value *TypeId =
            llvm::MetadataAsValue::get(CGF.getLLVMContext(), MD);
 
        llvm::Value *TypeTest =
            Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
                               {NonVirtualFn, TypeId});
        Bit = Builder.CreateOr(Bit, TypeTest);
      }
 
      CGF.EmitCheck(std::make_pair(Bit, SanitizerKind::SO_CFIMFCall),
                    SanitizerHandler::CFICheckFail, StaticData,
                    {NonVirtualFn, llvm::UndefValue::get(CGF.IntPtrTy)});
 
      FnNonVirtual = Builder.GetInsertBlock();
    }
  }
 
  // We're done.
  CGF.EmitBlock(FnEnd);
  llvm::PHINode *CalleePtr = Builder.CreatePHI(CGF.UnqualPtrTy, 2);
  CalleePtr->addIncoming(VirtualFn, FnVirtual);
  CalleePtr->addIncoming(NonVirtualFn, FnNonVirtual);
 
  CGPointerAuthInfo PointerAuth;
 
  if (const auto &Schema =
          CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers) {
    llvm::PHINode *DiscriminatorPHI = Builder.CreatePHI(CGF.IntPtrTy, 2);
    DiscriminatorPHI->addIncoming(llvm::ConstantInt::get(CGF.IntPtrTy, 0),
                                  FnVirtual);
    const auto &AuthInfo =
        CGM.getMemberFunctionPointerAuthInfo(QualType(MPT, 0));
    assert(Schema.getKey() == AuthInfo.getKey() &&
           "Keys for virtual and non-virtual member functions must match");
    auto *NonVirtualDiscriminator = AuthInfo.getDiscriminator();
    DiscriminatorPHI->addIncoming(NonVirtualDiscriminator, FnNonVirtual);
    PointerAuth = CGPointerAuthInfo(
        Schema.getKey(), Schema.getAuthenticationMode(), Schema.isIsaPointer(),
        Schema.authenticatesNullValues(), DiscriminatorPHI);
  }
 
  CGCallee Callee(FPT, CalleePtr, PointerAuth);
  return Callee;
}
 
/// Compute an l-value by applying the given pointer-to-member to a
/// base object.
llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(
    CodeGenFunction &CGF, const Expr *E, Address Base, llvm::Value *MemPtr,
    const MemberPointerType *MPT) {
  assert(MemPtr->getType() == CGM.PtrDiffTy);
 
  CGBuilderTy &Builder = CGF.Builder;
 
  // Apply the offset, which we assume is non-null.
  return Builder.CreateInBoundsGEP(CGF.Int8Ty, Base.emitRawPointer(CGF), MemPtr,
                                   "memptr.offset");
}
 
// See if it's possible to return a constant signed pointer.
static llvm::Constant *pointerAuthResignConstant(
    llvm::Value *Ptr, const CGPointerAuthInfo &CurAuthInfo,
    const CGPointerAuthInfo &NewAuthInfo, CodeGenModule &CGM) {
  const auto *CPA = dyn_cast<llvm::ConstantPtrAuth>(Ptr);
 
  if (!CPA)
    return nullptr;
 
  assert(CPA->getKey()->getZExtValue() == CurAuthInfo.getKey() &&
         CPA->getAddrDiscriminator()->isZeroValue() &&
         CPA->getDiscriminator() == CurAuthInfo.getDiscriminator() &&
         "unexpected key or discriminators");
 
  return CGM.getConstantSignedPointer(
      CPA->getPointer(), NewAuthInfo.getKey(), nullptr,
      cast<llvm::ConstantInt>(NewAuthInfo.getDiscriminator()));
}
 
/// Perform a bitcast, derived-to-base, or base-to-derived member pointer
/// conversion.
///
/// Bitcast conversions are always a no-op under Itanium.
///
/// Obligatory offset/adjustment diagram:
///         <-- offset -->          <-- adjustment -->
///   |--------------------------|----------------------|--------------------|
///   ^Derived address point     ^Base address point    ^Member address point
///
/// So when converting a base member pointer to a derived member pointer,
/// we add the offset to the adjustment because the address point has
/// decreased;  and conversely, when converting a derived MP to a base MP
/// we subtract the offset from the adjustment because the address point
/// has increased.
///
/// The standard forbids (at compile time) conversion to and from
/// virtual bases, which is why we don't have to consider them here.
///
/// The standard forbids (at run time) casting a derived MP to a base
/// MP when the derived MP does not point to a member of the base.
/// This is why -1 is a reasonable choice for null data member
/// pointers.
llvm::Value *
ItaniumCXXABI::EmitMemberPointerConversion(CodeGenFunction &CGF,
                                           const CastExpr *E,
                                           llvm::Value *src) {
  // Use constant emission if we can.
  if (isa<llvm::Constant>(src))
    return EmitMemberPointerConversion(E, cast<llvm::Constant>(src));
 
  assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
         E->getCastKind() == CK_BaseToDerivedMemberPointer ||
         E->getCastKind() == CK_ReinterpretMemberPointer);
 
  CGBuilderTy &Builder = CGF.Builder;
  QualType DstType = E->getType();
 
  if (DstType->isMemberFunctionPointerType()) {
    if (const auto &NewAuthInfo =
            CGM.getMemberFunctionPointerAuthInfo(DstType)) {
      QualType SrcType = E->getSubExpr()->getType();
      assert(SrcType->isMemberFunctionPointerType());
      const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(SrcType);
      llvm::Value *MemFnPtr = Builder.CreateExtractValue(src, 0, "memptr.ptr");
      llvm::Type *OrigTy = MemFnPtr->getType();
 
      llvm::BasicBlock *StartBB = Builder.GetInsertBlock();
      llvm::BasicBlock *ResignBB = CGF.createBasicBlock("resign");
      llvm::BasicBlock *MergeBB = CGF.createBasicBlock("merge");
 
      // Check whether we have a virtual offset or a pointer to a function.
      assert(UseARMMethodPtrABI && "ARM ABI expected");
      llvm::Value *Adj = Builder.CreateExtractValue(src, 1, "memptr.adj");
      llvm::Constant *Ptrdiff_1 = llvm::ConstantInt::get(CGM.PtrDiffTy, 1);
      llvm::Value *AndVal = Builder.CreateAnd(Adj, Ptrdiff_1);
      llvm::Value *IsVirtualOffset =
          Builder.CreateIsNotNull(AndVal, "is.virtual.offset");
      Builder.CreateCondBr(IsVirtualOffset, MergeBB, ResignBB);
 
      CGF.EmitBlock(ResignBB);
      llvm::Type *PtrTy = llvm::PointerType::getUnqual(CGM.Int8Ty);
      MemFnPtr = Builder.CreateIntToPtr(MemFnPtr, PtrTy);
      MemFnPtr =
          CGF.emitPointerAuthResign(MemFnPtr, SrcType, CurAuthInfo, NewAuthInfo,
                                    isa<llvm::Constant>(src));
      MemFnPtr = Builder.CreatePtrToInt(MemFnPtr, OrigTy);
      llvm::Value *ResignedVal = Builder.CreateInsertValue(src, MemFnPtr, 0);
      ResignBB = Builder.GetInsertBlock();
 
      CGF.EmitBlock(MergeBB);
      llvm::PHINode *NewSrc = Builder.CreatePHI(src->getType(), 2);
      NewSrc->addIncoming(src, StartBB);
      NewSrc->addIncoming(ResignedVal, ResignBB);
      src = NewSrc;
    }
  }
 
  // Under Itanium, reinterprets don't require any additional processing.
  if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
 
  llvm::Constant *adj = getMemberPointerAdjustment(E);
  if (!adj) return src;
 
  bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
 
  const MemberPointerType *destTy =
    E->getType()->castAs<MemberPointerType>();
 
  // For member data pointers, this is just a matter of adding the
  // offset if the source is non-null.
  if (destTy->isMemberDataPointer()) {
    llvm::Value *dst;
    if (isDerivedToBase)
      dst = Builder.CreateNSWSub(src, adj, "adj");
    else
      dst = Builder.CreateNSWAdd(src, adj, "adj");
 
    // Null check.
    llvm::Value *null = llvm::Constant::getAllOnesValue(src->getType());
    llvm::Value *isNull = Builder.CreateICmpEQ(src, null, "memptr.isnull");
    return Builder.CreateSelect(isNull, src, dst);
  }
 
  // The this-adjustment is left-shifted by 1 on ARM.
  if (UseARMMethodPtrABI) {
    uint64_t offset = cast<llvm::ConstantInt>(adj)->getZExtValue();
    offset <<= 1;
    adj = llvm::ConstantInt::get(adj->getType(), offset);
  }
 
  llvm::Value *srcAdj = Builder.CreateExtractValue(src, 1, "src.adj");
  llvm::Value *dstAdj;
  if (isDerivedToBase)
    dstAdj = Builder.CreateNSWSub(srcAdj, adj, "adj");
  else
    dstAdj = Builder.CreateNSWAdd(srcAdj, adj, "adj");
 
  return Builder.CreateInsertValue(src, dstAdj, 1);
}
 
static llvm::Constant *
pointerAuthResignMemberFunctionPointer(llvm::Constant *Src, QualType DestType,
                                       QualType SrcType, CodeGenModule &CGM) {
  assert(DestType->isMemberFunctionPointerType() &&
         SrcType->isMemberFunctionPointerType() &&
         "member function pointers expected");
  if (DestType == SrcType)
    return Src;
 
  const auto &NewAuthInfo = CGM.getMemberFunctionPointerAuthInfo(DestType);
  const auto &CurAuthInfo = CGM.getMemberFunctionPointerAuthInfo(SrcType);
 
  if (!NewAuthInfo && !CurAuthInfo)
    return Src;
 
  llvm::Constant *MemFnPtr = Src->getAggregateElement(0u);
  if (MemFnPtr->getNumOperands() == 0) {
    // src must be a pair of null pointers.
    assert(isa<llvm::ConstantInt>(MemFnPtr) && "constant int expected");
    return Src;
  }
 
  llvm::Constant *ConstPtr = pointerAuthResignConstant(
      cast<llvm::User>(MemFnPtr)->getOperand(0), CurAuthInfo, NewAuthInfo, CGM);
  ConstPtr = llvm::ConstantExpr::getPtrToInt(ConstPtr, MemFnPtr->getType());
  return ConstantFoldInsertValueInstruction(Src, ConstPtr, 0);
}
 
llvm::Constant *
ItaniumCXXABI::EmitMemberPointerConversion(const CastExpr *E,
                                           llvm::Constant *src) {
  assert(E->getCastKind() == CK_DerivedToBaseMemberPointer ||
         E->getCastKind() == CK_BaseToDerivedMemberPointer ||
         E->getCastKind() == CK_ReinterpretMemberPointer);
 
  QualType DstType = E->getType();
 
  if (DstType->isMemberFunctionPointerType())
    src = pointerAuthResignMemberFunctionPointer(
        src, DstType, E->getSubExpr()->getType(), CGM);
 
  // Under Itanium, reinterprets don't require any additional processing.
  if (E->getCastKind() == CK_ReinterpretMemberPointer) return src;
 
  // If the adjustment is trivial, we don't need to do anything.
  llvm::Constant *adj = getMemberPointerAdjustment(E);
  if (!adj) return src;
 
  bool isDerivedToBase = (E->getCastKind() == CK_DerivedToBaseMemberPointer);
 
  const MemberPointerType *destTy =
    E->getType()->castAs<MemberPointerType>();
 
  // For member data pointers, this is just a matter of adding the
  // offset if the source is non-null.
  if (destTy->isMemberDataPointer()) {
    // null maps to null.
    if (src->isAllOnesValue()) return src;
 
    if (isDerivedToBase)
      return llvm::ConstantExpr::getNSWSub(src, adj);
    else
      return llvm::ConstantExpr::getNSWAdd(src, adj);
  }
 
  // The this-adjustment is left-shifted by 1 on ARM.
  if (UseARMMethodPtrABI) {
    uint64_t offset = cast<llvm::ConstantInt>(adj)->getZExtValue();
    offset <<= 1;
    adj = llvm::ConstantInt::get(adj->getType(), offset);
  }
 
  llvm::Constant *srcAdj = src->getAggregateElement(1);
  llvm::Constant *dstAdj;
  if (isDerivedToBase)
    dstAdj = llvm::ConstantExpr::getNSWSub(srcAdj, adj);
  else
    dstAdj = llvm::ConstantExpr::getNSWAdd(srcAdj, adj);
 
  llvm::Constant *res = ConstantFoldInsertValueInstruction(src, dstAdj, 1);
  assert(res != nullptr && "Folding must succeed");
  return res;
}
 
llvm::Constant *
ItaniumCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
  // Itanium C++ ABI 2.3:
  //   A NULL pointer is represented as -1.
  if (MPT->isMemberDataPointer())
    return llvm::ConstantInt::get(CGM.PtrDiffTy, -1ULL, /*isSigned=*/true);
 
  llvm::Constant *Zero = llvm::ConstantInt::get(CGM.PtrDiffTy, 0);
  llvm::Constant *Values[2] = { Zero, Zero };
  return llvm::ConstantStruct::getAnon(Values);
}
 
llvm::Constant *
ItaniumCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
                                     CharUnits offset) {
  // Itanium C++ ABI 2.3:
  //   A pointer to data member is an offset from the base address of
  //   the class object containing it, represented as a ptrdiff_t
  return llvm::ConstantInt::get(CGM.PtrDiffTy, offset.getQuantity());
}
 
llvm::Constant *
ItaniumCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
  return BuildMemberPointer(MD, CharUnits::Zero());
}
 
llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD,
                                                  CharUnits ThisAdjustment) {
  assert(MD->isInstance() && "Member function must not be static!");
 
  CodeGenTypes &Types = CGM.getTypes();
 
  // Get the function pointer (or index if this is a virtual function).
  llvm::Constant *MemPtr[2];
  if (MD->isVirtual()) {
    uint64_t Index = CGM.getItaniumVTableContext().getMethodVTableIndex(MD);
    uint64_t VTableOffset;
    if (CGM.getItaniumVTableContext().isRelativeLayout()) {
      // Multiply by 4-byte relative offsets.
      VTableOffset = Index * 4;
    } else {
      const ASTContext &Context = getContext();
      CharUnits PointerWidth = Context.toCharUnitsFromBits(
          Context.getTargetInfo().getPointerWidth(LangAS::Default));
      VTableOffset = Index * PointerWidth.getQuantity();
    }
 
    if (UseARMMethodPtrABI) {
      // ARM C++ ABI 3.2.1:
      //   This ABI specifies that adj contains twice the this
      //   adjustment, plus 1 if the member function is virtual. The
      //   least significant bit of adj then makes exactly the same
      //   discrimination as the least significant bit of ptr does for
      //   Itanium.
 
      // We cannot use the Itanium ABI's representation for virtual member
      // function pointers under pointer authentication because it would
      // require us to store both the virtual offset and the constant
      // discriminator in the pointer, which would be immediately vulnerable
      // to attack.  Instead we introduce a thunk that does the virtual dispatch
      // and store it as if it were a non-virtual member function.  This means
      // that virtual function pointers may not compare equal anymore, but
      // fortunately they aren't required to by the standard, and we do make
      // a best-effort attempt to re-use the thunk.
      //
      // To support interoperation with code in which pointer authentication
      // is disabled, derefencing a member function pointer must still handle
      // the virtual case, but it can use a discriminator which should never
      // be valid.
      const auto &Schema =
          CGM.getCodeGenOpts().PointerAuth.CXXMemberFunctionPointers;
      if (Schema)
        MemPtr[0] = llvm::ConstantExpr::getPtrToInt(
            getSignedVirtualMemberFunctionPointer(MD), CGM.PtrDiffTy);
      else
        MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset);
      // Don't set the LSB of adj to 1 if pointer authentication for member
      // function pointers is enabled.
      MemPtr[1] = llvm::ConstantInt::get(
          CGM.PtrDiffTy, 2 * ThisAdjustment.getQuantity() + !Schema);
    } else {
      // Itanium C++ ABI 2.3:
      //   For a virtual function, [the pointer field] is 1 plus the
      //   virtual table offset (in bytes) of the function,
      //   represented as a ptrdiff_t.
      MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset + 1);
      MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy,
                                         ThisAdjustment.getQuantity());
    }
  } else {
    const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
    llvm::Type *Ty;
    // Check whether the function has a computable LLVM signature.
    if (Types.isFuncTypeConvertible(FPT)) {
      // The function has a computable LLVM signature; use the correct type.
      Ty = Types.GetFunctionType(Types.arrangeCXXMethodDeclaration(MD));
    } else {
      // Use an arbitrary non-function type to tell GetAddrOfFunction that the
      // function type is incomplete.
      Ty = CGM.PtrDiffTy;
    }
    llvm::Constant *addr = CGM.getMemberFunctionPointer(MD, Ty);
 
    MemPtr[0] = llvm::ConstantExpr::getPtrToInt(addr, CGM.PtrDiffTy);
    MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy,
                                       (UseARMMethodPtrABI ? 2 : 1) *
                                       ThisAdjustment.getQuantity());
  }
 
  return llvm::ConstantStruct::getAnon(MemPtr);
}
 
llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP,
                                                 QualType MPType) {
  const MemberPointerType *MPT = MPType->castAs<MemberPointerType>();
  const ValueDecl *MPD = MP.getMemberPointerDecl();
  if (!MPD)
    return EmitNullMemberPointer(MPT);
 
  CharUnits ThisAdjustment = getContext().getMemberPointerPathAdjustment(MP);
 
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) {
    llvm::Constant *Src = BuildMemberPointer(MD, ThisAdjustment);
    QualType SrcType = getContext().getMemberPointerType(
        MD->getType(), MD->getParent()->getTypeForDecl());
    return pointerAuthResignMemberFunctionPointer(Src, MPType, SrcType, CGM);
  }
 
  CharUnits FieldOffset =
    getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD));
  return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset);
}
 
/// The comparison algorithm is pretty easy: the member pointers are
/// the same if they're either bitwise identical *or* both null.
///
/// ARM is different here only because null-ness is more complicated.
llvm::Value *
ItaniumCXXABI::EmitMemberPointerComparison(CodeGenFunction &CGF,
                                           llvm::Value *L,
                                           llvm::Value *R,
                                           const MemberPointerType *MPT,
                                           bool Inequality) {
  CGBuilderTy &Builder = CGF.Builder;
 
  llvm::ICmpInst::Predicate Eq;
  llvm::Instruction::BinaryOps And, Or;
  if (Inequality) {
    Eq = llvm::ICmpInst::ICMP_NE;
    And = llvm::Instruction::Or;
    Or = llvm::Instruction::And;
  } else {
    Eq = llvm::ICmpInst::ICMP_EQ;
    And = llvm::Instruction::And;
    Or = llvm::Instruction::Or;
  }
 
  // Member data pointers are easy because there's a unique null
  // value, so it just comes down to bitwise equality.
  if (MPT->isMemberDataPointer())
    return Builder.CreateICmp(Eq, L, R);
 
  // For member function pointers, the tautologies are more complex.
  // The Itanium tautology is:
  //   (L == R) <==> (L.ptr == R.ptr && (L.ptr == 0 || L.adj == R.adj))
  // The ARM tautology is:
  //   (L == R) <==> (L.ptr == R.ptr &&
  //                  (L.adj == R.adj ||
  //                   (L.ptr == 0 && ((L.adj|R.adj) & 1) == 0)))
  // The inequality tautologies have exactly the same structure, except
  // applying De Morgan's laws.
 
  llvm::Value *LPtr = Builder.CreateExtractValue(L, 0, "lhs.memptr.ptr");
  llvm::Value *RPtr = Builder.CreateExtractValue(R, 0, "rhs.memptr.ptr");
 
  // This condition tests whether L.ptr == R.ptr.  This must always be
  // true for equality to hold.
  llvm::Value *PtrEq = Builder.CreateICmp(Eq, LPtr, RPtr, "cmp.ptr");
 
  // This condition, together with the assumption that L.ptr == R.ptr,
  // tests whether the pointers are both null.  ARM imposes an extra
  // condition.
  llvm::Value *Zero = llvm::Constant::getNullValue(LPtr->getType());
  llvm::Value *EqZero = Builder.CreateICmp(Eq, LPtr, Zero, "cmp.ptr.null");
 
  // This condition tests whether L.adj == R.adj.  If this isn't
  // true, the pointers are unequal unless they're both null.
  llvm::Value *LAdj = Builder.CreateExtractValue(L, 1, "lhs.memptr.adj");
  llvm::Value *RAdj = Builder.CreateExtractValue(R, 1, "rhs.memptr.adj");
  llvm::Value *AdjEq = Builder.CreateICmp(Eq, LAdj, RAdj, "cmp.adj");
 
  // Null member function pointers on ARM clear the low bit of Adj,
  // so the zero condition has to check that neither low bit is set.
  if (UseARMMethodPtrABI) {
    llvm::Value *One = llvm::ConstantInt::get(LPtr->getType(), 1);
 
    // Compute (l.adj | r.adj) & 1 and test it against zero.
    llvm::Value *OrAdj = Builder.CreateOr(LAdj, RAdj, "or.adj");
    llvm::Value *OrAdjAnd1 = Builder.CreateAnd(OrAdj, One);
    llvm::Value *OrAdjAnd1EqZero = Builder.CreateICmp(Eq, OrAdjAnd1, Zero,
                                                      "cmp.or.adj");
    EqZero = Builder.CreateBinOp(And, EqZero, OrAdjAnd1EqZero);
  }
 
  // Tie together all our conditions.
  llvm::Value *Result = Builder.CreateBinOp(Or, EqZero, AdjEq);
  Result = Builder.CreateBinOp(And, PtrEq, Result,
                               Inequality ? "memptr.ne" : "memptr.eq");
  return Result;
}
 
llvm::Value *
ItaniumCXXABI::EmitMemberPointerIsNotNull(CodeGenFunction &CGF,
                                          llvm::Value *MemPtr,
                                          const MemberPointerType *MPT) {
  CGBuilderTy &Builder = CGF.Builder;
 
  /// For member data pointers, this is just a check against -1.
  if (MPT->isMemberDataPointer()) {
    assert(MemPtr->getType() == CGM.PtrDiffTy);
    llvm::Value *NegativeOne =
      llvm::Constant::getAllOnesValue(MemPtr->getType());
    return Builder.CreateICmpNE(MemPtr, NegativeOne, "memptr.tobool");
  }
 
  // In Itanium, a member function pointer is not null if 'ptr' is not null.
  llvm::Value *Ptr = Builder.CreateExtractValue(MemPtr, 0, "memptr.ptr");
 
  llvm::Constant *Zero = llvm::ConstantInt::get(Ptr->getType(), 0);
  llvm::Value *Result = Builder.CreateICmpNE(Ptr, Zero, "memptr.tobool");
 
  // On ARM, a member function pointer is also non-null if the low bit of 'adj'
  // (the virtual bit) is set.
  if (UseARMMethodPtrABI) {
    llvm::Constant *One = llvm::ConstantInt::get(Ptr->getType(), 1);
    llvm::Value *Adj = Builder.CreateExtractValue(MemPtr, 1, "memptr.adj");
    llvm::Value *VirtualBit = Builder.CreateAnd(Adj, One, "memptr.virtualbit");
    llvm::Value *IsVirtual = Builder.CreateICmpNE(VirtualBit, Zero,
                                                  "memptr.isvirtual");
    Result = Builder.CreateOr(Result, IsVirtual);
  }
 
  return Result;
}
 
bool ItaniumCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
  const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
  if (!RD)
    return false;
 
  // If C++ prohibits us from making a copy, return by address.
  if (!RD->canPassInRegisters()) {
    auto Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
    FI.getReturnInfo() = ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
    return true;
  }
  return false;
}
 
/// The Itanium ABI requires non-zero initialization only for data
/// member pointers, for which '0' is a valid offset.
bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
  return MPT->isMemberFunctionPointer();
}
 
/// The Itanium ABI always places an offset to the complete object
/// at entry -2 in the vtable.
void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
                                            const CXXDeleteExpr *DE,
                                            Address Ptr,
                                            QualType ElementType,
                                            const CXXDestructorDecl *Dtor) {
  bool UseGlobalDelete = DE->isGlobalDelete();
  if (UseGlobalDelete) {
    // Derive the complete-object pointer, which is what we need
    // to pass to the deallocation function.
 
    // Grab the vtable pointer as an intptr_t*.
    auto *ClassDecl =
        cast<CXXRecordDecl>(ElementType->castAs<RecordType>()->getDecl());
    llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.UnqualPtrTy, ClassDecl);
 
    // Track back to entry -2 and pull out the offset there.
    llvm::Value *OffsetPtr = CGF.Builder.CreateConstInBoundsGEP1_64(
        CGF.IntPtrTy, VTable, -2, "complete-offset.ptr");
    llvm::Value *Offset = CGF.Builder.CreateAlignedLoad(CGF.IntPtrTy, OffsetPtr,
                                                        CGF.getPointerAlign());
 
    // Apply the offset.
    llvm::Value *CompletePtr = Ptr.emitRawPointer(CGF);
    CompletePtr =
        CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, CompletePtr, Offset);
 
    // If we're supposed to call the global delete, make sure we do so
    // even if the destructor throws.
    CGF.pushCallObjectDeleteCleanup(DE->getOperatorDelete(), CompletePtr,
                                    ElementType);
  }
 
  // FIXME: Provide a source location here even though there's no
  // CXXMemberCallExpr for dtor call.
  CXXDtorType DtorType = UseGlobalDelete ? Dtor_Complete : Dtor_Deleting;
  EmitVirtualDestructorCall(CGF, Dtor, DtorType, Ptr, DE,
                            /*CallOrInvoke=*/nullptr);
 
  if (UseGlobalDelete)
    CGF.PopCleanupBlock();
}
 
void ItaniumCXXABI::emitRethrow(CodeGenFunction &CGF, bool isNoReturn) {
  // void __cxa_rethrow();
 
  llvm::FunctionType *FTy =
    llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
 
  llvm::FunctionCallee Fn = CGM.CreateRuntimeFunction(FTy, "__cxa_rethrow");
 
  if (isNoReturn)
    CGF.EmitNoreturnRuntimeCallOrInvoke(Fn, {});
  else
    CGF.EmitRuntimeCallOrInvoke(Fn);
}
 
static llvm::FunctionCallee getAllocateExceptionFn(CodeGenModule &CGM) {
  // void *__cxa_allocate_exception(size_t thrown_size);
 
  llvm::FunctionType *FTy =
    llvm::FunctionType::get(CGM.Int8PtrTy, CGM.SizeTy, /*isVarArg=*/false);
 
  return CGM.CreateRuntimeFunction(FTy, "__cxa_allocate_exception");
}
 
static llvm::FunctionCallee getThrowFn(CodeGenModule &CGM) {
  // void __cxa_throw(void *thrown_exception, std::type_info *tinfo,
  //                  void (*dest) (void *));
 
  llvm::Type *Args[3] = { CGM.Int8PtrTy, CGM.GlobalsInt8PtrTy, CGM.Int8PtrTy };
  llvm::FunctionType *FTy =
    llvm::FunctionType::get(CGM.VoidTy, Args, /*isVarArg=*/false);
 
  return CGM.CreateRuntimeFunction(FTy, "__cxa_throw");
}
 
void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
  QualType ThrowType = E->getSubExpr()->getType();
  // Now allocate the exception object.
  llvm::Type *SizeTy = CGF.ConvertType(getContext().getSizeType());
  uint64_t TypeSize = getContext().getTypeSizeInChars(ThrowType).getQuantity();
 
  llvm::FunctionCallee AllocExceptionFn = getAllocateExceptionFn(CGM);
  llvm::CallInst *ExceptionPtr = CGF.EmitNounwindRuntimeCall(
      AllocExceptionFn, llvm::ConstantInt::get(SizeTy, TypeSize), "exception");
 
  CharUnits ExnAlign = CGF.getContext().getExnObjectAlignment();
  CGF.EmitAnyExprToExn(
      E->getSubExpr(), Address(ExceptionPtr, CGM.Int8Ty, ExnAlign));
 
  // Now throw the exception.
  llvm::Constant *TypeInfo = CGM.GetAddrOfRTTIDescriptor(ThrowType,
                                                         /*ForEH=*/true);
 
  // The address of the destructor.  If the exception type has a
  // trivial destructor (or isn't a record), we just pass null.
  llvm::Constant *Dtor = nullptr;
  if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) {
    CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl());
    if (!Record->hasTrivialDestructor()) {
      // __cxa_throw is declared to take its destructor as void (*)(void *). We
      // must match that if function pointers can be authenticated with a
      // discriminator based on their type.
      const ASTContext &Ctx = getContext();
      QualType DtorTy = Ctx.getFunctionType(Ctx.VoidTy, {Ctx.VoidPtrTy},
                                            FunctionProtoType::ExtProtoInfo());
 
      CXXDestructorDecl *DtorD = Record->getDestructor();
      Dtor = CGM.getAddrOfCXXStructor(GlobalDecl(DtorD, Dtor_Complete));
      Dtor = CGM.getFunctionPointer(Dtor, DtorTy);
    }
  }
  if (!Dtor) Dtor = llvm::Constant::getNullValue(CGM.Int8PtrTy);
 
  llvm::Value *args[] = { ExceptionPtr, TypeInfo, Dtor };
  CGF.EmitNoreturnRuntimeCallOrInvoke(getThrowFn(CGM), args);
}
 
static llvm::FunctionCallee getItaniumDynamicCastFn(CodeGenFunction &CGF) {
  // void *__dynamic_cast(const void *sub,
  //                      global_as const abi::__class_type_info *src,
  //                      global_as const abi::__class_type_info *dst,
  //                      std::ptrdiff_t src2dst_offset);
 
  llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
  llvm::Type *GlobInt8PtrTy = CGF.GlobalsInt8PtrTy;
  llvm::Type *PtrDiffTy =
    CGF.ConvertType(CGF.getContext().getPointerDiffType());
 
  llvm::Type *Args[4] = { Int8PtrTy, GlobInt8PtrTy, GlobInt8PtrTy, PtrDiffTy };
 
  llvm::FunctionType *FTy = llvm::FunctionType::get(Int8PtrTy, Args, false);
 
  // Mark the function as nounwind willreturn readonly.
  llvm::AttrBuilder FuncAttrs(CGF.getLLVMContext());
  FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
  FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
  FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly());
  llvm::AttributeList Attrs = llvm::AttributeList::get(
      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, FuncAttrs);
 
  return CGF.CGM.CreateRuntimeFunction(FTy, "__dynamic_cast", Attrs);
}
 
static llvm::FunctionCallee getBadCastFn(CodeGenFunction &CGF) {
  // void __cxa_bad_cast();
  llvm::FunctionType *FTy = llvm::FunctionType::get(CGF.VoidTy, false);
  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_cast");
}
 
/// Compute the src2dst_offset hint as described in the
/// Itanium C++ ABI [2.9.7]
static CharUnits computeOffsetHint(ASTContext &Context,
                                   const CXXRecordDecl *Src,
                                   const CXXRecordDecl *Dst) {
  CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
                     /*DetectVirtual=*/false);
 
  // If Dst is not derived from Src we can skip the whole computation below and
  // return that Src is not a public base of Dst.  Record all inheritance paths.
  if (!Dst->isDerivedFrom(Src, Paths))
    return CharUnits::fromQuantity(-2ULL);
 
  unsigned NumPublicPaths = 0;
  CharUnits Offset;
 
  // Now walk all possible inheritance paths.
  for (const CXXBasePath &Path : Paths) {
    if (Path.Access != AS_public)  // Ignore non-public inheritance.
      continue;
 
    ++NumPublicPaths;
 
    for (const CXXBasePathElement &PathElement : Path) {
      // If the path contains a virtual base class we can't give any hint.
      // -1: no hint.
      if (PathElement.Base->isVirtual())
        return CharUnits::fromQuantity(-1ULL);
 
      if (NumPublicPaths > 1) // Won't use offsets, skip computation.
        continue;
 
      // Accumulate the base class offsets.
      const ASTRecordLayout &L = Context.getASTRecordLayout(PathElement.Class);
      Offset += L.getBaseClassOffset(
          PathElement.Base->getType()->getAsCXXRecordDecl());
    }
  }
 
  // -2: Src is not a public base of Dst.
  if (NumPublicPaths == 0)
    return CharUnits::fromQuantity(-2ULL);
 
  // -3: Src is a multiple public base type but never a virtual base type.
  if (NumPublicPaths > 1)
    return CharUnits::fromQuantity(-3ULL);
 
  // Otherwise, the Src type is a unique public nonvirtual base type of Dst.
  // Return the offset of Src from the origin of Dst.
  return Offset;
}
 
static llvm::FunctionCallee getBadTypeidFn(CodeGenFunction &CGF) {
  // void __cxa_bad_typeid();
  llvm::FunctionType *FTy = llvm::FunctionType::get(CGF.VoidTy, false);
 
  return CGF.CGM.CreateRuntimeFunction(FTy, "__cxa_bad_typeid");
}
 
bool ItaniumCXXABI::shouldTypeidBeNullChecked(QualType SrcRecordTy) {
  return true;
}
 
void ItaniumCXXABI::EmitBadTypeidCall(CodeGenFunction &CGF) {
  llvm::FunctionCallee Fn = getBadTypeidFn(CGF);
  llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(Fn);
  Call->setDoesNotReturn();
  CGF.Builder.CreateUnreachable();
}
 
llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
                                       QualType SrcRecordTy,
                                       Address ThisPtr,
                                       llvm::Type *StdTypeInfoPtrTy) {
  auto *ClassDecl =
      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
  llvm::Value *Value = CGF.GetVTablePtr(ThisPtr, CGM.GlobalsInt8PtrTy,
                                        ClassDecl);
 
  if (CGM.getItaniumVTableContext().isRelativeLayout()) {
    // Load the type info.
    Value = CGF.Builder.CreateCall(
        CGM.getIntrinsic(llvm::Intrinsic::load_relative, {CGM.Int32Ty}),
        {Value, llvm::ConstantInt::get(CGM.Int32Ty, -4)});
  } else {
    // Load the type info.
    Value =
        CGF.Builder.CreateConstInBoundsGEP1_64(StdTypeInfoPtrTy, Value, -1ULL);
  }
  return CGF.Builder.CreateAlignedLoad(StdTypeInfoPtrTy, Value,
                                       CGF.getPointerAlign());
}
 
bool ItaniumCXXABI::shouldDynamicCastCallBeNullChecked(bool SrcIsPtr,
                                                       QualType SrcRecordTy) {
  return SrcIsPtr;
}
 
llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
    CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
    QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastEnd) {
  llvm::Type *PtrDiffLTy =
      CGF.ConvertType(CGF.getContext().getPointerDiffType());
 
  llvm::Value *SrcRTTI =
      CGF.CGM.GetAddrOfRTTIDescriptor(SrcRecordTy.getUnqualifiedType());
  llvm::Value *DestRTTI =
      CGF.CGM.GetAddrOfRTTIDescriptor(DestRecordTy.getUnqualifiedType());
 
  // Compute the offset hint.
  const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
  const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
  llvm::Value *OffsetHint = llvm::ConstantInt::get(
      PtrDiffLTy,
      computeOffsetHint(CGF.getContext(), SrcDecl, DestDecl).getQuantity());
 
  // Emit the call to __dynamic_cast.
  llvm::Value *Value = ThisAddr.emitRawPointer(CGF);
  if (CGM.getCodeGenOpts().PointerAuth.CXXVTablePointers) {
    // We perform a no-op load of the vtable pointer here to force an
    // authentication. In environments that do not support pointer
    // authentication this is a an actual no-op that will be elided. When
    // pointer authentication is supported and enforced on vtable pointers this
    // load can trap.
    llvm::Value *Vtable =
        CGF.GetVTablePtr(ThisAddr, CGM.Int8PtrTy, SrcDecl,
                         CodeGenFunction::VTableAuthMode::MustTrap);
    assert(Vtable);
    (void)Vtable;
  }
 
  llvm::Value *args[] = {Value, SrcRTTI, DestRTTI, OffsetHint};
  Value = CGF.EmitNounwindRuntimeCall(getItaniumDynamicCastFn(CGF), args);
 
  /// C++ [expr.dynamic.cast]p9:
  ///   A failed cast to reference type throws std::bad_cast
  if (DestTy->isReferenceType()) {
    llvm::BasicBlock *BadCastBlock =
        CGF.createBasicBlock("dynamic_cast.bad_cast");
 
    llvm::Value *IsNull = CGF.Builder.CreateIsNull(Value);
    CGF.Builder.CreateCondBr(IsNull, BadCastBlock, CastEnd);
 
    CGF.EmitBlock(BadCastBlock);
    EmitBadCastCall(CGF);
  }
 
  return Value;
}
 
llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
    CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
    QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastSuccess,
    llvm::BasicBlock *CastFail) {
  ASTContext &Context = getContext();
 
  // Find all the inheritance paths.
  const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
  const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
  CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
                     /*DetectVirtual=*/false);
  (void)DestDecl->isDerivedFrom(SrcDecl, Paths);
 
  // Find an offset within `DestDecl` where a `SrcDecl` instance and its vptr
  // might appear.
  std::optional<CharUnits> Offset;
  for (const CXXBasePath &Path : Paths) {
    // dynamic_cast only finds public inheritance paths.
    if (Path.Access != AS_public)
      continue;
 
    CharUnits PathOffset;
    for (const CXXBasePathElement &PathElement : Path) {
      // Find the offset along this inheritance step.
      const CXXRecordDecl *Base =
          PathElement.Base->getType()->getAsCXXRecordDecl();
      if (PathElement.Base->isVirtual()) {
        // For a virtual base class, we know that the derived class is exactly
        // DestDecl, so we can use the vbase offset from its layout.
        const ASTRecordLayout &L = Context.getASTRecordLayout(DestDecl);
        PathOffset = L.getVBaseClassOffset(Base);
      } else {
        const ASTRecordLayout &L =
            Context.getASTRecordLayout(PathElement.Class);
        PathOffset += L.getBaseClassOffset(Base);
      }
    }
 
    if (!Offset)
      Offset = PathOffset;
    else if (Offset != PathOffset) {
      // Base appears in at least two different places. Find the most-derived
      // object and see if it's a DestDecl. Note that the most-derived object
      // must be at least as aligned as this base class subobject, and must
      // have a vptr at offset 0.
      ThisAddr = Address(emitDynamicCastToVoid(CGF, ThisAddr, SrcRecordTy),
                         CGF.VoidPtrTy, ThisAddr.getAlignment());
      SrcDecl = DestDecl;
      Offset = CharUnits::Zero();
      break;
    }
  }
 
  if (!Offset) {
    // If there are no public inheritance paths, the cast always fails.
    CGF.EmitBranch(CastFail);
    return llvm::PoisonValue::get(CGF.VoidPtrTy);
  }
 
  // Compare the vptr against the expected vptr for the destination type at
  // this offset. Note that we do not know what type ThisAddr points to in
  // the case where the derived class multiply inherits from the base class
  // so we can't use GetVTablePtr, so we load the vptr directly instead.
  llvm::Instruction *VPtr = CGF.Builder.CreateLoad(
      ThisAddr.withElementType(CGF.VoidPtrPtrTy), "vtable");
  CGM.DecorateInstructionWithTBAA(
      VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy));
  llvm::Value *Success = CGF.Builder.CreateICmpEQ(
      VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl));
  llvm::Value *Result = ThisAddr.emitRawPointer(CGF);
  if (!Offset->isZero())
    Result = CGF.Builder.CreateInBoundsGEP(
        CGF.CharTy, Result,
        {llvm::ConstantInt::get(CGF.PtrDiffTy, -Offset->getQuantity())});
  CGF.Builder.CreateCondBr(Success, CastSuccess, CastFail);
  return Result;
}
 
llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
                                                  Address ThisAddr,
                                                  QualType SrcRecordTy) {
  auto *ClassDecl =
      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
  llvm::Value *OffsetToTop;
  if (CGM.getItaniumVTableContext().isRelativeLayout()) {
    // Get the vtable pointer.
    llvm::Value *VTable =
        CGF.GetVTablePtr(ThisAddr, CGF.UnqualPtrTy, ClassDecl);
 
    // Get the offset-to-top from the vtable.
    OffsetToTop =
        CGF.Builder.CreateConstInBoundsGEP1_32(CGM.Int32Ty, VTable, -2U);
    OffsetToTop = CGF.Builder.CreateAlignedLoad(
        CGM.Int32Ty, OffsetToTop, CharUnits::fromQuantity(4), "offset.to.top");
  } else {
    llvm::Type *PtrDiffLTy =
        CGF.ConvertType(CGF.getContext().getPointerDiffType());
 
    // Get the vtable pointer.
    llvm::Value *VTable =
        CGF.GetVTablePtr(ThisAddr, CGF.UnqualPtrTy, ClassDecl);
 
    // Get the offset-to-top from the vtable.
    OffsetToTop =
        CGF.Builder.CreateConstInBoundsGEP1_64(PtrDiffLTy, VTable, -2ULL);
    OffsetToTop = CGF.Builder.CreateAlignedLoad(
        PtrDiffLTy, OffsetToTop, CGF.getPointerAlign(), "offset.to.top");
  }
  // Finally, add the offset to the pointer.
  return CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ThisAddr.emitRawPointer(CGF),
                                       OffsetToTop);
}
 
bool ItaniumCXXABI::EmitBadCastCall(CodeGenFunction &CGF) {
  llvm::FunctionCallee Fn = getBadCastFn(CGF);
  llvm::CallBase *Call = CGF.EmitRuntimeCallOrInvoke(Fn);
  Call->setDoesNotReturn();
  CGF.Builder.CreateUnreachable();
  return true;
}
 
llvm::Value *
ItaniumCXXABI::GetVirtualBaseClassOffset(CodeGenFunction &CGF,
                                         Address This,
                                         const CXXRecordDecl *ClassDecl,
                                         const CXXRecordDecl *BaseClassDecl) {
  llvm::Value *VTablePtr = CGF.GetVTablePtr(This, CGM.Int8PtrTy, ClassDecl);
  CharUnits VBaseOffsetOffset =
      CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl,
                                                               BaseClassDecl);
  llvm::Value *VBaseOffsetPtr =
    CGF.Builder.CreateConstGEP1_64(
        CGF.Int8Ty, VTablePtr, VBaseOffsetOffset.getQuantity(),
        "vbase.offset.ptr");
 
  llvm::Value *VBaseOffset;
  if (CGM.getItaniumVTableContext().isRelativeLayout()) {
    VBaseOffset = CGF.Builder.CreateAlignedLoad(
        CGF.Int32Ty, VBaseOffsetPtr, CharUnits::fromQuantity(4),
        "vbase.offset");
  } else {
    VBaseOffset = CGF.Builder.CreateAlignedLoad(
        CGM.PtrDiffTy, VBaseOffsetPtr, CGF.getPointerAlign(), "vbase.offset");
  }
  return VBaseOffset;
}
 
void ItaniumCXXABI::EmitCXXConstructors(const CXXConstructorDecl *D) {
  // Just make sure we're in sync with TargetCXXABI.
  assert(CGM.getTarget().getCXXABI().hasConstructorVariants());
 
  // The constructor used for constructing this as a base class;
  // ignores virtual bases.
  CGM.EmitGlobal(GlobalDecl(D, Ctor_Base));
 
  // The constructor used for constructing this as a complete class;
  // constructs the virtual bases, then calls the base constructor.
  if (!D->getParent()->isAbstract()) {
    // We don't need to emit the complete ctor if the class is abstract.
    CGM.EmitGlobal(GlobalDecl(D, Ctor_Complete));
  }
}
 
CGCXXABI::AddedStructorArgCounts
ItaniumCXXABI::buildStructorSignature(GlobalDecl GD,
                                      SmallVectorImpl<CanQualType> &ArgTys) {
  ASTContext &Context = getContext();
 
  // All parameters are already in place except VTT, which goes after 'this'.
  // These are Clang types, so we don't need to worry about sret yet.
 
  // Check if we need to add a VTT parameter (which has type global void **).
  if ((isa<CXXConstructorDecl>(GD.getDecl()) ? GD.getCtorType() == Ctor_Base
                                             : GD.getDtorType() == Dtor_Base) &&
      cast<CXXMethodDecl>(GD.getDecl())->getParent()->getNumVBases() != 0) {
    LangAS AS = CGM.GetGlobalVarAddressSpace(nullptr);
    QualType Q = Context.getAddrSpaceQualType(Context.VoidPtrTy, AS);
    ArgTys.insert(ArgTys.begin() + 1,
                  Context.getPointerType(CanQualType::CreateUnsafe(Q)));
    return AddedStructorArgCounts::prefix(1);
  }
  return AddedStructorArgCounts{};
}
 
void ItaniumCXXABI::EmitCXXDestructors(const CXXDestructorDecl *D) {
  // The destructor used for destructing this as a base class; ignores
  // virtual bases.
  CGM.EmitGlobal(GlobalDecl(D, Dtor_Base));
 
  // The destructor used for destructing this as a most-derived class;
  // call the base destructor and then destructs any virtual bases.
  CGM.EmitGlobal(GlobalDecl(D, Dtor_Complete));
 
  // The destructor in a virtual table is always a 'deleting'
  // destructor, which calls the complete destructor and then uses the
  // appropriate operator delete.
  if (D->isVirtual())
    CGM.EmitGlobal(GlobalDecl(D, Dtor_Deleting));
}
 
void ItaniumCXXABI::addImplicitStructorParams(CodeGenFunction &CGF,
                                              QualType &ResTy,
                                              FunctionArgList &Params) {
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(CGF.CurGD.getDecl());
  assert(isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD));
 
  // Check if we need a VTT parameter as well.
  if (NeedsVTTParameter(CGF.CurGD)) {
    ASTContext &Context = getContext();
 
    // FIXME: avoid the fake decl
    LangAS AS = CGM.GetGlobalVarAddressSpace(nullptr);
    QualType Q = Context.getAddrSpaceQualType(Context.VoidPtrTy, AS);
    QualType T = Context.getPointerType(Q);
    auto *VTTDecl = ImplicitParamDecl::Create(
        Context, /*DC=*/nullptr, MD->getLocation(), &Context.Idents.get("vtt"),
        T, ImplicitParamKind::CXXVTT);
    Params.insert(Params.begin() + 1, VTTDecl);
    getStructorImplicitParamDecl(CGF) = VTTDecl;
  }
}
 
void ItaniumCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
  // Naked functions have no prolog.
  if (CGF.CurFuncDecl && CGF.CurFuncDecl->hasAttr<NakedAttr>())
    return;
 
  /// Initialize the 'this' slot. In the Itanium C++ ABI, no prologue
  /// adjustments are required, because they are all handled by thunks.
  setCXXABIThisValue(CGF, loadIncomingCXXThis(CGF));
 
  /// Initialize the 'vtt' slot if needed.
  if (getStructorImplicitParamDecl(CGF)) {
    getStructorImplicitParamValue(CGF) = CGF.Builder.CreateLoad(
        CGF.GetAddrOfLocalVar(getStructorImplicitParamDecl(CGF)), "vtt");
  }
 
  /// If this is a function that the ABI specifies returns 'this', initialize
  /// the return slot to 'this' at the start of the function.
  ///
  /// Unlike the setting of return types, this is done within the ABI
  /// implementation instead of by clients of CGCXXABI because:
  /// 1) getThisValue is currently protected
  /// 2) in theory, an ABI could implement 'this' returns some other way;
  ///    HasThisReturn only specifies a contract, not the implementation
  if (HasThisReturn(CGF.CurGD))
    CGF.Builder.CreateStore(getThisValue(CGF), CGF.ReturnValue);
}
 
CGCXXABI::AddedStructorArgs ItaniumCXXABI::getImplicitConstructorArgs(
    CodeGenFunction &CGF, const CXXConstructorDecl *D, CXXCtorType Type,
    bool ForVirtualBase, bool Delegating) {
  if (!NeedsVTTParameter(GlobalDecl(D, Type)))
    return AddedStructorArgs{};
 
  // Insert the implicit 'vtt' argument as the second argument. Make sure to
  // correctly reflect its address space, which can differ from generic on
  // some targets.
  llvm::Value *VTT =
      CGF.GetVTTParameter(GlobalDecl(D, Type), ForVirtualBase, Delegating);
  LangAS AS = CGM.GetGlobalVarAddressSpace(nullptr);
  QualType Q = getContext().getAddrSpaceQualType(getContext().VoidPtrTy, AS);
  QualType VTTTy = getContext().getPointerType(Q);
  return AddedStructorArgs::prefix({{VTT, VTTTy}});
}
 
llvm::Value *ItaniumCXXABI::getCXXDestructorImplicitParam(
    CodeGenFunction &CGF, const CXXDestructorDecl *DD, CXXDtorType Type,
    bool ForVirtualBase, bool Delegating) {
  GlobalDecl GD(DD, Type);
  return CGF.GetVTTParameter(GD, ForVirtualBase, Delegating);
}
 
void ItaniumCXXABI::EmitDestructorCall(CodeGenFunction &CGF,
                                       const CXXDestructorDecl *DD,
                                       CXXDtorType Type, bool ForVirtualBase,
                                       bool Delegating, Address This,
                                       QualType ThisTy) {
  GlobalDecl GD(DD, Type);
  llvm::Value *VTT =
      getCXXDestructorImplicitParam(CGF, DD, Type, ForVirtualBase, Delegating);
  QualType VTTTy = getContext().getPointerType(getContext().VoidPtrTy);
 
  CGCallee Callee;
  if (getContext().getLangOpts().AppleKext &&
      Type != Dtor_Base && DD->isVirtual())
    Callee = CGF.BuildAppleKextVirtualDestructorCall(DD, Type, DD->getParent());
  else
    Callee = CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD), GD);
 
  CGF.EmitCXXDestructorCall(GD, Callee, CGF.getAsNaturalPointerTo(This, ThisTy),
                            ThisTy, VTT, VTTTy, nullptr);
}
 
// Check if any non-inline method has the specified attribute.
template <typename T>
static bool CXXRecordNonInlineHasAttr(const CXXRecordDecl *RD) {
  for (const auto *D : RD->noload_decls()) {
    if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
      if (FD->isInlined() || FD->doesThisDeclarationHaveABody() ||
          FD->isPureVirtual())
        continue;
      if (D->hasAttr<T>())
        return true;
    }
  }
 
  return false;
}
 
static void setVTableSelectiveDLLImportExport(CodeGenModule &CGM,
                                              llvm::GlobalVariable *VTable,
                                              const CXXRecordDecl *RD) {
  if (VTable->getDLLStorageClass() !=
          llvm::GlobalVariable::DefaultStorageClass ||
      RD->hasAttr<DLLImportAttr>() || RD->hasAttr<DLLExportAttr>())
    return;
 
  if (CGM.getVTables().isVTableExternal(RD)) {
    if (CXXRecordNonInlineHasAttr<DLLImportAttr>(RD))
      VTable->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
  } else if (CXXRecordNonInlineHasAttr<DLLExportAttr>(RD))
    VTable->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
}
 
void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
                                          const CXXRecordDecl *RD) {
  llvm::GlobalVariable *VTable = getAddrOfVTable(RD, CharUnits());
  if (VTable->hasInitializer())
    return;
 
  ItaniumVTableContext &VTContext = CGM.getItaniumVTableContext();
  const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
  llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
  llvm::Constant *RTTI =
      CGM.GetAddrOfRTTIDescriptor(CGM.getContext().getTagDeclType(RD));
 
  // Create and set the initializer.
  ConstantInitBuilder builder(CGM);
  auto components = builder.beginStruct();
  CGVT.createVTableInitializer(components, VTLayout, RTTI,
                               llvm::GlobalValue::isLocalLinkage(Linkage));
  components.finishAndSetAsInitializer(VTable);
 
  // Set the correct linkage.
  VTable->setLinkage(Linkage);
 
  if (CGM.supportsCOMDAT() && VTable->isWeakForLinker())
    VTable->setComdat(CGM.getModule().getOrInsertComdat(VTable->getName()));
 
  if (CGM.getTarget().hasPS4DLLImportExport())
    setVTableSelectiveDLLImportExport(CGM, VTable, RD);
 
  // Set the right visibility.
  CGM.setGVProperties(VTable, RD);
 
  // If this is the magic class __cxxabiv1::__fundamental_type_info,
  // we will emit the typeinfo for the fundamental types. This is the
  // same behaviour as GCC.
  const DeclContext *DC = RD->getDeclContext();
  if (RD->getIdentifier() &&
      RD->getIdentifier()->isStr("__fundamental_type_info") &&
      isa<NamespaceDecl>(DC) && cast<NamespaceDecl>(DC)->getIdentifier() &&
      cast<NamespaceDecl>(DC)->getIdentifier()->isStr("__cxxabiv1") &&
      DC->getParent()->isTranslationUnit())
    EmitFundamentalRTTIDescriptors(RD);
 
  // Always emit type metadata on non-available_externally definitions, and on
  // available_externally definitions if we are performing whole program
  // devirtualization. For WPD we need the type metadata on all vtable
  // definitions to ensure we associate derived classes with base classes
  // defined in headers but with a strong definition only in a shared library.
  if (!VTable->isDeclarationForLinker() ||
      CGM.getCodeGenOpts().WholeProgramVTables) {
    CGM.EmitVTableTypeMetadata(RD, VTable, VTLayout);
    // For available_externally definitions, add the vtable to
    // @llvm.compiler.used so that it isn't deleted before whole program
    // analysis.
    if (VTable->isDeclarationForLinker()) {
      assert(CGM.getCodeGenOpts().WholeProgramVTables);
      CGM.addCompilerUsedGlobal(VTable);
    }
  }
 
  if (VTContext.isRelativeLayout()) {
    CGVT.RemoveHwasanMetadata(VTable);
    if (!VTable->isDSOLocal())
      CGVT.GenerateRelativeVTableAlias(VTable, VTable->getName());
  }
}
 
bool ItaniumCXXABI::isVirtualOffsetNeededForVTableField(
    CodeGenFunction &CGF, CodeGenFunction::VPtr Vptr) {
  if (Vptr.NearestVBase == nullptr)
    return false;
  return NeedsVTTParameter(CGF.CurGD);
}
 
llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructor(
    CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
    const CXXRecordDecl *NearestVBase) {
 
  if ((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
      NeedsVTTParameter(CGF.CurGD)) {
    return getVTableAddressPointInStructorWithVTT(CGF, VTableClass, Base,
                                                  NearestVBase);
  }
  return getVTableAddressPoint(Base, VTableClass);
}
 
llvm::Constant *
ItaniumCXXABI::getVTableAddressPoint(BaseSubobject Base,
                                     const CXXRecordDecl *VTableClass) {
  llvm::GlobalValue *VTable = getAddrOfVTable(VTableClass, CharUnits());
 
  // Find the appropriate vtable within the vtable group, and the address point
  // within that vtable.
  const VTableLayout &Layout =
      CGM.getItaniumVTableContext().getVTableLayout(VTableClass);
  VTableLayout::AddressPointLocation AddressPoint =
      Layout.getAddressPoint(Base);
  llvm::Value *Indices[] = {
    llvm::ConstantInt::get(CGM.Int32Ty, 0),
    llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.VTableIndex),
    llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.AddressPointIndex),
  };
 
  // Add inrange attribute to indicate that only the VTableIndex can be
  // accessed.
  unsigned ComponentSize =
      CGM.getDataLayout().getTypeAllocSize(CGM.getVTableComponentType());
  unsigned VTableSize =
      ComponentSize * Layout.getVTableSize(AddressPoint.VTableIndex);
  unsigned Offset = ComponentSize * AddressPoint.AddressPointIndex;
  llvm::ConstantRange InRange(
      llvm::APInt(32, (int)-Offset, true),
      llvm::APInt(32, (int)(VTableSize - Offset), true));
  return llvm::ConstantExpr::getGetElementPtr(
      VTable->getValueType(), VTable, Indices, /*InBounds=*/true, InRange);
}
 
llvm::Value *ItaniumCXXABI::getVTableAddressPointInStructorWithVTT(
    CodeGenFunction &CGF, const CXXRecordDecl *VTableClass, BaseSubobject Base,
    const CXXRecordDecl *NearestVBase) {
  assert((Base.getBase()->getNumVBases() || NearestVBase != nullptr) &&
         NeedsVTTParameter(CGF.CurGD) && "This class doesn't have VTT");
 
  // Get the secondary vpointer index.
  uint64_t VirtualPointerIndex =
      CGM.getVTables().getSecondaryVirtualPointerIndex(VTableClass, Base);