//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file implements some simple delegations needed for call lowering.
///
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/GlobalISel/CallLowering.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#define DEBUG_TYPE "call-lowering"
using namespace llvm;
void CallLowering::anchor() {}
bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs,
Register SwiftErrorVReg,
std::function<unsigned()> GetCalleeReg) const {
CallLoweringInfo Info;
auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout();
// First step is to marshall all the function's parameters into the correct
// physregs and memory locations. Gather the sequence of argument types that
// we'll pass to the assigner function.
unsigned i = 0;
unsigned NumFixedArgs = CS.getFunctionType()->getNumParams();
for (auto &Arg : CS.args()) {
ArgInfo OrigArg{ArgRegs[i], Arg->getType(), ISD::ArgFlagsTy{},
i < NumFixedArgs};
setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CS);
Info.OrigArgs.push_back(OrigArg);
++i;
}
if (const Function *F = CS.getCalledFunction())
Info.Callee = MachineOperand::CreateGA(F, 0);
else
Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
Info.OrigRet = ArgInfo{ResRegs, CS.getType(), ISD::ArgFlagsTy{}};
if (!Info.OrigRet.Ty->isVoidTy())
setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CS);
Info.KnownCallees =
CS.getInstruction()->getMetadata(LLVMContext::MD_callees);
Info.CallConv = CS.getCallingConv();
Info.SwiftErrorVReg = SwiftErrorVReg;
Info.IsMustTailCall = CS.isMustTailCall();
Info.IsTailCall = CS.isTailCall() &&
isInTailCallPosition(CS, MIRBuilder.getMF().getTarget()) &&
(MIRBuilder.getMF()
.getFunction()
.getFnAttribute("disable-tail-calls")
.getValueAsString() != "true");
Info.IsVarArg = CS.getFunctionType()->isVarArg();
return lowerCall(MIRBuilder, Info);
}
template <typename FuncInfoTy>
void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,
const DataLayout &DL,
const FuncInfoTy &FuncInfo) const {
auto &Flags = Arg.Flags[0];
const AttributeList &Attrs = FuncInfo.getAttributes();
if (Attrs.hasAttribute(OpIdx, Attribute::ZExt))
Flags.setZExt();
if (Attrs.hasAttribute(OpIdx, Attribute::SExt))
Flags.setSExt();
if (Attrs.hasAttribute(OpIdx, Attribute::InReg))
Flags.setInReg();
if (Attrs.hasAttribute(OpIdx, Attribute::StructRet))
Flags.setSRet();
if (Attrs.hasAttribute(OpIdx, Attribute::SwiftSelf))
Flags.setSwiftSelf();
if (Attrs.hasAttribute(OpIdx, Attribute::SwiftError))
Flags.setSwiftError();
if (Attrs.hasAttribute(OpIdx, Attribute::ByVal))
Flags.setByVal();
if (Attrs.hasAttribute(OpIdx, Attribute::InAlloca))
Flags.setInAlloca();
if (Flags.isByVal() || Flags.isInAlloca()) {
Type *ElementTy = cast<PointerType>(Arg.Ty)->getElementType();
auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType();
Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy));
// For ByVal, alignment should be passed from FE. BE will guess if
// this info is not there but there are cases it cannot get right.
unsigned FrameAlign;
if (FuncInfo.getParamAlignment(OpIdx - 2))
FrameAlign = FuncInfo.getParamAlignment(OpIdx - 2);
else
FrameAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);
Flags.setByValAlign(Align(FrameAlign));
}
if (Attrs.hasAttribute(OpIdx, Attribute::Nest))
Flags.setNest();
Flags.setOrigAlign(Align(DL.getABITypeAlignment(Arg.Ty)));
}
template void
CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
const DataLayout &DL,
const Function &FuncInfo) const;
template void
CallLowering::setArgFlags<CallInst>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
const DataLayout &DL,
const CallInst &FuncInfo) const;
Register CallLowering::packRegs(ArrayRef<Register> SrcRegs, Type *PackedTy,
MachineIRBuilder &MIRBuilder) const {
assert(SrcRegs.size() > 1 && "Nothing to pack");
const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
LLT PackedLLT = getLLTForType(*PackedTy, DL);
SmallVector<LLT, 8> LLTs;
SmallVector<uint64_t, 8> Offsets;
computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch");
Register Dst = MRI->createGenericVirtualRegister(PackedLLT);
MIRBuilder.buildUndef(Dst);
for (unsigned i = 0; i < SrcRegs.size(); ++i) {
Register NewDst = MRI->createGenericVirtualRegister(PackedLLT);
MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]);
Dst = NewDst;
}
return Dst;
}
void CallLowering::unpackRegs(ArrayRef<Register> DstRegs, Register SrcReg,
Type *PackedTy,
MachineIRBuilder &MIRBuilder) const {
assert(DstRegs.size() > 1 && "Nothing to unpack");
const DataLayout &DL = MIRBuilder.getMF().getDataLayout();
SmallVector<LLT, 8> LLTs;
SmallVector<uint64_t, 8> Offsets;
computeValueLLTs(DL, *PackedTy, LLTs, &Offsets);
assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch");
for (unsigned i = 0; i < DstRegs.size(); ++i)
MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]);
}
bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder,
SmallVectorImpl<ArgInfo> &Args,
ValueHandler &Handler) const {
MachineFunction &MF = MIRBuilder.getMF();
const Function &F = MF.getFunction();
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler);
}
bool CallLowering::handleAssignments(CCState &CCInfo,
SmallVectorImpl<CCValAssign> &ArgLocs,
MachineIRBuilder &MIRBuilder,
SmallVectorImpl<ArgInfo> &Args,
ValueHandler &Handler) const {
MachineFunction &MF = MIRBuilder.getMF();
const Function &F = MF.getFunction();
const DataLayout &DL = F.getParent()->getDataLayout();
unsigned NumArgs = Args.size();
for (unsigned i = 0; i != NumArgs; ++i) {
MVT CurVT = MVT::getVT(Args[i].Ty);
if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i],
Args[i].Flags[0], CCInfo)) {
if (!CurVT.isValid())
return false;
MVT NewVT = TLI->getRegisterTypeForCallingConv(
F.getContext(), F.getCallingConv(), EVT(CurVT));
// If we need to split the type over multiple regs, check it's a scenario
// we currently support.
unsigned NumParts = TLI->getNumRegistersForCallingConv(
F.getContext(), F.getCallingConv(), CurVT);
if (NumParts > 1) {
// For now only handle exact splits.
if (NewVT.getSizeInBits() * NumParts != CurVT.getSizeInBits())
return false;
}
// For incoming arguments (physregs to vregs), we could have values in
// physregs (or memlocs) which we want to extract and copy to vregs.
// During this, we might have to deal with the LLT being split across
// multiple regs, so we have to record this information for later.
//
// If we have outgoing args, then we have the opposite case. We have a
// vreg with an LLT which we want to assign to a physical location, and
// we might have to record that the value has to be split later.
if (Handler.isIncomingArgumentHandler()) {
if (NumParts == 1) {
// Try to use the register type if we couldn't assign the VT.
if (Handler.assignArg(i, NewVT, NewVT, CCValAssign::Full, Args[i],
Args[i].Flags[0], CCInfo))
return false;
} else {
// We're handling an incoming arg which is split over multiple regs.
// E.g. passing an s128 on AArch64.
ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
Args[i].OrigRegs.push_back(Args[i].Regs[0]);
Args[i].Regs.clear();
Args[i].Flags.clear();
LLT NewLLT = getLLTForMVT(NewVT);
// For each split register, create and assign a vreg that will store
// the incoming component of the larger value. These will later be
// merged to form the final vreg.
for (unsigned Part = 0; Part < NumParts; ++Part) {
Register Reg =
MIRBuilder.getMRI()->createGenericVirtualRegister(NewLLT);
ISD::ArgFlagsTy Flags = OrigFlags;
if (Part == 0) {
Flags.setSplit();
} else {
Flags.setOrigAlign(Align::None());
if (Part == NumParts - 1)
Flags.setSplitEnd();
}
Args[i].Regs.push_back(Reg);
Args[i].Flags.push_back(Flags);
if (Handler.assignArg(i + Part, NewVT, NewVT, CCValAssign::Full,
Args[i], Args[i].Flags[Part], CCInfo)) {
// Still couldn't assign this smaller part type for some reason.
return false;
}
}
}
} else {
// Handling an outgoing arg that might need to be split.
if (NumParts < 2)
return false; // Don't know how to deal with this type combination.
// This type is passed via multiple registers in the calling convention.
// We need to extract the individual parts.
Register LargeReg = Args[i].Regs[0];
LLT SmallTy = LLT::scalar(NewVT.getSizeInBits());
auto Unmerge = MIRBuilder.buildUnmerge(SmallTy, LargeReg);
assert(Unmerge->getNumOperands() == NumParts + 1);
ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];
// We're going to replace the regs and flags with the split ones.
Args[i].Regs.clear();
Args[i].Flags.clear();
for (unsigned PartIdx = 0; PartIdx < NumParts; ++PartIdx) {
ISD::ArgFlagsTy Flags = OrigFlags;
if (PartIdx == 0) {
Flags.setSplit();
} else {
Flags.setOrigAlign(Align::None());
if (PartIdx == NumParts - 1)
Flags.setSplitEnd();
}
Args[i].Regs.push_back(Unmerge.getReg(PartIdx));
Args[i].Flags.push_back(Flags);
if (Handler.assignArg(i + PartIdx, NewVT, NewVT, CCValAssign::Full,
Args[i], Args[i].Flags[PartIdx], CCInfo))
return false;
}
}
}
}
for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) {
assert(j < ArgLocs.size() && "Skipped too many arg locs");
CCValAssign &VA = ArgLocs[j];
assert(VA.getValNo() == i && "Location doesn't correspond to current arg");
if (VA.needsCustom()) {
j += Handler.assignCustomValue(Args[i], makeArrayRef(ArgLocs).slice(j));
continue;
}
// FIXME: Pack registers if we have more than one.
Register ArgReg = Args[i].Regs[0];
MVT OrigVT = MVT::getVT(Args[i].Ty);
MVT VAVT = VA.getValVT();
if (VA.isRegLoc()) {
if (Handler.isIncomingArgumentHandler() && VAVT != OrigVT) {
if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) {
// Expected to be multiple regs for a single incoming arg.
unsigned NumArgRegs = Args[i].Regs.size();
if (NumArgRegs < 2)
return false;
assert((j + (NumArgRegs - 1)) < ArgLocs.size() &&
"Too many regs for number of args");
for (unsigned Part = 0; Part < NumArgRegs; ++Part) {
// There should be Regs.size() ArgLocs per argument.
VA = ArgLocs[j + Part];
Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
}
j += NumArgRegs - 1;
// Merge the split registers into the expected larger result vreg
// of the original call.
MIRBuilder.buildMerge(Args[i].OrigRegs[0], Args[i].Regs);
continue;
}
const LLT VATy(VAVT);
Register NewReg =
MIRBuilder.getMRI()->createGenericVirtualRegister(VATy);
Handler.assignValueToReg(NewReg, VA.getLocReg(), VA);
// If it's a vector type, we either need to truncate the elements
// or do an unmerge to get the lower block of elements.
if (VATy.isVector() &&
VATy.getNumElements() > OrigVT.getVectorNumElements()) {
const LLT OrigTy(OrigVT);
// Just handle the case where the VA type is 2 * original type.
if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) {
LLVM_DEBUG(dbgs()
<< "Incoming promoted vector arg has too many elts");
return false;
}
auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg});
MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0));
} else {
MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0);
}
} else if (!Handler.isIncomingArgumentHandler()) {
assert((j + (Args[i].Regs.size() - 1)) < ArgLocs.size() &&
"Too many regs for number of args");
// This is an outgoing argument that might have been split.
for (unsigned Part = 0; Part < Args[i].Regs.size(); ++Part) {
// There should be Regs.size() ArgLocs per argument.
VA = ArgLocs[j + Part];
Handler.assignValueToReg(Args[i].Regs[Part], VA.getLocReg(), VA);
}
j += Args[i].Regs.size() - 1;
} else {
Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);
}
} else if (VA.isMemLoc()) {
// Don't currently support loading/storing a type that needs to be split
// to the stack. Should be easy, just not implemented yet.
if (Args[i].Regs.size() > 1) {
LLVM_DEBUG(
dbgs()
<< "Load/store a split arg to/from the stack not implemented yet");
return false;
}
MVT VT = MVT::getVT(Args[i].Ty);
unsigned Size = VT == MVT::iPTR ? DL.getPointerSize()
: alignTo(VT.getSizeInBits(), 8) / 8;
unsigned Offset = VA.getLocMemOffset();
MachinePointerInfo MPO;
Register StackAddr = Handler.getStackAddress(Size, Offset, MPO);
Handler.assignValueToAddress(ArgReg, StackAddr, Size, MPO, VA);
} else {
// FIXME: Support byvals and other weirdness
return false;
}
}
return true;
}
bool CallLowering::analyzeArgInfo(CCState &CCState,
SmallVectorImpl<ArgInfo> &Args,
CCAssignFn &AssignFnFixed,
CCAssignFn &AssignFnVarArg) const {
for (unsigned i = 0, e = Args.size(); i < e; ++i) {
MVT VT = MVT::getVT(Args[i].Ty);
CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg;
if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) {
// Bail out on anything we can't handle.
LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString()
<< " (arg number = " << i << "\n");
return false;
}
}
return true;
}
bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
MachineFunction &MF,
SmallVectorImpl<ArgInfo> &InArgs,
CCAssignFn &CalleeAssignFnFixed,
CCAssignFn &CalleeAssignFnVarArg,
CCAssignFn &CallerAssignFnFixed,
CCAssignFn &CallerAssignFnVarArg) const {
const Function &F = MF.getFunction();
CallingConv::ID CalleeCC = Info.CallConv;
CallingConv::ID CallerCC = F.getCallingConv();
if (CallerCC == CalleeCC)
return true;
SmallVector<CCValAssign, 16> ArgLocs1;
CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext());
if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed,
CalleeAssignFnVarArg))
return false;
SmallVector<CCValAssign, 16> ArgLocs2;
CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext());
if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed,
CalleeAssignFnVarArg))
return false;
// We need the argument locations to match up exactly. If there's more in
// one than the other, then we are done.
if (ArgLocs1.size() != ArgLocs2.size())
return false;
// Make sure that each location is passed in exactly the same way.
for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {
const CCValAssign &Loc1 = ArgLocs1[i];
const CCValAssign &Loc2 = ArgLocs2[i];
// We need both of them to be the same. So if one is a register and one
// isn't, we're done.
if (Loc1.isRegLoc() != Loc2.isRegLoc())
return false;
if (Loc1.isRegLoc()) {
// If they don't have the same register location, we're done.
if (Loc1.getLocReg() != Loc2.getLocReg())
return false;
// They matched, so we can move to the next ArgLoc.
continue;
}
// Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.
if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())
return false;
}
return true;
}
Register CallLowering::ValueHandler::extendRegister(Register ValReg,
CCValAssign &VA) {
LLT LocTy{VA.getLocVT()};
if (LocTy.getSizeInBits() == MRI.getType(ValReg).getSizeInBits())
return ValReg;
switch (VA.getLocInfo()) {
default: break;
case CCValAssign::Full:
case CCValAssign::BCvt:
// FIXME: bitconverting between vector types may or may not be a
// nop in big-endian situations.
return ValReg;
case CCValAssign::AExt: {
auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);
return MIB->getOperand(0).getReg();
}
case CCValAssign::SExt: {
Register NewReg = MRI.createGenericVirtualRegister(LocTy);
MIRBuilder.buildSExt(NewReg, ValReg);
return NewReg;
}
case CCValAssign::ZExt: {
Register NewReg = MRI.createGenericVirtualRegister(LocTy);
MIRBuilder.buildZExt(NewReg, ValReg);
return NewReg;
}
}
llvm_unreachable("unable to extend register");
}
void CallLowering::ValueHandler::anchor() {}