1360 lines
49 KiB
C++
1360 lines
49 KiB
C++
// [AsmJit]
|
|
// Complete x86/x64 JIT and Remote Assembler for C++.
|
|
//
|
|
// [License]
|
|
// Zlib - See LICENSE.md file in the package.
|
|
|
|
// [Export]
|
|
#define ASMJIT_EXPORTS
|
|
|
|
// [Guard]
|
|
#include "../asmjit_build.h"
|
|
#if defined(ASMJIT_BUILD_X86)
|
|
|
|
// [Dependencies]
|
|
#include "../x86/x86internal_p.h"
|
|
|
|
// [Api-Begin]
|
|
#include "../asmjit_apibegin.h"
|
|
|
|
namespace asmjit {
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - Helpers]
|
|
// ============================================================================
|
|
|
|
static ASMJIT_INLINE uint32_t x86GetXmmMovInst(const FuncFrameLayout& layout) {
|
|
bool avx = layout.isAvxEnabled();
|
|
bool aligned = layout.hasAlignedVecSR();
|
|
|
|
return aligned ? (avx ? X86Inst::kIdVmovaps : X86Inst::kIdMovaps)
|
|
: (avx ? X86Inst::kIdVmovups : X86Inst::kIdMovups);
|
|
}
|
|
|
|
static ASMJIT_INLINE uint32_t x86VecTypeIdToRegType(uint32_t typeId) noexcept {
|
|
return typeId <= TypeId::_kVec128End ? X86Reg::kRegXmm :
|
|
typeId <= TypeId::_kVec256End ? X86Reg::kRegYmm :
|
|
X86Reg::kRegZmm ;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86FuncArgsContext]
|
|
// ============================================================================
|
|
|
|
// Used by both, `Utils::argsToFrameInfo()` and `Utils::allocArgs()`.
|
|
class X86FuncArgsContext {
|
|
public:
|
|
typedef FuncDetail::Value SrcArg;
|
|
typedef FuncArgsMapper::Value DstArg;
|
|
|
|
enum { kMaxVRegKinds = Globals::kMaxVRegKinds };
|
|
|
|
struct WorkData {
|
|
uint32_t archRegs; //!< Architecture provided and allocable regs.
|
|
uint32_t workRegs; //!< Registers that can be used by shuffler.
|
|
uint32_t usedRegs; //!< Only registers used to pass arguments.
|
|
uint32_t srcRegs; //!< Source registers that need shuffling.
|
|
uint32_t dstRegs; //!< Destination registers that need shuffling.
|
|
uint8_t numOps; //!< Number of operations to finish.
|
|
uint8_t numSwaps; //!< Number of register swaps.
|
|
uint8_t numStackArgs; //!< Number of stack loads.
|
|
uint8_t reserved[9]; //!< Reserved (only used as padding).
|
|
uint8_t argIndex[32]; //!< Only valid if a corresponding bit in `userRegs` is true.
|
|
};
|
|
|
|
X86FuncArgsContext() noexcept;
|
|
Error initWorkData(const FuncArgsMapper& args, const uint32_t* dirtyRegs, bool preservedFP) noexcept;
|
|
|
|
Error markRegsForSwaps(FuncFrameInfo& ffi) noexcept;
|
|
Error markDstRegsDirty(FuncFrameInfo& ffi) noexcept;
|
|
Error markStackArgsReg(FuncFrameInfo& ffi) noexcept;
|
|
|
|
// --------------------------------------------------------------------------
|
|
// [Members]
|
|
// --------------------------------------------------------------------------
|
|
|
|
WorkData _workData[kMaxVRegKinds];
|
|
bool _hasStackArgs;
|
|
bool _hasRegSwaps;
|
|
};
|
|
|
|
X86FuncArgsContext::X86FuncArgsContext() noexcept {
|
|
::memset(_workData, 0, sizeof(_workData));
|
|
_hasStackArgs = false;
|
|
_hasRegSwaps = false;
|
|
}
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::initWorkData(const FuncArgsMapper& args, const uint32_t* dirtyRegs, bool preservedFP) noexcept {
|
|
// This code has to be updated if this changes.
|
|
ASMJIT_ASSERT(kMaxVRegKinds == 4);
|
|
|
|
uint32_t i;
|
|
const FuncDetail& func = *args.getFuncDetail();
|
|
|
|
uint32_t archType = func.getCallConv().getArchType();
|
|
uint32_t count = (archType == ArchInfo::kTypeX86) ? 8 : 16;
|
|
|
|
// Initialize WorkData::archRegs.
|
|
_workData[X86Reg::kKindGp ].archRegs = Utils::bits(count) & ~Utils::mask(X86Gp::kIdSp);
|
|
_workData[X86Reg::kKindMm ].archRegs = Utils::bits(8);
|
|
_workData[X86Reg::kKindK ].archRegs = Utils::bits(8);
|
|
_workData[X86Reg::kKindVec].archRegs = Utils::bits(count);
|
|
|
|
if (preservedFP)
|
|
_workData[X86Reg::kKindGp].archRegs &= ~Utils::mask(X86Gp::kIdBp);
|
|
|
|
// Initialize WorkData::workRegs.
|
|
for (i = 0; i < kMaxVRegKinds; i++)
|
|
_workData[i].workRegs = _workData[i].archRegs & (dirtyRegs[i] | ~func.getCallConv().getPreservedRegs(i));
|
|
|
|
// Build WorkData.
|
|
for (i = 0; i < kFuncArgCountLoHi; i++) {
|
|
const DstArg& dstArg = args.getArg(i);
|
|
if (!dstArg.isAssigned()) continue;
|
|
|
|
const SrcArg& srcArg = func.getArg(i);
|
|
if (ASMJIT_UNLIKELY(!srcArg.isAssigned()))
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
|
|
uint32_t dstRegType = dstArg.getRegType();
|
|
if (ASMJIT_UNLIKELY(dstRegType >= X86Reg::kRegCount))
|
|
return DebugUtils::errored(kErrorInvalidRegType);
|
|
|
|
uint32_t dstRegKind = X86Reg::kindOf(dstRegType);
|
|
if (ASMJIT_UNLIKELY(dstRegKind >= kMaxVRegKinds))
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
|
|
WorkData& dstData = _workData[dstRegKind];
|
|
uint32_t dstRegId = dstArg.getRegId();
|
|
if (ASMJIT_UNLIKELY(dstRegId >= 32 || !(dstData.archRegs & Utils::mask(dstRegId))))
|
|
return DebugUtils::errored(kErrorInvalidPhysId);
|
|
|
|
uint32_t dstRegMask = Utils::mask(dstRegId);
|
|
if (ASMJIT_UNLIKELY(dstData.usedRegs & dstRegMask))
|
|
return DebugUtils::errored(kErrorOverlappedRegs);
|
|
|
|
dstData.usedRegs |= dstRegMask;
|
|
dstData.argIndex[dstRegId] = static_cast<uint8_t>(i);
|
|
|
|
if (srcArg.byReg()) {
|
|
uint32_t srcRegKind = X86Reg::kindOf(srcArg.getRegType());
|
|
uint32_t srcRegId = srcArg.getRegId();
|
|
uint32_t srcRegMask = Utils::mask(srcRegId);
|
|
|
|
if (dstRegKind == srcRegKind) {
|
|
// The best case, register is allocated where it is expected to be.
|
|
if (dstRegId == srcRegId) continue;
|
|
|
|
// Detect a register swap.
|
|
if (dstData.usedRegs & srcRegMask) {
|
|
const SrcArg& ref = func.getArg(dstData.argIndex[srcRegId]);
|
|
if (ref.byReg() && X86Reg::kindOf(ref.getRegType()) == dstRegKind && ref.getRegId() == dstRegId) {
|
|
dstData.numSwaps++;
|
|
_hasRegSwaps = true;
|
|
}
|
|
}
|
|
dstData.srcRegs |= srcRegMask;
|
|
}
|
|
else {
|
|
if (ASMJIT_UNLIKELY(srcRegKind >= kMaxVRegKinds))
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
|
|
WorkData& srcData = _workData[srcRegKind];
|
|
srcData.srcRegs |= srcRegMask;
|
|
}
|
|
}
|
|
else {
|
|
dstData.numStackArgs++;
|
|
_hasStackArgs = true;
|
|
}
|
|
|
|
dstData.numOps++;
|
|
dstData.dstRegs |= dstRegMask;
|
|
}
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markDstRegsDirty(FuncFrameInfo& ffi) noexcept {
|
|
for (uint32_t i = 0; i < kMaxVRegKinds; i++) {
|
|
WorkData& wd = _workData[i];
|
|
uint32_t regs = wd.usedRegs | wd.dstRegs;
|
|
|
|
wd.workRegs |= regs;
|
|
ffi.addDirtyRegs(i, regs);
|
|
}
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markRegsForSwaps(FuncFrameInfo& ffi) noexcept {
|
|
if (!_hasRegSwaps)
|
|
return kErrorOk;
|
|
|
|
// If some registers require swapping then select one dirty register that
|
|
// can be used as a temporary. We can do it also without it (by using xors),
|
|
// but using temporary is always safer and also faster approach.
|
|
for (uint32_t i = 0; i < kMaxVRegKinds; i++) {
|
|
// Skip all register kinds where swapping is natively supported (GP regs).
|
|
if (i == X86Reg::kKindGp) continue;
|
|
|
|
// Skip all register kinds that don't require swapping.
|
|
WorkData& wd = _workData[i];
|
|
if (!wd.numSwaps) continue;
|
|
|
|
// Initially, pick some clobbered or dirty register.
|
|
uint32_t workRegs = wd.workRegs;
|
|
uint32_t regs = workRegs & ~(wd.usedRegs | wd.dstRegs);
|
|
|
|
// If that didn't work out pick some register which is not in 'used'.
|
|
if (!regs) regs = workRegs & ~wd.usedRegs;
|
|
|
|
// If that didn't work out pick any other register that is allocable.
|
|
// This last resort case will, however, result in marking one more
|
|
// register dirty.
|
|
if (!regs) regs = wd.archRegs & ~workRegs;
|
|
|
|
// If that didn't work out we will have to use xors instead of moves.
|
|
if (!regs) continue;
|
|
|
|
uint32_t regMask = Utils::mask(Utils::findFirstBit(regs));
|
|
wd.workRegs |= regMask;
|
|
ffi.addDirtyRegs(i, regMask);
|
|
}
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markStackArgsReg(FuncFrameInfo& ffi) noexcept {
|
|
if (!_hasStackArgs)
|
|
return kErrorOk;
|
|
|
|
// Decide which register to use to hold the stack base address.
|
|
if (!ffi.hasPreservedFP()) {
|
|
WorkData& wd = _workData[X86Reg::kKindGp];
|
|
uint32_t saRegId = ffi.getStackArgsRegId();
|
|
uint32_t usedRegs = wd.usedRegs;
|
|
|
|
if (saRegId != Globals::kInvalidRegId) {
|
|
// Check if the user chosen SA register doesn't overlap with others.
|
|
// However, it's fine if it overlaps with some 'dstMove' register.
|
|
if (usedRegs & Utils::mask(saRegId))
|
|
return DebugUtils::errored(kErrorOverlappingStackRegWithRegArg);
|
|
}
|
|
else {
|
|
// Initially, pick some clobbered or dirty register that is neither
|
|
// in 'used' and neither in 'dstMove'. That's the safest bet as the
|
|
// register won't collide with anything right now.
|
|
uint32_t regs = wd.workRegs & ~(usedRegs | wd.dstRegs);
|
|
|
|
// If that didn't work out pick some register which is not in 'used'.
|
|
if (!regs) regs = wd.workRegs & ~usedRegs;
|
|
|
|
// If that didn't work out then we have to make one more register dirty.
|
|
if (!regs) regs = wd.archRegs & ~wd.workRegs;
|
|
|
|
// If that didn't work out we can't continue.
|
|
if (ASMJIT_UNLIKELY(!regs))
|
|
return DebugUtils::errored(kErrorNoMorePhysRegs);
|
|
|
|
saRegId = Utils::findFirstBit(regs);
|
|
ffi.setStackArgsRegId(saRegId);
|
|
}
|
|
}
|
|
else {
|
|
ffi.setStackArgsRegId(X86Gp::kIdBp);
|
|
}
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - CallConv]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::initCallConv(CallConv& cc, uint32_t ccId) noexcept {
|
|
const uint32_t kKindGp = X86Reg::kKindGp;
|
|
const uint32_t kKindVec = X86Reg::kKindVec;
|
|
const uint32_t kKindMm = X86Reg::kKindMm;
|
|
const uint32_t kKindK = X86Reg::kKindK;
|
|
|
|
const uint32_t kZax = X86Gp::kIdAx;
|
|
const uint32_t kZbx = X86Gp::kIdBx;
|
|
const uint32_t kZcx = X86Gp::kIdCx;
|
|
const uint32_t kZdx = X86Gp::kIdDx;
|
|
const uint32_t kZsp = X86Gp::kIdSp;
|
|
const uint32_t kZbp = X86Gp::kIdBp;
|
|
const uint32_t kZsi = X86Gp::kIdSi;
|
|
const uint32_t kZdi = X86Gp::kIdDi;
|
|
|
|
switch (ccId) {
|
|
case CallConv::kIdX86StdCall:
|
|
cc.setFlags(CallConv::kFlagCalleePopsStack);
|
|
goto X86CallConv;
|
|
|
|
case CallConv::kIdX86MsThisCall:
|
|
cc.setFlags(CallConv::kFlagCalleePopsStack);
|
|
cc.setPassedOrder(kKindGp, kZcx);
|
|
goto X86CallConv;
|
|
|
|
case CallConv::kIdX86MsFastCall:
|
|
case CallConv::kIdX86GccFastCall:
|
|
cc.setFlags(CallConv::kFlagCalleePopsStack);
|
|
cc.setPassedOrder(kKindGp, kZcx, kZdx);
|
|
goto X86CallConv;
|
|
|
|
case CallConv::kIdX86GccRegParm1:
|
|
cc.setPassedOrder(kKindGp, kZax);
|
|
goto X86CallConv;
|
|
|
|
case CallConv::kIdX86GccRegParm2:
|
|
cc.setPassedOrder(kKindGp, kZax, kZdx);
|
|
goto X86CallConv;
|
|
|
|
case CallConv::kIdX86GccRegParm3:
|
|
cc.setPassedOrder(kKindGp, kZax, kZdx, kZcx);
|
|
goto X86CallConv;
|
|
|
|
case CallConv::kIdX86CDecl:
|
|
X86CallConv:
|
|
cc.setNaturalStackAlignment(4);
|
|
cc.setArchType(ArchInfo::kTypeX86);
|
|
cc.setPreservedRegs(kKindGp, Utils::mask(kZbx, kZsp, kZbp, kZsi, kZdi));
|
|
break;
|
|
|
|
case CallConv::kIdX86Win64:
|
|
cc.setArchType(ArchInfo::kTypeX64);
|
|
cc.setAlgorithm(CallConv::kAlgorithmWin64);
|
|
cc.setFlags(CallConv::kFlagPassFloatsByVec | CallConv::kFlagIndirectVecArgs);
|
|
cc.setNaturalStackAlignment(16);
|
|
cc.setSpillZoneSize(32);
|
|
cc.setPassedOrder(kKindGp, kZcx, kZdx, 8, 9);
|
|
cc.setPassedOrder(kKindVec, 0, 1, 2, 3);
|
|
cc.setPreservedRegs(kKindGp, Utils::mask(kZbx, kZsp, kZbp, kZsi, kZdi, 12, 13, 14, 15));
|
|
cc.setPreservedRegs(kKindVec, Utils::mask(6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
|
|
break;
|
|
|
|
case CallConv::kIdX86SysV64:
|
|
cc.setArchType(ArchInfo::kTypeX64);
|
|
cc.setFlags(CallConv::kFlagPassFloatsByVec);
|
|
cc.setNaturalStackAlignment(16);
|
|
cc.setRedZoneSize(128);
|
|
cc.setPassedOrder(kKindGp, kZdi, kZsi, kZdx, kZcx, 8, 9);
|
|
cc.setPassedOrder(kKindVec, 0, 1, 2, 3, 4, 5, 6, 7);
|
|
cc.setPreservedRegs(kKindGp, Utils::mask(kZbx, kZsp, kZbp, 12, 13, 14, 15));
|
|
break;
|
|
|
|
case CallConv::kIdX86FastEval2:
|
|
case CallConv::kIdX86FastEval3:
|
|
case CallConv::kIdX86FastEval4: {
|
|
uint32_t n = ccId - CallConv::kIdX86FastEval2;
|
|
|
|
cc.setArchType(ArchInfo::kTypeX86);
|
|
cc.setFlags(CallConv::kFlagPassFloatsByVec);
|
|
cc.setNaturalStackAlignment(16);
|
|
cc.setPassedOrder(kKindGp, kZax, kZdx, kZcx, kZsi, kZdi);
|
|
cc.setPassedOrder(kKindMm, 0, 1, 2, 3, 4, 5, 6, 7);
|
|
cc.setPassedOrder(kKindVec, 0, 1, 2, 3, 4, 5, 6, 7);
|
|
|
|
cc.setPreservedRegs(kKindGp , Utils::bits(8));
|
|
cc.setPreservedRegs(kKindVec, Utils::bits(8) & ~Utils::bits(n));
|
|
cc.setPreservedRegs(kKindMm , Utils::bits(8));
|
|
cc.setPreservedRegs(kKindK , Utils::bits(8));
|
|
break;
|
|
}
|
|
|
|
case CallConv::kIdX64FastEval2:
|
|
case CallConv::kIdX64FastEval3:
|
|
case CallConv::kIdX64FastEval4: {
|
|
uint32_t n = ccId - CallConv::kIdX64FastEval2;
|
|
|
|
cc.setArchType(ArchInfo::kTypeX64);
|
|
cc.setFlags(CallConv::kFlagPassFloatsByVec);
|
|
cc.setNaturalStackAlignment(16);
|
|
cc.setPassedOrder(kKindGp, kZax, kZdx, kZcx, kZsi, kZdi);
|
|
cc.setPassedOrder(kKindMm, 0, 1, 2, 3, 4, 5, 6, 7);
|
|
cc.setPassedOrder(kKindVec, 0, 1, 2, 3, 4, 5, 6, 7);
|
|
|
|
cc.setPreservedRegs(kKindGp , Utils::bits(16));
|
|
cc.setPreservedRegs(kKindVec,~Utils::bits(n));
|
|
cc.setPreservedRegs(kKindMm , Utils::bits(8));
|
|
cc.setPreservedRegs(kKindK , Utils::bits(8));
|
|
break;
|
|
}
|
|
|
|
default:
|
|
return DebugUtils::errored(kErrorInvalidArgument);
|
|
}
|
|
|
|
cc.setId(ccId);
|
|
return kErrorOk;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - FuncDetail]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::initFuncDetail(FuncDetail& func, const FuncSignature& sign, uint32_t gpSize) noexcept {
|
|
const CallConv& cc = func.getCallConv();
|
|
uint32_t archType = cc.getArchType();
|
|
|
|
uint32_t i;
|
|
uint32_t argCount = func.getArgCount();
|
|
|
|
if (func.getRetCount() != 0) {
|
|
uint32_t typeId = func._rets[0].getTypeId();
|
|
switch (typeId) {
|
|
case TypeId::kI64:
|
|
case TypeId::kU64: {
|
|
if (archType == ArchInfo::kTypeX86) {
|
|
// Convert a 64-bit return to two 32-bit returns.
|
|
func._retCount = 2;
|
|
typeId -= 2;
|
|
|
|
// 64-bit value is returned in EDX:EAX on X86.
|
|
func._rets[0].initReg(typeId, X86Gp::kRegGpd, X86Gp::kIdAx);
|
|
func._rets[1].initReg(typeId, X86Gp::kRegGpd, X86Gp::kIdDx);
|
|
break;
|
|
}
|
|
else {
|
|
func._rets[0].initReg(typeId, X86Gp::kRegGpq, X86Gp::kIdAx);
|
|
}
|
|
break;
|
|
}
|
|
|
|
case TypeId::kI8:
|
|
case TypeId::kU8:
|
|
case TypeId::kI16:
|
|
case TypeId::kU16:
|
|
case TypeId::kI32:
|
|
case TypeId::kU32: {
|
|
func._rets[0].assignToReg(X86Gp::kRegGpd, X86Gp::kIdAx);
|
|
break;
|
|
}
|
|
|
|
case TypeId::kF32:
|
|
case TypeId::kF64: {
|
|
uint32_t regType = (archType == ArchInfo::kTypeX86) ? X86Reg::kRegFp : X86Reg::kRegXmm;
|
|
func._rets[0].assignToReg(regType, 0);
|
|
break;
|
|
}
|
|
|
|
case TypeId::kF80: {
|
|
// 80-bit floats are always returned by FP0.
|
|
func._rets[0].assignToReg(X86Reg::kRegFp, 0);
|
|
break;
|
|
}
|
|
|
|
case TypeId::kMmx32:
|
|
case TypeId::kMmx64: {
|
|
// On X64 MM register(s) are returned through XMM or GPQ (Win64).
|
|
uint32_t regType = X86Reg::kRegMm;
|
|
if (archType != ArchInfo::kTypeX86)
|
|
regType = cc.getAlgorithm() == CallConv::kAlgorithmDefault ? X86Reg::kRegXmm : X86Reg::kRegGpq;
|
|
|
|
func._rets[0].assignToReg(regType, 0);
|
|
break;
|
|
}
|
|
|
|
default: {
|
|
func._rets[0].assignToReg(x86VecTypeIdToRegType(typeId), 0);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
uint32_t stackBase = gpSize;
|
|
uint32_t stackOffset = stackBase + cc._spillZoneSize;
|
|
|
|
if (cc.getAlgorithm() == CallConv::kAlgorithmDefault) {
|
|
uint32_t gpzPos = 0;
|
|
uint32_t vecPos = 0;
|
|
|
|
for (i = 0; i < argCount; i++) {
|
|
FuncDetail::Value& arg = func._args[i];
|
|
uint32_t typeId = arg.getTypeId();
|
|
|
|
if (TypeId::isInt(typeId)) {
|
|
uint32_t regId = gpzPos < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindGp].id[gpzPos] : Globals::kInvalidRegId;
|
|
if (regId != Globals::kInvalidRegId) {
|
|
uint32_t regType = (typeId <= TypeId::kU32)
|
|
? X86Reg::kRegGpd
|
|
: X86Reg::kRegGpq;
|
|
arg.assignToReg(regType, regId);
|
|
func.addUsedRegs(X86Reg::kKindGp, Utils::mask(regId));
|
|
gpzPos++;
|
|
}
|
|
else {
|
|
uint32_t size = std::max<uint32_t>(TypeId::sizeOf(typeId), gpSize);
|
|
arg.assignToStack(stackOffset);
|
|
stackOffset += size;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (TypeId::isFloat(typeId) || TypeId::isVec(typeId)) {
|
|
uint32_t regId = vecPos < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindVec].id[vecPos] : Globals::kInvalidRegId;
|
|
|
|
// If this is a float, but `floatByVec` is false, we have to pass by stack.
|
|
if (TypeId::isFloat(typeId) && !cc.hasFlag(CallConv::kFlagPassFloatsByVec))
|
|
regId = Globals::kInvalidRegId;
|
|
|
|
if (regId != Globals::kInvalidRegId) {
|
|
arg.initReg(typeId, x86VecTypeIdToRegType(typeId), regId);
|
|
func.addUsedRegs(X86Reg::kKindVec, Utils::mask(regId));
|
|
vecPos++;
|
|
}
|
|
else {
|
|
int32_t size = TypeId::sizeOf(typeId);
|
|
arg.assignToStack(stackOffset);
|
|
stackOffset += size;
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (cc.getAlgorithm() == CallConv::kAlgorithmWin64) {
|
|
for (i = 0; i < argCount; i++) {
|
|
FuncDetail::Value& arg = func._args[i];
|
|
|
|
uint32_t typeId = arg.getTypeId();
|
|
uint32_t size = TypeId::sizeOf(typeId);
|
|
|
|
if (TypeId::isInt(typeId) || TypeId::isMmx(typeId)) {
|
|
uint32_t regId = i < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindGp].id[i] : Globals::kInvalidRegId;
|
|
if (regId != Globals::kInvalidRegId) {
|
|
uint32_t regType = (size <= 4 && !TypeId::isMmx(typeId))
|
|
? X86Reg::kRegGpd
|
|
: X86Reg::kRegGpq;
|
|
|
|
arg.assignToReg(regType, regId);
|
|
func.addUsedRegs(X86Reg::kKindGp, Utils::mask(regId));
|
|
}
|
|
else {
|
|
arg.assignToStack(stackOffset);
|
|
stackOffset += gpSize;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (TypeId::isFloat(typeId) || TypeId::isVec(typeId)) {
|
|
uint32_t regId = i < CallConv::kNumRegArgsPerKind ? cc._passedOrder[X86Reg::kKindVec].id[i] : Globals::kInvalidRegId;
|
|
if (regId != Globals::kInvalidRegId && (TypeId::isFloat(typeId) || cc.hasFlag(CallConv::kFlagVectorCall))) {
|
|
uint32_t regType = x86VecTypeIdToRegType(typeId);
|
|
uint32_t regId = cc._passedOrder[X86Reg::kKindVec].id[i];
|
|
|
|
arg.assignToReg(regType, regId);
|
|
func.addUsedRegs(X86Reg::kKindVec, Utils::mask(regId));
|
|
}
|
|
else {
|
|
arg.assignToStack(stackOffset);
|
|
stackOffset += 8; // Always 8 bytes (float/double).
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
func._argStackSize = stackOffset - stackBase;
|
|
return kErrorOk;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - FrameLayout]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::initFrameLayout(FuncFrameLayout& layout, const FuncDetail& func, const FuncFrameInfo& ffi) noexcept {
|
|
layout.reset();
|
|
|
|
uint32_t kind;
|
|
uint32_t gpSize = (func.getCallConv().getArchType() == ArchInfo::kTypeX86) ? 4 : 8;
|
|
|
|
// Calculate a bit-mask of all registers that must be saved & restored.
|
|
for (kind = 0; kind < Globals::kMaxVRegKinds; kind++)
|
|
layout._savedRegs[kind] = (ffi.getDirtyRegs(kind) & ~func.getPassedRegs(kind)) & func.getPreservedRegs(kind);
|
|
|
|
// Include EBP|RBP if the function preserves the frame-pointer.
|
|
if (ffi.hasPreservedFP()) {
|
|
layout._preservedFP = true;
|
|
layout._savedRegs[X86Reg::kKindGp] |= Utils::mask(X86Gp::kIdBp);
|
|
}
|
|
|
|
// Exclude ESP/RSP - this register is never included in saved-regs.
|
|
layout._savedRegs[X86Reg::kKindGp] &= ~Utils::mask(X86Gp::kIdSp);
|
|
|
|
// Calculate the final stack alignment.
|
|
uint32_t stackAlignment =
|
|
std::max<uint32_t>(
|
|
std::max<uint32_t>(
|
|
ffi.getStackFrameAlignment(),
|
|
ffi.getCallFrameAlignment()),
|
|
func.getCallConv().getNaturalStackAlignment());
|
|
layout._stackAlignment = static_cast<uint8_t>(stackAlignment);
|
|
|
|
// Calculate if dynamic stack alignment is required. If true the function has
|
|
// to align stack dynamically to match `_stackAlignment` and would require to
|
|
// access its stack-based arguments through `_stackArgsRegId`.
|
|
bool dsa = stackAlignment > func.getCallConv().getNaturalStackAlignment() && stackAlignment >= 16;
|
|
layout._dynamicAlignment = dsa;
|
|
|
|
// This flag describes if the prolog inserter must store the previous ESP|RSP
|
|
// to stack so the epilog inserter can load the stack from it before returning.
|
|
bool dsaSlotUsed = dsa && !ffi.hasPreservedFP();
|
|
layout._dsaSlotUsed = dsaSlotUsed;
|
|
|
|
// These two are identical if the function doesn't align its stack dynamically.
|
|
uint32_t stackArgsRegId = ffi.getStackArgsRegId();
|
|
if (stackArgsRegId == Globals::kInvalidRegId)
|
|
stackArgsRegId = X86Gp::kIdSp;
|
|
|
|
// Fix stack arguments base-register from ESP|RSP to EBP|RBP in case it was
|
|
// not picked before and the function performs dynamic stack alignment.
|
|
if (dsa && stackArgsRegId == X86Gp::kIdSp)
|
|
stackArgsRegId = X86Gp::kIdBp;
|
|
|
|
if (stackArgsRegId != X86Gp::kIdSp)
|
|
layout._savedRegs[X86Reg::kKindGp] |= Utils::mask(stackArgsRegId) & func.getPreservedRegs(X86Gp::kKindGp);
|
|
|
|
layout._stackBaseRegId = X86Gp::kIdSp;
|
|
layout._stackArgsRegId = static_cast<uint8_t>(stackArgsRegId);
|
|
|
|
// Setup stack size used to save preserved registers.
|
|
layout._gpStackSize = Utils::bitCount(layout.getSavedRegs(X86Reg::kKindGp )) * gpSize;
|
|
layout._vecStackSize = Utils::bitCount(layout.getSavedRegs(X86Reg::kKindVec)) * 16 +
|
|
Utils::bitCount(layout.getSavedRegs(X86Reg::kKindMm )) * 8 ;
|
|
|
|
uint32_t v = 0; // The beginning of the stack frame, aligned to CallFrame alignment.
|
|
v += ffi._callFrameSize; // Count '_callFrameSize' <- This is used to call functions.
|
|
v = Utils::alignTo(v, stackAlignment);// Align to function's SA
|
|
|
|
layout._stackBaseOffset = v; // Store '_stackBaseOffset'<- Function's own stack starts here..
|
|
v += ffi._stackFrameSize; // Count '_stackFrameSize' <- Function's own stack ends here.
|
|
|
|
// If the function is aligned, calculate the alignment necessary to store
|
|
// vector registers, and set `FuncFrameInfo::kX86FlagAlignedVecSR` to inform
|
|
// PrologEpilog inserter that it can use instructions to perform aligned
|
|
// stores/loads to save/restore VEC registers.
|
|
if (stackAlignment >= 16 && layout._vecStackSize) {
|
|
v = Utils::alignTo(v, 16); // Align '_vecStackOffset'.
|
|
layout._alignedVecSR = true;
|
|
}
|
|
|
|
layout._vecStackOffset = v; // Store '_vecStackOffset' <- Functions VEC Save|Restore starts here.
|
|
v += layout._vecStackSize; // Count '_vecStackSize' <- Functions VEC Save|Restore ends here.
|
|
|
|
if (dsaSlotUsed) {
|
|
layout._dsaSlot = v; // Store '_dsaSlot' <- Old stack pointer is stored here.
|
|
v += gpSize;
|
|
}
|
|
|
|
// The return address should be stored after GP save/restore regs. It has
|
|
// the same size as `gpSize` (basically the native register/pointer size).
|
|
// We don't adjust it now as `v` now contains the exact size that the
|
|
// function requires to adjust (call frame + stack frame, vec stack size).
|
|
// The stack (if we consider this size) is misaligned now, as it's always
|
|
// aligned before the function call - when `call()` is executed it pushes
|
|
// the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes
|
|
// (depending on the architecture). So count number of bytes needed to align
|
|
// it up to the function's CallFrame (the beginning).
|
|
if (v || ffi.hasCalls())
|
|
v += Utils::alignDiff(v + layout._gpStackSize + gpSize, stackAlignment);
|
|
|
|
layout._stackAdjustment = v; // Store '_stackAdjustment'<- SA used by 'add zsp, SA' and 'sub zsp, SA'.
|
|
layout._gpStackOffset = v; // Store '_gpStackOffset' <- Functions GP Save|Restore starts here.
|
|
v += layout._gpStackSize; // Count '_gpStackSize' <- Functions GP Save|Restore ends here.
|
|
|
|
v += gpSize; // Count 'ReturnAddress'.
|
|
v += func.getSpillZoneSize(); // Count 'SpillZoneSize'.
|
|
|
|
// Calculate where function arguments start, relative to the stackArgsRegId.
|
|
// If the register that will be used to access arguments passed by stack is
|
|
// ESP|RSP then it's exactly where we are now, otherwise we must calculate
|
|
// how many 'push regs' we did and adjust it based on that.
|
|
uint32_t stackArgsOffset = v;
|
|
if (stackArgsRegId != X86Gp::kIdSp) {
|
|
if (ffi.hasPreservedFP())
|
|
stackArgsOffset = gpSize;
|
|
else
|
|
stackArgsOffset = layout._gpStackSize;
|
|
}
|
|
layout._stackArgsOffset = stackArgsOffset;
|
|
|
|
// If the function does dynamic stack adjustment then the stack-adjustment
|
|
// must be aligned.
|
|
if (dsa)
|
|
layout._stackAdjustment = Utils::alignTo(layout._stackAdjustment, stackAlignment);
|
|
|
|
// Initialize variables based on CallConv flags.
|
|
if (func.hasFlag(CallConv::kFlagCalleePopsStack))
|
|
layout._calleeStackCleanup = static_cast<uint16_t>(func.getArgStackSize());
|
|
|
|
// Initialize variables based on FFI flags.
|
|
layout._mmxCleanup = ffi.hasMmxCleanup();
|
|
layout._avxEnabled = ffi.isAvxEnabled();
|
|
layout._avxCleanup = ffi.hasAvxCleanup();
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - ArgsToFrameInfo]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::argsToFrameInfo(const FuncArgsMapper& args, FuncFrameInfo& ffi) noexcept {
|
|
X86FuncArgsContext ctx;
|
|
ASMJIT_PROPAGATE(ctx.initWorkData(args, ffi._dirtyRegs, ffi.hasPreservedFP()));
|
|
|
|
ASMJIT_PROPAGATE(ctx.markDstRegsDirty(ffi));
|
|
ASMJIT_PROPAGATE(ctx.markRegsForSwaps(ffi));
|
|
ASMJIT_PROPAGATE(ctx.markStackArgsReg(ffi));
|
|
return kErrorOk;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - Emit Helpers]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::emitRegMove(X86Emitter* emitter,
|
|
const Operand_& dst_,
|
|
const Operand_& src_, uint32_t typeId, bool avxEnabled, const char* comment) {
|
|
|
|
// Invalid or abstract TypeIds are not allowed.
|
|
ASMJIT_ASSERT(TypeId::isValid(typeId) && !TypeId::isAbstract(typeId));
|
|
|
|
Operand dst(dst_);
|
|
Operand src(src_);
|
|
|
|
uint32_t instId = Inst::kIdNone;
|
|
uint32_t memFlags = 0;
|
|
|
|
enum MemFlags {
|
|
kDstMem = 0x1,
|
|
kSrcMem = 0x2
|
|
};
|
|
|
|
// Detect memory operands and patch them to have the same size as the register.
|
|
// CodeCompiler always sets memory size of allocs and spills, so it shouldn't
|
|
// be really necessary, however, after this function was separated from Compiler
|
|
// it's better to make sure that the size is always specified, as we can use
|
|
// 'movzx' and 'movsx' that rely on it.
|
|
if (dst.isMem()) { memFlags |= kDstMem; dst.as<X86Mem>().setSize(src.getSize()); }
|
|
if (src.isMem()) { memFlags |= kSrcMem; src.as<X86Mem>().setSize(dst.getSize()); }
|
|
|
|
switch (typeId) {
|
|
case TypeId::kI8:
|
|
case TypeId::kU8:
|
|
case TypeId::kI16:
|
|
case TypeId::kU16:
|
|
// Special case - 'movzx' load.
|
|
if (memFlags & kSrcMem) {
|
|
instId = X86Inst::kIdMovzx;
|
|
dst.setSignature(X86RegTraits<X86Reg::kRegGpd>::kSignature);
|
|
}
|
|
else if (!memFlags) {
|
|
// Change both destination and source registers to GPD (safer, no dependencies).
|
|
dst.setSignature(X86RegTraits<X86Reg::kRegGpd>::kSignature);
|
|
src.setSignature(X86RegTraits<X86Reg::kRegGpd>::kSignature);
|
|
}
|
|
ASMJIT_FALLTHROUGH;
|
|
|
|
case TypeId::kI32:
|
|
case TypeId::kU32:
|
|
case TypeId::kI64:
|
|
case TypeId::kU64:
|
|
instId = X86Inst::kIdMov;
|
|
break;
|
|
|
|
case TypeId::kMmx32:
|
|
instId = X86Inst::kIdMovd;
|
|
if (memFlags) break;
|
|
ASMJIT_FALLTHROUGH;
|
|
case TypeId::kMmx64 : instId = X86Inst::kIdMovq ; break;
|
|
case TypeId::kMask8 : instId = X86Inst::kIdKmovb; break;
|
|
case TypeId::kMask16: instId = X86Inst::kIdKmovw; break;
|
|
case TypeId::kMask32: instId = X86Inst::kIdKmovd; break;
|
|
case TypeId::kMask64: instId = X86Inst::kIdKmovq; break;
|
|
|
|
default: {
|
|
uint32_t elementTypeId = TypeId::elementOf(typeId);
|
|
if (TypeId::isVec32(typeId) && memFlags) {
|
|
if (elementTypeId == TypeId::kF32)
|
|
instId = avxEnabled ? X86Inst::kIdVmovss : X86Inst::kIdMovss;
|
|
else
|
|
instId = avxEnabled ? X86Inst::kIdVmovd : X86Inst::kIdMovd;
|
|
break;
|
|
}
|
|
|
|
if (TypeId::isVec64(typeId) && memFlags) {
|
|
if (elementTypeId == TypeId::kF64)
|
|
instId = avxEnabled ? X86Inst::kIdVmovsd : X86Inst::kIdMovsd;
|
|
else
|
|
instId = avxEnabled ? X86Inst::kIdVmovq : X86Inst::kIdMovq;
|
|
break;
|
|
}
|
|
|
|
if (elementTypeId == TypeId::kF32)
|
|
instId = avxEnabled ? X86Inst::kIdVmovaps : X86Inst::kIdMovaps;
|
|
else if (elementTypeId == TypeId::kF64)
|
|
instId = avxEnabled ? X86Inst::kIdVmovapd : X86Inst::kIdMovapd;
|
|
else if (typeId <= TypeId::_kVec256End)
|
|
instId = avxEnabled ? X86Inst::kIdVmovdqa : X86Inst::kIdMovdqa;
|
|
else if (elementTypeId <= TypeId::kU32)
|
|
instId = X86Inst::kIdVmovdqa32;
|
|
else
|
|
instId = X86Inst::kIdVmovdqa64;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!instId)
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
|
|
emitter->setInlineComment(comment);
|
|
return emitter->emit(instId, dst, src);
|
|
}
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::emitArgMove(X86Emitter* emitter,
|
|
const X86Reg& dst_, uint32_t dstTypeId,
|
|
const Operand_& src_, uint32_t srcTypeId, bool avxEnabled, const char* comment) {
|
|
|
|
// Deduce optional `dstTypeId`, which may be `TypeId::kVoid` in some cases.
|
|
if (!dstTypeId) dstTypeId = x86OpData.archRegs.regTypeToTypeId[dst_.getType()];
|
|
|
|
// Invalid or abstract TypeIds are not allowed.
|
|
ASMJIT_ASSERT(TypeId::isValid(dstTypeId) && !TypeId::isAbstract(dstTypeId));
|
|
ASMJIT_ASSERT(TypeId::isValid(srcTypeId) && !TypeId::isAbstract(srcTypeId));
|
|
|
|
X86Reg dst(dst_);
|
|
Operand src(src_);
|
|
|
|
uint32_t dstSize = TypeId::sizeOf(dstTypeId);
|
|
uint32_t srcSize = TypeId::sizeOf(srcTypeId);
|
|
|
|
int32_t instId = Inst::kIdNone;
|
|
|
|
// Not a real loop, just 'break' is nicer than 'goto'.
|
|
for (;;) {
|
|
if (TypeId::isInt(dstTypeId)) {
|
|
if (TypeId::isInt(srcTypeId)) {
|
|
instId = X86Inst::kIdMovsx;
|
|
uint32_t typeOp = (dstTypeId << 8) | srcTypeId;
|
|
|
|
// Sign extend by using 'movsx'.
|
|
if (typeOp == ((TypeId::kI16 << 8) | TypeId::kI8 ) ||
|
|
typeOp == ((TypeId::kI32 << 8) | TypeId::kI8 ) ||
|
|
typeOp == ((TypeId::kI32 << 8) | TypeId::kI16) ||
|
|
typeOp == ((TypeId::kI64 << 8) | TypeId::kI8 ) ||
|
|
typeOp == ((TypeId::kI64 << 8) | TypeId::kI16)) break;
|
|
|
|
// Sign extend by using 'movsxd'.
|
|
instId = X86Inst::kIdMovsxd;
|
|
if (typeOp == ((TypeId::kI64 << 8) | TypeId::kI32)) break;
|
|
}
|
|
|
|
if (TypeId::isInt(srcTypeId) || src_.isMem()) {
|
|
// Zero extend by using 'movzx' or 'mov'.
|
|
if (dstSize <= 4 && srcSize < 4) {
|
|
instId = X86Inst::kIdMovzx;
|
|
dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
|
|
}
|
|
else {
|
|
// We should have caught all possibilities where `srcSize` is less
|
|
// than 4, so we don't have to worry about 'movzx' anymore. Minimum
|
|
// size is enough to determine if we want 32-bit or 64-bit move.
|
|
instId = X86Inst::kIdMov;
|
|
srcSize = std::min(srcSize, dstSize);
|
|
|
|
dst.setSignature(srcSize == 4 ? X86Reg::signatureOfT<X86Reg::kRegGpd>()
|
|
: X86Reg::signatureOfT<X86Reg::kRegGpq>());
|
|
if (src.isReg()) src.setSignature(dst.getSignature());
|
|
}
|
|
break;
|
|
}
|
|
|
|
// NOTE: The previous branch caught all memory sources, from here it's
|
|
// always register to register conversion, so catch the remaining cases.
|
|
srcSize = std::min(srcSize, dstSize);
|
|
|
|
if (TypeId::isMmx(srcTypeId)) {
|
|
// 64-bit move.
|
|
instId = X86Inst::kIdMovq;
|
|
if (srcSize == 8) break;
|
|
|
|
// 32-bit move.
|
|
instId = X86Inst::kIdMovd;
|
|
dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
|
|
break;
|
|
}
|
|
|
|
if (TypeId::isMask(srcTypeId)) {
|
|
instId = X86Inst::kmovIdFromSize(srcSize);
|
|
dst.setSignature(srcSize <= 4 ? X86Reg::signatureOfT<X86Reg::kRegGpd>()
|
|
: X86Reg::signatureOfT<X86Reg::kRegGpq>());
|
|
break;
|
|
}
|
|
|
|
if (TypeId::isVec(srcTypeId)) {
|
|
// 64-bit move.
|
|
instId = avxEnabled ? X86Inst::kIdVmovq : X86Inst::kIdMovq;
|
|
if (srcSize == 8) break;
|
|
|
|
// 32-bit move.
|
|
instId = avxEnabled ? X86Inst::kIdVmovd : X86Inst::kIdMovd;
|
|
dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (TypeId::isMmx(dstTypeId)) {
|
|
instId = X86Inst::kIdMovq;
|
|
srcSize = std::min(srcSize, dstSize);
|
|
|
|
if (TypeId::isInt(srcTypeId) || src.isMem()) {
|
|
// 64-bit move.
|
|
if (srcSize == 8) break;
|
|
|
|
// 32-bit move.
|
|
instId = X86Inst::kIdMovd;
|
|
if (src.isReg()) src.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
|
|
break;
|
|
}
|
|
|
|
if (TypeId::isMmx(srcTypeId)) break;
|
|
|
|
// NOTE: This will hurt if `avxEnabled`.
|
|
instId = X86Inst::kIdMovdq2q;
|
|
if (TypeId::isVec(srcTypeId)) break;
|
|
}
|
|
|
|
if (TypeId::isMask(dstTypeId)) {
|
|
srcSize = std::min(srcSize, dstSize);
|
|
|
|
if (TypeId::isInt(srcTypeId) || TypeId::isMask(srcTypeId) || src.isMem()) {
|
|
instId = X86Inst::kmovIdFromSize(srcSize);
|
|
if (X86Reg::isGp(src) && srcSize <= 4) src.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (TypeId::isVec(dstTypeId)) {
|
|
// By default set destination to XMM, will be set to YMM|ZMM if needed.
|
|
dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegXmm>());
|
|
|
|
// NOTE: This will hurt if `avxEnabled`.
|
|
if (X86Reg::isMm(src)) {
|
|
// 64-bit move.
|
|
instId = X86Inst::kIdMovq2dq;
|
|
break;
|
|
}
|
|
|
|
// Argument conversion.
|
|
uint32_t dstElement = TypeId::elementOf(dstTypeId);
|
|
uint32_t srcElement = TypeId::elementOf(srcTypeId);
|
|
|
|
if (dstElement == TypeId::kF32 && srcElement == TypeId::kF64) {
|
|
srcSize = std::min(dstSize * 2, srcSize);
|
|
dstSize = srcSize / 2;
|
|
|
|
if (srcSize <= 8)
|
|
instId = avxEnabled ? X86Inst::kIdVcvtss2sd : X86Inst::kIdCvtss2sd;
|
|
else
|
|
instId = avxEnabled ? X86Inst::kIdVcvtps2pd : X86Inst::kIdCvtps2pd;
|
|
|
|
if (dstSize == 32)
|
|
dst.setSignature(X86Reg::signatureOfT<X86Reg::kRegYmm>());
|
|
if (src.isReg())
|
|
src.setSignature(X86Reg::signatureOfVecBySize(srcSize));
|
|
break;
|
|
}
|
|
|
|
if (dstElement == TypeId::kF64 && srcElement == TypeId::kF32) {
|
|
srcSize = std::min(dstSize, srcSize * 2) / 2;
|
|
dstSize = srcSize * 2;
|
|
|
|
if (srcSize <= 4)
|
|
instId = avxEnabled ? X86Inst::kIdVcvtsd2ss : X86Inst::kIdCvtsd2ss;
|
|
else
|
|
instId = avxEnabled ? X86Inst::kIdVcvtpd2ps : X86Inst::kIdCvtpd2ps;
|
|
|
|
dst.setSignature(X86Reg::signatureOfVecBySize(dstSize));
|
|
if (src.isReg() && srcSize >= 32)
|
|
src.setSignature(X86Reg::signatureOfT<X86Reg::kRegYmm>());
|
|
break;
|
|
}
|
|
|
|
srcSize = std::min(srcSize, dstSize);
|
|
if (X86Reg::isGp(src) || src.isMem()) {
|
|
// 32-bit move.
|
|
if (srcSize <= 4) {
|
|
instId = avxEnabled ? X86Inst::kIdVmovd : X86Inst::kIdMovd;
|
|
if (src.isReg()) src.setSignature(X86Reg::signatureOfT<X86Reg::kRegGpd>());
|
|
break;
|
|
}
|
|
|
|
// 64-bit move.
|
|
if (srcSize == 8) {
|
|
instId = avxEnabled ? X86Inst::kIdVmovq : X86Inst::kIdMovq;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (X86Reg::isVec(src) || src.isMem()) {
|
|
instId = avxEnabled ? X86Inst::kIdVmovaps : X86Inst::kIdMovaps;
|
|
uint32_t sign = X86Reg::signatureOfVecBySize(srcSize);
|
|
|
|
dst.setSignature(sign);
|
|
if (src.isReg()) src.setSignature(sign);
|
|
break;
|
|
}
|
|
}
|
|
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
}
|
|
|
|
if (src.isMem())
|
|
src.as<X86Mem>().setSize(srcSize);
|
|
|
|
emitter->setInlineComment(comment);
|
|
return emitter->emit(instId, dst, src);
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - Emit Prolog & Epilog]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::emitProlog(X86Emitter* emitter, const FuncFrameLayout& layout) {
|
|
uint32_t gpSaved = layout.getSavedRegs(X86Reg::kKindGp);
|
|
|
|
X86Gp zsp = emitter->zsp(); // ESP|RSP register.
|
|
X86Gp zbp = emitter->zsp(); // EBP|RBP register.
|
|
zbp.setId(X86Gp::kIdBp);
|
|
|
|
X86Gp gpReg = emitter->zsp(); // General purpose register (temporary).
|
|
X86Gp saReg = emitter->zsp(); // Stack-arguments base register.
|
|
|
|
// Emit: 'push zbp'
|
|
// 'mov zbp, zsp'.
|
|
if (layout.hasPreservedFP()) {
|
|
gpSaved &= ~Utils::mask(X86Gp::kIdBp);
|
|
ASMJIT_PROPAGATE(emitter->push(zbp));
|
|
ASMJIT_PROPAGATE(emitter->mov(zbp, zsp));
|
|
}
|
|
|
|
// Emit: 'push gp' sequence.
|
|
if (gpSaved) {
|
|
for (uint32_t i = gpSaved, regId = 0; i; i >>= 1, regId++) {
|
|
if (!(i & 0x1)) continue;
|
|
gpReg.setId(regId);
|
|
ASMJIT_PROPAGATE(emitter->push(gpReg));
|
|
}
|
|
}
|
|
|
|
// Emit: 'mov saReg, zsp'.
|
|
uint32_t stackArgsRegId = layout.getStackArgsRegId();
|
|
if (stackArgsRegId != Globals::kInvalidRegId && stackArgsRegId != X86Gp::kIdSp) {
|
|
saReg.setId(stackArgsRegId);
|
|
if (!(layout.hasPreservedFP() && stackArgsRegId == X86Gp::kIdBp))
|
|
ASMJIT_PROPAGATE(emitter->mov(saReg, zsp));
|
|
}
|
|
|
|
// Emit: 'and zsp, StackAlignment'.
|
|
if (layout.hasDynamicAlignment())
|
|
ASMJIT_PROPAGATE(emitter->and_(zsp, -static_cast<int32_t>(layout.getStackAlignment())));
|
|
|
|
// Emit: 'sub zsp, StackAdjustment'.
|
|
if (layout.hasStackAdjustment())
|
|
ASMJIT_PROPAGATE(emitter->sub(zsp, layout.getStackAdjustment()));
|
|
|
|
// Emit: 'mov [zsp + dsaSlot], saReg'.
|
|
if (layout.hasDynamicAlignment() && layout.hasDsaSlotUsed()) {
|
|
X86Mem saMem = x86::ptr(zsp, layout._dsaSlot);
|
|
ASMJIT_PROPAGATE(emitter->mov(saMem, saReg));
|
|
}
|
|
|
|
// Emit 'movaps|movups [zsp + X], xmm0..15'.
|
|
uint32_t xmmSaved = layout.getSavedRegs(X86Reg::kKindVec);
|
|
if (xmmSaved) {
|
|
X86Mem vecBase = x86::ptr(zsp, layout.getVecStackOffset());
|
|
X86Reg vecReg = x86::xmm(0);
|
|
|
|
uint32_t vecInst = x86GetXmmMovInst(layout);
|
|
uint32_t vecSize = 16;
|
|
|
|
for (uint32_t i = xmmSaved, regId = 0; i; i >>= 1, regId++) {
|
|
if (!(i & 0x1)) continue;
|
|
vecReg.setId(regId);
|
|
ASMJIT_PROPAGATE(emitter->emit(vecInst, vecBase, vecReg));
|
|
vecBase.addOffsetLo32(static_cast<int32_t>(vecSize));
|
|
}
|
|
}
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::emitEpilog(X86Emitter* emitter, const FuncFrameLayout& layout) {
|
|
uint32_t i;
|
|
uint32_t regId;
|
|
|
|
uint32_t gpSize = emitter->getGpSize();
|
|
uint32_t gpSaved = layout.getSavedRegs(X86Reg::kKindGp);
|
|
|
|
X86Gp zsp = emitter->zsp(); // ESP|RSP register.
|
|
X86Gp zbp = emitter->zsp(); // EBP|RBP register.
|
|
zbp.setId(X86Gp::kIdBp);
|
|
|
|
X86Gp gpReg = emitter->zsp(); // General purpose register (temporary).
|
|
|
|
// Don't emit 'pop zbp' in the pop sequence, this case is handled separately.
|
|
if (layout.hasPreservedFP()) gpSaved &= ~Utils::mask(X86Gp::kIdBp);
|
|
|
|
// Emit 'movaps|movups xmm0..15, [zsp + X]'.
|
|
uint32_t xmmSaved = layout.getSavedRegs(X86Reg::kKindVec);
|
|
if (xmmSaved) {
|
|
X86Mem vecBase = x86::ptr(zsp, layout.getVecStackOffset());
|
|
X86Reg vecReg = x86::xmm(0);
|
|
|
|
uint32_t vecInst = x86GetXmmMovInst(layout);
|
|
uint32_t vecSize = 16;
|
|
|
|
for (i = xmmSaved, regId = 0; i; i >>= 1, regId++) {
|
|
if (!(i & 0x1)) continue;
|
|
vecReg.setId(regId);
|
|
ASMJIT_PROPAGATE(emitter->emit(vecInst, vecReg, vecBase));
|
|
vecBase.addOffsetLo32(static_cast<int32_t>(vecSize));
|
|
}
|
|
}
|
|
|
|
// Emit 'emms' and 'vzeroupper'.
|
|
if (layout.hasMmxCleanup()) ASMJIT_PROPAGATE(emitter->emms());
|
|
if (layout.hasAvxCleanup()) ASMJIT_PROPAGATE(emitter->vzeroupper());
|
|
|
|
if (layout.hasPreservedFP()) {
|
|
// Emit 'mov zsp, zbp' or 'lea zsp, [zbp - x]'
|
|
int32_t count = static_cast<int32_t>(layout.getGpStackSize() - gpSize);
|
|
if (!count)
|
|
ASMJIT_PROPAGATE(emitter->mov(zsp, zbp));
|
|
else
|
|
ASMJIT_PROPAGATE(emitter->lea(zsp, x86::ptr(zbp, -count)));
|
|
}
|
|
else {
|
|
if (layout.hasDynamicAlignment() && layout.hasDsaSlotUsed()) {
|
|
// Emit 'mov zsp, [zsp + DsaSlot]'.
|
|
X86Mem saMem = x86::ptr(zsp, layout._dsaSlot);
|
|
ASMJIT_PROPAGATE(emitter->mov(zsp, saMem));
|
|
}
|
|
else if (layout.hasStackAdjustment()) {
|
|
// Emit 'add zsp, StackAdjustment'.
|
|
ASMJIT_PROPAGATE(emitter->add(zsp, static_cast<int32_t>(layout.getStackAdjustment())));
|
|
}
|
|
}
|
|
|
|
// Emit 'pop gp' sequence.
|
|
if (gpSaved) {
|
|
i = gpSaved;
|
|
regId = 16;
|
|
|
|
do {
|
|
regId--;
|
|
if (i & 0x8000) {
|
|
gpReg.setId(regId);
|
|
ASMJIT_PROPAGATE(emitter->pop(gpReg));
|
|
}
|
|
i <<= 1;
|
|
} while (regId != 0);
|
|
}
|
|
|
|
// Emit 'pop zbp'.
|
|
if (layout.hasPreservedFP()) ASMJIT_PROPAGATE(emitter->pop(zbp));
|
|
|
|
// Emit 'ret' or 'ret x'.
|
|
if (layout.hasCalleeStackCleanup())
|
|
ASMJIT_PROPAGATE(emitter->emit(X86Inst::kIdRet, static_cast<int>(layout.getCalleeStackCleanup())));
|
|
else
|
|
ASMJIT_PROPAGATE(emitter->emit(X86Inst::kIdRet));
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
// ============================================================================
|
|
// [asmjit::X86Internal - AllocArgs]
|
|
// ============================================================================
|
|
|
|
ASMJIT_FAVOR_SIZE Error X86Internal::allocArgs(X86Emitter* emitter, const FuncFrameLayout& layout, const FuncArgsMapper& args) {
|
|
typedef X86FuncArgsContext::SrcArg SrcArg;
|
|
typedef X86FuncArgsContext::DstArg DstArg;
|
|
typedef X86FuncArgsContext::WorkData WorkData;
|
|
enum { kMaxVRegKinds = Globals::kMaxVRegKinds };
|
|
|
|
uint32_t i;
|
|
const FuncDetail& func = *args.getFuncDetail();
|
|
|
|
X86FuncArgsContext ctx;
|
|
ASMJIT_PROPAGATE(ctx.initWorkData(args, layout._savedRegs, layout.hasPreservedFP()));
|
|
|
|
// We must honor AVX if it's enabled.
|
|
bool avxEnabled = layout.isAvxEnabled();
|
|
|
|
// Free registers that can be used as temporaries and during shuffling.
|
|
// We initialize them to match all workRegs (registers that can be used
|
|
// by the function) except source regs, which are used to pass arguments.
|
|
// Free registers are changed during shuffling - when an argument is moved
|
|
// to the final register then the register itself is removed from freeRegs
|
|
// (it can't be altered anymore during shuffling).
|
|
uint32_t freeRegs[kMaxVRegKinds];
|
|
for (i = 0; i < kMaxVRegKinds; i++)
|
|
freeRegs[i] = ctx._workData[i].workRegs & ~ctx._workData[i].srcRegs;
|
|
|
|
// This is an iterative process that runs until there is a work to do. When
|
|
// one register is moved it can create space for another move. Such moves can
|
|
// depend on each other so the algorithm may run multiple times before all
|
|
// arguments are in place. This part does only register-to-register work,
|
|
// arguments moved from stack-to-register area handled later.
|
|
for (;;) {
|
|
bool hasWork = false; // Do we have a work to do?
|
|
bool didWork = false; // If we did something...
|
|
|
|
uint32_t dstRegKind = kMaxVRegKinds;
|
|
do {
|
|
WorkData& wd = ctx._workData[--dstRegKind];
|
|
if (wd.numOps > wd.numStackArgs) {
|
|
hasWork = true;
|
|
|
|
// Iterate over all destination regs and check if we can do something.
|
|
// We always go from destination to source, never the opposite.
|
|
uint32_t regsToDo = wd.dstRegs;
|
|
do {
|
|
// If there is a work to do there has to be at least one dstReg.
|
|
ASMJIT_ASSERT(regsToDo != 0);
|
|
uint32_t dstRegId = Utils::findFirstBit(regsToDo);
|
|
uint32_t dstRegMask = Utils::mask(dstRegId);
|
|
|
|
uint32_t argIndex = wd.argIndex[dstRegId];
|
|
const DstArg& dstArg = args.getArg(argIndex);
|
|
const SrcArg& srcArg = func.getArg(argIndex);
|
|
|
|
if (srcArg.byReg()) {
|
|
uint32_t srcRegType = srcArg.getRegType();
|
|
uint32_t srcRegKind = X86Reg::kindOf(srcRegType);
|
|
|
|
if (freeRegs[dstRegKind] & dstRegMask) {
|
|
X86Reg dstReg(X86Reg::fromTypeAndId(dstArg.getRegType(), dstRegId));
|
|
X86Reg srcReg(X86Reg::fromTypeAndId(srcRegType, srcArg.getRegId()));
|
|
|
|
ASMJIT_PROPAGATE(
|
|
emitArgMove(emitter,
|
|
dstReg, dstArg.getTypeId(),
|
|
srcReg, srcArg.getTypeId(), avxEnabled));
|
|
freeRegs[dstRegKind] ^= dstRegMask; // Make the DST reg occupied.
|
|
freeRegs[srcRegKind] |= Utils::mask(srcArg.getRegId()); // Make the SRC reg free.
|
|
|
|
ASMJIT_ASSERT(wd.numOps >= 1);
|
|
wd.numOps--;
|
|
didWork = true;
|
|
}
|
|
else {
|
|
// Check if this is a swap operation.
|
|
if (dstRegKind == srcRegKind) {
|
|
uint32_t srcRegId = srcArg.getRegId();
|
|
|
|
uint32_t otherIndex = wd.argIndex[srcRegId];
|
|
const DstArg& otherArg = args.getArg(otherIndex);
|
|
|
|
if (otherArg.getRegId() == srcRegId && X86Reg::kindOf(otherArg.getRegType()) == dstRegKind) {
|
|
// If this is GP reg it can be handled by 'xchg'.
|
|
if (dstRegKind == X86Reg::kKindGp) {
|
|
uint32_t highestType = std::max(dstArg.getRegType(), srcRegType);
|
|
|
|
X86Reg dstReg = x86::gpd(dstRegId);
|
|
X86Reg srcReg = x86::gpd(srcRegId);
|
|
|
|
if (highestType == X86Reg::kRegGpq) {
|
|
dstReg.setSignature(X86RegTraits<X86Reg::kRegGpq>::kSignature);
|
|
srcReg.setSignature(X86RegTraits<X86Reg::kRegGpq>::kSignature);
|
|
}
|
|
ASMJIT_PROPAGATE(emitter->emit(X86Inst::kIdXchg, dstReg, srcReg));
|
|
regsToDo &= ~Utils::mask(srcRegId);
|
|
freeRegs[dstRegKind] &= ~(Utils::mask(srcRegId) | dstRegMask);
|
|
|
|
ASMJIT_ASSERT(wd.numOps >= 2);
|
|
ASMJIT_ASSERT(wd.numSwaps >= 1);
|
|
wd.numOps-=2;
|
|
wd.numSwaps--;
|
|
didWork = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Clear the reg in `regsToDo` and continue if there are more.
|
|
regsToDo ^= dstRegMask;
|
|
} while (regsToDo);
|
|
}
|
|
} while (dstRegKind);
|
|
|
|
if (!hasWork)
|
|
break;
|
|
|
|
if (!didWork)
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
}
|
|
|
|
// Load arguments passed by stack into registers. This is pretty simple and
|
|
// it never requires multiple iterations like the previous phase.
|
|
if (ctx._hasStackArgs) {
|
|
// Base address of all arguments passed by stack.
|
|
X86Mem saBase = x86::ptr(emitter->gpz(layout.getStackArgsRegId()), layout.getStackArgsOffset());
|
|
|
|
uint32_t dstRegKind = kMaxVRegKinds;
|
|
do {
|
|
WorkData& wd = ctx._workData[--dstRegKind];
|
|
if (wd.numStackArgs) {
|
|
// Iterate over all destination regs and check if we can do something.
|
|
// We always go from destination to source, never the opposite.
|
|
uint32_t regsToDo = wd.dstRegs;
|
|
do {
|
|
// If there is a work to do there has to be at least one dstReg.
|
|
ASMJIT_ASSERT(regsToDo != 0);
|
|
ASMJIT_ASSERT(wd.numOps > 0);
|
|
|
|
uint32_t dstRegId = Utils::findFirstBit(regsToDo);
|
|
uint32_t dstRegMask = Utils::mask(dstRegId);
|
|
|
|
uint32_t argIndex = wd.argIndex[dstRegId];
|
|
const DstArg& dstArg = args.getArg(argIndex);
|
|
const SrcArg& srcArg = func.getArg(argIndex);
|
|
|
|
// Only arguments passed by stack should remain, also the destination
|
|
// registers must be free now (otherwise the first part of the algorithm
|
|
// failed). Ideally this should be assert, but it's much safer to enforce
|
|
// this in release as well.
|
|
if (!srcArg.byStack() || !(freeRegs[dstRegKind] & dstRegMask))
|
|
return DebugUtils::errored(kErrorInvalidState);
|
|
|
|
X86Reg dstReg = X86Reg::fromTypeAndId(dstArg.getRegType(), dstRegId);
|
|
X86Mem srcMem = saBase.adjusted(srcArg.getStackOffset());
|
|
|
|
ASMJIT_PROPAGATE(
|
|
emitArgMove(emitter,
|
|
dstReg, dstArg.getTypeId(),
|
|
srcMem, srcArg.getTypeId(), avxEnabled));
|
|
|
|
freeRegs[dstRegKind] ^= dstRegMask;
|
|
regsToDo ^= dstRegMask;
|
|
wd.numOps--;
|
|
} while (regsToDo);
|
|
}
|
|
} while (dstRegKind);
|
|
}
|
|
|
|
return kErrorOk;
|
|
}
|
|
|
|
} // asmjit namespace
|
|
|
|
// [Api-End]
|
|
#include "../asmjit_apiend.h"
|
|
|
|
// [Guard]
|
|
#endif // ASMJIT_BUILD_X86
|