diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 648e2acc65acdc..be01a5095bc33b 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -933,7 +933,7 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block) // Assumptions: // The mask contains one and only one register. -inline regNumber genRegNumFromMask(regMaskTP mask) +inline regNumber genRegNumFromMask(const regMaskTP& mask) { assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero @@ -947,6 +947,27 @@ inline regNumber genRegNumFromMask(regMaskTP mask) return regNum; } +//------------------------------------------------------------------------------ +// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number. +// +// Arguments: +// mask - the register mask +// +// Return Value: +// The number of the first register contained in the mask. +// + +inline regNumber genFirstRegNumFromMask(const regMaskTP& mask) +{ + assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero + + /* Convert the mask to a register number */ + + regNumber regNum = (regNumber)BitScanForward(mask); + + return regNum; +} + //------------------------------------------------------------------------------ // genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a // register number and also toggle the bit in the `mask`. @@ -964,30 +985,33 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask) /* Convert the mask to a register number */ - regNumber regNum = (regNumber)BitScanForward(mask); + regNumber regNum = (regNumber)genFirstRegNumFromMask(mask); - mask ^= genRegMask(regNum); + mask ^= regNum; return regNum; } //------------------------------------------------------------------------------ -// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number. -// +// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a +// register number and also toggle the bit in the `mask`. // Arguments: // mask - the register mask // // Return Value: -// The number of the first register contained in the mask. +// The number of the first register contained in the mask and updates the `mask` to toggle +// the bit. // -inline regNumber genFirstRegNumFromMask(regMaskTP mask) +inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask) { - assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero + assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero /* Convert the mask to a register number */ - regNumber regNum = (regNumber)BitScanForward(mask); + regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); + + mask ^= genRegMask(regNum); return regNum; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b642d59bca05a2..b437c34dac874f 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -4022,7 +4022,7 @@ void LinearScan::processKills(RefPosition* killRefPosition) { RefPosition* nextKill = killRefPosition->nextRefPosition; - regMaskTP killedRegs = killRefPosition->registerAssignment; + SingleTypeRegSet killedRegs = killRefPosition->registerAssignment; while (killedRegs != RBM_NONE) { regNumber killedReg = genFirstRegNumFromMaskAndToggle(killedRegs); @@ -4064,7 +4064,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) { // For each physical register that can hold a GC type, // if it is occupied by an interval of a GC type, spill that interval. - regMaskTP candidateRegs = killRefPosition->registerAssignment; + SingleTypeRegSet candidateRegs = killRefPosition->registerAssignment; INDEBUG(bool killedRegs = false); while (candidateRegs != RBM_NONE) { @@ -4157,9 +4157,10 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar { // If we're rotating the register locations at block boundaries, try to use // the next higher register number of the appropriate register type. - regMaskTP candidateRegs = allRegs(interval->registerType) & availableRegs; - regNumber firstReg = REG_NA; - regNumber newReg = REG_NA; + SingleTypeRegSet candidateRegs = + allRegs(interval->registerType) & availableRegs.GetRegSetForType(interval->registerType); + regNumber firstReg = REG_NA; + regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); @@ -12182,7 +12183,7 @@ void LinearScan::verifyFinalAllocation() // However, we will assert that, at resolution time, no registers contain GC refs. { DBEXEC(VERBOSE, printf(" ")); - regMaskTP candidateRegs = currentRefPosition.registerAssignment; + SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 3be436c53b0187..e416f5e40c98aa 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2825,14 +2825,23 @@ void LinearScan::buildIntervals() availableRegCount = REG_INT_COUNT; } - if (availableRegCount < (sizeof(regMaskTP) * 8)) + static_assert(sizeof(regMaskTP) == 2 * sizeof(regMaskSmall)); + + if (availableRegCount < (sizeof(regMaskSmall) * 8)) + { + // Mask out the bits that are between (8 * regMaskSmall) ~ availableRegCount + actualRegistersMask = regMaskTP((1ULL << availableRegCount) - 1); + } + else if (availableRegCount < (sizeof(regMaskTP) * 8)) { - // Mask out the bits that are between 64 ~ availableRegCount - actualRegistersMask = (1ULL << availableRegCount) - 1; + // Mask out the bits that are between (8 * regMaskTP) ~ availableRegCount + // Subtract one extra for stack. + unsigned topRegCount = availableRegCount - sizeof(regMaskSmall) * 8 - 1; + actualRegistersMask = regMaskTP(~RBM_NONE, (1ULL << topRegCount) - 1); } else { - actualRegistersMask = ~RBM_NONE; + actualRegistersMask = regMaskTP(~RBM_NONE, ~RBM_NONE); } #ifdef DEBUG diff --git a/src/coreclr/jit/regset.h b/src/coreclr/jit/regset.h index 99c7f8be6bc556..49cd4dd2e6ecda 100644 --- a/src/coreclr/jit/regset.h +++ b/src/coreclr/jit/regset.h @@ -119,7 +119,7 @@ class RegSet bool rsRegsModified(regMaskTP mask) const { assert(rsModifiedRegsMaskInitialized); - return (rsModifiedRegsMask & mask) != 0; + return (rsModifiedRegsMask & mask).IsNonEmpty(); } void verifyRegUsed(regNumber reg); diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 7e144eb9c5b213..3e3e4591281372 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -229,14 +229,25 @@ typedef uint64_t regMaskSmall; #define REG_MASK_ALL_FMT "%016llX" #endif -typedef regMaskSmall SingleTypeRegSet; +typedef regMaskSmall SingleTypeRegSet; +inline SingleTypeRegSet genRegMask(regNumber reg); +inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); struct regMaskTP { private: regMaskSmall low; + regMaskSmall high; + public: - constexpr regMaskTP(regMaskSmall regMask) + + regMaskTP(regMaskSmall lowMask, regMaskSmall highMask) + : low(lowMask) + , high(highMask) + { + } + + regMaskTP(regMaskSmall regMask) : low(regMask) { } @@ -278,12 +289,12 @@ struct regMaskTP return low; } - bool IsEmpty() + bool IsEmpty() const { return low == RBM_NONE; } - bool IsNonEmpty() + bool IsNonEmpty() const { return !IsEmpty(); } @@ -296,50 +307,52 @@ struct regMaskTP void RemoveRegNumFromMask(regNumber reg); bool IsRegNumInMask(regNumber reg); + + void operator|=(const regMaskTP& second) + { + low |= second.getLow(); + } + + void operator^=(const regMaskTP& second) + { + low ^= second.getLow(); + } + + void operator^=(const regNumber reg) + { + low ^= genRegMask(reg); + } + + void operator&=(const regMaskTP& second) + { + low &= second.getLow(); + } }; -static regMaskTP operator^(regMaskTP first, regMaskTP second) +static regMaskTP operator^(const regMaskTP& first, const regMaskTP& second) { regMaskTP result(first.getLow() ^ second.getLow()); return result; } -static regMaskTP operator&(regMaskTP first, regMaskTP second) +static regMaskTP operator&(const regMaskTP& first, const regMaskTP& second) { regMaskTP result(first.getLow() & second.getLow()); return result; } -static regMaskTP operator|(regMaskTP first, regMaskTP second) +static regMaskTP operator|(const regMaskTP& first, const regMaskTP& second) { regMaskTP result(first.getLow() | second.getLow()); return result; } -static regMaskTP& operator|=(regMaskTP& first, regMaskTP second) -{ - first = first | second; - return first; -} - -static regMaskTP& operator^=(regMaskTP& first, regMaskTP second) -{ - first = first ^ second; - return first; -} - -static regMaskTP& operator&=(regMaskTP& first, regMaskTP second) -{ - first = first & second; - return first; -} - -static bool operator==(regMaskTP first, regMaskTP second) +static bool operator==(const regMaskTP& first, const regMaskTP& second) { return (first.getLow() == second.getLow()); } -static bool operator!=(regMaskTP first, regMaskTP second) +static bool operator!=(const regMaskTP& first, const regMaskTP& second) { return !(first == second); } @@ -375,18 +388,18 @@ static regMaskTP& operator<<=(regMaskTP& first, const int b) } #endif -static regMaskTP operator~(regMaskTP first) +static regMaskTP operator~(const regMaskTP& first) { regMaskTP result(~first.getLow()); return result; } -static uint32_t PopCount(regMaskTP value) +static uint32_t PopCount(const regMaskTP& value) { return BitOperations::PopCount(value.getLow()); } -static uint32_t BitScanForward(regMaskTP mask) +static uint32_t BitScanForward(const regMaskTP& mask) { return BitOperations::BitScanForward(mask.getLow()); } @@ -508,9 +521,6 @@ inline bool isByteReg(regNumber reg) } #endif -inline SingleTypeRegSet genRegMask(regNumber reg); -inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); - /***************************************************************************** * Return true if the register number is valid */