Generate calls to interface methods through resolve helper

eng/pipelines/coreclr/jit-cfg.yml

@@ -16,8 +16,6 @@ extends:
   template:  /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml
   parameters:
     platforms:
-    - linux_arm64
-    - linux_x64
     - windows_x64
     - windows_arm64
     testGroup: jit-cfg

src/coreclr/inc/corinfo.h

@@ -573,6 +573,7 @@ enum CorInfoHelpFunc
     CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS, // Transition to preemptive mode and track transitions in reverse P/Invoke prolog.
 
     CORINFO_HELP_GVMLOOKUP_FOR_SLOT,        // Resolve a generic virtual method target from this pointer and runtime method handle
+    CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT,  // Resolve a non-generic interface method from this pointer and dispatch cell
 
     CORINFO_HELP_STACK_PROBE,               // Probes each page of the allocated stack frame
 

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* d1188f9b-d81e-487a-b525-cedb10d15c8e */
-    0xd1188f9b,
-    0xd81e,
-    0x487a,
-    {0xb5, 0x25, 0xce, 0xdb, 0x10, 0xd1, 0x5c, 0x8e}
+constexpr GUID JITEEVersionIdentifier = { /* 7EA1AD2F-5830-4802-AD99-A6DA4A5269AB */
+    0x7ea1ad2f,
+    0x5830,
+    0x4802,
+    {0xad, 0x99, 0xa6, 0xda, 0x4a, 0x52, 0x69, 0xab}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/inc/jithelpers.h

@@ -332,6 +332,11 @@
     JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS,  JIT_ReversePInvokeExitTrackTransitions,  METHOD__NIL)
 
     JITHELPER(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, NULL, METHOD__NIL)
+#ifdef FEATURE_RESOLVE_HELPER_DISPATCH
+    JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, JIT_InterfaceLookupForSlot, METHOD__NIL)
+#else
+    JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, NULL, METHOD__NIL)
+#endif
 
 #if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64)
     JITHELPER(CORINFO_HELP_STACK_PROBE, JIT_StackProbe, METHOD__NIL)

src/coreclr/inc/switches.h

@@ -170,6 +170,12 @@
 #define FEATURE_PORTABLE_SHUFFLE_THUNKS
 #endif
 
+// Dispatch interface calls via resolve helper followed by an indirect call.
+// Slow functional implementation, only used for stress-testing of DOTNET_JitForceControlFlowGuard=1.
+#if defined(TARGET_WINDOWS) && (defined(TARGET_AMD64) || defined(TARGET_ARM64))
+#define FEATURE_RESOLVE_HELPER_DISPATCH
+#endif
+
 // If this is uncommented, leaves a file "StubLog_<pid>.log" with statistics on the behavior
 // of stub-based interface dispatch.
 //#define STUB_LOGGING

src/coreclr/jit/codegencommon.cpp

@@ -766,6 +766,11 @@ regMaskTP Compiler::compHelperCallKillSet(CorInfoHelpFunc helper)
         case CORINFO_HELP_VALIDATE_INDIRECT_CALL:
             return RBM_VALIDATE_INDIRECT_CALL_TRASH;
 
+#ifdef RBM_INTERFACELOOKUP_FOR_SLOT_TRASH
+        case CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT:
+            return RBM_INTERFACELOOKUP_FOR_SLOT_TRASH;
+#endif
+
         default:
             return RBM_CALLEE_TRASH;
     }
@@ -5849,6 +5854,7 @@ void CodeGen::genDefinePendingCallLabel(GenTreeCall* call)
         {
             case CORINFO_HELP_VALIDATE_INDIRECT_CALL:
             case CORINFO_HELP_VIRTUAL_FUNC_PTR:
+            case CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT:
             case CORINFO_HELP_MEMSET:
             case CORINFO_HELP_MEMCPY:
                 return;

src/coreclr/jit/emit.cpp

@@ -10690,13 +10690,13 @@ const char* emitter::emitOffsetToLabel(unsigned offs)
 regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd)
 {
     // Is it a helper with a special saved set?
-    bool isNoGCHelper = emitNoGChelper(methHnd);
+    bool            isNoGCHelper = emitNoGChelper(methHnd);
+    CorInfoHelpFunc helper       = Compiler::eeGetHelperNum(methHnd);
+
     if (isNoGCHelper)
     {
-        CorInfoHelpFunc helpFunc = Compiler::eeGetHelperNum(methHnd);
-
         // Get the set of registers that this call kills and remove it from the saved set.
-        regMaskTP savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc);
+        regMaskTP savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helper);
 
 #ifdef DEBUG
         if (emitComp->verbose)
@@ -10709,6 +10709,13 @@ regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd)
 #endif
         return savedSet;
     }
+#ifdef RBM_INTERFACELOOKUP_FOR_SLOT_TRASH
+    else if (helper == CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT)
+    {
+        // This one is not no-gc, but it preserves arg registers.
+        return RBM_ALLINT & ~RBM_INTERFACELOOKUP_FOR_SLOT_TRASH;
+    }
+#endif
     else
     {
         // This is the saved set of registers after a normal call.

src/coreclr/jit/gentree.cpp

@@ -1897,6 +1897,47 @@ void CallArgs::RemoveUnsafe(CallArg* arg)
     assert(!"Did not find arg to remove in CallArgs::Remove");
 }
 
+//---------------------------------------------------------------
+// RemoveLate: Remove an argument from the argument list and late argument list.
+//
+// Parameters:
+//   arg - The arg to remove.
+//
+// Remarks:
+//   This can be used to remove arguments after ABI determination and after morph.
+//   It removes the argument from both the early and late list. However, no ABI
+//   information is updated. Caller needs to know what they are doing.
+//
+void CallArgs::RemoveLate(CallArg* arg)
+{
+    CallArg** slot = &m_lateHead;
+    while (*slot != nullptr)
+    {
+        if (*slot == arg)
+        {
+            *slot = arg->GetLateNext();
+            break;
+        }
+
+        slot = &(*slot)->LateNextRef();
+    }
+
+    slot = &m_head;
+    while (*slot != nullptr)
+    {
+        if (*slot == arg)
+        {
+            *slot = arg->GetNext();
+            RemovedWellKnownArg(arg->GetWellKnownArg());
+            return;
+        }
+
+        slot = &(*slot)->NextRef();
+    }
+
+    assert(!"Did not find arg to remove in CallArgs::RemoveLate");
+}
+
 #ifdef TARGET_XARCH
 //---------------------------------------------------------------
 // NeedsVzeroupper: Determines if the call needs a vzeroupper emitted before it is invoked

src/coreclr/jit/gentree.h

@@ -4840,6 +4840,7 @@ class CallArgs
     CallArg* InsertAfterThisOrFirst(Compiler* comp, const NewCallArg& arg);
     void     PushLateBack(CallArg* arg);
     void     Remove(CallArg* arg);
+    void     RemoveLate(CallArg* arg);
     void     RemoveUnsafe(CallArg* arg);
 
     template <typename CopyNodeFunc>

src/coreclr/jit/lower.cpp

@@ -3627,8 +3627,99 @@ void Lowering::LowerCFGCall(GenTreeCall* call)
     {
         return;
     }
+    auto cloneUse = [=](LIR::Use& use, bool cloneConsts) -> GenTree* {
+        bool canClone = cloneConsts && use.Def()->IsCnsIntOrI();
+        if (!canClone && use.Def()->OperIs(GT_LCL_VAR))
+        {
+            canClone = !comp->lvaGetDesc(use.Def()->AsLclVarCommon())->IsAddressExposed();
+        }
+
+        if (canClone)
+        {
+            return comp->gtCloneExpr(use.Def());
+        }
+        else
+        {
+            unsigned newLcl = use.ReplaceWithLclVar(comp);
+            return comp->gtNewLclvNode(newLcl, TYP_I_IMPL);
+        }
+    };
 
     GenTree* callTarget = call->gtCallType == CT_INDIRECT ? call->gtCallAddr : call->gtControlExpr;
+
+    if (call->IsVirtualStub())
+    {
+        // VSDs go through a resolver instead which skips double validation and
+        // indirection.
+        CallArg* vsdCellArg = call->gtArgs.FindWellKnownArg(WellKnownArg::VirtualStubCell);
+        CallArg* thisArg    = call->gtArgs.GetThisArg();
+
+        assert((vsdCellArg != nullptr) && (thisArg != nullptr));
+        assert(thisArg->GetNode()->OperIs(GT_PUTARG_REG));
+        LIR::Use thisArgUse(BlockRange(), &thisArg->GetNode()->AsOp()->gtOp1, thisArg->GetNode());
+        GenTree* thisArgClone = cloneUse(thisArgUse, true);
+
+        // The VSD cell is not needed for the original call when going through the resolver.
+        // It can be removed without further fixups because it has fixed ABI assignment.
+        call->gtArgs.RemoveLate(vsdCellArg);
+        assert(vsdCellArg->GetNode()->OperIs(GT_PUTARG_REG));
+        // Also PUTARG_REG can be removed.
+        BlockRange().Remove(vsdCellArg->GetNode());
+        // The actual cell we need for the resolver.
+        GenTree* vsdCellArgNode = vsdCellArg->GetNode()->gtGetOp1();
+
+        GenTreeCall* resolve = comp->gtNewHelperCallNode(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, TYP_I_IMPL);
+
+        // Use a placeholder for the cell since the cell is already inserted in
+        // LIR.
+        GenTree* vsdCellPlaceholder = comp->gtNewZeroConNode(TYP_I_IMPL);
+        resolve->gtArgs.PushFront(comp,
+                                  NewCallArg::Primitive(vsdCellPlaceholder).WellKnown(WellKnownArg::VirtualStubCell));
+
+        // 'this' arg clone is not inserted, so no need to use a placeholder for that.
+        resolve->gtArgs.PushFront(comp, NewCallArg::Primitive(thisArgClone));
+
+        comp->fgMorphTree(resolve);
+
+        LIR::Range resolveRange = LIR::SeqTree(comp, resolve);
+        GenTree*   resolveFirst = resolveRange.FirstNode();
+        GenTree*   resolveLast  = resolveRange.LastNode();
+        // Resolution comes with a null check, so it must happen after all
+        // arguments are evaluated, hence we insert it right before the call.
+        BlockRange().InsertBefore(call, std::move(resolveRange));
+
+        // Swap out the VSD cell argument.
+        LIR::Use vsdCellUse;
+        bool     gotUse = BlockRange().TryGetUse(vsdCellPlaceholder, &vsdCellUse);
+        assert(gotUse);
+        vsdCellUse.ReplaceWith(vsdCellArgNode);
+        vsdCellPlaceholder->SetUnusedValue();
+
+        // Now we can lower the resolver.
+        LowerRange(resolveFirst, resolveLast);
+
+        // That inserted new PUTARG nodes right before the call, so we need to
+        // legalize the existing call's PUTARG_REG nodes.
+        MovePutArgNodesUpToCall(call);
+
+        // Finally update the call target
+        call->gtCallType = CT_INDIRECT;
+        call->gtFlags &= ~GTF_CALL_VIRT_STUB;
+        call->gtCallAddr   = resolve;
+        call->gtCallCookie = nullptr;
+#ifdef FEATURE_READYTORUN
+        call->gtEntryPoint.addr       = nullptr;
+        call->gtEntryPoint.accessType = IAT_VALUE;
+#endif
+
+        if (callTarget != nullptr)
+        {
+            callTarget->SetUnusedValue();
+        }
+
+        callTarget = resolve;
+    }
+
     if (callTarget == nullptr)
     {
         assert((call->gtCallType != CT_INDIRECT) && (!call->IsVirtual() || call->IsVirtualStubRelativeIndir()));
@@ -3655,7 +3746,7 @@ void Lowering::LowerCFGCall(GenTreeCall* call)
         cloneConsts = true;
 #endif
 
-        GenTree* indirCellClone;
+        GenTree* indirCellClone = cloneUse(indirCellArgUse, cloneConsts);
 
         if (indirCellArgUse.Def()->OperIs(GT_LCL_VAR) || (cloneConsts && indirCellArgUse.Def()->IsCnsIntOrI()))
         {
@@ -7352,7 +7443,7 @@ GenTree* Lowering::LowerVirtualStubCall(GenTreeCall* call)
         // fgMorphArgs will have created trees to pass the address in VirtualStubParam.reg.
         // All we have to do here is add an indirection to generate the actual call target.
 
-        GenTree* ind = Ind(call->gtCallAddr);
+        GenTree* ind = comp->gtNewIndir(TYP_I_IMPL, call->gtCallAddr, GTF_IND_NONFAULTING);
         BlockRange().InsertAfter(call->gtCallAddr, ind);
         call->gtCallAddr = ind;
 
@@ -7394,7 +7485,7 @@ GenTree* Lowering::LowerVirtualStubCall(GenTreeCall* call)
 
             if (!shouldOptimizeVirtualStubCall)
             {
-                result = Ind(addr);
+                result = comp->gtNewIndir(TYP_I_IMPL, addr, GTF_IND_NONFAULTING);
             }
         }
     }

src/coreclr/jit/morph.cpp

@@ -1021,12 +1021,12 @@ void CallArgs::ArgsComplete(Compiler* comp, GenTreeCall* call)
         }
     }
 
-    // When CFG is enabled and this is a delegate call or vtable call we must
+    // When CFG is enabled and this is a delegate call or virtual call we must
     // compute the call target before all late args. However this will
     // effectively null-check 'this', which should happen only after all
     // arguments are evaluated. Thus we must evaluate all args with side
     // effects to a temp.
-    if (comp->opts.IsCFGEnabled() && (call->IsVirtualVtable() || call->IsDelegateInvoke()))
+    if (comp->opts.IsCFGEnabled() && (call->IsVirtual() || call->IsDelegateInvoke()))
     {
         // Always evaluate 'this' to temp.
         assert(HasThisPointer());

src/coreclr/jit/targetamd64.h

@@ -540,6 +540,8 @@
   // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper.
   #define RBM_INIT_PINVOKE_FRAME_TRASH  RBM_CALLEE_TRASH
 
+  #define RBM_INTERFACELOOKUP_FOR_SLOT_TRASH (RBM_RAX | RBM_R10 | RBM_R11)
+
   #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R10 | RBM_RCX))
   #define RBM_VALIDATE_INDIRECT_CALL_TRASH_ALL (RBM_INT_CALLEE_TRASH_ALL & ~(RBM_R10 | RBM_RCX))
   #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX

src/coreclr/jit/targetarm64.h

@@ -263,6 +263,8 @@
   // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper.
   #define RBM_INIT_PINVOKE_FRAME_TRASH  RBM_CALLEE_TRASH
 
+  #define RBM_INTERFACELOOKUP_FOR_SLOT_TRASH (REG_R0 | REG_R12 | REG_R13 | REG_R14 | REG_R15)
+
   #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R15))
   #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15
   #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9

src/coreclr/nativeaot/Runtime/AsmOffsets.h

@@ -72,8 +72,10 @@ ASM_OFFSET(    4,     8, InterfaceDispatchCell, m_pCache)
 #ifdef INTERFACE_DISPATCH_CACHE_HAS_CELL_BACKPOINTER
 ASM_OFFSET(    8,     0, InterfaceDispatchCache, m_pCell)
 #endif
+ASM_OFFSET(    C,    18, InterfaceDispatchCache, m_cEntries)
 ASM_OFFSET(   10,    20, InterfaceDispatchCache, m_rgEntries)
 ASM_SIZEOF(    8,    10, InterfaceDispatchCacheEntry)
+ASM_CONST(     3,     3, IDC_CACHE_POINTER_MASK)
 #endif
 
 // Undefine macros that are only used in this header for convenience.

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -241,6 +241,14 @@ if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)
   )
 endif ()
 
+if(CLR_CMAKE_TARGET_WIN32)
+  if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)
+    list(APPEND RUNTIME_SOURCES_ARCH_ASM
+      ${ARCH_SOURCES_DIR}/DispatchResolve.${ASM_SUFFIX}
+    )
+  endif()
+endif()
+
 # Add architecture specific folder for looking up headers.
 convert_to_absolute_path(ARCH_SOURCES_DIR ${ARCH_SOURCES_DIR})
 include_directories(${ARCH_SOURCES_DIR})

src/coreclr/nativeaot/Runtime/EHHelpers.cpp

@@ -276,6 +276,9 @@ static bool InWriteBarrierHelper(uintptr_t faultingIP)
     return false;
 }
 
+#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS)
+EXTERN_C CODE_LOCATION RhpResolveInterfaceMethodFast;
+#endif
 EXTERN_C CODE_LOCATION RhpInitialInterfaceDispatch;
 EXTERN_C CODE_LOCATION RhpInterfaceDispatchAVLocation1;
 EXTERN_C CODE_LOCATION RhpInterfaceDispatchAVLocation2;
@@ -290,6 +293,9 @@ static bool InInterfaceDispatchHelper(uintptr_t faultingIP)
 #ifndef FEATURE_PORTABLE_HELPERS
     static uintptr_t interfaceDispatchAVLocations[] =
     {
+#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS)
+        (uintptr_t)&RhpResolveInterfaceMethodFast,
+#endif
         (uintptr_t)&RhpInitialInterfaceDispatch,
         (uintptr_t)&RhpInterfaceDispatchAVLocation1,
         (uintptr_t)&RhpInterfaceDispatchAVLocation2,

src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp

@@ -33,6 +33,9 @@
 
 #if !defined(FEATURE_PORTABLE_HELPERS) // @TODO: these are (currently) only implemented in assembly helpers
 EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionTailCall;
+#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS)
+EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnResult;
+#endif
 
 EXTERN_C CODE_LOCATION RhpCallCatchFunclet2;
 EXTERN_C CODE_LOCATION RhpCallFinallyFunclet2;
@@ -2231,6 +2234,12 @@ StackFrameIterator::ReturnAddressCategory StackFrameIterator::CategorizeUnadjust
     {
         return InUniversalTransitionThunk;
     }
+#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS)
+    if (EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnResult))
+    {
+        return InUniversalTransitionThunk;
+    }
+#endif
 
     if (EQUALS_RETURN_ADDRESS(returnAddress, RhpThrowEx2) ||
         EQUALS_RETURN_ADDRESS(returnAddress, RhpThrowHwEx2) ||