From 91d24b01b2729204dac537a4f562c2ce1550fe52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Thu, 18 Dec 2025 14:30:17 +0100 Subject: [PATCH 01/22] Generate calls to interface methods through resolve helper Successor of https://github.com/dotnet/runtime/pull/112406. That branch has way too many merge conflicts, so leaving the PR as-is for posterity and starting a new one with a clean slate (I basically manually ported everything). This changes interface calls when CFG is enabled to use a resolve helper instead of the stub dispatcher. I've also extended the CFG testing a bit to cover more of interface calls in the smoke test. ```csharp static int Call(IFoo f, int x, int y) => f.Call(x, y); ``` Before this change: ``` 00007FF605971D50 push rbx 00007FF605971D51 sub rsp,20h 00007FF605971D55 mov qword ptr [rsp+30h],rcx 00007FF605971D5A mov r10d,edx 00007FF605971D5D mov ebx,r8d 00007FF605971D60 mov rcx,qword ptr [__InterfaceDispatchCell_repro_Program_IFoo__Call_repro_Program__Call (07FF6059BEBD0h)] 00007FF605971D67 call qword ptr [__guard_check_icall_fptr (07FF6059CD558h)] 00007FF605971D6D mov rax,rcx 00007FF605971D70 mov rcx,qword ptr [rsp+30h] 00007FF605971D75 mov r8d,ebx 00007FF605971D78 mov edx,r10d 00007FF605971D7B lea r10,[__InterfaceDispatchCell_repro_Program_IFoo__Call_repro_Program__Call (07FF6059BEBD0h)] 00007FF605971D82 call rax 00007FF605971D84 nop 00007FF605971D85 add rsp,20h 00007FF605971D89 pop rbx 00007FF605971D8A ret ``` After this change: ``` 00007FF704181CF0 sub rsp,28h 00007FF704181CF4 lea r10,[__InterfaceDispatchCell_repro_Program_IFoo__Call_repro_Program__Call (07FF7041CEBD0h)] 00007FF704181CFB call RhpResolveInterfaceMethodFast (07FF703FFF5A0h) 00007FF704181D00 call qword ptr [__guard_dispatch_icall_fptr (07FF7041DD560h)] 00007FF704181D06 nop 00007FF704181D07 add rsp,28h 00007FF704181D0B ret ``` --- src/coreclr/inc/corinfo.h | 1 + src/coreclr/inc/jiteeversionguid.h | 10 +- src/coreclr/inc/jithelpers.h | 5 + src/coreclr/jit/codegencommon.cpp | 6 + src/coreclr/jit/emit.cpp | 15 ++- src/coreclr/jit/gentree.cpp | 41 ++++++ src/coreclr/jit/gentree.h | 1 + src/coreclr/jit/lower.cpp | 97 ++++++++++++++- src/coreclr/jit/morph.cpp | 4 +- src/coreclr/jit/targetamd64.h | 2 + src/coreclr/jit/targetarm64.h | 2 + src/coreclr/nativeaot/Runtime/AsmOffsets.h | 2 + src/coreclr/nativeaot/Runtime/CMakeLists.txt | 8 ++ src/coreclr/nativeaot/Runtime/EHHelpers.cpp | 6 + .../nativeaot/Runtime/StackFrameIterator.cpp | 12 +- .../Runtime/amd64/DispatchResolve.asm | 63 ++++++++++ .../Runtime/amd64/UniversalTransition.asm | 10 +- .../Runtime/arm64/DispatchResolve.asm | 60 +++++++++ .../Runtime/arm64/UniversalTransition.asm | 14 ++- src/coreclr/nativeaot/Runtime/inc/rhbinder.h | 2 + .../Common/JitInterface/CorInfoHelpFunc.cs | 1 + .../JitInterface/CorInfoImpl.RyuJit.cs | 2 + src/coreclr/vm/amd64/AsmHelpers.asm | 1 - src/coreclr/vm/amd64/VirtualCallStubAMD64.asm | 21 ++++ src/coreclr/vm/amd64/cgenamd64.cpp | 26 ++++ src/coreclr/vm/contractimpl.h | 1 - src/coreclr/vm/jithelpers.cpp | 4 +- src/coreclr/vm/virtualcallstub.cpp | 93 ++++++++++++++ src/coreclr/vm/virtualcallstub.h | 1 + .../ControlFlowGuard/ControlFlowGuard.cs | 117 ++++++++++++++++++ 30 files changed, 603 insertions(+), 25 deletions(-) create mode 100644 src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm create mode 100644 src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index a4ba0a37c235b1..2383adbb13de22 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -573,6 +573,7 @@ enum CorInfoHelpFunc CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS, // Transition to preemptive mode and track transitions in reverse P/Invoke prolog. CORINFO_HELP_GVMLOOKUP_FOR_SLOT, // Resolve a generic virtual method target from this pointer and runtime method handle + CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, // Resolve a non-generic interface method from this pointer and dispatch cell CORINFO_HELP_STACK_PROBE, // Probes each page of the allocated stack frame diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index d426f8003599d3..26ab011000ff25 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* d1188f9b-d81e-487a-b525-cedb10d15c8e */ - 0xd1188f9b, - 0xd81e, - 0x487a, - {0xb5, 0x25, 0xce, 0xdb, 0x10, 0xd1, 0x5c, 0x8e} +constexpr GUID JITEEVersionIdentifier = { /* 7EA1AD2F-5830-4802-AD99-A6DA4A5269AB */ + 0x7ea1ad2f, + 0x5830, + 0x4802, + {0xad, 0x99, 0xa6, 0xda, 0x4a, 0x52, 0x69, 0xab} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 24ce3b00085129..6eb8d9d8f46c98 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -332,6 +332,11 @@ JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS, JIT_ReversePInvokeExitTrackTransitions, METHOD__NIL) JITHELPER(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, NULL, METHOD__NIL) +#if defined(TARGET_AMD64) + JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, JIT_InterfaceLookupForSlot, METHOD__NIL) +#else + JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, NULL, METHOD__NIL) +#endif #if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) JITHELPER(CORINFO_HELP_STACK_PROBE, JIT_StackProbe, METHOD__NIL) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 4113e40f1d44be..fd85663bd45b8d 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -766,6 +766,11 @@ regMaskTP Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) case CORINFO_HELP_VALIDATE_INDIRECT_CALL: return RBM_VALIDATE_INDIRECT_CALL_TRASH; +#ifdef RBM_INTERFACELOOKUP_FOR_SLOT_TRASH + case CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT: + return RBM_INTERFACELOOKUP_FOR_SLOT_TRASH; +#endif + default: return RBM_CALLEE_TRASH; } @@ -5849,6 +5854,7 @@ void CodeGen::genDefinePendingCallLabel(GenTreeCall* call) { case CORINFO_HELP_VALIDATE_INDIRECT_CALL: case CORINFO_HELP_VIRTUAL_FUNC_PTR: + case CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT: case CORINFO_HELP_MEMSET: case CORINFO_HELP_MEMCPY: return; diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 2482c1b22c6ce8..595d8606c05175 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -10690,13 +10690,13 @@ const char* emitter::emitOffsetToLabel(unsigned offs) regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { // Is it a helper with a special saved set? - bool isNoGCHelper = emitNoGChelper(methHnd); + bool isNoGCHelper = emitNoGChelper(methHnd); + CorInfoHelpFunc helper = Compiler::eeGetHelperNum(methHnd); + if (isNoGCHelper) { - CorInfoHelpFunc helpFunc = Compiler::eeGetHelperNum(methHnd); - // Get the set of registers that this call kills and remove it from the saved set. - regMaskTP savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helpFunc); + regMaskTP savedSet = RBM_ALLINT & ~emitGetGCRegsKilledByNoGCCall(helper); #ifdef DEBUG if (emitComp->verbose) @@ -10709,6 +10709,13 @@ regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) #endif return savedSet; } +#ifdef RBM_INTERFACELOOKUP_FOR_SLOT_TRASH + else if (helper == CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT) + { + // This one is not no-gc, but it preserves arg registers. + return RBM_ALLINT & ~RBM_INTERFACELOOKUP_FOR_SLOT_TRASH; + } +#endif else { // This is the saved set of registers after a normal call. diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 05edb2adb5f170..ffde65198e6e37 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1897,6 +1897,47 @@ void CallArgs::RemoveUnsafe(CallArg* arg) assert(!"Did not find arg to remove in CallArgs::Remove"); } +//--------------------------------------------------------------- +// RemoveLate: Remove an argument from the argument list and late argument list. +// +// Parameters: +// arg - The arg to remove. +// +// Remarks: +// This can be used to remove arguments after ABI determination and after morph. +// It removes the argument from both the early and late list. However, no ABI +// information is updated. Caller needs to know what they are doing. +// +void CallArgs::RemoveLate(CallArg* arg) +{ + CallArg** slot = &m_lateHead; + while (*slot != nullptr) + { + if (*slot == arg) + { + *slot = arg->GetLateNext(); + break; + } + + slot = &(*slot)->LateNextRef(); + } + + slot = &m_head; + while (*slot != nullptr) + { + if (*slot == arg) + { + *slot = arg->GetNext(); + RemovedWellKnownArg(arg->GetWellKnownArg()); + return; + } + + slot = &(*slot)->NextRef(); + } + + assert(!"Did not find arg to remove in CallArgs::RemoveLate"); +} + #ifdef TARGET_XARCH //--------------------------------------------------------------- // NeedsVzeroupper: Determines if the call needs a vzeroupper emitted before it is invoked diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 0726b9b9f8846f..c5156f53c15a3d 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4840,6 +4840,7 @@ class CallArgs CallArg* InsertAfterThisOrFirst(Compiler* comp, const NewCallArg& arg); void PushLateBack(CallArg* arg); void Remove(CallArg* arg); + void RemoveLate(CallArg* arg); void RemoveUnsafe(CallArg* arg); template diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 118d7b75b320e1..d01b47982692d3 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -3627,8 +3627,99 @@ void Lowering::LowerCFGCall(GenTreeCall* call) { return; } + auto cloneUse = [=](LIR::Use& use, bool cloneConsts) -> GenTree* { + bool canClone = cloneConsts && use.Def()->IsCnsIntOrI(); + if (!canClone && use.Def()->OperIs(GT_LCL_VAR)) + { + canClone = !comp->lvaGetDesc(use.Def()->AsLclVarCommon())->IsAddressExposed(); + } + + if (canClone) + { + return comp->gtCloneExpr(use.Def()); + } + else + { + unsigned newLcl = use.ReplaceWithLclVar(comp); + return comp->gtNewLclvNode(newLcl, TYP_I_IMPL); + } + }; GenTree* callTarget = call->gtCallType == CT_INDIRECT ? call->gtCallAddr : call->gtControlExpr; + + if (call->IsVirtualStub()) + { + // VSDs go through a resolver instead which skips double validation and + // indirection. + CallArg* vsdCellArg = call->gtArgs.FindWellKnownArg(WellKnownArg::VirtualStubCell); + CallArg* thisArg = call->gtArgs.GetThisArg(); + + assert((vsdCellArg != nullptr) && (thisArg != nullptr)); + assert(thisArg->GetNode()->OperIs(GT_PUTARG_REG)); + LIR::Use thisArgUse(BlockRange(), &thisArg->GetNode()->AsOp()->gtOp1, thisArg->GetNode()); + GenTree* thisArgClone = cloneUse(thisArgUse, true); + + // The VSD cell is not needed for the original call when going through the resolver. + // It can be removed without further fixups because it has fixed ABI assignment. + call->gtArgs.RemoveLate(vsdCellArg); + assert(vsdCellArg->GetNode()->OperIs(GT_PUTARG_REG)); + // Also PUTARG_REG can be removed. + BlockRange().Remove(vsdCellArg->GetNode()); + // The actual cell we need for the resolver. + GenTree* vsdCellArgNode = vsdCellArg->GetNode()->gtGetOp1(); + + GenTreeCall* resolve = comp->gtNewHelperCallNode(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, TYP_I_IMPL); + + // Use a placeholder for the cell since the cell is already inserted in + // LIR. + GenTree* vsdCellPlaceholder = comp->gtNewZeroConNode(TYP_I_IMPL); + resolve->gtArgs.PushFront(comp, + NewCallArg::Primitive(vsdCellPlaceholder).WellKnown(WellKnownArg::VirtualStubCell)); + + // 'this' arg clone is not inserted, so no need to use a placeholder for that. + resolve->gtArgs.PushFront(comp, NewCallArg::Primitive(thisArgClone)); + + comp->fgMorphTree(resolve); + + LIR::Range resolveRange = LIR::SeqTree(comp, resolve); + GenTree* resolveFirst = resolveRange.FirstNode(); + GenTree* resolveLast = resolveRange.LastNode(); + // Resolution comes with a null check, so it must happen after all + // arguments are evaluated, hence we insert it right before the call. + BlockRange().InsertBefore(call, std::move(resolveRange)); + + // Swap out the VSD cell argument. + LIR::Use vsdCellUse; + bool gotUse = BlockRange().TryGetUse(vsdCellPlaceholder, &vsdCellUse); + assert(gotUse); + vsdCellUse.ReplaceWith(vsdCellArgNode); + vsdCellPlaceholder->SetUnusedValue(); + + // Now we can lower the resolver. + LowerRange(resolveFirst, resolveLast); + + // That inserted new PUTARG nodes right before the call, so we need to + // legalize the existing call's PUTARG_REG nodes. + MovePutArgNodesUpToCall(call); + + // Finally update the call target + call->gtCallType = CT_INDIRECT; + call->gtFlags &= ~GTF_CALL_VIRT_STUB; + call->gtCallAddr = resolve; + call->gtCallCookie = nullptr; +#ifdef FEATURE_READYTORUN + call->gtEntryPoint.addr = nullptr; + call->gtEntryPoint.accessType = IAT_VALUE; +#endif + + if (callTarget != nullptr) + { + callTarget->SetUnusedValue(); + } + + callTarget = resolve; + } + if (callTarget == nullptr) { assert((call->gtCallType != CT_INDIRECT) && (!call->IsVirtual() || call->IsVirtualStubRelativeIndir())); @@ -3655,7 +3746,7 @@ void Lowering::LowerCFGCall(GenTreeCall* call) cloneConsts = true; #endif - GenTree* indirCellClone; + GenTree* indirCellClone = cloneUse(indirCellArgUse, cloneConsts); if (indirCellArgUse.Def()->OperIs(GT_LCL_VAR) || (cloneConsts && indirCellArgUse.Def()->IsCnsIntOrI())) { @@ -7352,7 +7443,7 @@ GenTree* Lowering::LowerVirtualStubCall(GenTreeCall* call) // fgMorphArgs will have created trees to pass the address in VirtualStubParam.reg. // All we have to do here is add an indirection to generate the actual call target. - GenTree* ind = Ind(call->gtCallAddr); + GenTree* ind = comp->gtNewIndir(TYP_I_IMPL, call->gtCallAddr, GTF_IND_NONFAULTING); BlockRange().InsertAfter(call->gtCallAddr, ind); call->gtCallAddr = ind; @@ -7394,7 +7485,7 @@ GenTree* Lowering::LowerVirtualStubCall(GenTreeCall* call) if (!shouldOptimizeVirtualStubCall) { - result = Ind(addr); + result = comp->gtNewIndir(TYP_I_IMPL, addr, GTF_IND_NONFAULTING); } } } diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index fd688eea61214b..452b31c3145527 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -1021,12 +1021,12 @@ void CallArgs::ArgsComplete(Compiler* comp, GenTreeCall* call) } } - // When CFG is enabled and this is a delegate call or vtable call we must + // When CFG is enabled and this is a delegate call or virtual call we must // compute the call target before all late args. However this will // effectively null-check 'this', which should happen only after all // arguments are evaluated. Thus we must evaluate all args with side // effects to a temp. - if (comp->opts.IsCFGEnabled() && (call->IsVirtualVtable() || call->IsDelegateInvoke())) + if (comp->opts.IsCFGEnabled() && (call->IsVirtual() || call->IsDelegateInvoke())) { // Always evaluate 'this' to temp. assert(HasThisPointer()); diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index a298b9c989fcdf..9b287dd6906605 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -540,6 +540,8 @@ // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define RBM_INIT_PINVOKE_FRAME_TRASH RBM_CALLEE_TRASH + #define RBM_INTERFACELOOKUP_FOR_SLOT_TRASH (RBM_RAX | RBM_R10 | RBM_R11) + #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R10 | RBM_RCX)) #define RBM_VALIDATE_INDIRECT_CALL_TRASH_ALL (RBM_INT_CALLEE_TRASH_ALL & ~(RBM_R10 | RBM_RCX)) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index c11844ddba5b18..ae25b459322922 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -263,6 +263,8 @@ // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define RBM_INIT_PINVOKE_FRAME_TRASH RBM_CALLEE_TRASH + #define RBM_INTERFACELOOKUP_FOR_SLOT_TRASH (REG_R0 | REG_R12 | REG_R13 | REG_R14 | REG_R15) + #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R15)) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15 #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9 diff --git a/src/coreclr/nativeaot/Runtime/AsmOffsets.h b/src/coreclr/nativeaot/Runtime/AsmOffsets.h index e903dec5fefaaa..62156959948f2d 100644 --- a/src/coreclr/nativeaot/Runtime/AsmOffsets.h +++ b/src/coreclr/nativeaot/Runtime/AsmOffsets.h @@ -72,8 +72,10 @@ ASM_OFFSET( 4, 8, InterfaceDispatchCell, m_pCache) #ifdef INTERFACE_DISPATCH_CACHE_HAS_CELL_BACKPOINTER ASM_OFFSET( 8, 0, InterfaceDispatchCache, m_pCell) #endif +ASM_OFFSET( C, 18, InterfaceDispatchCache, m_cEntries) ASM_OFFSET( 10, 20, InterfaceDispatchCache, m_rgEntries) ASM_SIZEOF( 8, 10, InterfaceDispatchCacheEntry) +ASM_CONST( 3, 3, IDC_CACHE_POINTER_MASK) #endif // Undefine macros that are only used in this header for convenience. diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index dbd19d083a16ee..1c2e24c0099ade 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -241,6 +241,14 @@ if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64) ) endif () +if(CLR_CMAKE_TARGET_WIN32) + if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64) + list(APPEND RUNTIME_SOURCES_ARCH_ASM + ${ARCH_SOURCES_DIR}/DispatchResolve.${ASM_SUFFIX} + ) + endif() +endif() + # Add architecture specific folder for looking up headers. convert_to_absolute_path(ARCH_SOURCES_DIR ${ARCH_SOURCES_DIR}) include_directories(${ARCH_SOURCES_DIR}) diff --git a/src/coreclr/nativeaot/Runtime/EHHelpers.cpp b/src/coreclr/nativeaot/Runtime/EHHelpers.cpp index 19ad512805012c..6f8d0873282a15 100644 --- a/src/coreclr/nativeaot/Runtime/EHHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/EHHelpers.cpp @@ -276,6 +276,9 @@ static bool InWriteBarrierHelper(uintptr_t faultingIP) return false; } +#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) +EXTERN_C CODE_LOCATION RhpResolveInterfaceMethodFast; +#endif EXTERN_C CODE_LOCATION RhpInitialInterfaceDispatch; EXTERN_C CODE_LOCATION RhpInterfaceDispatchAVLocation1; EXTERN_C CODE_LOCATION RhpInterfaceDispatchAVLocation2; @@ -290,6 +293,9 @@ static bool InInterfaceDispatchHelper(uintptr_t faultingIP) #ifndef FEATURE_PORTABLE_HELPERS static uintptr_t interfaceDispatchAVLocations[] = { +#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) + (uintptr_t)&RhpResolveInterfaceMethodFast, +#endif (uintptr_t)&RhpInitialInterfaceDispatch, (uintptr_t)&RhpInterfaceDispatchAVLocation1, (uintptr_t)&RhpInterfaceDispatchAVLocation2, diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp index 5762e817a88904..949a94fc239a81 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp @@ -36,6 +36,10 @@ #if defined(FEATURE_DYNAMIC_CODE) EXTERN_C CODE_LOCATION ReturnFromUniversalTransition; EXTERN_C CODE_LOCATION ReturnFromUniversalTransition_DebugStepTailCall; +#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) +EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnResult; +EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnResult_DebugStepTailCall; +#endif #endif EXTERN_C CODE_LOCATION RhpCallCatchFunclet2; @@ -2233,7 +2237,13 @@ StackFrameIterator::ReturnAddressCategory StackFrameIterator::CategorizeUnadjust #if defined(FEATURE_DYNAMIC_CODE) if (EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransition) || - EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransition_DebugStepTailCall)) + EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransition_DebugStepTailCall) +#if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) + || + EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnResult) || + EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnResult_DebugStepTailCall) +#endif + ) { return InUniversalTransitionThunk; } diff --git a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm new file mode 100644 index 00000000000000..a3925f9e2e0ff7 --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm @@ -0,0 +1,63 @@ +;; Licensed to the .NET Foundation under one or more agreements. +;; The .NET Foundation licenses this file to you under the MIT license. + +include AsmMacros.inc + + +ifdef FEATURE_CACHED_INTERFACE_DISPATCH + +EXTERN RhpCidResolve : PROC +EXTERN RhpUniversalTransitionReturnResult_DebugStepTailCall : PROC + +;; Fast version of RhpResolveInterfaceMethod +LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT + + ;; Load the MethodTable from the object instance in rcx. + ;; Trigger an AV if we're dispatching on a null this. + ;; The exception handling infrastructure is aware of the fact that this is the first + ;; instruction of RhpResolveInterfaceMethodFast and uses it to translate an AV here + ;; to a NullReferenceException at the callsite. + mov rax, [rcx] + + ;; r11 currently contains the indirection cell address. + ;; load r10 to point to the cache block. + mov r10, [r11 + OFFSETOF__InterfaceDispatchCell__m_pCache] + test r10b, IDC_CACHE_POINTER_MASK + jne RhpResolveInterfaceMethodFast_SlowPath + + lea r10, [r10 + OFFSETOF__InterfaceDispatchCache__m_rgEntries] + cmp qword ptr [r10], rax + jne RhpResolveInterfaceMethodFast_Polymorphic + mov rax, qword ptr [r10 + 8] + ret + + RhpResolveInterfaceMethodFast_Polymorphic: + ;; load the count of cache entries into edx + ;; r11 points to the first cache entry so to get to m_cEntries, we need to subtract m_rgEntries first + push rdx + mov edx, dword ptr [r10 - OFFSETOF__InterfaceDispatchCache__m_rgEntries + OFFSETOF__InterfaceDispatchCache__m_cEntries] + + RhpResolveInterfaceMethodFast_NextEntry: + add r10, SIZEOF__InterfaceDispatchCacheEntry + dec edx + jz RhpResolveInterfaceMethodFast_SlowPath_Pop + + cmp qword ptr [r10], rax + jne RhpResolveInterfaceMethodFast_NextEntry + + mov rax, qword ptr [r10 + 8] + pop rdx + ret + + RhpResolveInterfaceMethodFast_SlowPath_Pop: + pop rdx + RhpResolveInterfaceMethodFast_SlowPath: + ;; r11 contains indirection cell address + lea r10, RhpCidResolve + jmp RhpUniversalTransitionReturnResult_DebugStepTailCall + +LEAF_END RhpResolveInterfaceMethodFast, _TEXT + +endif ;; FEATURE_CACHED_INTERFACE_DISPATCH + +end diff --git a/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm index 777ef9d2de620c..91eaf95a48eda0 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm @@ -84,7 +84,7 @@ DISTANCE_FROM_CHILDSP_TO_CALLERSP equ DISTANCE_FROM_CHILDSP_TO_RET ; everything between the base of the ReturnBlock and the top of the StackPassedArgs. ; -UNIVERSAL_TRANSITION macro FunctionName +UNIVERSAL_TRANSITION macro FunctionName, ExitSequence NESTED_ENTRY Rhp&FunctionName, _TEXT @@ -146,7 +146,7 @@ ALTERNATE_ENTRY ReturnFrom&FunctionName ; Pop the space that was allocated between the ChildSP and the caller return address. add rsp, DISTANCE_FROM_CHILDSP_TO_RETADDR - TAILJMP_RAX + ExitSequence NESTED_END Rhp&FunctionName, _TEXT @@ -155,8 +155,10 @@ NESTED_END Rhp&FunctionName, _TEXT ; To enable proper step-in behavior in the debugger, we need to have two instances ; of the thunk. For the first one, the debugger steps into the call in the function, ; for the other, it steps over it. - UNIVERSAL_TRANSITION UniversalTransition - UNIVERSAL_TRANSITION UniversalTransition_DebugStepTailCall + UNIVERSAL_TRANSITION UniversalTransition, TAILJMP_RAX + UNIVERSAL_TRANSITION UniversalTransition_DebugStepTailCall, TAILJMP_RAX + UNIVERSAL_TRANSITION UniversalTransitionReturnResult, ret + UNIVERSAL_TRANSITION UniversalTransitionReturnResult_DebugStepTailCall, ret endif diff --git a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm new file mode 100644 index 00000000000000..e5e467800a36c3 --- /dev/null +++ b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm @@ -0,0 +1,60 @@ +;; Licensed to the .NET Foundation under one or more agreements. +;; The .NET Foundation licenses this file to you under the MIT license. + +#include "AsmMacros.h" + + TEXTAREA + +#ifdef FEATURE_CACHED_INTERFACE_DISPATCH + + EXTERN RhpCidResolve + EXTERN RhpUniversalTransitionReturnResult_DebugStepTailCall + + NESTED_ENTRY RhpResolveInterfaceMethodFast, _TEXT + + ;; Load the MethodTable from the object instance in x0. + ;; Trigger an AV if we're dispatching on a null this. + ;; The exception handling infrastructure is aware of the fact that this is the first + ;; instruction of RhpResolveInterfaceMethodFast and uses it to translate an AV here + ;; to a NullReferenceException at the callsite. + ldr x12, [x0] + + ;; x11 currently contains the indirection cell address. + ;; load x13 to point to the cache block. + ldr x13, [x11, #OFFSETOF__InterfaceDispatchCell__m_pCache] + and x14, x13, #IDC_CACHE_POINTER_MASK + cbnz x14, RhpResolveInterfaceMethodFast_SlowPath + + add x14, x13, #OFFSETOF__InterfaceDispatchCache__m_rgEntries + ldr x15, [x14] + cmp x15, x12 + bne RhpResolveInterfaceMethodFast_Polymorphic + ldur x0, [x14, #8] + ret + +RhpResolveInterfaceMethodFast_Polymorphic + ldr w13, [x13, #OFFSETOF__InterfaceDispatchCache__m_cEntries] + +RhpResolveInterfaceMethodFast_NextEntry + add x14, x14, #SIZEOF__InterfaceDispatchCacheEntry + sub w13, w13, #1 + cmp w13, #0 + beq RhpResolveInterfaceMethodFast_SlowPath + + ldr x15, [x14] + cmp x15, x12 + bne RhpResolveInterfaceMethodFast_NextEntry + + ldur x0, [x14, #8] + ret + +RhpResolveInterfaceMethodFast_SlowPath + ldr xip0, =RhpCidResolve + mov xip1, x11 + b RhpUniversalTransitionReturnResult_DebugStepTailCall + + NESTED_END RhpResolveInterfaceMethodFast + +#endif // FEATURE_CACHED_INTERFACE_DISPATCH + + END diff --git a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm index 7d3607f27a01eb..2de3903b528cba 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm @@ -89,7 +89,7 @@ TEXTAREA MACRO - UNIVERSAL_TRANSITION $FunctionName + UNIVERSAL_TRANSITION $FunctionName, $ReturnResult NESTED_ENTRY Rhp$FunctionName @@ -142,8 +142,14 @@ ;; Restore FP and LR registers, and free the allocated stack block EPILOG_RESTORE_REG_PAIR fp, lr, #STACK_SIZE! + IF $ReturnResult == 0 ;; Tailcall to the target address. EPILOG_NOP br x12 + ELSE + ;; Return target address + EPILOG_NOP mov x0, x12 + ret + ENDIF NESTED_END Rhp$FunctionName @@ -152,7 +158,9 @@ ; To enable proper step-in behavior in the debugger, we need to have two instances ; of the thunk. For the first one, the debugger steps into the call in the function, ; for the other, it steps over it. - UNIVERSAL_TRANSITION UniversalTransition - UNIVERSAL_TRANSITION UniversalTransition_DebugStepTailCall + UNIVERSAL_TRANSITION UniversalTransition, 0 + UNIVERSAL_TRANSITION UniversalTransition_DebugStepTailCall, 0 + UNIVERSAL_TRANSITION UniversalTransitionReturnResult, 1 + UNIVERSAL_TRANSITION UniversalTransitionReturnResult_DebugStepTailCall, 1 END diff --git a/src/coreclr/nativeaot/Runtime/inc/rhbinder.h b/src/coreclr/nativeaot/Runtime/inc/rhbinder.h index 963942ce6d846b..12f6565d06982c 100644 --- a/src/coreclr/nativeaot/Runtime/inc/rhbinder.h +++ b/src/coreclr/nativeaot/Runtime/inc/rhbinder.h @@ -241,6 +241,8 @@ struct InterfaceDispatchCell } }; +#define IDC_CACHE_POINTER_MASK (InterfaceDispatchCell::Flags::IDC_CachePointerMask) + #endif // FEATURE_CACHED_INTERFACE_DISPATCH #ifdef TARGET_ARM diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index 55faa2e79b64de..87c743f1680b76 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -264,6 +264,7 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS, // Transition to preemptive mode and track transitions in reverse P/Invoke prolog. CORINFO_HELP_GVMLOOKUP_FOR_SLOT, // Resolve a generic virtual method target from this pointer and runtime method handle + CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, // Resolve a non-generic interface method from this pointer and dispatch cell CORINFO_HELP_STACK_PROBE, // Probes each page of the allocated stack frame diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs index 4f705b6453141a..5080c068e3b70b 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs @@ -772,6 +772,8 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_GVMLOOKUP_FOR_SLOT: id = ReadyToRunHelper.GVMLookupForSlot; break; + case CorInfoHelpFunc.CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT: + return _compilation.NodeFactory.ExternFunctionSymbol("RhpResolveInterfaceMethodFast"); case CorInfoHelpFunc.CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL: id = ReadyToRunHelper.TypeHandleToRuntimeType; diff --git a/src/coreclr/vm/amd64/AsmHelpers.asm b/src/coreclr/vm/amd64/AsmHelpers.asm index 83e53a633ded54..125143377f0ecf 100644 --- a/src/coreclr/vm/amd64/AsmHelpers.asm +++ b/src/coreclr/vm/amd64/AsmHelpers.asm @@ -405,7 +405,6 @@ NESTED_ENTRY JIT_Patchpoint, _TEXT call JIT_PatchpointWorkerWorkerWithPolicy EPILOG_WITH_TRANSITION_BLOCK_RETURN - TAILJMP_RAX NESTED_END JIT_Patchpoint, _TEXT ; first arg register holds iloffset, which needs to be moved to the second register, and the first register filled with NULL diff --git a/src/coreclr/vm/amd64/VirtualCallStubAMD64.asm b/src/coreclr/vm/amd64/VirtualCallStubAMD64.asm index 6170adf6d4f20b..249c5dd91f5539 100644 --- a/src/coreclr/vm/amd64/VirtualCallStubAMD64.asm +++ b/src/coreclr/vm/amd64/VirtualCallStubAMD64.asm @@ -9,6 +9,7 @@ ifdef FEATURE_VIRTUAL_STUB_DISPATCH CHAIN_SUCCESS_COUNTER equ g_dispatch_cache_chain_success_counter extern VSD_ResolveWorker:proc + extern VSD_ResolveWorkerForInterfaceLookupSlot:proc extern CHAIN_SUCCESS_COUNTER:dword BACKPATCH_FLAG equ 1 ;; Also known as SDF_ResolveBackPatch in the EE @@ -85,5 +86,25 @@ Fail: LEAF_END ResolveWorkerChainLookupAsmStub, _TEXT +;; On Input: +;; rcx contains object 'this' pointer +;; r11 contains the address of the indirection cell (with the flags in the low bits) +;; +;; Preserves all argument registers +NESTED_ENTRY JIT_InterfaceLookupForSlot, _TEXT + + PROLOG_WITH_TRANSITION_BLOCK + + lea rcx, [rsp + __PWTB_TransitionBlock] ; pTransitionBlock + mov rdx, r11 ; indirection cell + + call VSD_ResolveWorkerForInterfaceLookupSlot + + RESTORE_FLOAT_ARGUMENT_REGISTERS __PWTB_FloatArgumentRegisters + RESTORE_ARGUMENT_REGISTERS __PWTB_ArgumentRegisters + EPILOG_WITH_TRANSITION_BLOCK_RETURN + +NESTED_END JIT_InterfaceLookupForSlot, _TEXT + endif ;; FEATURE_VIRTUAL_STUB_DISPATCH end diff --git a/src/coreclr/vm/amd64/cgenamd64.cpp b/src/coreclr/vm/amd64/cgenamd64.cpp index aed4226fcbf6d7..6a9296b57009f1 100644 --- a/src/coreclr/vm/amd64/cgenamd64.cpp +++ b/src/coreclr/vm/amd64/cgenamd64.cpp @@ -43,6 +43,27 @@ void UpdateRegDisplayFromCalleeSavedRegisters(REGDISPLAY * pRD, CalleeSavedRegis #undef CALLEE_SAVED_REGISTER } +#ifdef TARGET_WINDOWS +void UpdateRegDisplayFromArgumentRegisters(REGDISPLAY * pRD, ArgumentRegisters* pRegs) +{ + LIMITED_METHOD_CONTRACT; + + // TODO: Fix ENUM_ARGUMENT_REGISTERS to have consistent casing for rcx and rdx + + T_CONTEXT * pContext = pRD->pCurrentContext; + pContext->Rcx = pRegs->RCX; + pContext->Rdx = pRegs->RDX; + pContext->R8 = pRegs->R8; + pContext->R9 = pRegs->R9; + + KNONVOLATILE_CONTEXT_POINTERS * pContextPointers = pRD->pCurrentContextPointers; + pContextPointers->Rcx = (PULONG64)&pRegs->RCX; + pContextPointers->Rdx = (PULONG64)&pRegs->RDX; + pContextPointers->R8 = (PULONG64)&pRegs->R8; + pContextPointers->R9 = (PULONG64)&pRegs->R9; +} +#endif + void ClearRegDisplayArgumentAndScratchRegisters(REGDISPLAY * pRD) { LIMITED_METHOD_CONTRACT; @@ -83,6 +104,11 @@ void TransitionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFl UpdateRegDisplayFromCalleeSavedRegisters(pRD, GetCalleeSavedRegisters()); ClearRegDisplayArgumentAndScratchRegisters(pRD); +#ifdef TARGET_WINDOWS + // TODO: Quick hack + UpdateRegDisplayFromArgumentRegisters(pRD, GetArgumentRegisters()); +#endif + SyncRegDisplayToCurrentContext(pRD); LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay_Impl(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP)); diff --git a/src/coreclr/vm/contractimpl.h b/src/coreclr/vm/contractimpl.h index 5b3a1f2ee53e8b..6ec975c9a80925 100644 --- a/src/coreclr/vm/contractimpl.h +++ b/src/coreclr/vm/contractimpl.h @@ -299,7 +299,6 @@ struct DispatchToken explicit DispatchToken(UINT_PTR token) { - CONSISTENCY_CHECK(token != INVALID_TOKEN); m_token = token; } diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 9edf5b97eb33a1..c44bad435d110f 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -2475,10 +2475,12 @@ HCIMPL1_RAW(void, JIT_ReversePInvokeExit, ReversePInvokeFrame* frame) } HCIMPLEND_RAW -// These two do take args but have a custom calling convention. +// These do take args but have a custom calling convention. EXTERN_C void JIT_ValidateIndirectCall(); EXTERN_C void JIT_DispatchIndirectCall(); +EXTERN_C void JIT_InterfaceLookupForSlot(); + //======================================================================== // // JIT HELPERS INITIALIZATION diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index 5ddf8e6438403e..d7ade3b1cf4ce4 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -1697,6 +1697,99 @@ PCODE VSD_ResolveWorker(TransitionBlock * pTransitionBlock, return target; } +PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock, TADDR siteAddrForRegisterIndirect) +{ + CONTRACTL { + THROWS; + GC_TRIGGERS; + INJECT_FAULT(COMPlusThrowOM();); + PRECONDITION(CheckPointer(pTransitionBlock)); + MODE_COOPERATIVE; + } CONTRACTL_END; + + MAKE_CURRENT_THREAD_AVAILABLE(); + +#ifdef _DEBUG + Thread::ObjectRefFlush(CURRENT_THREAD); +#endif + + StubDispatchFrame frame(pTransitionBlock); + StubDispatchFrame * pSDFrame = &frame; + + PCODE returnAddress = pSDFrame->GetUnadjustedReturnAddress(); + + StubCallSite callSite(siteAddrForRegisterIndirect, returnAddress); + + OBJECTREF *protectedObj = pSDFrame->GetThisPtr(); + _ASSERTE(protectedObj != NULL); + OBJECTREF pObj = *protectedObj; + + PCODE target = (PCODE)NULL; + + bool propagateExceptionToNativeCode = IsCallDescrWorkerInternalReturnAddress(pTransitionBlock->m_ReturnAddress); + + if (pObj == NULL) { + pSDFrame->Push(CURRENT_THREAD); + INSTALL_MANAGED_EXCEPTION_DISPATCHER_EX; + INSTALL_UNWIND_AND_CONTINUE_HANDLER_EX; + COMPlusThrow(kNullReferenceException); + UNINSTALL_UNWIND_AND_CONTINUE_HANDLER_EX(propagateExceptionToNativeCode); + UNINSTALL_MANAGED_EXCEPTION_DISPATCHER_EX(propagateExceptionToNativeCode); + _ASSERTE(!"Throw returned"); + } + + pSDFrame->SetCallSite(NULL, (TADDR)callSite.GetIndirectCell()); + + DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStub(callSite.GetSiteTarget(), NULL)); + + MethodTable * pRepresentativeMT = pObj->GetMethodTable(); + if (representativeToken.IsTypedToken()) + { + pRepresentativeMT = AppDomain::GetCurrentDomain()->LookupType(representativeToken.GetTypeID()); + CONSISTENCY_CHECK(CheckPointer(pRepresentativeMT)); + } + + pSDFrame->Push(CURRENT_THREAD); + + INSTALL_MANAGED_EXCEPTION_DISPATCHER_EX; + INSTALL_UNWIND_AND_CONTINUE_HANDLER_EX; + + GCPROTECT_BEGIN(*protectedObj); + + // For Virtual Delegates the m_siteAddr is a field of a managed object + // Thus we have to report it as an interior pointer, + // so that it is updated during a gc + GCPROTECT_BEGININTERIOR( *(callSite.GetIndirectCellAddress()) ); + + GCStress::MaybeTriggerAndProtect(pObj); + + PCODE callSiteTarget = callSite.GetSiteTarget(); + CONSISTENCY_CHECK(callSiteTarget != (PCODE)NULL); + + StubCodeBlockKind stubKind = STUB_CODE_BLOCK_UNKNOWN; + VirtualCallStubManager *pMgr = VirtualCallStubManager::FindStubManager(callSiteTarget, &stubKind); + _ASSERTE(pMgr != NULL); + + target = pMgr->ResolveWorker(&callSite, protectedObj, representativeToken, stubKind); + +#if _DEBUG + if (pSDFrame->GetGCRefMap() != NULL) + { + GCX_PREEMP(); + _ASSERTE(CheckGCRefMapEqual(pSDFrame->GetGCRefMap(), pSDFrame->GetFunction(), true)); + } +#endif // _DEBUG + + GCPROTECT_END(); + GCPROTECT_END(); + + UNINSTALL_UNWIND_AND_CONTINUE_HANDLER_EX(propagateExceptionToNativeCode); + UNINSTALL_MANAGED_EXCEPTION_DISPATCHER_EX(propagateExceptionToNativeCode); + pSDFrame->Pop(CURRENT_THREAD); + + return target; +} + void VirtualCallStubManager::BackPatchWorkerStatic(PCODE returnAddress, TADDR siteAddrForRegisterIndirect) { CONTRACTL { diff --git a/src/coreclr/vm/virtualcallstub.h b/src/coreclr/vm/virtualcallstub.h index bfa3280f4597a7..243fb311ed1278 100644 --- a/src/coreclr/vm/virtualcallstub.h +++ b/src/coreclr/vm/virtualcallstub.h @@ -44,6 +44,7 @@ extern "C" PCODE STDCALL VSD_ResolveWorker(TransitionBlock * pTransitionBlock, #endif ); +extern "C" PCODE STDCALL VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock, TADDR siteAddrForRegisterIndirect); ///////////////////////////////////////////////////////////////////////////////////// #if defined(TARGET_X86) || defined(TARGET_AMD64) diff --git a/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs b/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs index 5e9189ac90a572..28c4a72e73a23f 100644 --- a/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs +++ b/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs @@ -35,6 +35,9 @@ static int Main(string[] args) // Are we running the control program? if (args.Length == 0) { + TestExceptionFromDispatch.Run(); + TestInterfaceDispatch.Run(); + // Dry run - execute all scenarios while s_armed is false. // // The replaced call target will not be considered invalid by CFG and none of this @@ -84,6 +87,120 @@ static int Main(string[] args) return 10; } + class TestExceptionFromDispatch + { + class CastableObject : IDynamicInterfaceCastable + { + public RuntimeTypeHandle GetInterfaceImplementation(RuntimeTypeHandle interfaceType) => throw new Exception(); + public bool IsInterfaceImplemented(RuntimeTypeHandle interfaceType, bool throwIfNotImplemented) => true; + } + + public static void Run() + { + bool caughtException = false; + + IDisposable obj = (IDisposable)new CastableObject(); + try + { + obj.Dispose(); + } + catch (Exception) + { + caughtException = true; + } + + if (!caughtException) + throw new Exception(); + } + } + + internal class TestInterfaceDispatch + { + interface IFoo + { + int Call(int x, int y); + } + + interface IFoo + { + int Call(int x, int y); + } + + class C1 : IFoo, IFoo + { + public int Call(int x, int y) => x + y; + } + + class C2 : IFoo, IFoo + { + public int Call(int x, int y) => x - y; + } + class C3 : IFoo, IFoo + { + public int Call(int x, int y) => x * y; + } + + class C4 : IFoo, IFoo + { + public int Call(int x, int y) => x / y; + } + + class C5 : IFoo, IFoo + { + public int Call(int x, int y) => x % y; + } + + public static void Run() + { + if (Call(new C1(), 10, 20) != (10 + 20)) + throw new Exception(); + if (Call(new C1(), 11, 22) != (11 + 22)) + throw new Exception(); + if (Call(new C2(), 10, 20) != (10 - 20)) + throw new Exception(); + if (Call(new C2(), 11, 22) != (11 - 22)) + throw new Exception(); + if (Call(new C3(), 10, 20) != (10 * 20)) + throw new Exception(); + if (Call(new C3(), 11, 22) != (11 * 22)) + throw new Exception(); + if (Call(new C4(), 10, 20) != (10 / 20)) + throw new Exception(); + if (Call(new C5(), 10, 20) != (10 % 20)) + throw new Exception(); + + if (CallGen(new C1(), 10, 20) != (10 + 20)) + throw new Exception(); + if (CallGen(new C2(), 11, 22) != (11 - 22)) + throw new Exception(); + if (CallGen(new C3(), 11, 22) != (11 * 22)) + throw new Exception(); + if (CallGen(new C4(), 10, 20) != (10 / 20)) + throw new Exception(); + if (CallGen(new C5(), 10, 20) != (10 % 20)) + throw new Exception(); + + bool caught = false; + try + { + Call(null, 10, 20); + } + catch (NullReferenceException) + { + caught = true; + } + + if (!caught) + throw new Exception(); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int Call(IFoo f, int x, int y) => f.Call(x, y); + + [MethodImpl(MethodImplOptions.NoInlining)] + static int CallGen(IFoo f, int x, int y) => f.Call(x, y); + } + class TestFunctionPointer { public static int Run() From 9531ff97f77bb6ba23e4a11e37688993eb2a33e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Thu, 18 Dec 2025 15:30:46 +0100 Subject: [PATCH 02/22] Fix non-Windows --- src/coreclr/inc/jithelpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 6eb8d9d8f46c98..0741d0e7bffd8f 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -332,7 +332,7 @@ JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS, JIT_ReversePInvokeExitTrackTransitions, METHOD__NIL) JITHELPER(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, NULL, METHOD__NIL) -#if defined(TARGET_AMD64) +#if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, JIT_InterfaceLookupForSlot, METHOD__NIL) #else JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, NULL, METHOD__NIL) From 4c77f75af8fba838db2478af0accac543a396038 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Thu, 18 Dec 2025 19:43:10 -0800 Subject: [PATCH 03/22] Formatting --- src/coreclr/jit/emit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 595d8606c05175..bd9a5dc655f638 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -10691,7 +10691,7 @@ regMaskTP emitter::emitGetGCRegsSavedOrModified(CORINFO_METHOD_HANDLE methHnd) { // Is it a helper with a special saved set? bool isNoGCHelper = emitNoGChelper(methHnd); - CorInfoHelpFunc helper = Compiler::eeGetHelperNum(methHnd); + CorInfoHelpFunc helper = Compiler::eeGetHelperNum(methHnd); if (isNoGCHelper) { From 09b0ae820a3d54b1125d9fa68e9e50ec74231c12 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Sat, 20 Dec 2025 21:22:04 -0800 Subject: [PATCH 04/22] Fix double reporting --- src/coreclr/vm/virtualcallstub.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index d7ade3b1cf4ce4..894718138be8a5 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -1754,14 +1754,14 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock INSTALL_MANAGED_EXCEPTION_DISPATCHER_EX; INSTALL_UNWIND_AND_CONTINUE_HANDLER_EX; - GCPROTECT_BEGIN(*protectedObj); + GCPROTECT_BEGIN(pObj); // For Virtual Delegates the m_siteAddr is a field of a managed object // Thus we have to report it as an interior pointer, // so that it is updated during a gc GCPROTECT_BEGININTERIOR( *(callSite.GetIndirectCellAddress()) ); - GCStress::MaybeTriggerAndProtect(pObj); + GCStress::MaybeTrigger(); PCODE callSiteTarget = callSite.GetSiteTarget(); CONSISTENCY_CHECK(callSiteTarget != (PCODE)NULL); @@ -1770,7 +1770,7 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock VirtualCallStubManager *pMgr = VirtualCallStubManager::FindStubManager(callSiteTarget, &stubKind); _ASSERTE(pMgr != NULL); - target = pMgr->ResolveWorker(&callSite, protectedObj, representativeToken, stubKind); + target = pMgr->ResolveWorker(&callSite, &pObj, representativeToken, stubKind); #if _DEBUG if (pSDFrame->GetGCRefMap() != NULL) From 9268892969f5cc162fb9dd1c67b480f19d9453a0 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Sun, 21 Dec 2025 10:09:28 -0800 Subject: [PATCH 05/22] Use a special frame to protect arguments --- src/coreclr/inc/jithelpers.h | 2 +- src/coreclr/inc/switches.h | 4 +++ src/coreclr/vm/FrameTypes.h | 3 ++ src/coreclr/vm/amd64/cgenamd64.cpp | 31 ++++++++++++++++ src/coreclr/vm/arm/virtualcallstubcpu.hpp | 2 -- src/coreclr/vm/frames.h | 43 +++++++++++++++++++++++ src/coreclr/vm/virtualcallstub.cpp | 20 +++-------- 7 files changed, 87 insertions(+), 18 deletions(-) diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 0741d0e7bffd8f..4f862c21105bb0 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -332,7 +332,7 @@ JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT_TRACK_TRANSITIONS, JIT_ReversePInvokeExitTrackTransitions, METHOD__NIL) JITHELPER(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, NULL, METHOD__NIL) -#if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, JIT_InterfaceLookupForSlot, METHOD__NIL) #else JITHELPER(CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT, NULL, METHOD__NIL) diff --git a/src/coreclr/inc/switches.h b/src/coreclr/inc/switches.h index 9d6f7083f53fa4..267542e8a8e97f 100644 --- a/src/coreclr/inc/switches.h +++ b/src/coreclr/inc/switches.h @@ -170,6 +170,10 @@ #define FEATURE_PORTABLE_SHUFFLE_THUNKS #endif +#if defined(TARGET_WINDOWS) && defined(TARGET_AMD64) +#define FEATURE_RESOLVE_HELPER_DISPATCH +#endif + // If this is uncommented, leaves a file "StubLog_.log" with statistics on the behavior // of stub-based interface dispatch. //#define STUB_LOGGING diff --git a/src/coreclr/vm/FrameTypes.h b/src/coreclr/vm/FrameTypes.h index 83a4e0aa6de455..1588290d9064cc 100644 --- a/src/coreclr/vm/FrameTypes.h +++ b/src/coreclr/vm/FrameTypes.h @@ -30,6 +30,9 @@ FRAME_TYPE_NAME(HijackFrame) FRAME_TYPE_NAME(PrestubMethodFrame) FRAME_TYPE_NAME(CallCountingHelperFrame) FRAME_TYPE_NAME(StubDispatchFrame) +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH +FRAME_TYPE_NAME(ResolveHelperFrame) +#endif // FEATURE_RESOLVE_HELPER_DISPATCH FRAME_TYPE_NAME(ExternalMethodFrame) FRAME_TYPE_NAME(DynamicHelperFrame) FRAME_TYPE_NAME(ProtectValueClassFrame) diff --git a/src/coreclr/vm/amd64/cgenamd64.cpp b/src/coreclr/vm/amd64/cgenamd64.cpp index 6a9296b57009f1..048af9039e5996 100644 --- a/src/coreclr/vm/amd64/cgenamd64.cpp +++ b/src/coreclr/vm/amd64/cgenamd64.cpp @@ -114,6 +114,37 @@ void TransitionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFl LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay_Impl(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP)); } +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH +void ResolveHelperFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats) +{ + LIMITED_METHOD_CONTRACT; + +#ifndef DACCESS_COMPILE + if (updateFloats) + { + UpdateFloatingPointRegisters(pRD, GetSP()); + _ASSERTE(pRD->pCurrentContext->Rip == GetReturnAddress()); + _ASSERTE(pRD->pCurrentContext->Rsp == GetSP()); + } +#endif // DACCESS_COMPILE + + pRD->IsCallerContextValid = FALSE; + pRD->IsCallerSPValid = FALSE; // Don't add usage of this field. This is only temporary. + + pRD->pCurrentContext->Rip = GetReturnAddress(); + pRD->pCurrentContext->Rsp = GetSP(); + + UpdateRegDisplayFromCalleeSavedRegisters(pRD, GetCalleeSavedRegisters()); + ClearRegDisplayArgumentAndScratchRegisters(pRD); + + UpdateRegDisplayFromArgumentRegisters(pRD, GetArgumentRegisters()); + + SyncRegDisplayToCurrentContext(pRD); + + LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay_Impl(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP)); +} +#endif // FEATURE_RESOLVE_HELPER_DISPATCH + void InlinedCallFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats) { CONTRACTL diff --git a/src/coreclr/vm/arm/virtualcallstubcpu.hpp b/src/coreclr/vm/arm/virtualcallstubcpu.hpp index 49023d7f84cd31..ede098dde691bf 100644 --- a/src/coreclr/vm/arm/virtualcallstubcpu.hpp +++ b/src/coreclr/vm/arm/virtualcallstubcpu.hpp @@ -366,8 +366,6 @@ extern size_t g_poly_call_counter; extern size_t g_poly_miss_counter; #endif -TADDR StubDispatchFrame_MethodFrameVPtr; - LookupHolder* LookupHolder::FromLookupEntry(PCODE lookupEntry) { lookupEntry = lookupEntry & ~THUMB_CODE; diff --git a/src/coreclr/vm/frames.h b/src/coreclr/vm/frames.h index 5491e238918411..e6a732b84caaf4 100644 --- a/src/coreclr/vm/frames.h +++ b/src/coreclr/vm/frames.h @@ -54,6 +54,12 @@ // | | to either a EE runtime helper function or // | | a framed method. // | | +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH +// | | +// | + ResolveHelperFrame - represents a call to interface resolve helper +// | | +#endif // FEATURE_RESOLVE_HELPER_DISPATCH +// | | // | +-FramedMethodFrame - this abstract frame represents a call to a method // | | that generates a full-fledged frame. // | | @@ -894,6 +900,43 @@ class TransitionFrame : public Frame #endif }; +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH + +typedef DPTR(class ResolveHelperFrame) PTR_ResolveHelperFrame; + +// Represents a call to interface resolve helper +// +// This frame saves all argument registers and leaves GC reporting them to the callsite. +// +class ResolveHelperFrame : public TransitionFrame +{ + TADDR m_pTransitionBlock; + +public: +#ifndef DACCESS_COMPILE + ResolveHelperFrame(TransitionBlock* pTransitionBlock) + : TransitionFrame(FrameIdentifier::ResolveHelperFrame), m_pTransitionBlock(dac_cast(pTransitionBlock)) + { + } +#endif + + TADDR GetTransitionBlock_Impl() + { + LIMITED_METHOD_DAC_CONTRACT; + return m_pTransitionBlock; + } + + unsigned GetFrameAttribs_Impl() + { + LIMITED_METHOD_DAC_CONTRACT; + return FRAME_ATTR_RESUMABLE; // Treat the next frame as the top frame. + } + + void UpdateRegDisplay_Impl(const PREGDISPLAY, bool updateFloats = false); +}; + +#endif // FEATURE_RESOLVE_HELPER_DISPATCH + //----------------------------------------------------------------------- // TransitionFrames for exceptions //----------------------------------------------------------------------- diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index 894718138be8a5..c86bdc4a79c8cf 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -1697,6 +1697,7 @@ PCODE VSD_ResolveWorker(TransitionBlock * pTransitionBlock, return target; } +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock, TADDR siteAddrForRegisterIndirect) { CONTRACTL { @@ -1713,12 +1714,10 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock Thread::ObjectRefFlush(CURRENT_THREAD); #endif - StubDispatchFrame frame(pTransitionBlock); - StubDispatchFrame * pSDFrame = &frame; - - PCODE returnAddress = pSDFrame->GetUnadjustedReturnAddress(); + ResolveHelperFrame frame(pTransitionBlock); + ResolveHelperFrame * pSDFrame = &frame; - StubCallSite callSite(siteAddrForRegisterIndirect, returnAddress); + StubCallSite callSite(siteAddrForRegisterIndirect, pTransitionBlock->m_ReturnAddress); OBJECTREF *protectedObj = pSDFrame->GetThisPtr(); _ASSERTE(protectedObj != NULL); @@ -1738,8 +1737,6 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock _ASSERTE(!"Throw returned"); } - pSDFrame->SetCallSite(NULL, (TADDR)callSite.GetIndirectCell()); - DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStub(callSite.GetSiteTarget(), NULL)); MethodTable * pRepresentativeMT = pObj->GetMethodTable(); @@ -1772,14 +1769,6 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock target = pMgr->ResolveWorker(&callSite, &pObj, representativeToken, stubKind); -#if _DEBUG - if (pSDFrame->GetGCRefMap() != NULL) - { - GCX_PREEMP(); - _ASSERTE(CheckGCRefMapEqual(pSDFrame->GetGCRefMap(), pSDFrame->GetFunction(), true)); - } -#endif // _DEBUG - GCPROTECT_END(); GCPROTECT_END(); @@ -1789,6 +1778,7 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock return target; } +#endif // FEATURE_RESOLVE_HELPER_DISPATCH void VirtualCallStubManager::BackPatchWorkerStatic(PCODE returnAddress, TADDR siteAddrForRegisterIndirect) { From 6915a0d462a0ba17f089f0dcec9e9552f7d9c29e Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Sun, 21 Dec 2025 11:21:56 -0800 Subject: [PATCH 06/22] Add comment --- src/coreclr/inc/switches.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/inc/switches.h b/src/coreclr/inc/switches.h index 267542e8a8e97f..e0298dfa001fa3 100644 --- a/src/coreclr/inc/switches.h +++ b/src/coreclr/inc/switches.h @@ -170,6 +170,8 @@ #define FEATURE_PORTABLE_SHUFFLE_THUNKS #endif +// Dispatch interface calls via resolve helper followed by an indirect call. +// Slow functional implementation, only used for stress-testing of DOTNET_JitForceControlFlowGuard=1. #if defined(TARGET_WINDOWS) && defined(TARGET_AMD64) #define FEATURE_RESOLVE_HELPER_DISPATCH #endif From 43e210131fd24a962f2be83192f2752d6d362ca2 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Sun, 21 Dec 2025 11:25:46 -0800 Subject: [PATCH 07/22] Cleanup --- src/coreclr/vm/amd64/cgenamd64.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/coreclr/vm/amd64/cgenamd64.cpp b/src/coreclr/vm/amd64/cgenamd64.cpp index 048af9039e5996..45dbd9a0b9b70b 100644 --- a/src/coreclr/vm/amd64/cgenamd64.cpp +++ b/src/coreclr/vm/amd64/cgenamd64.cpp @@ -48,7 +48,7 @@ void UpdateRegDisplayFromArgumentRegisters(REGDISPLAY * pRD, ArgumentRegisters* { LIMITED_METHOD_CONTRACT; - // TODO: Fix ENUM_ARGUMENT_REGISTERS to have consistent casing for rcx and rdx + // Bridge inconsistent casing between ArgumentRegisters and T_CONTEXT T_CONTEXT * pContext = pRD->pCurrentContext; pContext->Rcx = pRegs->RCX; @@ -104,11 +104,6 @@ void TransitionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFl UpdateRegDisplayFromCalleeSavedRegisters(pRD, GetCalleeSavedRegisters()); ClearRegDisplayArgumentAndScratchRegisters(pRD); -#ifdef TARGET_WINDOWS - // TODO: Quick hack - UpdateRegDisplayFromArgumentRegisters(pRD, GetArgumentRegisters()); -#endif - SyncRegDisplayToCurrentContext(pRD); LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay_Impl(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP)); From c726f083f78b0224facd98b90f8e2bb17c6b9e2e Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Sun, 21 Dec 2025 17:07:55 -0800 Subject: [PATCH 08/22] Run cfg tests on Windows only --- eng/pipelines/coreclr/jit-cfg.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/coreclr/jit-cfg.yml b/eng/pipelines/coreclr/jit-cfg.yml index a99a9edb89db66..5cc80c787c1f67 100644 --- a/eng/pipelines/coreclr/jit-cfg.yml +++ b/eng/pipelines/coreclr/jit-cfg.yml @@ -16,8 +16,6 @@ extends: template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: platforms: - - linux_arm64 - - linux_x64 - windows_x64 - windows_arm64 testGroup: jit-cfg \ No newline at end of file From 6646b0e4c980d5d91de14529eba3380bdf989668 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Sun, 21 Dec 2025 21:27:44 -0800 Subject: [PATCH 09/22] Add Win Arm64 impl --- src/coreclr/inc/switches.h | 2 +- src/coreclr/vm/amd64/cgenamd64.cpp | 4 +-- src/coreclr/vm/arm64/asmhelpers.asm | 22 ++++++++++++ src/coreclr/vm/arm64/stubs.cpp | 52 +++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/src/coreclr/inc/switches.h b/src/coreclr/inc/switches.h index e0298dfa001fa3..a5ccfee195e612 100644 --- a/src/coreclr/inc/switches.h +++ b/src/coreclr/inc/switches.h @@ -172,7 +172,7 @@ // Dispatch interface calls via resolve helper followed by an indirect call. // Slow functional implementation, only used for stress-testing of DOTNET_JitForceControlFlowGuard=1. -#if defined(TARGET_WINDOWS) && defined(TARGET_AMD64) +#if defined(TARGET_WINDOWS) && (defined(TARGET_AMD64) || defined(TARGET_ARM64)) #define FEATURE_RESOLVE_HELPER_DISPATCH #endif diff --git a/src/coreclr/vm/amd64/cgenamd64.cpp b/src/coreclr/vm/amd64/cgenamd64.cpp index 45dbd9a0b9b70b..977c97ed77b808 100644 --- a/src/coreclr/vm/amd64/cgenamd64.cpp +++ b/src/coreclr/vm/amd64/cgenamd64.cpp @@ -48,8 +48,6 @@ void UpdateRegDisplayFromArgumentRegisters(REGDISPLAY * pRD, ArgumentRegisters* { LIMITED_METHOD_CONTRACT; - // Bridge inconsistent casing between ArgumentRegisters and T_CONTEXT - T_CONTEXT * pContext = pRD->pCurrentContext; pContext->Rcx = pRegs->RCX; pContext->Rdx = pRegs->RDX; @@ -136,7 +134,7 @@ void ResolveHelperFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updat SyncRegDisplayToCurrentContext(pRD); - LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay_Impl(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP)); + LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK ResolveHelperFrame::UpdateRegDisplay_Impl(rip:%p, rsp:%p)\n", pRD->ControlPC, pRD->SP)); } #endif // FEATURE_RESOLVE_HELPER_DISPATCH diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index 281ca3bd0e85bc..f0d0cc4f26c1f7 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -10,6 +10,7 @@ IMPORT PInvokeImportWorker #ifdef FEATURE_VIRTUAL_STUB_DISPATCH IMPORT VSD_ResolveWorker + IMPORT VSD_ResolveWorkerForInterfaceLookupSlot #endif IMPORT ComPreStubWorker IMPORT COMToCLRWorker @@ -760,6 +761,27 @@ Fail EPILOG_BRANCH_REG x9 NESTED_END + + +;; On Input: +;; r0 contains object 'this' pointer +;; r11 contains the address of the indirection cell (with the flags in the low bits) +;; +;; Preserves all argument registers + NESTED_ENTRY JIT_InterfaceLookupForSlot + + PROLOG_WITH_TRANSITION_BLOCK + + add x0, sp, #__PWTB_TransitionBlock ; pTransitionBlock + mov x1, x11 ; indirection cell + + bl VSD_ResolveWorkerForInterfaceLookupSlot + + EPILOG_WITH_TRANSITION_BLOCK_TAILCALL + EPILOG_RETURN + + NESTED_END + #endif // FEATURE_VIRTUAL_STUB_DISPATCH #ifdef FEATURE_READYTORUN diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index a14d52045f34ee..67ad33bde99a0a 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -193,6 +193,23 @@ void UpdateRegDisplayFromCalleeSavedRegisters(REGDISPLAY * pRD, CalleeSavedRegis pContextPointers->Lr = (PDWORD64)&pCalleeSaved->x30; } +void UpdateRegDisplayFromArgumentRegisters(REGDISPLAY * pRD, ArgumentRegisters* pRegs) +{ + LIMITED_METHOD_CONTRACT; + + T_CONTEXT * pContext = pRD->pCurrentContext; + for (int i = 0; i < 8; i++) + { + pContext->X[i] = pRegs->x[i]; + } + + Arm64VolatileContextPointer * pContextPointers = &pRD->volatileCurrContextPointers; + for (int i = 0; i < 8; i++) + { + pContextPointers->X[i] = (PDWORD64)&pRegs->x[i]; + } +} + void TransitionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats) { #ifndef DACCESS_COMPILE @@ -224,6 +241,41 @@ void TransitionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFl LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK TransitionFrame::UpdateRegDisplay_Impl(pc:%p, sp:%p)\n", pRD->ControlPC, pRD->SP)); } +#ifdef FEATURE_RESOLVE_HELPER_DISPATCH +void ResolveHelperFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats) +{ +#ifndef DACCESS_COMPILE + if (updateFloats) + { + UpdateFloatingPointRegisters(pRD, GetSP()); + _ASSERTE(pRD->pCurrentContext->Pc == GetReturnAddress()); + } +#endif // DACCESS_COMPILE + + pRD->IsCallerContextValid = FALSE; + pRD->IsCallerSPValid = FALSE; // Don't add usage of this field. This is only temporary. + + // copy the callee saved regs + CalleeSavedRegisters *pCalleeSaved = GetCalleeSavedRegisters(); + UpdateRegDisplayFromCalleeSavedRegisters(pRD, pCalleeSaved); + + ClearRegDisplayArgumentAndScratchRegisters(pRD); + + // copy the control registers + pRD->pCurrentContext->Fp = pCalleeSaved->x29; + pRD->pCurrentContext->Lr = pCalleeSaved->x30; + pRD->pCurrentContext->Pc = GetReturnAddress(); + pRD->pCurrentContext->Sp = this->GetSP(); + + UpdateRegDisplayFromArgumentRegisters(pRD, GetArgumentRegisters()); + + // Finally, syncup the regdisplay with the context + SyncRegDisplayToCurrentContext(pRD); + + LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK ResolveHelperFrame::UpdateRegDisplay_Impl(pc:%p, sp:%p)\n", pRD->ControlPC, pRD->SP)); +} +#endif // FEATURE_RESOLVE_HELPER_DISPATCH + void FaultingExceptionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats) { LIMITED_METHOD_DAC_CONTRACT; From 41d22be9a76e1c3048d717c0225f3aea77a49453 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Mon, 22 Dec 2025 06:08:29 -0800 Subject: [PATCH 10/22] Fix build breaks --- .../nativeaot/Runtime/amd64/DispatchResolve.asm | 4 ++-- .../nativeaot/Runtime/arm64/DispatchResolve.asm | 4 ++-- .../JitInterface/CorInfoImpl.RyuJit.cs | 2 +- src/coreclr/vm/arm64/asmhelpers.asm | 13 ++++++++++--- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm index a3925f9e2e0ff7..d311fc5bf0994d 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm @@ -7,7 +7,7 @@ include AsmMacros.inc ifdef FEATURE_CACHED_INTERFACE_DISPATCH EXTERN RhpCidResolve : PROC -EXTERN RhpUniversalTransitionReturnResult_DebugStepTailCall : PROC +EXTERN RhpUniversalTransitionReturnResult : PROC ;; Fast version of RhpResolveInterfaceMethod LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT @@ -54,7 +54,7 @@ LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT RhpResolveInterfaceMethodFast_SlowPath: ;; r11 contains indirection cell address lea r10, RhpCidResolve - jmp RhpUniversalTransitionReturnResult_DebugStepTailCall + jmp RhpUniversalTransitionReturnResult LEAF_END RhpResolveInterfaceMethodFast, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm index e5e467800a36c3..338bba7eeceac1 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm @@ -8,7 +8,7 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH EXTERN RhpCidResolve - EXTERN RhpUniversalTransitionReturnResult_DebugStepTailCall + EXTERN RhpUniversalTransitionReturnResult NESTED_ENTRY RhpResolveInterfaceMethodFast, _TEXT @@ -51,7 +51,7 @@ RhpResolveInterfaceMethodFast_NextEntry RhpResolveInterfaceMethodFast_SlowPath ldr xip0, =RhpCidResolve mov xip1, x11 - b RhpUniversalTransitionReturnResult_DebugStepTailCall + b RhpUniversalTransitionReturnResult NESTED_END RhpResolveInterfaceMethodFast diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs index 0cccf27d4c5f65..43e116ade2c554 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs @@ -773,7 +773,7 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) id = ReadyToRunHelper.GVMLookupForSlot; break; case CorInfoHelpFunc.CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT: - return _compilation.NodeFactory.ExternFunctionSymbol("RhpResolveInterfaceMethodFast"); + return _compilation.NodeFactory.ExternFunctionSymbol(new Utf8String("RhpResolveInterfaceMethodFast"u8)); case CorInfoHelpFunc.CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL: id = ReadyToRunHelper.TypeHandleToRuntimeType; diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index f0d0cc4f26c1f7..6245881ee0cef1 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -764,8 +764,8 @@ Fail ;; On Input: -;; r0 contains object 'this' pointer -;; r11 contains the address of the indirection cell (with the flags in the low bits) +;; x0 contains object 'this' pointer +;; x11 contains the address of the indirection cell (with the flags in the low bits) ;; ;; Preserves all argument registers NESTED_ENTRY JIT_InterfaceLookupForSlot @@ -775,9 +775,16 @@ Fail add x0, sp, #__PWTB_TransitionBlock ; pTransitionBlock mov x1, x11 ; indirection cell - bl VSD_ResolveWorkerForInterfaceLookupSlot + bl VSD_ResolveWorkerForInterfaceLookupSlot + + ;; Move the result (the target address) to x12 so it doesn't get overridden when we restore the + ;; argument registers. + mov x12, x0 EPILOG_WITH_TRANSITION_BLOCK_TAILCALL + + EPILOG_NOP mov x0, x12 + EPILOG_RETURN NESTED_END From 124785a6569869ec7c4105c4ccf7ea0099c7bd88 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Mon, 22 Dec 2025 10:12:13 -0800 Subject: [PATCH 11/22] x8 too --- src/coreclr/vm/arm64/stubs.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 67ad33bde99a0a..ec6377012cef5a 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -202,12 +202,14 @@ void UpdateRegDisplayFromArgumentRegisters(REGDISPLAY * pRD, ArgumentRegisters* { pContext->X[i] = pRegs->x[i]; } + pContext->X[8] = *(&pRegs->x[0] - 1); // m_x8RetBuffReg Arm64VolatileContextPointer * pContextPointers = &pRD->volatileCurrContextPointers; for (int i = 0; i < 8; i++) { pContextPointers->X[i] = (PDWORD64)&pRegs->x[i]; } + pContextPointers->X[8] = (PDWORD64)(&pRegs->x[0] - 1); // m_x8RetBuffReg } void TransitionFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats) From c2ca3d25d8c18032b321e4017cd8a8bc4d3101f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 31 Dec 2025 05:56:15 +0100 Subject: [PATCH 12/22] wip --- src/coreclr/vm/virtualcallstub.cpp | 39 +++++++++++++++++++++++++++++- src/coreclr/vm/virtualcallstub.h | 3 +++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index c86bdc4a79c8cf..00d2bc7a968b01 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -1297,6 +1297,43 @@ size_t VirtualCallStubManager::GetTokenFromStub(PCODE stub, T_CONTEXT *pContext) #endif } +size_t VirtualCallStubManager::GetTokenFromStubAndIndirectionCell(PCODE stub, TADDR indcell) +{ + CONTRACTL + { + NOTHROW; + GC_NOTRIGGER; + FORBID_FAULT; + } + CONTRACTL_END + +#ifdef FEATURE_CACHED_INTERFACE_DISPATCH + if (isCachedInterfaceDispatchStub(stub)) + { + VirtualCallStubManagerIterator it = + VirtualCallStubManagerManager::GlobalManager()->IterateVirtualCallStubManagers(); + while (it.Next()) + { + if (it.Current()->indcell_rangeList.IsInRange(indcell)) + { + InterfaceDispatchCell * pCell = (InterfaceDispatchCell *)indcell; + return pCell->GetDispatchCellInfo().Token.To_SIZE_T(); + } + } + } +#endif // FEATURE_CACHED_INTERFACE_DISPATCH + +#ifdef FEATURE_VIRTUAL_STUB_DISPATCH + _ASSERTE(stub != (PCODE)NULL); + StubCodeBlockKind stubKind = STUB_CODE_BLOCK_UNKNOWN; + VirtualCallStubManager * pMgr = FindStubManager(stub, &stubKind); + + return GetTokenFromStubQuick(pMgr, stub, stubKind); +#else + return 0; +#endif +} + #ifdef FEATURE_VIRTUAL_STUB_DISPATCH size_t VirtualCallStubManager::GetTokenFromStubQuick(VirtualCallStubManager * pMgr, PCODE stub, StubCodeBlockKind kind) { @@ -1737,7 +1774,7 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock _ASSERTE(!"Throw returned"); } - DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStub(callSite.GetSiteTarget(), NULL)); + DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStubAndIndirectionCell(callSite.GetSiteTarget(), (TADDR)callSite.GetIndirectCell())); MethodTable * pRepresentativeMT = pObj->GetMethodTable(); if (representativeToken.IsTypedToken()) diff --git a/src/coreclr/vm/virtualcallstub.h b/src/coreclr/vm/virtualcallstub.h index 243fb311ed1278..5ec5bd58582473 100644 --- a/src/coreclr/vm/virtualcallstub.h +++ b/src/coreclr/vm/virtualcallstub.h @@ -456,6 +456,9 @@ class VirtualCallStubManager : public StubManager //This is used to get the token out of a stub static size_t GetTokenFromStub(PCODE stub, T_CONTEXT *pContext); + //This is used to get the token out of a stub and indirection cell + static size_t GetTokenFromStubAndIndirectionCell(PCODE stub, TADDR indcell); + #ifdef FEATURE_VIRTUAL_STUB_DISPATCH //This is used to get the token out of a stub and we know the stub manager and stub kind static size_t GetTokenFromStubQuick(VirtualCallStubManager * pMgr, PCODE stub, StubCodeBlockKind kind); From 480cea947bd107975197c2da43238c47588f3b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 31 Dec 2025 05:56:23 +0100 Subject: [PATCH 13/22] Revert "wip" This reverts commit c2ca3d25d8c18032b321e4017cd8a8bc4d3101f1. --- src/coreclr/vm/virtualcallstub.cpp | 39 +----------------------------- src/coreclr/vm/virtualcallstub.h | 3 --- 2 files changed, 1 insertion(+), 41 deletions(-) diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index 00d2bc7a968b01..c86bdc4a79c8cf 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -1297,43 +1297,6 @@ size_t VirtualCallStubManager::GetTokenFromStub(PCODE stub, T_CONTEXT *pContext) #endif } -size_t VirtualCallStubManager::GetTokenFromStubAndIndirectionCell(PCODE stub, TADDR indcell) -{ - CONTRACTL - { - NOTHROW; - GC_NOTRIGGER; - FORBID_FAULT; - } - CONTRACTL_END - -#ifdef FEATURE_CACHED_INTERFACE_DISPATCH - if (isCachedInterfaceDispatchStub(stub)) - { - VirtualCallStubManagerIterator it = - VirtualCallStubManagerManager::GlobalManager()->IterateVirtualCallStubManagers(); - while (it.Next()) - { - if (it.Current()->indcell_rangeList.IsInRange(indcell)) - { - InterfaceDispatchCell * pCell = (InterfaceDispatchCell *)indcell; - return pCell->GetDispatchCellInfo().Token.To_SIZE_T(); - } - } - } -#endif // FEATURE_CACHED_INTERFACE_DISPATCH - -#ifdef FEATURE_VIRTUAL_STUB_DISPATCH - _ASSERTE(stub != (PCODE)NULL); - StubCodeBlockKind stubKind = STUB_CODE_BLOCK_UNKNOWN; - VirtualCallStubManager * pMgr = FindStubManager(stub, &stubKind); - - return GetTokenFromStubQuick(pMgr, stub, stubKind); -#else - return 0; -#endif -} - #ifdef FEATURE_VIRTUAL_STUB_DISPATCH size_t VirtualCallStubManager::GetTokenFromStubQuick(VirtualCallStubManager * pMgr, PCODE stub, StubCodeBlockKind kind) { @@ -1774,7 +1737,7 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock _ASSERTE(!"Throw returned"); } - DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStubAndIndirectionCell(callSite.GetSiteTarget(), (TADDR)callSite.GetIndirectCell())); + DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStub(callSite.GetSiteTarget(), NULL)); MethodTable * pRepresentativeMT = pObj->GetMethodTable(); if (representativeToken.IsTypedToken()) diff --git a/src/coreclr/vm/virtualcallstub.h b/src/coreclr/vm/virtualcallstub.h index 5ec5bd58582473..243fb311ed1278 100644 --- a/src/coreclr/vm/virtualcallstub.h +++ b/src/coreclr/vm/virtualcallstub.h @@ -456,9 +456,6 @@ class VirtualCallStubManager : public StubManager //This is used to get the token out of a stub static size_t GetTokenFromStub(PCODE stub, T_CONTEXT *pContext); - //This is used to get the token out of a stub and indirection cell - static size_t GetTokenFromStubAndIndirectionCell(PCODE stub, TADDR indcell); - #ifdef FEATURE_VIRTUAL_STUB_DISPATCH //This is used to get the token out of a stub and we know the stub manager and stub kind static size_t GetTokenFromStubQuick(VirtualCallStubManager * pMgr, PCODE stub, StubCodeBlockKind kind); From 9bb5978e870ddd2ab60ef88a302c695ff0f0cb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Thu, 1 Jan 2026 14:41:16 -0800 Subject: [PATCH 14/22] Which test runs into this? --- src/coreclr/vm/contractimpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/vm/contractimpl.h b/src/coreclr/vm/contractimpl.h index 6ec975c9a80925..5b3a1f2ee53e8b 100644 --- a/src/coreclr/vm/contractimpl.h +++ b/src/coreclr/vm/contractimpl.h @@ -299,6 +299,7 @@ struct DispatchToken explicit DispatchToken(UINT_PTR token) { + CONSISTENCY_CHECK(token != INVALID_TOKEN); m_token = token; } From c37c693c746f76477de30205d75a187828c6e1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Fri, 2 Jan 2026 13:41:39 +0100 Subject: [PATCH 15/22] Fix with UseCachedInterfaceDispatch --- src/coreclr/vm/virtualcallstub.cpp | 33 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/coreclr/vm/virtualcallstub.cpp b/src/coreclr/vm/virtualcallstub.cpp index c86bdc4a79c8cf..ac49aaea29a51a 100644 --- a/src/coreclr/vm/virtualcallstub.cpp +++ b/src/coreclr/vm/virtualcallstub.cpp @@ -1737,15 +1737,6 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock _ASSERTE(!"Throw returned"); } - DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStub(callSite.GetSiteTarget(), NULL)); - - MethodTable * pRepresentativeMT = pObj->GetMethodTable(); - if (representativeToken.IsTypedToken()) - { - pRepresentativeMT = AppDomain::GetCurrentDomain()->LookupType(representativeToken.GetTypeID()); - CONSISTENCY_CHECK(CheckPointer(pRepresentativeMT)); - } - pSDFrame->Push(CURRENT_THREAD); INSTALL_MANAGED_EXCEPTION_DISPATCHER_EX; @@ -1760,14 +1751,26 @@ PCODE VSD_ResolveWorkerForInterfaceLookupSlot(TransitionBlock * pTransitionBlock GCStress::MaybeTrigger(); - PCODE callSiteTarget = callSite.GetSiteTarget(); - CONSISTENCY_CHECK(callSiteTarget != (PCODE)NULL); +#ifdef FEATURE_CACHED_INTERFACE_DISPATCH + if (UseCachedInterfaceDispatch()) + { + DispatchToken representativeToken(((InterfaceDispatchCell*)siteAddrForRegisterIndirect)->GetDispatchCellInfo().Token); + target = CachedInterfaceDispatchResolveWorker(&callSite, protectedObj, representativeToken); + } + else +#endif // FEATURE_CACHED_INTERFACE_DISPATCH + { + DispatchToken representativeToken = DispatchToken(VirtualCallStubManager::GetTokenFromStub(callSite.GetSiteTarget(), NULL)); - StubCodeBlockKind stubKind = STUB_CODE_BLOCK_UNKNOWN; - VirtualCallStubManager *pMgr = VirtualCallStubManager::FindStubManager(callSiteTarget, &stubKind); - _ASSERTE(pMgr != NULL); + PCODE callSiteTarget = callSite.GetSiteTarget(); + CONSISTENCY_CHECK(callSiteTarget != (PCODE)NULL); - target = pMgr->ResolveWorker(&callSite, &pObj, representativeToken, stubKind); + StubCodeBlockKind stubKind = STUB_CODE_BLOCK_UNKNOWN; + VirtualCallStubManager *pMgr = VirtualCallStubManager::FindStubManager(callSiteTarget, &stubKind); + _ASSERTE(pMgr != NULL); + + target = pMgr->ResolveWorker(&callSite, &pObj, representativeToken, stubKind); + } GCPROTECT_END(); GCPROTECT_END(); From 1e29576bc2f1fb8f4669854a6f11c7e3652fd257 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 2 Jan 2026 14:26:54 +0100 Subject: [PATCH 16/22] Use existing helper --- src/coreclr/jit/gentree.cpp | 43 +------------------------------------ src/coreclr/jit/lower.cpp | 2 +- 2 files changed, 2 insertions(+), 43 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index ffde65198e6e37..b5e1d3e5cac18a 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1598,7 +1598,7 @@ bool CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension cc, We break; -#ifdef REG_DISPATCH_INDIRECT_CELL_ADDR +#ifdef REG_DISPATCH_INDIRECT_CALL_ADDR case WellKnownArg::DispatchIndirectCallTarget: *reg = REG_DISPATCH_INDIRECT_CALL_ADDR; return true; @@ -1897,47 +1897,6 @@ void CallArgs::RemoveUnsafe(CallArg* arg) assert(!"Did not find arg to remove in CallArgs::Remove"); } -//--------------------------------------------------------------- -// RemoveLate: Remove an argument from the argument list and late argument list. -// -// Parameters: -// arg - The arg to remove. -// -// Remarks: -// This can be used to remove arguments after ABI determination and after morph. -// It removes the argument from both the early and late list. However, no ABI -// information is updated. Caller needs to know what they are doing. -// -void CallArgs::RemoveLate(CallArg* arg) -{ - CallArg** slot = &m_lateHead; - while (*slot != nullptr) - { - if (*slot == arg) - { - *slot = arg->GetLateNext(); - break; - } - - slot = &(*slot)->LateNextRef(); - } - - slot = &m_head; - while (*slot != nullptr) - { - if (*slot == arg) - { - *slot = arg->GetNext(); - RemovedWellKnownArg(arg->GetWellKnownArg()); - return; - } - - slot = &(*slot)->NextRef(); - } - - assert(!"Did not find arg to remove in CallArgs::RemoveLate"); -} - #ifdef TARGET_XARCH //--------------------------------------------------------------- // NeedsVzeroupper: Determines if the call needs a vzeroupper emitted before it is invoked diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index d01b47982692d3..fd32b48c025ce6 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -3661,7 +3661,7 @@ void Lowering::LowerCFGCall(GenTreeCall* call) // The VSD cell is not needed for the original call when going through the resolver. // It can be removed without further fixups because it has fixed ABI assignment. - call->gtArgs.RemoveLate(vsdCellArg); + call->gtArgs.RemoveUnsafe(vsdCellArg); assert(vsdCellArg->GetNode()->OperIs(GT_PUTARG_REG)); // Also PUTARG_REG can be removed. BlockRange().Remove(vsdCellArg->GetNode()); From 2681592a1d36fd119a12134610b9ea6ba8e0b65a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 2 Jan 2026 14:54:54 +0100 Subject: [PATCH 17/22] Return target address x15 --- src/coreclr/jit/codegenarmarch.cpp | 7 +++++++ src/coreclr/jit/lsraarmarch.cpp | 7 +++++++ src/coreclr/jit/targetarm64.h | 4 ++-- src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm | 4 ++-- .../nativeaot/Runtime/arm64/UniversalTransition.asm | 2 +- src/coreclr/vm/arm64/asmhelpers.asm | 7 ++++--- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index a9e6aeec4a97be..da444bdefdfa85 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3169,6 +3169,13 @@ void CodeGen::genCall(GenTreeCall* call) } else #endif // TARGET_ARM +#ifdef TARGET_ARM64 + if (call->IsHelperCall(compiler, CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT)) + { + returnReg = genFirstRegNumFromMask(RBM_INTERFACELOOKUP_FOR_SLOT_RETURN); + } + else +#endif if (varTypeUsesFloatArgReg(returnType)) { returnReg = REG_FLOATRET; diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index cb062b91cc1b8e..f965758ecf920a 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -231,6 +231,13 @@ int LinearScan::BuildCall(GenTreeCall* call) } else #endif // TARGET_ARM +#ifdef TARGET_ARM64 + if (call->IsHelperCall(compiler, CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT)) + { + singleDstCandidates = RBM_INTERFACELOOKUP_FOR_SLOT_RETURN.GetIntRegSet(); + } + else +#endif if (!hasMultiRegRetVal) { if (varTypeUsesFloatArgReg(registerType)) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index ae25b459322922..70b21b9288c378 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -263,8 +263,8 @@ // The registers trashed by the CORINFO_HELP_INIT_PINVOKE_FRAME helper. #define RBM_INIT_PINVOKE_FRAME_TRASH RBM_CALLEE_TRASH - #define RBM_INTERFACELOOKUP_FOR_SLOT_TRASH (REG_R0 | REG_R12 | REG_R13 | REG_R14 | REG_R15) - + #define RBM_INTERFACELOOKUP_FOR_SLOT_TRASH (RBM_R12 | RBM_R13 | RBM_R14 | RBM_R15) + #define RBM_INTERFACELOOKUP_FOR_SLOT_RETURN RBM_R15 #define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~(RBM_R0 | RBM_R1 | RBM_R2 | RBM_R3 | RBM_R4 | RBM_R5 | RBM_R6 | RBM_R7 | RBM_R8 | RBM_R15)) #define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15 #define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9 diff --git a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm index 338bba7eeceac1..fcd885fc98efa9 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm @@ -29,7 +29,7 @@ ldr x15, [x14] cmp x15, x12 bne RhpResolveInterfaceMethodFast_Polymorphic - ldur x0, [x14, #8] + ldur x15, [x14, #8] ret RhpResolveInterfaceMethodFast_Polymorphic @@ -45,7 +45,7 @@ RhpResolveInterfaceMethodFast_NextEntry cmp x15, x12 bne RhpResolveInterfaceMethodFast_NextEntry - ldur x0, [x14, #8] + ldur x15, [x14, #8] ret RhpResolveInterfaceMethodFast_SlowPath diff --git a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm index 994c8e0c8c8f78..eb9314598cedb4 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm @@ -147,7 +147,7 @@ EPILOG_NOP br x12 ELSE ;; Return target address - EPILOG_NOP mov x0, x12 + EPILOG_NOP mov x15, x12 ret ENDIF diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm index 6245881ee0cef1..437ab1a527016d 100644 --- a/src/coreclr/vm/arm64/asmhelpers.asm +++ b/src/coreclr/vm/arm64/asmhelpers.asm @@ -767,6 +767,9 @@ Fail ;; x0 contains object 'this' pointer ;; x11 contains the address of the indirection cell (with the flags in the low bits) ;; +;; On Output: +;; x15 contains the target address +;; ;; Preserves all argument registers NESTED_ENTRY JIT_InterfaceLookupForSlot @@ -779,12 +782,10 @@ Fail ;; Move the result (the target address) to x12 so it doesn't get overridden when we restore the ;; argument registers. - mov x12, x0 + mov x15, x0 EPILOG_WITH_TRANSITION_BLOCK_TAILCALL - EPILOG_NOP mov x0, x12 - EPILOG_RETURN NESTED_END From 31d400f5f7711caac7d87e27e6e5768be579e4ac Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 2 Jan 2026 14:56:00 +0100 Subject: [PATCH 18/22] Run jit-format --- src/coreclr/jit/codegenarmarch.cpp | 22 +++++++++++----------- src/coreclr/jit/lsraarmarch.cpp | 26 +++++++++++++------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index da444bdefdfa85..00cd452f8c88f6 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -3171,19 +3171,19 @@ void CodeGen::genCall(GenTreeCall* call) #endif // TARGET_ARM #ifdef TARGET_ARM64 if (call->IsHelperCall(compiler, CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT)) - { - returnReg = genFirstRegNumFromMask(RBM_INTERFACELOOKUP_FOR_SLOT_RETURN); - } - else + { + returnReg = genFirstRegNumFromMask(RBM_INTERFACELOOKUP_FOR_SLOT_RETURN); + } + else #endif if (varTypeUsesFloatArgReg(returnType)) - { - returnReg = REG_FLOATRET; - } - else - { - returnReg = REG_INTRET; - } + { + returnReg = REG_FLOATRET; + } + else + { + returnReg = REG_INTRET; + } if (call->GetRegNum() != returnReg) { diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index f965758ecf920a..f85cdc3c7ee8c9 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -232,27 +232,27 @@ int LinearScan::BuildCall(GenTreeCall* call) else #endif // TARGET_ARM #ifdef TARGET_ARM64 - if (call->IsHelperCall(compiler, CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT)) + if (call->IsHelperCall(compiler, CORINFO_HELP_INTERFACELOOKUP_FOR_SLOT)) { singleDstCandidates = RBM_INTERFACELOOKUP_FOR_SLOT_RETURN.GetIntRegSet(); } else #endif if (!hasMultiRegRetVal) + { + if (varTypeUsesFloatArgReg(registerType)) { - if (varTypeUsesFloatArgReg(registerType)) - { - singleDstCandidates = RBM_FLOATRET.GetFloatRegSet(); - } - else if (registerType == TYP_LONG) - { - singleDstCandidates = RBM_LNGRET.GetIntRegSet(); - } - else - { - singleDstCandidates = RBM_INTRET.GetIntRegSet(); - } + singleDstCandidates = RBM_FLOATRET.GetFloatRegSet(); + } + else if (registerType == TYP_LONG) + { + singleDstCandidates = RBM_LNGRET.GetIntRegSet(); } + else + { + singleDstCandidates = RBM_INTRET.GetIntRegSet(); + } + } srcCount += BuildCallArgUses(call); From aa9116cf4ecd409edf94600369498c3c20a08461 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 5 Jan 2026 11:34:00 +0100 Subject: [PATCH 19/22] Mark GCStressIncompatible and reduce array size --- src/tests/async/pgo/pgo.cs | 2 +- src/tests/async/pgo/pgo.csproj | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tests/async/pgo/pgo.cs b/src/tests/async/pgo/pgo.cs index 78488ac054c68e..02e57b925cd2bc 100644 --- a/src/tests/async/pgo/pgo.cs +++ b/src/tests/async/pgo/pgo.cs @@ -19,7 +19,7 @@ public static void EntryPoint() internal static async Task AsyncEntryPoint() { - int[] arr = Enumerable.Range(0, TestLibrary.Utilities.IsCoreClrInterpreter ? 100 : 100_000).ToArray(); + int[] arr = Enumerable.Range(0, TestLibrary.Utilities.IsCoreClrInterpreter ? 100 : 10_000).ToArray(); int sum = 0; int jCount = TestLibrary.Utilities.IsCoreClrInterpreter ? 10 : 100; diff --git a/src/tests/async/pgo/pgo.csproj b/src/tests/async/pgo/pgo.csproj index ff94a80928aef7..8d460faebada27 100644 --- a/src/tests/async/pgo/pgo.csproj +++ b/src/tests/async/pgo/pgo.csproj @@ -1,4 +1,9 @@ + + + true + true + From 3948ebf8aaf8b401a4beb647bc5e48e10ff7bd27 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Mon, 5 Jan 2026 23:35:39 -0800 Subject: [PATCH 20/22] Include call target validation in the resolve helper --- src/coreclr/jit/lower.cpp | 3 ++- .../nativeaot/Runtime/StackFrameIterator.cpp | 4 ++-- .../Runtime/amd64/DispatchResolve.asm | 10 ++++++---- .../Runtime/amd64/UniversalTransition.asm | 4 +++- .../Runtime/arm64/DispatchResolve.asm | 12 ++++++++---- .../Runtime/arm64/UniversalTransition.asm | 18 ++++++++++-------- 6 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index fd32b48c025ce6..cb19e453105468 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -3717,7 +3717,8 @@ void Lowering::LowerCFGCall(GenTreeCall* call) callTarget->SetUnusedValue(); } - callTarget = resolve; + // The call target validation is done by the resolve helper. + return; } if (callTarget == nullptr) diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp index a7db02a03eeb82..adf7bfaac0442d 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp @@ -34,7 +34,7 @@ #if !defined(FEATURE_PORTABLE_HELPERS) // @TODO: these are (currently) only implemented in assembly helpers EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionTailCall; #if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) -EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnResult; +EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnValidatedCodeAddress; #endif EXTERN_C CODE_LOCATION RhpCallCatchFunclet2; @@ -2235,7 +2235,7 @@ StackFrameIterator::ReturnAddressCategory StackFrameIterator::CategorizeUnadjust return InUniversalTransitionThunk; } #if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) - if (EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnResult)) + if (EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnValidatedCodeAddress)) { return InUniversalTransitionThunk; } diff --git a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm index d311fc5bf0994d..fb2a8ea842e92e 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm @@ -7,7 +7,9 @@ include AsmMacros.inc ifdef FEATURE_CACHED_INTERFACE_DISPATCH EXTERN RhpCidResolve : PROC -EXTERN RhpUniversalTransitionReturnResult : PROC +EXTERN RhpUniversalTransitionReturnValidatedCodeAddress : PROC + +EXTERN __guard_check_icall_fptr : QWORD ;; Fast version of RhpResolveInterfaceMethod LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT @@ -29,7 +31,7 @@ LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT cmp qword ptr [r10], rax jne RhpResolveInterfaceMethodFast_Polymorphic mov rax, qword ptr [r10 + 8] - ret + jmp [__guard_check_icall_fptr] RhpResolveInterfaceMethodFast_Polymorphic: ;; load the count of cache entries into edx @@ -47,14 +49,14 @@ LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT mov rax, qword ptr [r10 + 8] pop rdx - ret + jmp [__guard_check_icall_fptr] RhpResolveInterfaceMethodFast_SlowPath_Pop: pop rdx RhpResolveInterfaceMethodFast_SlowPath: ;; r11 contains indirection cell address lea r10, RhpCidResolve - jmp RhpUniversalTransitionReturnResult + jmp RhpUniversalTransitionReturnValidatedCodeAddress LEAF_END RhpResolveInterfaceMethodFast, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm index f70a0c6f65708e..452b88e8f1d79b 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm @@ -3,6 +3,8 @@ include AsmMacros.inc +EXTERN __guard_check_icall_fptr : QWORD + ifdef _DEBUG TRASH_SAVED_ARGUMENT_REGISTERS equ 1 else @@ -151,6 +153,6 @@ NESTED_END Rhp&FunctionName, _TEXT endm UNIVERSAL_TRANSITION UniversalTransitionTailCall, TAILJMP_RAX - UNIVERSAL_TRANSITION UniversalTransitionReturnResult, ret + UNIVERSAL_TRANSITION UniversalTransitionReturnValidatedCodeAddress, jmp [__guard_check_icall_fptr] end diff --git a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm index fcd885fc98efa9..fd9d9c638b5094 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm @@ -7,8 +7,10 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH + EXTERN __guard_check_icall_fptr + EXTERN RhpCidResolve - EXTERN RhpUniversalTransitionReturnResult + EXTERN RhpUniversalTransitionReturnValidatedCodeAddress NESTED_ENTRY RhpResolveInterfaceMethodFast, _TEXT @@ -29,8 +31,9 @@ ldr x15, [x14] cmp x15, x12 bne RhpResolveInterfaceMethodFast_Polymorphic + PREPARE_EXTERNAL_VAR_INDIRECT x12, __guard_check_icall_fptr ldur x15, [x14, #8] - ret + br x12 RhpResolveInterfaceMethodFast_Polymorphic ldr w13, [x13, #OFFSETOF__InterfaceDispatchCache__m_cEntries] @@ -45,13 +48,14 @@ RhpResolveInterfaceMethodFast_NextEntry cmp x15, x12 bne RhpResolveInterfaceMethodFast_NextEntry + PREPARE_EXTERNAL_VAR_INDIRECT x12, __guard_check_icall_fptr ldur x15, [x14, #8] - ret + br x12 RhpResolveInterfaceMethodFast_SlowPath ldr xip0, =RhpCidResolve mov xip1, x11 - b RhpUniversalTransitionReturnResult + b RhpUniversalTransitionReturnValidatedCodeAddress NESTED_END RhpResolveInterfaceMethodFast diff --git a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm index eb9314598cedb4..23489f1d80a66a 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm @@ -3,6 +3,8 @@ #include "AsmMacros.h" + EXTERN __guard_check_icall_fptr + #ifdef _DEBUG #define TRASH_SAVED_ARGUMENT_REGISTERS #endif @@ -89,7 +91,7 @@ TEXTAREA MACRO - UNIVERSAL_TRANSITION $FunctionName, $ReturnResult + UNIVERSAL_TRANSITION $FunctionName, $ReturnValidatedCodeAddress NESTED_ENTRY Rhp$FunctionName @@ -122,9 +124,15 @@ ALTERNATE_ENTRY ReturnFrom$FunctionName + IF $ReturnValidatedCodeAddress == 0 ;; Move the result (the target address) to x12 so it doesn't get overridden when we restore the ;; argument registers. mov x12, x0 + ELSE + ;; Move the result (the target address) to x15 where it is expected by the validator + mov x15, x0 + PREPARE_EXTERNAL_VAR_INDIRECT x12, __guard_check_icall_fptr + ENDIF ;; Restore floating point registers ldp q0, q1, [sp, #(FLOAT_ARG_OFFSET )] @@ -142,20 +150,14 @@ ;; Restore FP and LR registers, and free the allocated stack block EPILOG_RESTORE_REG_PAIR fp, lr, #STACK_SIZE! - IF $ReturnResult == 0 ;; Tailcall to the target address. EPILOG_NOP br x12 - ELSE - ;; Return target address - EPILOG_NOP mov x15, x12 - ret - ENDIF NESTED_END Rhp$FunctionName MEND UNIVERSAL_TRANSITION UniversalTransitionTailCall, 0 - UNIVERSAL_TRANSITION UniversalTransitionReturnResult, 1 + UNIVERSAL_TRANSITION UniversalTransitionReturnValidatedCodeAddress, 1 END From d5b6fa275276c001856f45cfbef5e0594d254bdd Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Tue, 6 Jan 2026 23:33:43 -0800 Subject: [PATCH 21/22] Revert "Include call target validation in the resolve helper" This reverts commit 3948ebf8aaf8b401a4beb647bc5e48e10ff7bd27. --- src/coreclr/jit/lower.cpp | 3 +-- .../nativeaot/Runtime/StackFrameIterator.cpp | 4 ++-- .../Runtime/amd64/DispatchResolve.asm | 10 ++++------ .../Runtime/amd64/UniversalTransition.asm | 4 +--- .../Runtime/arm64/DispatchResolve.asm | 12 ++++-------- .../Runtime/arm64/UniversalTransition.asm | 18 ++++++++---------- 6 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index cb19e453105468..fd32b48c025ce6 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -3717,8 +3717,7 @@ void Lowering::LowerCFGCall(GenTreeCall* call) callTarget->SetUnusedValue(); } - // The call target validation is done by the resolve helper. - return; + callTarget = resolve; } if (callTarget == nullptr) diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp index adf7bfaac0442d..a7db02a03eeb82 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp @@ -34,7 +34,7 @@ #if !defined(FEATURE_PORTABLE_HELPERS) // @TODO: these are (currently) only implemented in assembly helpers EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionTailCall; #if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) -EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnValidatedCodeAddress; +EXTERN_C CODE_LOCATION ReturnFromUniversalTransitionReturnResult; #endif EXTERN_C CODE_LOCATION RhpCallCatchFunclet2; @@ -2235,7 +2235,7 @@ StackFrameIterator::ReturnAddressCategory StackFrameIterator::CategorizeUnadjust return InUniversalTransitionThunk; } #if (defined(HOST_AMD64) || defined(HOST_ARM64)) && defined(HOST_WINDOWS) - if (EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnValidatedCodeAddress)) + if (EQUALS_RETURN_ADDRESS(returnAddress, ReturnFromUniversalTransitionReturnResult)) { return InUniversalTransitionThunk; } diff --git a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm index fb2a8ea842e92e..d311fc5bf0994d 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/DispatchResolve.asm @@ -7,9 +7,7 @@ include AsmMacros.inc ifdef FEATURE_CACHED_INTERFACE_DISPATCH EXTERN RhpCidResolve : PROC -EXTERN RhpUniversalTransitionReturnValidatedCodeAddress : PROC - -EXTERN __guard_check_icall_fptr : QWORD +EXTERN RhpUniversalTransitionReturnResult : PROC ;; Fast version of RhpResolveInterfaceMethod LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT @@ -31,7 +29,7 @@ LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT cmp qword ptr [r10], rax jne RhpResolveInterfaceMethodFast_Polymorphic mov rax, qword ptr [r10 + 8] - jmp [__guard_check_icall_fptr] + ret RhpResolveInterfaceMethodFast_Polymorphic: ;; load the count of cache entries into edx @@ -49,14 +47,14 @@ LEAF_ENTRY RhpResolveInterfaceMethodFast, _TEXT mov rax, qword ptr [r10 + 8] pop rdx - jmp [__guard_check_icall_fptr] + ret RhpResolveInterfaceMethodFast_SlowPath_Pop: pop rdx RhpResolveInterfaceMethodFast_SlowPath: ;; r11 contains indirection cell address lea r10, RhpCidResolve - jmp RhpUniversalTransitionReturnValidatedCodeAddress + jmp RhpUniversalTransitionReturnResult LEAF_END RhpResolveInterfaceMethodFast, _TEXT diff --git a/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm index 452b88e8f1d79b..f70a0c6f65708e 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/UniversalTransition.asm @@ -3,8 +3,6 @@ include AsmMacros.inc -EXTERN __guard_check_icall_fptr : QWORD - ifdef _DEBUG TRASH_SAVED_ARGUMENT_REGISTERS equ 1 else @@ -153,6 +151,6 @@ NESTED_END Rhp&FunctionName, _TEXT endm UNIVERSAL_TRANSITION UniversalTransitionTailCall, TAILJMP_RAX - UNIVERSAL_TRANSITION UniversalTransitionReturnValidatedCodeAddress, jmp [__guard_check_icall_fptr] + UNIVERSAL_TRANSITION UniversalTransitionReturnResult, ret end diff --git a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm index fd9d9c638b5094..fcd885fc98efa9 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/DispatchResolve.asm @@ -7,10 +7,8 @@ #ifdef FEATURE_CACHED_INTERFACE_DISPATCH - EXTERN __guard_check_icall_fptr - EXTERN RhpCidResolve - EXTERN RhpUniversalTransitionReturnValidatedCodeAddress + EXTERN RhpUniversalTransitionReturnResult NESTED_ENTRY RhpResolveInterfaceMethodFast, _TEXT @@ -31,9 +29,8 @@ ldr x15, [x14] cmp x15, x12 bne RhpResolveInterfaceMethodFast_Polymorphic - PREPARE_EXTERNAL_VAR_INDIRECT x12, __guard_check_icall_fptr ldur x15, [x14, #8] - br x12 + ret RhpResolveInterfaceMethodFast_Polymorphic ldr w13, [x13, #OFFSETOF__InterfaceDispatchCache__m_cEntries] @@ -48,14 +45,13 @@ RhpResolveInterfaceMethodFast_NextEntry cmp x15, x12 bne RhpResolveInterfaceMethodFast_NextEntry - PREPARE_EXTERNAL_VAR_INDIRECT x12, __guard_check_icall_fptr ldur x15, [x14, #8] - br x12 + ret RhpResolveInterfaceMethodFast_SlowPath ldr xip0, =RhpCidResolve mov xip1, x11 - b RhpUniversalTransitionReturnValidatedCodeAddress + b RhpUniversalTransitionReturnResult NESTED_END RhpResolveInterfaceMethodFast diff --git a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm index 23489f1d80a66a..eb9314598cedb4 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/UniversalTransition.asm @@ -3,8 +3,6 @@ #include "AsmMacros.h" - EXTERN __guard_check_icall_fptr - #ifdef _DEBUG #define TRASH_SAVED_ARGUMENT_REGISTERS #endif @@ -91,7 +89,7 @@ TEXTAREA MACRO - UNIVERSAL_TRANSITION $FunctionName, $ReturnValidatedCodeAddress + UNIVERSAL_TRANSITION $FunctionName, $ReturnResult NESTED_ENTRY Rhp$FunctionName @@ -124,15 +122,9 @@ ALTERNATE_ENTRY ReturnFrom$FunctionName - IF $ReturnValidatedCodeAddress == 0 ;; Move the result (the target address) to x12 so it doesn't get overridden when we restore the ;; argument registers. mov x12, x0 - ELSE - ;; Move the result (the target address) to x15 where it is expected by the validator - mov x15, x0 - PREPARE_EXTERNAL_VAR_INDIRECT x12, __guard_check_icall_fptr - ENDIF ;; Restore floating point registers ldp q0, q1, [sp, #(FLOAT_ARG_OFFSET )] @@ -150,14 +142,20 @@ ;; Restore FP and LR registers, and free the allocated stack block EPILOG_RESTORE_REG_PAIR fp, lr, #STACK_SIZE! + IF $ReturnResult == 0 ;; Tailcall to the target address. EPILOG_NOP br x12 + ELSE + ;; Return target address + EPILOG_NOP mov x15, x12 + ret + ENDIF NESTED_END Rhp$FunctionName MEND UNIVERSAL_TRANSITION UniversalTransitionTailCall, 0 - UNIVERSAL_TRANSITION UniversalTransitionReturnValidatedCodeAddress, 1 + UNIVERSAL_TRANSITION UniversalTransitionReturnResult, 1 END From a41e103f0a090c13cdc8bcddf89559814c44d3fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Wed, 7 Jan 2026 08:54:10 +0100 Subject: [PATCH 22/22] Delete redundant testing not actually testing CFG --- .../ControlFlowGuard/ControlFlowGuard.cs | 117 ------------------ 1 file changed, 117 deletions(-) diff --git a/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs b/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs index 28c4a72e73a23f..5e9189ac90a572 100644 --- a/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs +++ b/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.cs @@ -35,9 +35,6 @@ static int Main(string[] args) // Are we running the control program? if (args.Length == 0) { - TestExceptionFromDispatch.Run(); - TestInterfaceDispatch.Run(); - // Dry run - execute all scenarios while s_armed is false. // // The replaced call target will not be considered invalid by CFG and none of this @@ -87,120 +84,6 @@ static int Main(string[] args) return 10; } - class TestExceptionFromDispatch - { - class CastableObject : IDynamicInterfaceCastable - { - public RuntimeTypeHandle GetInterfaceImplementation(RuntimeTypeHandle interfaceType) => throw new Exception(); - public bool IsInterfaceImplemented(RuntimeTypeHandle interfaceType, bool throwIfNotImplemented) => true; - } - - public static void Run() - { - bool caughtException = false; - - IDisposable obj = (IDisposable)new CastableObject(); - try - { - obj.Dispose(); - } - catch (Exception) - { - caughtException = true; - } - - if (!caughtException) - throw new Exception(); - } - } - - internal class TestInterfaceDispatch - { - interface IFoo - { - int Call(int x, int y); - } - - interface IFoo - { - int Call(int x, int y); - } - - class C1 : IFoo, IFoo - { - public int Call(int x, int y) => x + y; - } - - class C2 : IFoo, IFoo - { - public int Call(int x, int y) => x - y; - } - class C3 : IFoo, IFoo - { - public int Call(int x, int y) => x * y; - } - - class C4 : IFoo, IFoo - { - public int Call(int x, int y) => x / y; - } - - class C5 : IFoo, IFoo - { - public int Call(int x, int y) => x % y; - } - - public static void Run() - { - if (Call(new C1(), 10, 20) != (10 + 20)) - throw new Exception(); - if (Call(new C1(), 11, 22) != (11 + 22)) - throw new Exception(); - if (Call(new C2(), 10, 20) != (10 - 20)) - throw new Exception(); - if (Call(new C2(), 11, 22) != (11 - 22)) - throw new Exception(); - if (Call(new C3(), 10, 20) != (10 * 20)) - throw new Exception(); - if (Call(new C3(), 11, 22) != (11 * 22)) - throw new Exception(); - if (Call(new C4(), 10, 20) != (10 / 20)) - throw new Exception(); - if (Call(new C5(), 10, 20) != (10 % 20)) - throw new Exception(); - - if (CallGen(new C1(), 10, 20) != (10 + 20)) - throw new Exception(); - if (CallGen(new C2(), 11, 22) != (11 - 22)) - throw new Exception(); - if (CallGen(new C3(), 11, 22) != (11 * 22)) - throw new Exception(); - if (CallGen(new C4(), 10, 20) != (10 / 20)) - throw new Exception(); - if (CallGen(new C5(), 10, 20) != (10 % 20)) - throw new Exception(); - - bool caught = false; - try - { - Call(null, 10, 20); - } - catch (NullReferenceException) - { - caught = true; - } - - if (!caught) - throw new Exception(); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - static int Call(IFoo f, int x, int y) => f.Call(x, y); - - [MethodImpl(MethodImplOptions.NoInlining)] - static int CallGen(IFoo f, int x, int y) => f.Call(x, y); - } - class TestFunctionPointer { public static int Run()