From e3bd1f03c3257f419ff8e1f09927a3733481049c Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Sun, 12 Oct 2025 22:41:03 -0700 Subject: [PATCH 1/4] Add -Zannotate-moves for profiler visibility of move/copy operations This implements a new unstable compiler flag `-Zannotate-moves` that makes move and copy operations visible in profilers by creating synthetic debug information. This is achieved with zero runtime cost by manipulating debug info scopes to make moves/copies appear as calls to `compiler_move` and `compiler_copy` marker functions in profiling tools. This allows developers to identify expensive move/copy operations in their code using standard profiling tools, without requiring specialized tooling or runtime instrumentation. The implementation works at codegen time. When processing MIR operands (`Operand::Move` and `Operand::Copy`), the codegen creates an `OperandRef` with an optional `move_annotation` field containing an `Instance` of the appropriate profiling marker function. When storing the operand, `store_with_annotation()` wraps the store operation in a synthetic debug scope that makes it appear inlined from the marker. Two marker functions (`compiler_move` and `compiler_copy`) are defined in `library/core/src/profiling.rs`. These are never actually called - they exist solely as debug info anchors. Operations are only annotated if the type: - Meets the size threshold (default: 65 bytes, configurable via `-Zannotate-moves=SIZE`) - Has a non-scalar backend representation (scalars use registers, not memcpy) This has a very small size impact on object file size. With the default limit it's well under 0.1%, and even with a very small limit of 8 bytes it's still ~1.5%. This could be enabled by default. --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_gcc/src/debuginfo.rs | 2 +- compiler/rustc_codegen_llvm/src/abi.rs | 2 +- compiler/rustc_codegen_llvm/src/builder.rs | 2 +- .../rustc_codegen_llvm/src/debuginfo/mod.rs | 53 ++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 14 +- compiler/rustc_codegen_ssa/src/mir/mod.rs | 2 + compiler/rustc_codegen_ssa/src/mir/operand.rs | 121 ++++++++++- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 47 +++-- .../rustc_codegen_ssa/src/traits/builder.rs | 2 +- .../rustc_codegen_ssa/src/traits/debuginfo.rs | 16 +- compiler/rustc_interface/src/tests.rs | 3 +- compiler/rustc_session/src/config.rs | 25 ++- compiler/rustc_session/src/options.rs | 28 +++ compiler/rustc_span/src/symbol.rs | 2 + library/core/src/lib.rs | 2 + library/core/src/profiling.rs | 31 +++ .../src/compiler-flags/annotate-moves.md | 79 +++++++ .../annotate-moves/call-arg-scope.rs | 114 +++++++++++ tests/codegen-llvm/annotate-moves/disabled.rs | 35 ++++ .../annotate-moves/integration.rs | 192 ++++++++++++++++++ .../codegen-llvm/annotate-moves/size-limit.rs | 74 +++++++ .../ui/annotate-moves/annotate-moves-basic.rs | 15 ++ .../annotate-moves-invalid-flag.rs | 10 + .../annotate-moves-invalid-flag.stderr | 2 + .../annotate-moves-size-limit-invalid.rs | 10 + .../annotate-moves-size-limit-invalid.stderr | 2 + 27 files changed, 839 insertions(+), 48 deletions(-) create mode 100644 library/core/src/profiling.rs create mode 100644 src/doc/unstable-book/src/compiler-flags/annotate-moves.md create mode 100644 tests/codegen-llvm/annotate-moves/call-arg-scope.rs create mode 100644 tests/codegen-llvm/annotate-moves/disabled.rs create mode 100644 tests/codegen-llvm/annotate-moves/integration.rs create mode 100644 tests/codegen-llvm/annotate-moves/size-limit.rs create mode 100644 tests/ui/annotate-moves/annotate-moves-basic.rs create mode 100644 tests/ui/annotate-moves/annotate-moves-invalid-flag.rs create mode 100644 tests/ui/annotate-moves/annotate-moves-invalid-flag.stderr create mode 100644 tests/ui/annotate-moves/annotate-moves-size-limit-invalid.rs create mode 100644 tests/ui/annotate-moves/annotate-moves-size-limit-invalid.stderr diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 5657620879ca1..86031a8a46342 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { OperandValue::Ref(place.val) }; - OperandRef { val, layout: place.layout } + OperandRef { val, layout: place.layout, move_annotation: None } } fn write_operand_repeatedly( diff --git a/compiler/rustc_codegen_gcc/src/debuginfo.rs b/compiler/rustc_codegen_gcc/src/debuginfo.rs index 0f015cc23f52f..3979f62987f2c 100644 --- a/compiler/rustc_codegen_gcc/src/debuginfo.rs +++ b/compiler/rustc_codegen_gcc/src/debuginfo.rs @@ -19,7 +19,7 @@ use crate::context::CodegenCx; pub(super) const UNKNOWN_LINE_NUMBER: u32 = 0; pub(super) const UNKNOWN_COLUMN_NUMBER: u32 = 0; -impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> { +impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // FIXME(eddyb) find a common convention for all of the debuginfo-related // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). fn dbg_var_addr( diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 680ad98593e7e..52f3d21036955 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -247,7 +247,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { ); bx.lifetime_end(llscratch, scratch_size); } - _ => { + PassMode::Pair(..) | PassMode::Direct { .. } => { OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst); } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index c082a82306848..27996d5c1fcde 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -752,7 +752,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { OperandValue::Ref(place.val) }; - OperandRef { val, layout: place.layout } + OperandRef { val, layout: place.layout, move_annotation: None } } fn write_operand_repeatedly( diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index b95ad03b70e03..c2c55ee2b64f9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> { } } -impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { +impl<'ll, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // FIXME(eddyb) find a common convention for all of the debuginfo-related // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). fn dbg_var_addr( @@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { llvm::set_value_name(value, name.as_bytes()); } } + + /// Annotate move/copy operations with debug info for profiling. + /// + /// This creates a temporary debug scope that makes the move/copy appear as an inlined call to + /// `compiler_move()` or `compiler_copy()`. The provided closure is executed + /// with this temporary debug location active. + /// + /// The `instance` parameter should be the monomorphized instance of the `compiler_move` or + /// `compiler_copy` function with the actual type and size. + fn with_move_annotation( + &mut self, + instance: ty::Instance<'tcx>, + f: impl FnOnce(&mut Self) -> R, + ) -> R { + // Save the current debug location + let saved_loc = self.get_dbg_loc(); + + // Create a DIScope for the compiler_move/compiler_copy function + // We use the function's FnAbi for debug info generation + let fn_abi = self + .cx() + .tcx + .fn_abi_of_instance( + self.cx().typing_env().as_query_input((instance, ty::List::empty())), + ) + .unwrap(); + + let di_scope = self.cx().dbg_scope_fn(instance, fn_abi, None); + + // Create an inlined debug location: + // - scope: the compiler_move/compiler_copy function + // - inlined_at: the current location (where the move/copy actually occurs) + // - span: use the function's definition span + let fn_span = self.cx().tcx.def_span(instance.def_id()); + let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span); + + // Set the temporary debug location + self.set_dbg_loc(inlined_loc); + + // Execute the closure (which will generate the memcpy) + let result = f(self); + + // Restore the original debug location + if let Some(loc) = saved_loc { + self.set_dbg_loc(loc); + } else { + self.clear_dbg_loc(); + } + + result + } } /// A source code location used to generate debug information. diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index e2241a77d186c..5e71bb74e8a66 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -557,9 +557,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let op = match self.locals[mir::RETURN_PLACE] { LocalRef::Operand(op) => op, LocalRef::PendingOperand => bug!("use of return before def"), - LocalRef::Place(cg_place) => { - OperandRef { val: Ref(cg_place.val), layout: cg_place.layout } - } + LocalRef::Place(cg_place) => OperandRef { + val: Ref(cg_place.val), + layout: cg_place.layout, + move_annotation: None, + }, LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), }; let llslot = match op.val { @@ -1145,7 +1147,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | (&mir::Operand::Constant(_), Ref(PlaceValue { llextra: None, .. })) => { let tmp = PlaceRef::alloca(bx, op.layout); bx.lifetime_start(tmp.val.llval, tmp.layout.size); - op.val.store(bx, tmp); + op.store_with_annotation(bx, tmp); op.val = Ref(tmp.val); lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size)); } @@ -1553,13 +1555,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align); bx.lifetime_start(scratch.llval, arg.layout.size); - op.val.store(bx, scratch.with_type(arg.layout)); + op.store_with_annotation(bx, scratch.with_type(arg.layout)); lifetime_ends_after_call.push((scratch.llval, arg.layout.size)); (scratch.llval, scratch.align, true) } PassMode::Cast { .. } => { let scratch = PlaceRef::alloca(bx, arg.layout); - op.val.store(bx, scratch); + op.store_with_annotation(bx, scratch); (scratch.val.llval, scratch.val.align, true) } _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false), diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 1670e2da71fb3..819abb9ae644d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -480,6 +480,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( return local(OperandRef { val: OperandValue::Pair(a, b), layout: arg.layout, + move_annotation: None, }); } _ => {} @@ -552,6 +553,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( fx.caller_location = Some(OperandRef { val: OperandValue::Immediate(bx.get_param(llarg_idx)), layout: arg.layout, + move_annotation: None, }); } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 88a8e2a844cbc..1bf5ab8df4c49 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -7,10 +7,11 @@ use rustc_abi::{ }; use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range}; use rustc_middle::mir::{self, ConstValue}; -use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; -use rustc_session::config::OptLevel; +use rustc_session::config::{AnnotateMoves, DebugInfo, OptLevel}; +use rustc_span::{Symbol, sym}; use tracing::{debug, instrument}; use super::place::{PlaceRef, PlaceValue}; @@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> { /// The layout of value, based on its Rust type. pub layout: TyAndLayout<'tcx>, + + /// Annotation for profiler visibility of move/copy operations. + /// When set, the store operation should appear as an inlined call to this function. + pub move_annotation: Option>, } impl fmt::Debug for OperandRef<'_, V> { @@ -142,7 +147,7 @@ impl fmt::Debug for OperandRef<'_, V> { impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { pub fn zero_sized(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, V> { assert!(layout.is_zst()); - OperandRef { val: OperandValue::ZeroSized, layout } + OperandRef { val: OperandValue::ZeroSized, layout, move_annotation: None } } pub(crate) fn from_const>( @@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { } }; - OperandRef { val, layout } + OperandRef { val, layout, move_annotation: None } } fn from_const_alloc>( @@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let size = s.size(bx); assert_eq!(size, layout.size, "abi::Scalar size does not match layout size"); let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout)); - OperandRef { val: OperandValue::Immediate(val), layout } + OperandRef { val: OperandValue::Immediate(val), layout, move_annotation: None } } BackendRepr::ScalarPair( a @ abi::Scalar::Initialized { .. }, @@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { b, bx.scalar_pair_element_backend_type(layout, 1, true), ); - OperandRef { val: OperandValue::Pair(a_val, b_val), layout } + OperandRef { val: OperandValue::Pair(a_val, b_val), layout, move_annotation: None } } _ if layout.is_zst() => OperandRef::zero_sized(layout), _ => { @@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { self.val.deref(layout.align.abi).with_type(layout) } + /// Store this operand into a place, applying move/copy annotation if present. + /// + /// This is the preferred method for storing operands, as it automatically + /// applies profiler annotations for tracked move/copy operations. + pub fn store_with_annotation>( + self, + bx: &mut Bx, + dest: PlaceRef<'tcx, V>, + ) { + if let Some(instance) = self.move_annotation { + bx.with_move_annotation(instance, |bx| self.val.store(bx, dest)) + } else { + self.val.store(bx, dest) + } + } + /// If this operand is a `Pair`, we return an aggregate with the two values. /// For other cases, see `immediate`. pub fn immediate_or_packed_pair>( @@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { } else { OperandValue::Immediate(llval) }; - OperandRef { val, layout } + OperandRef { val, layout, move_annotation: None } } pub(crate) fn extract_field>( @@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { }) }; - OperandRef { val, layout: field } + OperandRef { val, layout: field, move_annotation: None } } /// Obtain the actual discriminant of a value. @@ -828,10 +849,13 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> { } }, }; - OperandRef { val, layout } + OperandRef { val, layout, move_annotation: None } } } +/// Default size limit for move/copy annotations (in bytes). +const MOVE_ANNOTATION_DEFAULT_LIMIT: u64 = 65; + impl<'a, 'tcx, V: CodegenObject> OperandValue { /// Returns an `OperandValue` that's generally UB to use in any way. /// @@ -961,7 +985,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { abi::Variants::Single { index: vidx }, ); let layout = o.layout.for_variant(bx.cx(), vidx); - o = OperandRef { val: o.val, layout } + o = OperandRef { layout, ..o } } _ => return None, } @@ -1014,7 +1038,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { - self.codegen_consume(bx, place.as_ref()) + let kind = match operand { + mir::Operand::Move(_) => sym::compiler_move, + mir::Operand::Copy(_) => sym::compiler_copy, + _ => unreachable!(), + }; + + // Check if we should annotate this move/copy for profiling + let move_annotation = self.move_copy_annotation_instance(bx, place.as_ref(), kind); + + OperandRef { move_annotation, ..self.codegen_consume(bx, place.as_ref()) } } mir::Operand::Constant(ref constant) => { @@ -1030,6 +1063,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return OperandRef { val: OperandValue::Immediate(llval), layout: bx.layout_of(ty), + move_annotation: None, }; } } @@ -1037,4 +1071,69 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } } + + /// Creates an `Instance` for annotating a move/copy operation at codegen time. + /// + /// Returns `Some(instance)` if the operation should be annotated with debug info, `None` + /// otherwise. The instance represents a monomorphized `compiler_move` or + /// `compiler_copy` function that can be used to create debug scopes. + /// + /// There are a number of conditions that must be met for an annotation to be created, but aside + /// from the basics (annotation is enabled, we're generating debuginfo), the primary concern is + /// moves/copies which could result in a real `memcpy`. So we check for the size limit, but also + /// that the underlying representation of the type is in memory. + fn move_copy_annotation_instance( + &self, + bx: &Bx, + place: mir::PlaceRef<'tcx>, + kind: Symbol, + ) -> Option> { + let tcx = bx.tcx(); + let sess = tcx.sess; + + // Skip if we're not generating debuginfo + if sess.opts.debuginfo == DebugInfo::None { + return None; + } + + // Check if annotation is enabled and get size limit (otherwise skip) + let size_limit = match sess.opts.unstable_opts.annotate_moves { + AnnotateMoves::Disabled => return None, + AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT, + AnnotateMoves::Enabled(Some(limit)) => limit, + }; + + let ty = self.monomorphized_place_ty(place); + let layout = bx.cx().layout_of(ty); + let ty_size = layout.size.bytes(); + + // Only annotate if type has a memory representation and exceeds size limit (and has a + // non-zero size) + if layout.is_zst() + || ty_size < size_limit + || !matches!(layout.backend_repr, BackendRepr::Memory { .. }) + { + return None; + } + + // Look up the DefId for compiler_move or compiler_copy (skip if it's missing for some + // reason). + let def_id = tcx.get_diagnostic_item(kind)?; + + // Create generic args: compiler_move or compiler_copy + let size_const = ty::Const::from_target_usize(tcx, ty_size); + let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]); + + // Create the Instance + let typing_env = self.mir.typing_env(tcx); + let instance = ty::Instance::expect_resolve( + tcx, + typing_env, + def_id, + generic_args, + rustc_span::DUMMY_SP, // span only used for error messages + ); + + Some(instance) + } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 640f7211dc994..e31dc86b926f2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -36,7 +36,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } // FIXME: consider not copying constants through stack. (Fixable by codegen'ing // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?) - cg_operand.val.store(bx, dest); + cg_operand.store_with_annotation(bx, dest); } mir::Rvalue::Cast( @@ -50,7 +50,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Into-coerce of a thin pointer to a wide pointer -- just // use the operand path. let temp = self.codegen_rvalue_operand(bx, rvalue); - temp.val.store(bx, dest); + temp.store_with_annotation(bx, dest); return; } @@ -70,7 +70,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { debug!("codegen_rvalue: creating ugly alloca"); let scratch = PlaceRef::alloca(bx, operand.layout); scratch.storage_live(bx); - operand.val.store(bx, scratch); + operand.store_with_annotation(bx, scratch); base::coerce_unsized_into(bx, scratch, dest); scratch.storage_dead(bx); } @@ -183,7 +183,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { variant_dest.project_field(bx, field_index.as_usize()) }; - op.val.store(bx, field); + op.store_with_annotation(bx, field); } } dest.codegen_set_discr(bx, variant_index); @@ -191,7 +191,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => { let temp = self.codegen_rvalue_operand(bx, rvalue); - temp.val.store(bx, dest); + temp.store_with_annotation(bx, dest); } } } @@ -221,7 +221,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Since in this path we have a place anyway, we can store or copy to it, // making sure we use the destination place's alignment even if the // source would normally have a higher one. - src.val.store(bx, dst.val.with_type(src.layout)); + src.store_with_annotation(bx, dst.val.with_type(src.layout)); } } @@ -320,7 +320,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let size = Ord::max(operand.layout.size, cast.size); let temp = PlaceValue::alloca(bx, size, align); bx.lifetime_start(temp.llval, size); - operand.val.store(bx, temp.with_type(operand.layout)); + operand.store_with_annotation(bx, temp.with_type(operand.layout)); let val = bx.load_operand(temp.with_type(cast)).val; bx.lifetime_end(temp.llval, size); val @@ -478,7 +478,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let to_backend_ty = bx.cx().immediate_backend_type(cast); if operand.layout.is_uninhabited() { let val = OperandValue::Immediate(bx.cx().const_poison(to_backend_ty)); - return OperandRef { val, layout: cast }; + return OperandRef { val, layout: cast, move_annotation: None }; } let abi::BackendRepr::Scalar(to_scalar) = cast.layout.backend_repr else { bug!("Found non-scalar for cast {cast:?}"); @@ -494,7 +494,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_transmute_operand(bx, operand, cast) } }; - OperandRef { val, layout: cast } + OperandRef { val, layout: cast, move_annotation: None } } mir::Rvalue::Ref(_, bk, place) => { @@ -525,7 +525,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ); let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty); let operand_ty = Ty::new_tup(bx.tcx(), &[val_ty, bx.tcx().types.bool]); - OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) } + OperandRef { + val: result, + layout: bx.cx().layout_of(operand_ty), + move_annotation: None, + } } mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => { let lhs = self.codegen_operand(bx, lhs); @@ -559,6 +563,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandRef { val: OperandValue::Immediate(llresult), layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)), + move_annotation: None, } } @@ -593,7 +598,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { val.is_expected_variant_for_type(self.cx, layout), "Made wrong variant {val:?} for type {layout:?}", ); - OperandRef { val, layout } + OperandRef { val, layout, move_annotation: None } } mir::Rvalue::Discriminant(ref place) => { @@ -604,6 +609,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandRef { val: OperandValue::Immediate(discr), layout: self.cx.layout_of(discr_ty), + move_annotation: None, } } @@ -631,6 +637,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandRef { val: OperandValue::Immediate(val), layout: self.cx.layout_of(null_op.ty(tcx)), + move_annotation: None, } } @@ -663,7 +670,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { bx.get_static(def_id) }; - OperandRef { val: OperandValue::Immediate(static_), layout } + OperandRef { val: OperandValue::Immediate(static_), layout, move_annotation: None } } mir::Rvalue::Use(ref operand) => self.codegen_operand(bx, operand), mir::Rvalue::Repeat(ref elem, len_const) => { @@ -675,7 +682,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let array_ty = self.monomorphize(array_ty); let array_layout = bx.layout_of(array_ty); assert!(array_layout.is_zst()); - OperandRef { val: OperandValue::ZeroSized, layout: array_layout } + OperandRef { + val: OperandValue::ZeroSized, + layout: array_layout, + move_annotation: None, + } } mir::Rvalue::Aggregate(ref kind, ref fields) => { let (variant_index, active_field_index) = match **kind { @@ -704,7 +715,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // more optimizability, if that turns out to be helpful. bx.abort(); let val = OperandValue::poison(bx, layout); - OperandRef { val, layout } + OperandRef { val, layout, move_annotation: None } } Ok(maybe_tag_value) => { if let Some((tag_field, tag_imm)) = maybe_tag_value { @@ -718,7 +729,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let operand = self.codegen_operand(bx, operand); let binder_ty = self.monomorphize(binder_ty); let layout = bx.cx().layout_of(binder_ty); - OperandRef { val: operand.val, layout } + OperandRef { val: operand.val, layout, move_annotation: None } } mir::Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"), mir::Rvalue::ShallowInitBox(..) => bug!("`ShallowInitBox` in codegen"), @@ -745,7 +756,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { "Address of place was unexpectedly {val:?} for pointee type {ty:?}", ); - OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) } + OperandRef { + val, + layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)), + move_annotation: None, + } } fn codegen_scalar_binop( diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 60296e36e0c1a..f57f9108f744b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -37,7 +37,7 @@ pub trait BuilderMethods<'a, 'tcx>: + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>> + Deref + CoverageInfoBuilderMethods<'tcx> - + DebugInfoBuilderMethods + + DebugInfoBuilderMethods<'tcx> + ArgAbiBuilderMethods<'tcx> + AbiBuilderMethods + IntrinsicCallBuilderMethods<'tcx> diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs index a4da6c915deec..268d863dcbd2c 100644 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -64,7 +64,7 @@ pub trait DebugInfoCodegenMethods<'tcx>: BackendTypes { ) -> Self::DIVariable; } -pub trait DebugInfoBuilderMethods: BackendTypes { +pub trait DebugInfoBuilderMethods<'tcx>: BackendTypes { // FIXME(eddyb) find a common convention for all of the debuginfo-related // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). fn dbg_var_addr( @@ -95,4 +95,18 @@ pub trait DebugInfoBuilderMethods: BackendTypes { fn clear_dbg_loc(&mut self); fn insert_reference_to_gdb_debug_scripts_section_global(&mut self); fn set_var_name(&mut self, value: Self::Value, name: &str); + + /// Hook to allow move/copy operations to be annotated for profiling. + /// + /// The `instance` parameter should be the monomorphized instance of the + /// `compiler_move` or `compiler_copy` function with the actual type and size. + /// + /// Default implementation does no annotation (just executes the closure). + fn with_move_annotation( + &mut self, + _instance: Instance<'tcx>, + f: impl FnOnce(&mut Self) -> R, + ) -> R { + f(self) + } } diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index be33db21d1df6..198ef1e41da41 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -10,7 +10,7 @@ use rustc_errors::emitter::HumanReadableErrorType; use rustc_errors::{ColorConfig, registry}; use rustc_hir::attrs::NativeLibKind; use rustc_session::config::{ - AutoDiff, BranchProtection, CFGuard, Cfg, CollapseMacroDebuginfo, CoverageLevel, + AnnotateMoves, AutoDiff, BranchProtection, CFGuard, Cfg, CollapseMacroDebuginfo, CoverageLevel, CoverageOptions, DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, FmtDebug, FunctionReturn, InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli, MirIncludeSpans, @@ -763,6 +763,7 @@ fn test_unstable_options_tracking_hash() { // tidy-alphabetical-start tracked!(allow_features, Some(vec![String::from("lang_items")])); tracked!(always_encode_mir, true); + tracked!(annotate_moves, AnnotateMoves::Enabled(Some(1234))); tracked!(assume_incomplete_release, true); tracked!(autodiff, vec![AutoDiff::Enable, AutoDiff::NoTT]); tracked!(binary_dep_depinfo, true); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 484a7d4217211..ce8dc807cec52 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -262,6 +262,16 @@ pub enum AutoDiff { NoTT, } +/// The different settings that the `-Z annotate-moves` flag can have. +#[derive(Clone, Copy, PartialEq, Hash, Debug)] +pub enum AnnotateMoves { + /// `-Z annotate-moves=no` (or `off`, `false` etc.) + Disabled, + /// `-Z annotate-moves` or `-Z annotate-moves=yes` (use default size limit) + /// `-Z annotate-moves=SIZE` (use specified size limit) + Enabled(Option), +} + /// Settings for `-Z instrument-xray` flag. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct InstrumentXRay { @@ -3239,13 +3249,13 @@ pub(crate) mod dep_tracking { }; use super::{ - AutoDiff, BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, CoverageOptions, - CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, - InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, - LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, OptLevel, OutFileName, - OutputType, OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, - ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, - SymbolManglingVersion, WasiExecModel, + AnnotateMoves, AutoDiff, BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, + CoverageOptions, CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, + FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, + LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, + OptLevel, OutFileName, OutputType, OutputTypes, PatchableFunctionEntry, Polonius, + RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, + SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3288,6 +3298,7 @@ pub(crate) mod dep_tracking { impl_dep_tracking_hash_via_hash!( (), + AnnotateMoves, AutoDiff, Offload, bool, diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 6dd90546de1b0..d86a4c4fa477e 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -864,6 +864,8 @@ mod desc { pub(crate) const parse_linker_features: &str = "a list of enabled (`+` prefix) and disabled (`-` prefix) features: `lld`"; pub(crate) const parse_polonius: &str = "either no value or `legacy` (the default), or `next`"; + pub(crate) const parse_annotate_moves: &str = + "either a boolean (`yes`, `no`, `on`, `off`, etc.), or a size limit in bytes"; pub(crate) const parse_stack_protector: &str = "one of (`none` (default), `basic`, `strong`, or `all`)"; pub(crate) const parse_branch_protection: &str = "a `,` separated combination of `bti`, `gcs`, `pac-ret`, (optionally with `pc`, `b-key`, `leaf` if `pac-ret` is set)"; @@ -950,6 +952,29 @@ pub mod parse { } } + pub(crate) fn parse_annotate_moves(slot: &mut AnnotateMoves, v: Option<&str>) -> bool { + let mut bslot = false; + let mut nslot = 0u64; + + *slot = match v { + // No value provided: -Z annotate-moves (enable with default limit) + None => AnnotateMoves::Enabled(None), + // Explicit boolean value provided: -Z annotate-moves=yes/no + s @ Some(_) if parse_bool(&mut bslot, s) => { + if bslot { + AnnotateMoves::Enabled(None) + } else { + AnnotateMoves::Disabled + } + } + // With numeric limit provided: -Z annotate-moves=1234 + s @ Some(_) if parse_number(&mut nslot, s) => AnnotateMoves::Enabled(Some(nslot)), + _ => return false, + }; + + true + } + /// Use this for any string option that has a static default. pub(crate) fn parse_string(slot: &mut String, v: Option<&str>) -> bool { match v { @@ -2196,6 +2221,9 @@ options! { "only allow the listed language features to be enabled in code (comma separated)"), always_encode_mir: bool = (false, parse_bool, [TRACKED], "encode MIR of all functions into the crate metadata (default: no)"), + annotate_moves: AnnotateMoves = (AnnotateMoves::Disabled, parse_annotate_moves, [TRACKED], + "emit debug info for compiler-generated move and copy operations \ + to make them visible in profilers. Can be a boolean or a size limit in bytes (default: disabled)"), assert_incr_state: Option = (None, parse_opt_string, [UNTRACKED], "assert that the incremental cache is in given state: \ either `loaded` or `not-loaded`."), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 223d818a2949b..cba2241aee6ac 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -694,7 +694,9 @@ symbols! { compile_error, compiler, compiler_builtins, + compiler_copy, compiler_fence, + compiler_move, concat, concat_bytes, concat_idents, diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index bc428c37a88f9..5edbf2bec8ece 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -281,6 +281,8 @@ pub mod num; pub mod hint; pub mod intrinsics; pub mod mem; +#[unstable(feature = "profiling_marker_api", issue = "none")] +pub mod profiling; pub mod ptr; #[unstable(feature = "ub_checks", issue = "none")] pub mod ub_checks; diff --git a/library/core/src/profiling.rs b/library/core/src/profiling.rs new file mode 100644 index 0000000000000..0d2260e88bd6e --- /dev/null +++ b/library/core/src/profiling.rs @@ -0,0 +1,31 @@ +//! Profiling markers for compiler instrumentation. + +/// Profiling marker for move operations. +/// +/// This function is never called at runtime. When `-Z annotate-moves` is enabled, +/// the compiler creates synthetic debug info that makes move operations appear as +/// calls to this function in profilers. +/// +/// The `SIZE` parameter encodes the size of the type being moved. +#[unstable(feature = "profiling_marker_api", issue = "none")] +#[rustc_diagnostic_item = "compiler_move"] +pub fn compiler_move(_src: *const T, _dst: *mut T) { + unreachable!( + "compiler_move marks where the compiler generated a copy; it is never actually called" + ) +} + +/// Profiling marker for copy operations. +/// +/// This function is never called at runtime. When `-Z annotate-moves` is enabled, +/// the compiler creates synthetic debug info that makes copy operations appear as +/// calls to this function in profilers. +/// +/// The `SIZE` parameter encodes the size of the type being copied. +#[unstable(feature = "profiling_marker_api", issue = "none")] +#[rustc_diagnostic_item = "compiler_copy"] +pub fn compiler_copy(_src: *const T, _dst: *mut T) { + unreachable!( + "compiler_copy marks where the compiler generated a copy; it is never actually called" + ) +} diff --git a/src/doc/unstable-book/src/compiler-flags/annotate-moves.md b/src/doc/unstable-book/src/compiler-flags/annotate-moves.md new file mode 100644 index 0000000000000..465a8430e4f62 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/annotate-moves.md @@ -0,0 +1,79 @@ +# `annotate-moves` + +The `-Z annotate-moves` flag enables annotation of compiler-generated +move and copy operations, making them visible in profilers and stack traces +for performance debugging. + +When enabled, the compiler manipulates debug info to make large move and copy +operations appear as if they were inlined calls to `core::profiling::compiler_move` +and `core::profiling::compiler_copy`. No actual function calls are generated - +this is purely a debug info transformation that makes expensive memory operations +visible in profilers and stack traces. + +## Syntax + +```bash +rustc -Z annotate-moves[=] +``` + +Where `` can be: +- A boolean: `true`, `false`, `yes`, `no`, `on`, `off` +- A number: size threshold in bytes (e.g., `128`) +- Omitted: enables with default threshold (65 bytes) + +## Options + +- `-Z annotate-moves` or `-Z annotate-moves=true`: Enable with default size limit +- `-Z annotate-moves=false`: Disable annotation +- `-Z annotate-moves=N`: Enable with custom size limit of N bytes + +## Examples + +```bash +# Enable annotation with default threshold (65 bytes) +rustc -Z annotate-moves main.rs + +# Enable with custom 128-byte threshold +rustc -Z annotate-moves=128 main.rs + +# Only annotate very large moves (1KB+) +rustc -Z annotate-moves=1024 main.rs + +# Explicitly disable +rustc -Z annotate-moves=false main.rs +``` + +## Behavior + +The annotation only applies to: +- Types larger than the specified size threshold +- Non-immediate types (those that would generate `memcpy`) +- Operations that actually move/copy data (not ZST types) + +Stack traces will show the operations: +```text +0: memcpy +1: core::profiling::compiler_move:: +2: my_function +``` + +## Example + +```rust +#[derive(Clone)] +struct LargeData { + buffer: [u8; 1000], +} + +fn example() { + let data = LargeData { buffer: [0; 1000] }; + let copy = data.clone(); // Shows as compiler_copy in profiler + let moved = data; // Shows as compiler_move in profiler +} +``` + +## Overhead + +This has no effect on generated code; it only adds debuginfo. The overhead is +typically very small; on rustc itself, the default limit of 65 bytes adds about +0.055% to the binary size. diff --git a/tests/codegen-llvm/annotate-moves/call-arg-scope.rs b/tests/codegen-llvm/annotate-moves/call-arg-scope.rs new file mode 100644 index 0000000000000..527ab0b09d7c8 --- /dev/null +++ b/tests/codegen-llvm/annotate-moves/call-arg-scope.rs @@ -0,0 +1,114 @@ +//@ compile-flags: -Z annotate-moves=8 -Copt-level=0 -g +// +// This test verifies that function call and return instructions use the correct debug scopes +// when passing/returning large values. The actual move/copy operations may be annotated, +// but the CALL and RETURN instructions themselves should reference the source location, +// NOT have an inlinedAt scope pointing to compiler_move/compiler_copy. + +#![crate_type = "lib"] + +#[derive(Clone, Copy)] +pub struct LargeStruct { + pub data: [u64; 20], // 160 bytes +} + +#[derive(Clone, Copy)] +pub struct MediumStruct { + pub data: [u64; 5], // 40 bytes +} + +pub struct SmallStruct { + pub x: u32, // 4 bytes +} + +// ============================================================================ +// Test 1: Single argument call +// ============================================================================ + +// CHECK-LABEL: call_arg_scope::test_call_with_single_arg +pub fn test_call_with_single_arg(s: LargeStruct) { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CALL1_ARG_LOC:]] + // CHECK: call {{.*}}@{{.*}}helper_single{{.*}}({{.*}}), !dbg ![[#CALL1_LOC:]] + helper_single(s); +} + +#[inline(never)] +fn helper_single(_s: LargeStruct) {} + +// ============================================================================ +// Test 2: Multiple arguments of different types +// ============================================================================ + +// CHECK-LABEL: call_arg_scope::test_call_with_multiple_args +pub fn test_call_with_multiple_args(large: LargeStruct, medium: MediumStruct, small: SmallStruct) { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CALL2_ARG1_LOC:]] + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CALL2_ARG2_LOC:]] + // CHECK: call {{.*}}@{{.*}}helper_multiple{{.*}}({{.*}}), !dbg ![[#CALL2_LOC:]] + helper_multiple(large, medium, small); +} + +#[inline(never)] +fn helper_multiple(_l: LargeStruct, _m: MediumStruct, _s: SmallStruct) {} + +// ============================================================================ +// Test 3: Return value +// ============================================================================ + +// CHECK-LABEL: call_arg_scope::test_return_large_value +pub fn test_return_large_value() -> LargeStruct { + let s = LargeStruct { data: [42; 20] }; + // CHECK: ret {{.*}}, !dbg ![[#RET1_LOC:]] + s +} + +// ============================================================================ +// Test 4: Calling a function that returns a large value +// ============================================================================ + +// CHECK-LABEL: call_arg_scope::test_call_returning_large +pub fn test_call_returning_large() { + // CHECK: call {{.*}}@{{.*}}make_large_struct{{.*}}({{.*}}), !dbg ![[#CALL3_LOC:]] + let _result = make_large_struct(); +} + +#[inline(never)] +fn make_large_struct() -> LargeStruct { + LargeStruct { data: [1; 20] } +} + +// ============================================================================ +// Test 5: Mixed scenario - passing and returning large values +// ============================================================================ + +// CHECK-LABEL: call_arg_scope::test_mixed_call +pub fn test_mixed_call(input: LargeStruct) -> LargeStruct { + // CHECK: call {{.*}}@{{.*}}transform_large{{.*}}({{.*}}), !dbg ![[#CALL4_LOC:]] + transform_large(input) +} + +#[inline(never)] +fn transform_large(mut s: LargeStruct) -> LargeStruct { + s.data[0] += 1; + s +} + +// CHECK-DAG: ![[#CALL1_ARG_LOC]] = !DILocation({{.*}}scope: ![[#CALL1_ARG_SCOPE:]] +// CHECK-DAG: ![[#CALL1_ARG_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy" +// CHECK-DAG: ![[#CALL1_LOC]] = !DILocation({{.*}}scope: ![[#CALL1_SCOPE:]] +// CHECK-DAG: ![[#CALL1_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "test_call_with_single_arg" + +// CHECK-DAG: ![[#CALL2_ARG1_LOC]] = !DILocation({{.*}}scope: ![[#CALL2_ARG1_SCOPE:]] +// CHECK-DAG: ![[#CALL2_ARG1_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy" +// CHECK-DAG: ![[#CALL2_ARG2_LOC]] = !DILocation({{.*}}scope: ![[#CALL2_ARG2_SCOPE:]] +// CHECK-DAG: ![[#CALL2_ARG2_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy" +// CHECK-DAG: ![[#CALL2_LOC]] = !DILocation({{.*}}scope: ![[#CALL2_SCOPE:]] +// CHECK-DAG: ![[#CALL2_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "test_call_with_multiple_args" + +// CHECK-DAG: ![[#CALL3_LOC]] = !DILocation({{.*}}scope: ![[#CALL3_SCOPE:]] +// CHECK-DAG: ![[#CALL3_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "test_call_returning_large" + +// CHECK-DAG: ![[#CALL4_LOC]] = !DILocation({{.*}}scope: ![[#CALL4_SCOPE:]] +// CHECK-DAG: ![[#CALL4_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "test_mixed_call" + +// CHECK-DAG: ![[#RET1_LOC]] = !DILocation({{.*}}scope: ![[#RET1_SCOPE:]] +// CHECK-DAG: ![[#RET1_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "test_return_large_value" diff --git a/tests/codegen-llvm/annotate-moves/disabled.rs b/tests/codegen-llvm/annotate-moves/disabled.rs new file mode 100644 index 0000000000000..2a0e786723183 --- /dev/null +++ b/tests/codegen-llvm/annotate-moves/disabled.rs @@ -0,0 +1,35 @@ +//@ compile-flags: -Zannotate-moves=no -Copt-level=0 -g +// Test that move/copy operations are NOT annotated when the flag is disabled + +#![crate_type = "lib"] + +struct LargeStruct { + data: [u64; 20], // 160 bytes - would normally trigger annotation +} + +impl Clone for LargeStruct { + // CHECK-LABEL: ::clone + fn clone(&self) -> Self { + // Should NOT be annotated when flag is disabled + // CHECK-NOT: compiler_copy + LargeStruct { data: self.data } + } +} + +// CHECK-LABEL: disabled::test_large_copy_no_annotation +pub fn test_large_copy_no_annotation() { + let large = LargeStruct { data: [42; 20] }; + // CHECK-NOT: compiler_copy + let _copy = large.clone(); +} + +// CHECK-LABEL: disabled::test_large_move_no_annotation +pub fn test_large_move_no_annotation() { + let large = LargeStruct { data: [42; 20] }; + // CHECK-NOT: compiler_move + let _moved = large; +} + +// Verify that no compiler_move or compiler_copy annotations exist anywhere +// CHECK-NOT: compiler_move +// CHECK-NOT: compiler_copy diff --git a/tests/codegen-llvm/annotate-moves/integration.rs b/tests/codegen-llvm/annotate-moves/integration.rs new file mode 100644 index 0000000000000..1e5d8b8db941f --- /dev/null +++ b/tests/codegen-llvm/annotate-moves/integration.rs @@ -0,0 +1,192 @@ +//@ compile-flags: -Z annotate-moves=1 -Copt-level=0 -g + +#![crate_type = "lib"] + +// Test with large array (non-struct type, Copy) +type LargeArray = [u64; 20]; // 160 bytes + +#[derive(Clone, Default)] +struct NonCopyU64(u64); + +// Test with Copy implementation +#[derive(Copy)] +struct ExplicitCopy { + data: [u64; 20], // 160 bytes +} + +impl Clone for ExplicitCopy { + // CHECK-LABEL: ::clone + fn clone(&self) -> Self { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#EXPLICIT_COPY_LOC:]] + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#EXPLICIT_RETURN_LOC:]] + Self { data: self.data } + } +} + +// Test with hand-implemented Clone (non-Copy) +struct NonCopyStruct { + data: [u64; 20], // 160 bytes +} + +impl Clone for NonCopyStruct { + // CHECK-LABEL: ::clone + fn clone(&self) -> Self { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CLONE_COPY_LOC:]] + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CLONE_RETURN_LOC:]] + NonCopyStruct { data: self.data } + } +} + +// CHECK-LABEL: integration::test_pure_assignment_move +pub fn test_pure_assignment_move() { + let arr: LargeArray = [42; 20]; + // Arrays are initialized with a loop + // CHECK-NOT: call void @llvm.memcpy{{.*}}, !dbg ![[#]] + let _moved = arr; +} + +// CHECK-LABEL: integration::test_pure_assignment_copy +pub fn test_pure_assignment_copy() { + let s = ExplicitCopy { data: [42; 20] }; + // Arrays are initialized with a loop + // CHECK-NOT: call void @llvm.memcpy{{.*}}, !dbg ![[#ASSIGN_COPY_LOC:]] + let _copied = s; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#ASSIGN_COPY2_LOC:]] + let _copied_2 = s; +} + +#[derive(Default)] +struct InitializeStruct { + field1: String, + field2: String, + field3: String, +} + +// CHECK-LABEL: integration::test_init_struct +pub fn test_init_struct() { + let mut s = InitializeStruct::default(); + + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#INIT_STRUCT_LOC:]] + s = InitializeStruct { + field1: String::from("Hello"), + field2: String::from("from"), + field3: String::from("Rust"), + }; +} + +// CHECK-LABEL: integration::test_tuple_of_scalars +pub fn test_tuple_of_scalars() { + // Tuple of scalars (even if large) may use scalar-pair repr, so may not be annotated + let t: (u64, u64, u64, u64) = (1, 2, 3, 4); // 32 bytes + // Copied with explicit stores + // CHECK-NOT: call void @llvm.memcpy{{.*}}, !dbg ![[#]] + let _moved = t; +} + +// CHECK-LABEL: integration::test_tuple_of_structs +pub fn test_tuple_of_structs() { + let s1 = NonCopyStruct { data: [1; 20] }; + let s2 = NonCopyStruct { data: [2; 20] }; + let tuple = (s1, s2); // Large tuple containing structs (320 bytes) + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#TUPLE_MOVE_LOC:]] + let _moved = tuple; +} + +// CHECK-LABEL: integration::test_tuple_mixed +pub fn test_tuple_mixed() { + let s = NonCopyStruct { data: [1; 20] }; + let tuple = (42u64, s); // Mixed tuple (168 bytes: 8 for u64 + 160 for struct) + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#MIXED_TUPLE_LOC:]] + let _moved = tuple; +} + +// CHECK-LABEL: integration::test_explicit_copy_assignment +pub fn test_explicit_copy_assignment() { + let c1 = ExplicitCopy { data: [1; 20] }; + // Initialized with loop + // CHECK-NOT: call void @llvm.memcpy{{.*}}, !dbg ![[#]] + let c2 = c1; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#COPY2_LOC:]] + let _c3 = c1; // Can still use c1 (it was copied) + let _ = c2; +} + +// CHECK-LABEL: integration::test_array_move +pub fn test_array_move() { + let arr: [String; 20] = std::array::from_fn(|i| i.to_string()); + + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#ARRAY_MOVE_LOC:]] + let _moved = arr; +} + +// CHECK-LABEL: integration::test_array_in_struct_field +pub fn test_array_in_struct_field() { + let s = NonCopyStruct { data: [1; 20] }; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#FIELD_MOVE_LOC:]] + let data = s.data; // Move array field out of struct + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#FIELD_MOVE2_LOC:]] + let _moved = data; +} + +// CHECK-LABEL: integration::test_clone_noncopy +pub fn test_clone_noncopy() { + let s = NonCopyStruct { data: [1; 20] }; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CALL_CLONE_NONCOPY_LOC:]] + let _cloned = s.clone(); // The copy happens inside the clone() impl above +} + +// CHECK-LABEL: integration::test_clone_explicit_copy +pub fn test_clone_explicit_copy() { + let c = ExplicitCopy { data: [1; 20] }; + // Derived Clone on Copy type - the copy happens inside the generated clone impl + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#CALL_CLONE_COPY_LOC:]] + let _cloned = c.clone(); +} + +// CHECK-LABEL: integration::test_copy_ref +pub fn test_copy_ref(x: &ExplicitCopy) { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LOCAL_COPY_LOC:]] + let _local = *x; +} + +// CHECK-DAG: ![[#EXPLICIT_COPY_LOC]] = !DILocation({{.*}}scope: ![[#EXPLICIT_COPY_SCOPE:]] +// CHECK-DAG: ![[#EXPLICIT_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u64; 20], 160>" +// CHECK-DAG: ![[#EXPLICIT_RETURN_LOC]] = !DILocation({{.*}}scope: ![[#EXPLICIT_RETURN_SCOPE:]] +// CHECK-DAG: ![[#EXPLICIT_RETURN_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#CLONE_COPY_LOC]] = !DILocation({{.*}}scope: ![[#CLONE_COPY_SCOPE:]] +// CHECK-DAG: ![[#CLONE_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u64; 20], 160>" +// CHECK-DAG: ![[#CLONE_RETURN_LOC]] = !DILocation({{.*}}scope: ![[#CLONE_RETURN_SCOPE:]] +// CHECK-DAG: ![[#CLONE_RETURN_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#ASSIGN_COPY2_LOC]] = !DILocation({{.*}}scope: ![[#ASSIGN_COPY2_SCOPE:]] +// CHECK-DAG: ![[#ASSIGN_COPY2_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#INIT_STRUCT_LOC]] = !DILocation({{.*}}scope: ![[#INIT_STRUCT_SCOPE:]] +// CHECK-DAG: ![[#INIT_STRUCT_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move" + +// CHECK-DAG: ![[#TUPLE_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#TUPLE_MOVE_SCOPE:]] +// CHECK-DAG: ![[#TUPLE_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#MIXED_TUPLE_LOC]] = !DILocation({{.*}}scope: ![[#MIXED_TUPLE_SCOPE:]] +// CHECK-DAG: ![[#MIXED_TUPLE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#COPY2_LOC]] = !DILocation({{.*}}scope: ![[#COPY2_SCOPE:]] +// CHECK-DAG: ![[#COPY2_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#ARRAY_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#ARRAY_MOVE_SCOPE:]] +// CHECK-DAG: ![[#ARRAY_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[alloc::string::String; 20], 480>" + +// CHECK-DAG: ![[#FIELD_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#FIELD_MOVE_SCOPE:]] +// CHECK-DAG: ![[#FIELD_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" +// CHECK-DAG: ![[#FIELD_MOVE2_LOC]] = !DILocation({{.*}}scope: ![[#FIELD_MOVE2_SCOPE:]] +// CHECK-DAG: ![[#FIELD_MOVE2_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u64; 20], 160>" + +// CHECK-DAG: ![[#CALL_CLONE_NONCOPY_LOC]] = !DILocation({{.*}}scope: ![[#CALL_CLONE_NONCOPY_SCOPE:]] +// CHECK-DAG: ![[#CALL_CLONE_NONCOPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#CALL_CLONE_COPY_LOC]] = !DILocation({{.*}}scope: ![[#CALL_CLONE_COPY_SCOPE:]] +// CHECK-DAG: ![[#CALL_CLONE_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// CHECK-DAG: ![[#LOCAL_COPY_LOC]] = !DILocation({{.*}}scope: ![[#LOCAL_COPY_SCOPE:]] +// CHECK-DAG: ![[#LOCAL_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy" diff --git a/tests/codegen-llvm/annotate-moves/size-limit.rs b/tests/codegen-llvm/annotate-moves/size-limit.rs new file mode 100644 index 0000000000000..c6265ef2ce861 --- /dev/null +++ b/tests/codegen-llvm/annotate-moves/size-limit.rs @@ -0,0 +1,74 @@ +//@ compile-flags: -Z annotate-moves=100 -Copt-level=0 -g +// Test that custom size limits work correctly +#![crate_type = "lib"] + +struct MediumStruct { + data: [u64; 10], // 80 bytes - below custom 100-byte threshold +} + +const _: () = { assert!(std::mem::size_of::() == 80) }; + +impl Clone for MediumStruct { + // CHECK-LABEL: ::clone + fn clone(&self) -> Self { + // Should NOT be annotated since 80 < 100 + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#MEDIUM_COPY_LOC:]] + MediumStruct { data: self.data } + } +} + +struct LargeStruct { + data: [u64; 20], // 160 bytes - above custom 100-byte threshold +} + +const _: () = { assert!(std::mem::size_of::() == 160) }; + +impl Clone for LargeStruct { + // CHECK-LABEL: ::clone + fn clone(&self) -> Self { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LARGE_COPY_LOC:]] + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LARGE_RETURN_LOC:]] + LargeStruct { data: self.data } + } +} + +// CHECK-LABEL: size_limit::test_medium_copy +pub fn test_medium_copy() { + let medium = MediumStruct { data: [42; 10] }; + let _copy = medium.clone(); +} + +// CHECK-LABEL: size_limit::test_large_copy +pub fn test_large_copy() { + let large = LargeStruct { data: [42; 20] }; + let _copy = large.clone(); +} + +// CHECK-LABEL: size_limit::test_medium_move +pub fn test_medium_move() { + let medium = MediumStruct { data: [42; 10] }; + // Should NOT be annotated + // CHECK-NOT: compiler_move + let _moved = medium; +} + +// CHECK-LABEL: size_limit::test_large_move +pub fn test_large_move() { + let large = LargeStruct { data: [42; 20] }; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LARGE_MOVE_LOC:]] + let _moved = large; +} + +// The scope for no-annotated is clone function itself +// CHECK-DAG: ![[#MEDIUM_COPY_LOC]] = !DILocation({{.*}}scope: ![[#MEDIUM_COPY_SCOPE:]] +// CHECK-DAG: ![[#MEDIUM_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "clone", + +// Clone itself is copy, but return is move. +// CHECK-DAG: ![[#LARGE_COPY_LOC]] = !DILocation({{.*}}scope: ![[#LARGE_COPY_SCOPE:]] +// CHECK-DAG: ![[#LARGE_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u64; 20], 160>" +// CHECK-DAG: ![[#LARGE_RETURN_LOC]] = !DILocation({{.*}}scope: ![[#LARGE_RETURN_SCOPE:]] +// CHECK-DAG: ![[#LARGE_RETURN_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" + +// Assignment is move +// CHECK-DAG: ![[#LARGE_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#LARGE_MOVE_SCOPE:]] +// CHECK-DAG: ![[#LARGE_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" diff --git a/tests/ui/annotate-moves/annotate-moves-basic.rs b/tests/ui/annotate-moves/annotate-moves-basic.rs new file mode 100644 index 0000000000000..645122113dab2 --- /dev/null +++ b/tests/ui/annotate-moves/annotate-moves-basic.rs @@ -0,0 +1,15 @@ +//@ check-pass +//@ compile-flags: -Z annotate-moves=100 + +// Test that valid annotate-moves flags are accepted + +#[derive(Clone)] +struct TestStruct { + data: [u64; 20], // 160 bytes +} + +fn main() { + let s = TestStruct { data: [42; 20] }; + let _copy = s.clone(); + let _moved = s; +} diff --git a/tests/ui/annotate-moves/annotate-moves-invalid-flag.rs b/tests/ui/annotate-moves/annotate-moves-invalid-flag.rs new file mode 100644 index 0000000000000..621b7861657ef --- /dev/null +++ b/tests/ui/annotate-moves/annotate-moves-invalid-flag.rs @@ -0,0 +1,10 @@ +//@ check-fail +//@ compile-flags: -Z annotate-moves=invalid + +// Test that invalid values for annotate-moves flag are rejected + +fn main() { + // This should fail at compile time due to invalid flag value +} + +//~? ERROR incorrect value `invalid` for unstable option `annotate-moves` diff --git a/tests/ui/annotate-moves/annotate-moves-invalid-flag.stderr b/tests/ui/annotate-moves/annotate-moves-invalid-flag.stderr new file mode 100644 index 0000000000000..5a323573a7770 --- /dev/null +++ b/tests/ui/annotate-moves/annotate-moves-invalid-flag.stderr @@ -0,0 +1,2 @@ +error: incorrect value `invalid` for unstable option `annotate-moves` - either a boolean (`yes`, `no`, `on`, `off`, etc.), or a size limit in bytes was expected + diff --git a/tests/ui/annotate-moves/annotate-moves-size-limit-invalid.rs b/tests/ui/annotate-moves/annotate-moves-size-limit-invalid.rs new file mode 100644 index 0000000000000..50a402102184a --- /dev/null +++ b/tests/ui/annotate-moves/annotate-moves-size-limit-invalid.rs @@ -0,0 +1,10 @@ +//@ check-fail +//@ compile-flags: -Z annotate-moves=-5 + +// Test that negative size limits are rejected + +fn main() { + // This should fail at compile time due to invalid negative size limit +} + +//~? ERROR incorrect value `-5` for unstable option `annotate-moves` diff --git a/tests/ui/annotate-moves/annotate-moves-size-limit-invalid.stderr b/tests/ui/annotate-moves/annotate-moves-size-limit-invalid.stderr new file mode 100644 index 0000000000000..24a3659a30a66 --- /dev/null +++ b/tests/ui/annotate-moves/annotate-moves-size-limit-invalid.stderr @@ -0,0 +1,2 @@ +error: incorrect value `-5` for unstable option `annotate-moves` - either a boolean (`yes`, `no`, `on`, `off`, etc.), or a size limit in bytes was expected + From 19dddf65bd2e1c0f2c0308d7c950402b87d0629b Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 23 Oct 2025 22:49:57 -0700 Subject: [PATCH 2/4] Use lang items rather than diagnostic items for profiling markers --- compiler/rustc_codegen_ssa/src/mir/operand.rs | 13 ++++++------- compiler/rustc_hir/src/lang_items.rs | 4 ++++ library/core/src/profiling.rs | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 1bf5ab8df4c49..e6c6e2af4343e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -5,13 +5,13 @@ use rustc_abi as abi; use rustc_abi::{ Align, BackendRepr, FIRST_VARIANT, FieldIdx, Primitive, Size, TagEncoding, VariantIdx, Variants, }; +use rustc_hir::LangItem; use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range}; use rustc_middle::mir::{self, ConstValue}; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_session::config::{AnnotateMoves, DebugInfo, OptLevel}; -use rustc_span::{Symbol, sym}; use tracing::{debug, instrument}; use super::place::{PlaceRef, PlaceValue}; @@ -1039,8 +1039,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { let kind = match operand { - mir::Operand::Move(_) => sym::compiler_move, - mir::Operand::Copy(_) => sym::compiler_copy, + mir::Operand::Move(_) => LangItem::CompilerMove, + mir::Operand::Copy(_) => LangItem::CompilerCopy, _ => unreachable!(), }; @@ -1086,7 +1086,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &self, bx: &Bx, place: mir::PlaceRef<'tcx>, - kind: Symbol, + kind: LangItem, ) -> Option> { let tcx = bx.tcx(); let sess = tcx.sess; @@ -1116,9 +1116,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return None; } - // Look up the DefId for compiler_move or compiler_copy (skip if it's missing for some - // reason). - let def_id = tcx.get_diagnostic_item(kind)?; + // Look up the DefId for compiler_move or compiler_copy lang item + let def_id = tcx.lang_items().get(kind)?; // Create generic args: compiler_move or compiler_copy let size_const = ty::Const::from_target_usize(tcx, ty_size); diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 9f7f4c7583412..e454ed58699c9 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -345,6 +345,10 @@ language_item_table! { EhPersonality, sym::eh_personality, eh_personality, Target::Fn, GenericRequirement::None; EhCatchTypeinfo, sym::eh_catch_typeinfo, eh_catch_typeinfo, Target::Static, GenericRequirement::None; + // Profiling markers for move/copy operations (used by -Z annotate-moves) + CompilerMove, sym::compiler_move, compiler_move_fn, Target::Fn, GenericRequirement::Exact(2); + CompilerCopy, sym::compiler_copy, compiler_copy_fn, Target::Fn, GenericRequirement::Exact(2); + OwnedBox, sym::owned_box, owned_box, Target::Struct, GenericRequirement::Minimum(1); GlobalAlloc, sym::global_alloc_ty, global_alloc_ty, Target::Struct, GenericRequirement::None; diff --git a/library/core/src/profiling.rs b/library/core/src/profiling.rs index 0d2260e88bd6e..3121c1f0eaea6 100644 --- a/library/core/src/profiling.rs +++ b/library/core/src/profiling.rs @@ -8,7 +8,7 @@ /// /// The `SIZE` parameter encodes the size of the type being moved. #[unstable(feature = "profiling_marker_api", issue = "none")] -#[rustc_diagnostic_item = "compiler_move"] +#[lang = "compiler_move"] pub fn compiler_move(_src: *const T, _dst: *mut T) { unreachable!( "compiler_move marks where the compiler generated a copy; it is never actually called" @@ -23,7 +23,7 @@ pub fn compiler_move(_src: *const T, _dst: *mut T) { /// /// The `SIZE` parameter encodes the size of the type being copied. #[unstable(feature = "profiling_marker_api", issue = "none")] -#[rustc_diagnostic_item = "compiler_copy"] +#[lang = "compiler_copy"] pub fn compiler_copy(_src: *const T, _dst: *mut T) { unreachable!( "compiler_copy marks where the compiler generated a copy; it is never actually called" From 136cb4a3fc39e59603192bbaacc94f727b78dbea Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 29 Oct 2025 23:05:20 -0700 Subject: [PATCH 3/4] More review updates --- compiler/rustc_codegen_ssa/src/mir/operand.rs | 4 +- library/core/src/profiling.rs | 10 +- .../src/compiler-flags/annotate-moves.md | 5 + .../annotate-moves/integration.rs | 2 +- .../codegen-llvm/annotate-moves/size-limit.rs | 122 ++++++++++++------ 5 files changed, 95 insertions(+), 48 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index e6c6e2af4343e..5a139702f81d9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -853,7 +853,9 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> { } } -/// Default size limit for move/copy annotations (in bytes). +/// Default size limit for move/copy annotations (in bytes). 64 bytes is a common size of a cache +/// line, and the assumption is that anything this size or below is very cheap to move/copy, so only +/// annotate copies larger than this. const MOVE_ANNOTATION_DEFAULT_LIMIT: u64 = 65; impl<'a, 'tcx, V: CodegenObject> OperandValue { diff --git a/library/core/src/profiling.rs b/library/core/src/profiling.rs index 3121c1f0eaea6..66c7ddcbaa6fd 100644 --- a/library/core/src/profiling.rs +++ b/library/core/src/profiling.rs @@ -6,12 +6,13 @@ /// the compiler creates synthetic debug info that makes move operations appear as /// calls to this function in profilers. /// -/// The `SIZE` parameter encodes the size of the type being moved. +/// The `SIZE` parameter encodes the size of the type being copied. It's the same as +/// `size_of::()`, and is only present for convenience. #[unstable(feature = "profiling_marker_api", issue = "none")] #[lang = "compiler_move"] pub fn compiler_move(_src: *const T, _dst: *mut T) { unreachable!( - "compiler_move marks where the compiler generated a copy; it is never actually called" + "compiler_move marks where the compiler-generated a memcpy for moves. It is never actually called." ) } @@ -21,11 +22,12 @@ pub fn compiler_move(_src: *const T, _dst: *mut T) { /// the compiler creates synthetic debug info that makes copy operations appear as /// calls to this function in profilers. /// -/// The `SIZE` parameter encodes the size of the type being copied. +/// The `SIZE` parameter encodes the size of the type being copied. It's the same as +/// `size_of::()`, and is only present for convenience. #[unstable(feature = "profiling_marker_api", issue = "none")] #[lang = "compiler_copy"] pub fn compiler_copy(_src: *const T, _dst: *mut T) { unreachable!( - "compiler_copy marks where the compiler generated a copy; it is never actually called" + "compiler_copy marks where the compiler-generated a memcpy for Copies. It is never actually called." ) } diff --git a/src/doc/unstable-book/src/compiler-flags/annotate-moves.md b/src/doc/unstable-book/src/compiler-flags/annotate-moves.md index 465a8430e4f62..d9d5d2bd72aeb 100644 --- a/src/doc/unstable-book/src/compiler-flags/annotate-moves.md +++ b/src/doc/unstable-book/src/compiler-flags/annotate-moves.md @@ -1,5 +1,10 @@ # `annotate-moves` +The tracking issue for this feature is: [#148197](https://github.com/rust-lang/rust/issues/148197). + +------------------------ + + The `-Z annotate-moves` flag enables annotation of compiler-generated move and copy operations, making them visible in profilers and stack traces for performance debugging. diff --git a/tests/codegen-llvm/annotate-moves/integration.rs b/tests/codegen-llvm/annotate-moves/integration.rs index 1e5d8b8db941f..5e5c2e05879bb 100644 --- a/tests/codegen-llvm/annotate-moves/integration.rs +++ b/tests/codegen-llvm/annotate-moves/integration.rs @@ -49,7 +49,7 @@ pub fn test_pure_assignment_move() { pub fn test_pure_assignment_copy() { let s = ExplicitCopy { data: [42; 20] }; // Arrays are initialized with a loop - // CHECK-NOT: call void @llvm.memcpy{{.*}}, !dbg ![[#ASSIGN_COPY_LOC:]] + // CHECK-NOT: call void @llvm.memcpy{{.*}}, !dbg ![[#]] let _copied = s; // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#ASSIGN_COPY2_LOC:]] let _copied_2 = s; diff --git a/tests/codegen-llvm/annotate-moves/size-limit.rs b/tests/codegen-llvm/annotate-moves/size-limit.rs index c6265ef2ce861..51bdd40d3cd31 100644 --- a/tests/codegen-llvm/annotate-moves/size-limit.rs +++ b/tests/codegen-llvm/annotate-moves/size-limit.rs @@ -2,73 +2,111 @@ // Test that custom size limits work correctly #![crate_type = "lib"] -struct MediumStruct { - data: [u64; 10], // 80 bytes - below custom 100-byte threshold +struct Struct99 { + data: [u8; 99], // just below custom 100-byte threshold } -const _: () = { assert!(std::mem::size_of::() == 80) }; +const _: () = { assert!(size_of::() == 99) }; -impl Clone for MediumStruct { - // CHECK-LABEL: ::clone +impl Clone for Struct99 { + // CHECK-LABEL: ::clone fn clone(&self) -> Self { // Should NOT be annotated since 80 < 100 - // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#MEDIUM_COPY_LOC:]] - MediumStruct { data: self.data } + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ99_COPY_LOC:]] + Struct99 { data: self.data } } } -struct LargeStruct { - data: [u64; 20], // 160 bytes - above custom 100-byte threshold +// CHECK-LABEL: size_limit::test_99_copy +pub fn test_99_copy() { + let sz99 = Struct99 { data: [42; 99] }; + let _copy = sz99.clone(); } -const _: () = { assert!(std::mem::size_of::() == 160) }; +// CHECK-LABEL: size_limit::test_99_move +pub fn test_99_move() { + let sz99 = Struct99 { data: [42; 99] }; + // Should NOT be annotated + // CHECK-NOT: compiler_move + let _moved = sz99; +} + +struct Struct100 { + data: [u8; 100], // 160 bytes - above custom 100-byte threshold +} -impl Clone for LargeStruct { - // CHECK-LABEL: ::clone +const _: () = { assert!(size_of::() == 100) }; + +impl Clone for Struct100 { + // CHECK-LABEL: ::clone fn clone(&self) -> Self { - // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LARGE_COPY_LOC:]] - // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LARGE_RETURN_LOC:]] - LargeStruct { data: self.data } + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ100_COPY_LOC:]] + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ100_RETURN_LOC:]] + Struct100 { data: self.data } } } -// CHECK-LABEL: size_limit::test_medium_copy -pub fn test_medium_copy() { - let medium = MediumStruct { data: [42; 10] }; - let _copy = medium.clone(); +// CHECK-LABEL: size_limit::test_100_copy +pub fn test_100_copy() { + let sz100 = Struct100 { data: [42; 100] }; + let _copy = sz100.clone(); } -// CHECK-LABEL: size_limit::test_large_copy -pub fn test_large_copy() { - let large = LargeStruct { data: [42; 20] }; - let _copy = large.clone(); +// CHECK-LABEL: size_limit::test_100_move +pub fn test_100_move() { + let sz100 = Struct100 { data: [42; 100] }; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ100_MOVE_LOC:]] + let _moved = sz100; } -// CHECK-LABEL: size_limit::test_medium_move -pub fn test_medium_move() { - let medium = MediumStruct { data: [42; 10] }; - // Should NOT be annotated - // CHECK-NOT: compiler_move - let _moved = medium; +struct Struct101 { + data: [u8; 101], // 160 bytes - above custom 101-byte threshold } -// CHECK-LABEL: size_limit::test_large_move -pub fn test_large_move() { - let large = LargeStruct { data: [42; 20] }; - // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#LARGE_MOVE_LOC:]] - let _moved = large; +const _: () = { assert!(size_of::() == 101) }; + +impl Clone for Struct101 { + // CHECK-LABEL: ::clone + fn clone(&self) -> Self { + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ101_COPY_LOC:]] + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ101_RETURN_LOC:]] + Struct101 { data: self.data } + } +} + +// CHECK-LABEL: size_limit::test_101_copy +pub fn test_101_copy() { + let sz101 = Struct101 { data: [42; 101] }; + let _copy = sz101.clone(); +} + +// CHECK-LABEL: size_limit::test_101_move +pub fn test_101_move() { + let sz101 = Struct101 { data: [42; 101] }; + // CHECK: call void @llvm.memcpy{{.*}}, !dbg ![[#SZ101_MOVE_LOC:]] + let _moved = sz101; } // The scope for no-annotated is clone function itself -// CHECK-DAG: ![[#MEDIUM_COPY_LOC]] = !DILocation({{.*}}scope: ![[#MEDIUM_COPY_SCOPE:]] -// CHECK-DAG: ![[#MEDIUM_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "clone", +// CHECK-DAG: ![[#SZ99_COPY_LOC]] = !DILocation({{.*}}scope: ![[#SZ99_COPY_SCOPE:]] +// CHECK-DAG: ![[#SZ99_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "clone", + +// Clone itself is copy, but return is move. +// CHECK-DAG: ![[#SZ100_COPY_LOC]] = !DILocation({{.*}}scope: ![[#SZ100_COPY_SCOPE:]] +// CHECK-DAG: ![[#SZ100_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u8; 100], 100>" +// CHECK-DAG: ![[#SZ100_RETURN_LOC]] = !DILocation({{.*}}scope: ![[#SZ100_RETURN_SCOPE:]] +// CHECK-DAG: ![[#SZ100_RETURN_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u8; 100], 100>" + +// Assignment is move +// CHECK-DAG: ![[#SZ100_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#SZ100_MOVE_SCOPE:]] +// CHECK-DAG: ![[#SZ100_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u8; 100], 100>" // Clone itself is copy, but return is move. -// CHECK-DAG: ![[#LARGE_COPY_LOC]] = !DILocation({{.*}}scope: ![[#LARGE_COPY_SCOPE:]] -// CHECK-DAG: ![[#LARGE_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u64; 20], 160>" -// CHECK-DAG: ![[#LARGE_RETURN_LOC]] = !DILocation({{.*}}scope: ![[#LARGE_RETURN_SCOPE:]] -// CHECK-DAG: ![[#LARGE_RETURN_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" +// CHECK-DAG: ![[#SZ101_COPY_LOC]] = !DILocation({{.*}}scope: ![[#SZ101_COPY_SCOPE:]] +// CHECK-DAG: ![[#SZ101_COPY_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_copy<[u8; 101], 101>" +// CHECK-DAG: ![[#SZ101_RETURN_LOC]] = !DILocation({{.*}}scope: ![[#SZ101_RETURN_SCOPE:]] +// CHECK-DAG: ![[#SZ101_RETURN_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u8; 101], 101>" // Assignment is move -// CHECK-DAG: ![[#LARGE_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#LARGE_MOVE_SCOPE:]] -// CHECK-DAG: ![[#LARGE_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u64; 20], 160>" +// CHECK-DAG: ![[#SZ101_MOVE_LOC]] = !DILocation({{.*}}scope: ![[#SZ101_MOVE_SCOPE:]] +// CHECK-DAG: ![[#SZ101_MOVE_SCOPE]] = {{(distinct )?}}!DISubprogram(name: "compiler_move<[u8; 101], 101>" From 3f7e412dcf1f6af50cd50da24bd2b99d789f37d5 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 30 Oct 2025 10:00:18 -0700 Subject: [PATCH 4/4] update tracking issue in `unstable` --- library/core/src/lib.rs | 2 +- library/core/src/profiling.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 5edbf2bec8ece..6678851d57738 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -281,7 +281,7 @@ pub mod num; pub mod hint; pub mod intrinsics; pub mod mem; -#[unstable(feature = "profiling_marker_api", issue = "none")] +#[unstable(feature = "profiling_marker_api", issue = "148197")] pub mod profiling; pub mod ptr; #[unstable(feature = "ub_checks", issue = "none")] diff --git a/library/core/src/profiling.rs b/library/core/src/profiling.rs index 66c7ddcbaa6fd..db4a62480a3a1 100644 --- a/library/core/src/profiling.rs +++ b/library/core/src/profiling.rs @@ -8,7 +8,7 @@ /// /// The `SIZE` parameter encodes the size of the type being copied. It's the same as /// `size_of::()`, and is only present for convenience. -#[unstable(feature = "profiling_marker_api", issue = "none")] +#[unstable(feature = "profiling_marker_api", issue = "148197")] #[lang = "compiler_move"] pub fn compiler_move(_src: *const T, _dst: *mut T) { unreachable!( @@ -24,7 +24,7 @@ pub fn compiler_move(_src: *const T, _dst: *mut T) { /// /// The `SIZE` parameter encodes the size of the type being copied. It's the same as /// `size_of::()`, and is only present for convenience. -#[unstable(feature = "profiling_marker_api", issue = "none")] +#[unstable(feature = "profiling_marker_api", issue = "148197")] #[lang = "compiler_copy"] pub fn compiler_copy(_src: *const T, _dst: *mut T) { unreachable!(