Add -Zannotate-moves for profiler visibility of move/copy operations

This implements a new unstable compiler flag `-Zannotate-moves` that makes
move and copy operations visible in profilers by creating synthetic debug
information. This is achieved with zero runtime cost by manipulating debug
info scopes to make moves/copies appear as calls to `compiler_move<T, SIZE>`
and `compiler_copy<T, SIZE>` marker functions in profiling tools.

This allows developers to identify expensive move/copy operations in their
code using standard profiling tools, without requiring specialized tooling
or runtime instrumentation.

The implementation works at codegen time. When processing MIR operands
(`Operand::Move` and `Operand::Copy`), the codegen creates an `OperandRef`
with an optional `move_annotation` field containing an `Instance` of the
appropriate profiling marker function. When storing the operand,
`store_with_annotation()` wraps the store operation in a synthetic debug
scope that makes it appear inlined from the marker.

Two marker functions (`compiler_move` and `compiler_copy`) are defined
in `library/core/src/profiling.rs`. These are never actually called -
they exist solely as debug info anchors.

Operations are only annotated if the type:
   - Meets the size threshold (default: 65 bytes, configurable via
     `-Zannotate-moves=SIZE`)
   - Has a non-scalar backend representation (scalars use registers,
     not memcpy)

This has a very small size impact on object file size. With the default
limit it's well under 0.1%, and even with a very small limit of 8 bytes
it's still ~1.5%. This could be enabled by default.

Author: jsgf

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1069,7 +1069,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
             OperandValue::Ref(place.val)
         };
 
-        OperandRef { val, layout: place.layout }
+        OperandRef { val, layout: place.layout, move_annotation: None }
     }
 
     fn write_operand_repeatedly(

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -19,7 +19,7 @@ use crate::context::CodegenCx;
 pub(super) const UNKNOWN_LINE_NUMBER: u32 = 0;
 pub(super) const UNKNOWN_COLUMN_NUMBER: u32 = 0;
 
-impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
+impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
     // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
     fn dbg_var_addr(

compiler/rustc_codegen_llvm/src/builder.rs

@@ -752,7 +752,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             OperandValue::Ref(place.val)
         };
 
-        OperandRef { val, layout: place.layout }
+        OperandRef { val, layout: place.layout, move_annotation: None }
     }
 
     fn write_operand_repeatedly(

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -146,7 +146,7 @@ impl<'ll> Builder<'_, 'll, '_> {
     }
 }
 
-impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
+impl<'ll, 'tcx> DebugInfoBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
     // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
     fn dbg_var_addr(
@@ -284,6 +284,57 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
             llvm::set_value_name(value, name.as_bytes());
         }
     }
+
+    /// Temporarily annotate move/copy operations with debug info for profiling.
+    ///
+    /// This creates a temporary debug scope that makes the move/copy appear as an
+    /// inlined call to `compiler_move<T, SIZE>()` or `compiler_copy<T, SIZE>()`.
+    /// The provided closure is executed with this temporary debug location active.
+    ///
+    /// The `instance` parameter should be the monomorphized instance of the
+    /// `compiler_move` or `compiler_copy` function with the actual type and size.
+    fn with_move_annotation<R>(
+        &mut self,
+        instance: ty::Instance<'tcx>,
+        f: impl FnOnce(&mut Self) -> R,
+    ) -> R {
+        // Save the current debug location
+        let saved_loc = self.get_dbg_loc();
+
+        // Create a DIScope for the compiler_move/compiler_copy function
+        // We use the function's FnAbi for debug info generation
+        let fn_abi = self
+            .cx()
+            .tcx
+            .fn_abi_of_instance(
+                self.cx().typing_env().as_query_input((instance, ty::List::empty())),
+            )
+            .unwrap();
+
+        let di_scope = self.cx().dbg_scope_fn(instance, fn_abi, None);
+
+        // Create an inlined debug location:
+        // - scope: the compiler_move/compiler_copy function
+        // - inlined_at: the current location (where the move/copy actually occurs)
+        // - span: use the function's definition span
+        let fn_span = self.cx().tcx.def_span(instance.def_id());
+        let inlined_loc = self.cx().dbg_loc(di_scope, saved_loc, fn_span);
+
+        // Set the temporary debug location
+        self.set_dbg_loc(inlined_loc);
+
+        // Execute the closure (which will generate the memcpy)
+        let result = f(self);
+
+        // Restore the original debug location
+        if let Some(loc) = saved_loc {
+            self.set_dbg_loc(loc);
+        } else {
+            self.clear_dbg_loc();
+        }
+
+        result
+    }
 }
 
 /// A source code location used to generate debug information.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -556,9 +556,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let op = match self.locals[mir::RETURN_PLACE] {
                     LocalRef::Operand(op) => op,
                     LocalRef::PendingOperand => bug!("use of return before def"),
-                    LocalRef::Place(cg_place) => {
-                        OperandRef { val: Ref(cg_place.val), layout: cg_place.layout }
-                    }
+                    LocalRef::Place(cg_place) => OperandRef {
+                        val: Ref(cg_place.val),
+                        layout: cg_place.layout,
+                        move_annotation: None,
+                    },
                     LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
                 };
                 let llslot = match op.val {

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -476,6 +476,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                         return local(OperandRef {
                             val: OperandValue::Pair(a, b),
                             layout: arg.layout,
+                            move_annotation: None,
                         });
                     }
                     _ => {}
@@ -548,6 +549,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         fx.caller_location = Some(OperandRef {
             val: OperandValue::Immediate(bx.get_param(llarg_idx)),
             layout: arg.layout,
+            move_annotation: None,
         });
     }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -7,10 +7,11 @@ use rustc_abi::{
 };
 use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range};
 use rustc_middle::mir::{self, ConstValue};
-use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_middle::ty::{self, Ty};
 use rustc_middle::{bug, span_bug};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{AnnotateMoves, DebugInfo, OptLevel};
+use rustc_span::{Symbol, sym};
 use tracing::{debug, instrument};
 
 use super::place::{PlaceRef, PlaceValue};
@@ -131,6 +132,10 @@ pub struct OperandRef<'tcx, V> {
 
     /// The layout of value, based on its Rust type.
     pub layout: TyAndLayout<'tcx>,
+
+    /// Annotation for profiler visibility of move/copy operations.
+    /// When set, the store operation should appear as an inlined call to this function.
+    pub move_annotation: Option<ty::Instance<'tcx>>,
 }
 
 impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
@@ -142,7 +147,7 @@ impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
 impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
     pub fn zero_sized(layout: TyAndLayout<'tcx>) -> OperandRef<'tcx, V> {
         assert!(layout.is_zst());
-        OperandRef { val: OperandValue::ZeroSized, layout }
+        OperandRef { val: OperandValue::ZeroSized, layout, move_annotation: None }
     }
 
     pub(crate) fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -180,7 +185,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             }
         };
 
-        OperandRef { val, layout }
+        OperandRef { val, layout, move_annotation: None }
     }
 
     fn from_const_alloc<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -214,7 +219,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 let size = s.size(bx);
                 assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
                 let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
-                OperandRef { val: OperandValue::Immediate(val), layout }
+                OperandRef { val: OperandValue::Immediate(val), layout, move_annotation: None }
             }
             BackendRepr::ScalarPair(
                 a @ abi::Scalar::Initialized { .. },
@@ -235,7 +240,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                     b,
                     bx.scalar_pair_element_backend_type(layout, 1, true),
                 );
-                OperandRef { val: OperandValue::Pair(a_val, b_val), layout }
+                OperandRef { val: OperandValue::Pair(a_val, b_val), layout, move_annotation: None }
             }
             _ if layout.is_zst() => OperandRef::zero_sized(layout),
             _ => {
@@ -285,6 +290,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         self.val.deref(layout.align.abi).with_type(layout)
     }
 
+    /// Store this operand into a place, applying move/copy annotation if present.
+    ///
+    /// This is the preferred method for storing operands, as it automatically
+    /// applies profiler annotations for tracked move/copy operations.
+    pub fn store_with_annotation<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+        self,
+        bx: &mut Bx,
+        dest: PlaceRef<'tcx, V>,
+    ) {
+        if let Some(instance) = self.move_annotation {
+            bx.with_move_annotation(instance, |bx| self.val.store(bx, dest))
+        } else {
+            self.val.store(bx, dest)
+        }
+    }
+
     /// If this operand is a `Pair`, we return an aggregate with the two values.
     /// For other cases, see `immediate`.
     pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -320,7 +341,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         } else {
             OperandValue::Immediate(llval)
         };
-        OperandRef { val, layout }
+        OperandRef { val, layout, move_annotation: None }
     }
 
     pub(crate) fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@@ -388,7 +409,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             })
         };
 
-        OperandRef { val, layout: field }
+        OperandRef { val, layout: field, move_annotation: None }
     }
 
     /// Obtain the actual discriminant of a value.
@@ -828,10 +849,13 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
                 }
             },
         };
-        OperandRef { val, layout }
+        OperandRef { val, layout, move_annotation: None }
     }
 }
 
+/// Default size limit for move/copy annotations (in bytes).
+const MOVE_ANNOTATION_DEFAULT_LIMIT: u64 = 65;
+
 impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
     /// Returns an `OperandValue` that's generally UB to use in any way.
     ///
@@ -961,7 +985,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 abi::Variants::Single { index: vidx },
                             );
                             let layout = o.layout.for_variant(bx.cx(), vidx);
-                            o = OperandRef { val: o.val, layout }
+                            o = OperandRef { layout, ..o }
                         }
                         _ => return None,
                     }
@@ -1014,7 +1038,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         match *operand {
             mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
-                self.codegen_consume(bx, place.as_ref())
+                let kind = match operand {
+                    mir::Operand::Move(_) => sym::compiler_move,
+                    mir::Operand::Copy(_) => sym::compiler_copy,
+                    _ => unreachable!(),
+                };
+                let ty = self.monomorphized_place_ty(place.as_ref());
+                let layout = bx.cx().layout_of(ty);
+
+                // Check if we should annotate this move/copy for profiling
+                let move_annotation = self.move_copy_annotation_instance(bx, ty, layout, kind);
+
+                OperandRef { move_annotation, ..self.codegen_consume(bx, place.as_ref()) }
             }
 
             mir::Operand::Constant(ref constant) => {
@@ -1030,11 +1065,77 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         return OperandRef {
                             val: OperandValue::Immediate(llval),
                             layout: bx.layout_of(ty),
+                            move_annotation: None,
                         };
                     }
                 }
                 self.eval_mir_constant_to_operand(bx, constant)
             }
         }
     }
+
+    /// Creates an `Instance` for annotating a move/copy operation at codegen time.
+    ///
+    /// Returns `Some(instance)` if the operation should be annotated with debug info,
+    /// `None` otherwise. The instance represents a monomorphized `compiler_move<T, SIZE>`
+    /// or `compiler_copy<T, SIZE>` function that can be used to create debug scopes.
+    ///
+    /// Annotation criteria:
+    /// - Codegen annotation flag must be enabled
+    /// - Debug info must be enabled
+    /// - Type must not be ZST
+    /// - Type size must meet threshold
+    /// - Type must not be scalar/scalar-pair/SIMD (no memcpy generated)
+    fn move_copy_annotation_instance(
+        &self,
+        bx: &Bx,
+        ty: Ty<'tcx>,
+        layout: TyAndLayout<'tcx>,
+        kind: Symbol,
+    ) -> Option<ty::Instance<'tcx>> {
+        let tcx = bx.tcx();
+        let sess = tcx.sess;
+
+        // Check if annotation is enabled and get size limit
+        let size_limit = match sess.opts.unstable_opts.annotate_moves {
+            AnnotateMoves::Disabled => return None,
+            AnnotateMoves::Enabled(None) => MOVE_ANNOTATION_DEFAULT_LIMIT,
+            AnnotateMoves::Enabled(Some(limit)) => limit,
+        };
+        let size = layout.size.bytes();
+        // Skip if we're not generating debug info, or if ty is zst, or the size limit isn't met, or
+        // if the layout is not moved by memcpy.
+        if sess.opts.debuginfo == DebugInfo::None
+            || layout.is_zst()
+            || size < size_limit
+            || matches!(
+                layout.backend_repr,
+                BackendRepr::Scalar(_)
+                    | BackendRepr::ScalarPair(_, _)
+                    | BackendRepr::SimdVector { .. }
+            )
+        {
+            return None;
+        }
+
+        // Look up the DefId for compiler_move or compiler_copy (skip if it's missing for some
+        // reason).
+        let def_id = tcx.get_diagnostic_item(kind)?;
+
+        // Create generic args: compiler_move<T, SIZE> or compiler_copy<T, SIZE>
+        let size_const = ty::Const::from_target_usize(tcx, size);
+        let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]);
+
+        // Create the Instance
+        let typing_env = self.mir.typing_env(tcx);
+        let instance = ty::Instance::expect_resolve(
+            tcx,
+            typing_env,
+            def_id,
+            generic_args,
+            rustc_span::DUMMY_SP, // span only used for error messages
+        );
+
+        Some(instance)
+    }
 }