Put move source locations for parameters into Call/TailCall

Rather than store parameter moves in a separate side structure, which
relies on MIR indexes not changing again, instead move the parameter
move source_info into TerminatorKind::Call/TailCall itself. This keeps
everything together as a parallel array of parameter move information.

If we aren't annotating moves, or no parameters need annotation, then
then remains as None, Otherwise it's a `Box<[Option<SourceInfo>]>`
recording the source location of the annotated parameters.

This ends up touching a lot of files with mostly one line changes just
because of the extra field in the Call/TailCall enums. I changed a lot
of the patterns to use `..` wildcard matching since in many cases most
fields were being skipped.

Author: jsgf

compiler/rustc_borrowck/src/lib.rs

@@ -904,22 +904,14 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
                     state,
                 );
             }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target: _,
-                unwind: _,
-                call_source: _,
-                fn_span: _,
-            } => {
+            TerminatorKind::Call { func, args, destination, .. } => {
                 self.consume_operand(loc, (func, span), state);
                 for arg in args {
                     self.consume_operand(loc, (&arg.node, arg.span), state);
                 }
                 self.mutate_place(loc, (*destination, span), Deep, state);
             }
-            TerminatorKind::TailCall { func, args, fn_span: _ } => {
+            TerminatorKind::TailCall { func, args, .. } => {
                 self.consume_operand(loc, (func, span), state);
                 for arg in args {
                     self.consume_operand(loc, (&arg.node, arg.span), state);

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -118,15 +118,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
                     LocalMutationIsAllowed::Yes,
                 );
             }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target: _,
-                unwind: _,
-                call_source: _,
-                fn_span: _,
-            } => {
+            TerminatorKind::Call { func, args, destination, .. } => {
                 self.consume_operand(location, func);
                 for arg in args {
                     self.consume_operand(location, &arg.node);

compiler/rustc_codegen_cranelift/src/base.rs

@@ -478,15 +478,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     switch.emit(&mut fx.bcx, discr, otherwise_block);
                 }
             }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target,
-                fn_span,
-                unwind,
-                call_source: _,
-            } => {
+            TerminatorKind::Call { func, args, destination, target, fn_span, unwind, .. } => {
                 fx.tcx.prof.generic_activity("codegen call").run(|| {
                     crate::abi::codegen_terminator_call(
                         fx,

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1152,13 +1152,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             // Look up stored SourceInfo for this argument if it exists (from annotate_moves pass)
-            let bb_info = self
-                .mir
-                .call_arg_move_source_info
-                .iter()
-                .find(|&&((block, idx), _)| block == helper.bb && idx == i);
-            if let Some((_, arg_source_info)) = bb_info {
-                self.set_debug_loc(bx, *arg_source_info);
+            let arg_source_info = match &terminator.kind {
+                mir::TerminatorKind::Call { arg_move_source_info, .. }
+                | mir::TerminatorKind::TailCall { arg_move_source_info, .. } => {
+                    arg_move_source_info.as_ref().and_then(|infos| infos.get(i).copied().flatten())
+                }
+                _ => None,
+            };
+
+            if let Some(arg_source_info) = arg_source_info {
+                self.set_debug_loc(bx, arg_source_info);
             }
 
             self.codegen_argument(
@@ -1454,8 +1457,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 destination,
                 target,
                 unwind,
-                call_source: _,
                 fn_span,
+                ..
             } => self.codegen_call_terminator(
                 helper,
                 bx,
@@ -1469,7 +1472,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 CallKind::Normal,
                 mergeable_succ(),
             ),
-            mir::TerminatorKind::TailCall { ref func, ref args, fn_span } => self
+            mir::TerminatorKind::TailCall { ref func, ref args, fn_span, .. } => self
                 .codegen_call_terminator(
                     helper,
                     bx,

compiler/rustc_const_eval/src/interpret/step.rs

@@ -544,15 +544,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.go_to_block(target_block);
             }
 
-            Call {
-                ref func,
-                ref args,
-                destination,
-                target,
-                unwind,
-                call_source: _,
-                fn_span: _,
-            } => {
+            Call { ref func, ref args, destination, target, unwind, .. } => {
                 let old_stack = self.frame_idx();
                 let old_loc = self.frame().loc;
 
@@ -576,7 +568,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 }
             }
 
-            TailCall { ref func, ref args, fn_span: _ } => {
+            TailCall { ref func, ref args, .. } => {
                 let old_frame_idx = self.frame_idx();
 
                 let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =

compiler/rustc_middle/src/mir/mod.rs

@@ -331,13 +331,6 @@ pub struct Body<'tcx> {
     #[type_foldable(identity)]
     #[type_visitable(ignore)]
     pub function_coverage_info: Option<Box<coverage::FunctionCoverageInfo>>,
-
-    /// Debug information for argument moves/copies in Call parameters. Stores pairs of
-    /// ((BasicBlock, argument_index), SourceInfo) for move/copy operations. Only populated when
-    /// `-Zannotate-moves` is enabled.
-    #[type_foldable(identity)]
-    #[type_visitable(ignore)]
-    pub call_arg_move_source_info: Vec<((BasicBlock, usize), SourceInfo)>,
 }
 
 impl<'tcx> Body<'tcx> {
@@ -381,7 +374,6 @@ impl<'tcx> Body<'tcx> {
             tainted_by_errors,
             coverage_info_hi: None,
             function_coverage_info: None,
-            call_arg_move_source_info: Vec::new(),
         };
         body.is_polymorphic = body.has_non_region_param();
         body
@@ -413,7 +405,6 @@ impl<'tcx> Body<'tcx> {
             tainted_by_errors: None,
             coverage_info_hi: None,
             function_coverage_info: None,
-            call_arg_move_source_info: Vec::new(),
         };
         body.is_polymorphic = body.has_non_region_param();
         body
@@ -1673,11 +1664,11 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 128);
+    static_assert_size!(BasicBlockData<'_>, 144);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 64);
     static_assert_size!(Statement<'_>, 32);
-    static_assert_size!(Terminator<'_>, 96);
+    static_assert_size!(Terminator<'_>, 112);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -16,7 +16,7 @@ use rustc_span::{Span, Symbol};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use smallvec::SmallVec;
 
-use super::{BasicBlock, Const, Local, UserTypeProjection};
+use super::{BasicBlock, Const, Local, SourceInfo, UserTypeProjection};
 use crate::mir::coverage::CoverageKind;
 use crate::ty::adjustment::PointerCoercion;
 use crate::ty::{self, GenericArgsRef, List, Region, Ty, UserTypeAnnotationIndex};
@@ -812,7 +812,7 @@ pub enum TerminatorKind<'tcx> {
     ///
     /// [#71117]: https://github.com/rust-lang/rust/issues/71117
     Call {
-        /// The function that’s being called.
+        /// The function that's being called.
         func: Operand<'tcx>,
         /// Arguments the function is called with.
         /// These are owned by the callee, which is free to modify them.
@@ -832,6 +832,13 @@ pub enum TerminatorKind<'tcx> {
         /// This `Span` is the span of the function, without the dot and receiver
         /// e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
+        /// Optional array of source info for move operations in the arguments.
+        /// This is used to make the move operations appear in profilers as if they
+        /// were inlined from compiler_move intrinsics.
+        /// If None, no special source info is used.
+        /// If Some, the array has the same length as args, with None for arguments
+        /// that don't need special source info.
+        arg_move_source_info: Option<Box<[Option<SourceInfo>]>>,
     },
 
     /// Tail call.
@@ -851,7 +858,7 @@ pub enum TerminatorKind<'tcx> {
     /// [`Call`]: TerminatorKind::Call
     /// [`Return`]: TerminatorKind::Return
     TailCall {
-        /// The function that’s being called.
+        /// The function that's being called.
         func: Operand<'tcx>,
         /// Arguments the function is called with.
         /// These are owned by the callee, which is free to modify them.
@@ -862,6 +869,13 @@ pub enum TerminatorKind<'tcx> {
         /// This `Span` is the span of the function, without the dot and receiver
         /// (e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
+        /// Optional array of source info for move operations in the arguments.
+        /// This is used to make the move operations appear in profilers as if they
+        /// were inlined from compiler_move intrinsics.
+        /// If None, no special source info is used.
+        /// If Some, the array has the same length as args, with None for arguments
+        /// that don't need special source info.
+        arg_move_source_info: Option<Box<[Option<SourceInfo>]>>,
     },
 
     /// Evaluates the operand, which must have type `bool`. If it is not equal to `expected`,
@@ -1734,6 +1748,6 @@ mod size_asserts {
     static_assert_size!(PlaceElem<'_>, 24);
     static_assert_size!(Rvalue<'_>, 40);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(TerminatorKind<'_>, 80);
+    static_assert_size!(TerminatorKind<'_>, 96);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/terminator.rs

@@ -779,15 +779,7 @@ impl<'tcx> TerminatorKind<'tcx> {
                 }
             }
 
-            Call {
-                unwind,
-                destination,
-                ref target,
-                func: _,
-                args: _,
-                fn_span: _,
-                call_source: _,
-            } => TerminatorEdges::AssignOnReturn {
+            Call { unwind, destination, ref target, .. } => TerminatorEdges::AssignOnReturn {
                 return_: target.as_ref().map(slice::from_ref).unwrap_or_default(),
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::Call(destination),

compiler/rustc_middle/src/mir/visit.rs

@@ -551,10 +551,8 @@ macro_rules! make_mir_visitor {
                         func,
                         args,
                         destination,
-                        target: _,
-                        unwind: _,
-                        call_source: _,
                         fn_span,
+                        ..
                     } => {
                         self.visit_span($(& $mutability)? *fn_span);
                         self.visit_operand(func, location);
@@ -568,7 +566,7 @@ macro_rules! make_mir_visitor {
                         );
                     }
 
-                    TerminatorKind::TailCall { func, args, fn_span } => {
+                    TerminatorKind::TailCall { func, args, fn_span, .. } => {
                         self.visit_span($(& $mutability)? *fn_span);
                         self.visit_operand(func, location);
                         for arg in args {

compiler/rustc_mir_build/src/builder/custom/mod.rs

@@ -62,7 +62,6 @@ pub(super) fn build_custom_mir<'tcx>(
         pass_count: 0,
         coverage_info_hi: None,
         function_coverage_info: None,
-        call_arg_move_source_info: Vec::new(),
     };
 
     body.local_decls.push(LocalDecl::new(return_ty, return_ty_span));