|
| 1 | +//! Instrumentation pass for move/copy operations. |
| 2 | +//! |
| 3 | +//! This pass modifies the source scopes of statements containing `Operand::Move` and `Operand::Copy` |
| 4 | +//! to make them appear as if they were inlined from `compiler_move()` and `compiler_copy()` intrinsic |
| 5 | +//! functions. This creates the illusion that moves/copies are function calls in debuggers and |
| 6 | +//! profilers, making them visible for performance analysis. |
| 7 | +//! |
| 8 | +//! The pass leverages the existing inlining infrastructure by creating synthetic `SourceScopeData` |
| 9 | +//! with the `inlined` field set to point to the appropriate intrinsic function. |
| 10 | +
|
| 11 | +use rustc_index::IndexVec; |
| 12 | +use rustc_middle::mir::*; |
| 13 | +use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypingEnv}; |
| 14 | +use rustc_session::config::DebugInfo; |
| 15 | +use rustc_span::sym; |
| 16 | + |
| 17 | +/// Default minimum size in bytes for move/copy operations to be instrumented. Set to 64+1 bytes |
| 18 | +/// (typical cache line size) to focus on potentially expensive operations. |
| 19 | +const DEFAULT_INSTRUMENT_MOVES_SIZE_LIMIT: u64 = 65; |
| 20 | + |
| 21 | +#[derive(Copy, Clone, Debug)] |
| 22 | +enum Operation { |
| 23 | + Move, |
| 24 | + Copy, |
| 25 | +} |
| 26 | + |
| 27 | +/// Bundle up parameters into a structure to make repeated calling neater |
| 28 | +struct Params<'a, 'tcx> { |
| 29 | + tcx: TyCtxt<'tcx>, |
| 30 | + source_scopes: &'a mut IndexVec<SourceScope, SourceScopeData<'tcx>>, |
| 31 | + local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>, |
| 32 | + typing_env: TypingEnv<'tcx>, |
| 33 | + size_limit: u64, |
| 34 | +} |
| 35 | + |
| 36 | +/// MIR transform that instruments move/copy operations for profiler visibility. |
| 37 | +pub(crate) struct InstrumentMoves; |
| 38 | + |
| 39 | +impl<'tcx> crate::MirPass<'tcx> for InstrumentMoves { |
| 40 | + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { |
| 41 | + sess.opts.unstable_opts.instrument_moves && sess.opts.debuginfo != DebugInfo::None |
| 42 | + } |
| 43 | + |
| 44 | + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
| 45 | + // Skip promoted MIR bodies to avoid recursion |
| 46 | + if body.source.promoted.is_some() { |
| 47 | + return; |
| 48 | + } |
| 49 | + |
| 50 | + let typing_env = body.typing_env(tcx); |
| 51 | + let size_limit = tcx |
| 52 | + .sess |
| 53 | + .opts |
| 54 | + .unstable_opts |
| 55 | + .instrument_moves_size_limit |
| 56 | + .unwrap_or(DEFAULT_INSTRUMENT_MOVES_SIZE_LIMIT); |
| 57 | + |
| 58 | + // Common params, including selectively borrowing the bits of Body we need to avoid |
| 59 | + // mut/non-mut aliasing problems. |
| 60 | + let mut params = Params { |
| 61 | + tcx, |
| 62 | + source_scopes: &mut body.source_scopes, |
| 63 | + local_decls: &body.local_decls, |
| 64 | + typing_env, |
| 65 | + size_limit, |
| 66 | + }; |
| 67 | + |
| 68 | + // Process each basic block |
| 69 | + for block_data in body.basic_blocks.as_mut() { |
| 70 | + for stmt in &mut block_data.statements { |
| 71 | + let source_info = &mut stmt.source_info; |
| 72 | + |
| 73 | + if let StatementKind::Assign(box (_, rvalue)) = &stmt.kind { |
| 74 | + match rvalue { |
| 75 | + Rvalue::Use(op) |
| 76 | + | Rvalue::Repeat(op, _) |
| 77 | + | Rvalue::Cast(_, op, _) |
| 78 | + | Rvalue::UnaryOp(_, op) => { |
| 79 | + self.annotate_move(&mut params, source_info, op); |
| 80 | + } |
| 81 | + Rvalue::BinaryOp(_, box (lop, rop)) => { |
| 82 | + self.annotate_move(&mut params, source_info, lop); |
| 83 | + self.annotate_move(&mut params, source_info, rop); |
| 84 | + } |
| 85 | + Rvalue::Aggregate(_, ops) => { |
| 86 | + for op in ops { |
| 87 | + self.annotate_move(&mut params, source_info, op); |
| 88 | + } |
| 89 | + } |
| 90 | + Rvalue::Ref(..) |
| 91 | + | Rvalue::ThreadLocalRef(..) |
| 92 | + | Rvalue::RawPtr(..) |
| 93 | + | Rvalue::NullaryOp(..) |
| 94 | + | Rvalue::Discriminant(..) |
| 95 | + | Rvalue::CopyForDeref(..) |
| 96 | + | Rvalue::ShallowInitBox(..) |
| 97 | + | Rvalue::WrapUnsafeBinder(..) => {} // No operands to instrument |
| 98 | + } |
| 99 | + } |
| 100 | + } |
| 101 | + |
| 102 | + // Process terminator operands |
| 103 | + if let Some(terminator) = &mut block_data.terminator { |
| 104 | + let source_info = &mut terminator.source_info; |
| 105 | + match &terminator.kind { |
| 106 | + TerminatorKind::Call { func, args, .. } => { |
| 107 | + // Instrument the function operand |
| 108 | + self.annotate_move(&mut params, source_info, func); |
| 109 | + // Instrument each argument |
| 110 | + for arg in &*args { |
| 111 | + self.annotate_move(&mut params, source_info, &arg.node); |
| 112 | + } |
| 113 | + } |
| 114 | + TerminatorKind::SwitchInt { discr, .. } => { |
| 115 | + self.annotate_move(&mut params, source_info, discr); |
| 116 | + } |
| 117 | + _ => {} // Other terminators don't have operands |
| 118 | + } |
| 119 | + } |
| 120 | + } |
| 121 | + } |
| 122 | + |
| 123 | + fn is_required(&self) -> bool { |
| 124 | + false // Optional optimization/instrumentation pass |
| 125 | + } |
| 126 | +} |
| 127 | + |
| 128 | +impl InstrumentMoves { |
| 129 | + /// If this is a Move or Copy of a concrete type, update its debug info to make it look like it |
| 130 | + /// was inlined from `core::profiling::compiler_move`/`compiler_copy`. |
| 131 | + fn annotate_move<'tcx>( |
| 132 | + &self, |
| 133 | + params: &mut Params<'_, 'tcx>, |
| 134 | + source_info: &mut SourceInfo, |
| 135 | + op: &Operand<'tcx>, |
| 136 | + ) { |
| 137 | + let (place, operation) = match op { |
| 138 | + Operand::Move(place) => (place, Operation::Move), |
| 139 | + Operand::Copy(place) => (place, Operation::Copy), |
| 140 | + _ => return, |
| 141 | + }; |
| 142 | + let Params { tcx, typing_env, local_decls, size_limit, source_scopes } = params; |
| 143 | + |
| 144 | + if let Some(type_size) = |
| 145 | + self.should_instrument_operation(*tcx, *typing_env, local_decls, place, *size_limit) |
| 146 | + { |
| 147 | + let ty = place.ty(*local_decls, *tcx).ty; |
| 148 | + source_info.scope = self.create_inlined_scope( |
| 149 | + *tcx, |
| 150 | + *typing_env, |
| 151 | + source_scopes, |
| 152 | + source_info, |
| 153 | + operation, |
| 154 | + ty, |
| 155 | + type_size, |
| 156 | + ); |
| 157 | + } |
| 158 | + } |
| 159 | + |
| 160 | + /// Determines if an operation should be instrumented based on type characteristics. |
| 161 | + /// Returns Some(size) if it should be instrumented, None otherwise. |
| 162 | + fn should_instrument_operation<'tcx>( |
| 163 | + &self, |
| 164 | + tcx: TyCtxt<'tcx>, |
| 165 | + typing_env: ty::TypingEnv<'tcx>, |
| 166 | + local_decls: &rustc_index::IndexVec<Local, LocalDecl<'tcx>>, |
| 167 | + place: &Place<'tcx>, |
| 168 | + size_limit: u64, |
| 169 | + ) -> Option<u64> { |
| 170 | + let ty = place.ty(local_decls, tcx).ty; |
| 171 | + let layout = match tcx.layout_of(typing_env.as_query_input(ty)) { |
| 172 | + Ok(layout) => layout, |
| 173 | + Err(err) => { |
| 174 | + tracing::info!("Failed to get layout of {ty:?}: {err}"); |
| 175 | + return None; |
| 176 | + } |
| 177 | + }; |
| 178 | + |
| 179 | + let size = layout.size.bytes(); |
| 180 | + |
| 181 | + // 1. Skip ZST types (no actual move/copy happens) |
| 182 | + if layout.is_zst() { |
| 183 | + return None; |
| 184 | + } |
| 185 | + |
| 186 | + // 2. Check size threshold (only instrument large moves/copies) |
| 187 | + if size < size_limit { |
| 188 | + return None; |
| 189 | + } |
| 190 | + |
| 191 | + // 3. Skip scalar/vector types that won't generate memcpy |
| 192 | + match layout.layout.backend_repr { |
| 193 | + rustc_abi::BackendRepr::Scalar(_) |
| 194 | + | rustc_abi::BackendRepr::ScalarPair(_, _) |
| 195 | + | rustc_abi::BackendRepr::SimdVector { .. } => None, |
| 196 | + _ => Some(size), |
| 197 | + } |
| 198 | + } |
| 199 | + |
| 200 | + /// Creates an inlined scope that makes operations appear to come from |
| 201 | + /// the specified compiler intrinsic function. |
| 202 | + fn create_inlined_scope<'tcx>( |
| 203 | + &self, |
| 204 | + tcx: TyCtxt<'tcx>, |
| 205 | + typing_env: TypingEnv<'tcx>, |
| 206 | + source_scopes: &mut IndexVec<SourceScope, SourceScopeData<'tcx>>, |
| 207 | + original_source_info: &SourceInfo, |
| 208 | + operation: Operation, |
| 209 | + ty: Ty<'tcx>, |
| 210 | + type_size: u64, |
| 211 | + ) -> SourceScope { |
| 212 | + let intrinsic_def_id = match operation { |
| 213 | + Operation::Move => tcx.get_diagnostic_item(sym::compiler_move), |
| 214 | + Operation::Copy => tcx.get_diagnostic_item(sym::compiler_copy), |
| 215 | + }; |
| 216 | + |
| 217 | + let Some(intrinsic_def_id) = intrinsic_def_id else { |
| 218 | + // Shouldn't happen, but just return original scope if it does |
| 219 | + return original_source_info.scope; |
| 220 | + }; |
| 221 | + |
| 222 | + // Monomorphize the intrinsic for the actual type being moved/copied + size const parameter |
| 223 | + // compiler_move<T, const SIZE: usize> or compiler_copy<T, const SIZE: usize> |
| 224 | + let size_const = ty::Const::from_target_usize(tcx, type_size); |
| 225 | + let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]); |
| 226 | + let intrinsic_instance = Instance::expect_resolve( |
| 227 | + tcx, |
| 228 | + typing_env, |
| 229 | + intrinsic_def_id, |
| 230 | + generic_args, |
| 231 | + original_source_info.span, |
| 232 | + ); |
| 233 | + |
| 234 | + // Create new inlined scope that makes the operation appear to come from the intrinsic |
| 235 | + let inlined_scope_data = SourceScopeData { |
| 236 | + span: original_source_info.span, |
| 237 | + parent_scope: Some(original_source_info.scope), |
| 238 | + |
| 239 | + // Pretend this op is inlined from the intrinsic |
| 240 | + inlined: Some((intrinsic_instance, original_source_info.span)), |
| 241 | + |
| 242 | + // Proper inlined scope chaining to maintain debug info hierarchy |
| 243 | + inlined_parent_scope: { |
| 244 | + let parent_scope = &source_scopes[original_source_info.scope]; |
| 245 | + if parent_scope.inlined.is_some() { |
| 246 | + // If parent is already inlined, chain through it |
| 247 | + Some(original_source_info.scope) |
| 248 | + } else { |
| 249 | + // Otherwise, use the parent's inlined_parent_scope |
| 250 | + parent_scope.inlined_parent_scope |
| 251 | + } |
| 252 | + }, |
| 253 | + |
| 254 | + local_data: ClearCrossCrate::Clear, |
| 255 | + }; |
| 256 | + |
| 257 | + // Add the new scope |
| 258 | + source_scopes.push(inlined_scope_data) |
| 259 | + } |
| 260 | +} |
0 commit comments