From de713a80cac60164f435a4e6a7ca710f5fdccd45 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 25 Jan 2021 15:37:49 +0100 Subject: [PATCH] Replace all uses of PassMode with ArgAbi --- src/abi/comments.rs | 5 +- src/abi/mod.rs | 61 +++--------- src/abi/pass_mode.rs | 230 +++++++++++++++++++++++++++---------------- src/abi/returning.rs | 95 ++++++++++++------ 4 files changed, 232 insertions(+), 159 deletions(-) diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 01073d26e832..af42e54451b1 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use rustc_middle::mir; +use rustc_target::abi::call::ArgAbi; use cranelift_codegen::entity::EntityRef; @@ -22,7 +23,7 @@ pub(super) fn add_arg_comment<'tcx>( local: Option, local_field: Option, params: EmptySinglePair, - pass_mode: PassMode, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ty: Ty<'tcx>, ) { let local = if let Some(local) = local { @@ -42,7 +43,7 @@ pub(super) fn add_arg_comment<'tcx>( Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)), }; - let pass_mode = format!("{:?}", pass_mode); + let pass_mode = format!("{:?}", arg_abi.mode); fx.add_global_comment(format!( "{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}", kind = kind, diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 76e1987459f8..ddbef5eadfec 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -6,9 +6,10 @@ mod pass_mode; mod returning; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_target::abi::call::PassMode as RustcPassMode; use rustc_target::spec::abi::Abi; -use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; +use cranelift_codegen::ir::AbiParam; use self::pass_mode::*; use crate::prelude::*; @@ -96,7 +97,6 @@ fn clif_sig_from_fn_sig<'tcx>( tcx: TyCtxt<'tcx>, triple: &target_lexicon::Triple, sig: FnSig<'tcx>, - span: Span, is_vtable_fn: bool, requires_caller_location: bool, ) -> Signature { @@ -147,54 +147,26 @@ fn clif_sig_from_fn_sig<'tcx>( .layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit()))) .unwrap(); } - let pass_mode = get_pass_mode(tcx, layout); + let mut arg_abi = get_arg_abi(tcx, layout); if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic { - match pass_mode { - PassMode::NoPass | PassMode::ByVal(_) => {} - PassMode::ByRef { size: Some(size) } => { - let purpose = ArgumentPurpose::StructArgument(u32::try_from(size.bytes()).expect("struct too big to pass on stack")); - return EmptySinglePair::Single(AbiParam::special(pointer_ty(tcx), purpose)).into_iter(); - } - PassMode::ByValPair(_, _) | PassMode::ByRef { size: None } => { - tcx.sess.span_warn( - span, - &format!( - "Argument of type `{:?}` with pass mode `{:?}` is not yet supported \ - for non-rust abi `{}`. Calling this function may result in a crash.", - layout.ty, - pass_mode, - abi, - ), - ); - } + match arg_abi.mode { + RustcPassMode::Indirect { + ref mut on_stack, .. + } => *on_stack = true, + _ => {} } } - pass_mode.get_param_ty(tcx).map(AbiParam::new).into_iter() + arg_abi.get_abi_param(tcx).into_iter() }) .flatten(); - let (mut params, returns): (Vec<_>, Vec<_>) = match get_pass_mode( + let return_arg_abi = get_arg_abi( tcx, tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(), - ) { - PassMode::NoPass => (inputs.collect(), vec![]), - PassMode::ByVal(ret_ty) => (inputs.collect(), vec![AbiParam::new(ret_ty)]), - PassMode::ByValPair(ret_ty_a, ret_ty_b) => ( - inputs.collect(), - vec![AbiParam::new(ret_ty_a), AbiParam::new(ret_ty_b)], - ), - PassMode::ByRef { size: Some(_) } => { - ( - Some(pointer_ty(tcx)) // First param is place to put return val - .into_iter() - .map(|ty| AbiParam::special(ty, ArgumentPurpose::StructReturn)) - .chain(inputs) - .collect(), - vec![], - ) - } - PassMode::ByRef { size: None } => todo!(), - }; + ); + let (return_ptr, returns) = return_arg_abi.get_abi_return(tcx); + // Sometimes the first param is an pointer to the place where the return value needs to be stored. + let mut params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); if requires_caller_location { params.push(AbiParam::new(pointer_ty(tcx))); @@ -226,7 +198,6 @@ pub(crate) fn get_function_name_and_sig<'tcx>( tcx, triple, fn_sig, - tcx.def_span(inst.def_id()), false, inst.def.requires_caller_location(tcx), ); @@ -584,7 +555,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( nop_inst, format!( "virtual call; self arg pass mode: {:?}", - get_pass_mode(fx.tcx, args[0].layout()) + get_arg_abi(fx.tcx, args[0].layout()).mode, ), ); } @@ -647,7 +618,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( fx.tcx, fx.triple(), fn_sig, - span, is_virtual_call, false, // calls through function pointers never pass the caller location ); @@ -723,7 +693,6 @@ pub(crate) fn codegen_drop<'tcx>( fx.tcx, fx.triple(), fn_sig, - span, true, false, // `drop_in_place` is never `#[track_caller]` ); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 2d2410f2d6ca..51fc4ecd1ef0 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -2,17 +2,10 @@ use crate::prelude::*; +use cranelift_codegen::ir::ArgumentPurpose; use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode}; pub(super) use EmptySinglePair::*; -#[derive(Copy, Clone, Debug)] -pub(super) enum PassMode { - NoPass, - ByVal(Type), - ByValPair(Type, Type), - ByRef { size: Option }, -} - #[derive(Copy, Clone, Debug)] pub(super) enum EmptySinglePair { Empty, @@ -67,19 +60,126 @@ impl EmptySinglePair { } } -impl PassMode { - pub(super) fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair { - match self { - PassMode::NoPass => Empty, - PassMode::ByVal(clif_type) => Single(clif_type), - PassMode::ByValPair(a, b) => Pair(a, b), - PassMode::ByRef { size: Some(_) } => Single(pointer_ty(tcx)), - PassMode::ByRef { size: None } => Pair(pointer_ty(tcx), pointer_ty(tcx)), +pub(super) trait ArgAbiExt<'tcx> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair; + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec); +} + +impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair { + match self.mode { + RustcPassMode::Ignore => EmptySinglePair::Empty, + RustcPassMode::Direct(_) => match &self.layout.abi { + Abi::Scalar(scalar) => { + EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))) + } + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + EmptySinglePair::Single(AbiParam::new(vector_ty)) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Pair(_, _) => match &self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + EmptySinglePair::Pair(AbiParam::new(a), AbiParam::new(b)) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack, + } => { + if on_stack { + let size = u32::try_from(self.layout.size.bytes()).unwrap(); + EmptySinglePair::Single(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructArgument(size), + )) + } else { + EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))) + } + } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack, + } => { + assert!(!on_stack); + EmptySinglePair::Pair( + AbiParam::new(pointer_ty(tcx)), + AbiParam::new(pointer_ty(tcx)), + ) + } + } + } + + fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec) { + match self.mode { + RustcPassMode::Ignore => (None, vec![]), + RustcPassMode::Direct(_) => match &self.layout.abi { + Abi::Scalar(scalar) => ( + None, + vec![AbiParam::new(scalar_to_clif_type( + tcx, + scalar.clone(), + ))], + ), + // FIXME implement Vector Abi in a cg_llvm compatible way + Abi::Vector { .. } => { + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + (None, vec![AbiParam::new(vector_ty)]) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Pair(_, _) => match &self.layout.abi { + Abi::ScalarPair(a, b) => { + let a = scalar_to_clif_type(tcx, a.clone()); + let b = scalar_to_clif_type(tcx, b.clone()); + ( + None, + vec![AbiParam::new(a), AbiParam::new(b)], + ) + } + _ => unreachable!("{:?}", self.layout.abi), + }, + RustcPassMode::Cast(_) => ( + Some(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructReturn, + )), + vec![], + ), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack, + } => { + assert!(!on_stack); + ( + Some(AbiParam::special( + pointer_ty(tcx), + ArgumentPurpose::StructReturn, + )), + vec![], + ) + } + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), } } } -pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode { +pub(super) fn get_arg_abi<'tcx>( + tcx: TyCtxt<'tcx>, + layout: TyAndLayout<'tcx>, +) -> ArgAbi<'tcx, Ty<'tcx>> { let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new()); if layout.is_zst() { // WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer @@ -88,7 +188,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) match arg_abi.mode { RustcPassMode::Ignore => {} RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(_) => {}, + Abi::Scalar(_) => {} // FIXME implement Vector Abi in a cg_llvm compatible way Abi::Vector { .. } => { if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() { @@ -99,7 +199,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) }; } } - _ => unreachable!("{:?}", arg_abi.layout.abi) + _ => unreachable!("{:?}", arg_abi.layout.abi), }, RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { Abi::ScalarPair(a, b) => { @@ -113,54 +213,11 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) }; } } - _ => unreachable!("{:?}", arg_abi.layout.abi) + _ => unreachable!("{:?}", arg_abi.layout.abi), }, _ => {} } - match arg_abi.mode { - RustcPassMode::Ignore => PassMode::NoPass, - RustcPassMode::Direct(_) => match &arg_abi.layout.abi { - Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())), - // FIXME implement Vector Abi in a cg_llvm compatible way - Abi::Vector { .. } => { - let vector_ty = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).unwrap(); - PassMode::ByVal(vector_ty) - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi { - Abi::ScalarPair(a, b) => { - let a = scalar_to_clif_type(tcx, a.clone()); - let b = scalar_to_clif_type(tcx, b.clone()); - PassMode::ByValPair(a, b) - } - _ => unreachable!("{:?}", arg_abi.layout.abi) - }, - RustcPassMode::Cast(_) | RustcPassMode::Indirect { - attrs: _, - extra_attrs: None, - on_stack: false, - } => PassMode::ByRef { - size: Some(arg_abi.layout.size), - }, - RustcPassMode::Indirect { - attrs: _, - extra_attrs, - on_stack: true, - } => { - assert!(extra_attrs.is_none()); - PassMode::ByRef { - size: Some(arg_abi.layout.size) - } - } - RustcPassMode::Indirect { - attrs: _, - extra_attrs: Some(_), - on_stack: false, - } => PassMode::ByRef { - size: None, - }, - } + arg_abi } /// Get a set of values to be passed as function arguments. @@ -168,14 +225,15 @@ pub(super) fn adjust_arg_for_abi<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Module>, arg: CValue<'tcx>, ) -> EmptySinglePair { - match get_pass_mode(fx.tcx, arg.layout()) { - PassMode::NoPass => Empty, - PassMode::ByVal(_) => Single(arg.load_scalar(fx)), - PassMode::ByValPair(_, _) => { + let arg_abi = get_arg_abi(fx.tcx, arg.layout()); + match arg_abi.mode { + RustcPassMode::Ignore => Empty, + RustcPassMode::Direct(_) => Single(arg.load_scalar(fx)), + RustcPassMode::Pair(_, _) => { let (a, b) = arg.load_scalar_pair(fx); Pair(a, b) } - PassMode::ByRef { size: _ } => match arg.force_stack(fx) { + RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => match arg.force_stack(fx) { (ptr, None) => Single(ptr.get_addr(fx)), (ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta), }, @@ -192,14 +250,11 @@ pub(super) fn cvalue_for_param<'tcx>( arg_ty: Ty<'tcx>, ) -> Option> { let layout = fx.layout_of(arg_ty); - let pass_mode = get_pass_mode(fx.tcx, layout); + let arg_abi = get_arg_abi(fx.tcx, layout); - if let PassMode::NoPass = pass_mode { - return None; - } - - let clif_types = pass_mode.get_param_ty(fx.tcx); - let block_params = clif_types.map(|t| fx.bcx.append_block_param(start_block, t)); + let clif_types = arg_abi.get_abi_param(fx.tcx); + let block_params = + clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type)); #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( @@ -208,22 +263,31 @@ pub(super) fn cvalue_for_param<'tcx>( local, local_field, block_params, - pass_mode, + &arg_abi, arg_ty, ); - match pass_mode { - PassMode::NoPass => unreachable!(), - PassMode::ByVal(_) => Some(CValue::by_val(block_params.assert_single(), layout)), - PassMode::ByValPair(_, _) => { + match arg_abi.mode { + RustcPassMode::Ignore => None, + RustcPassMode::Direct(_) => Some(CValue::by_val(block_params.assert_single(), layout)), + RustcPassMode::Pair(_, _) => { let (a, b) = block_params.assert_pair(); Some(CValue::by_val_pair(a, b, layout)) } - PassMode::ByRef { size: Some(_) } => Some(CValue::by_ref( + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => Some(CValue::by_ref( Pointer::new(block_params.assert_single()), layout, )), - PassMode::ByRef { size: None } => { + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => { let (ptr, meta) = block_params.assert_pair(); Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout)) } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index f6d40c880d09..9edaa1dd879f 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -3,6 +3,8 @@ use crate::abi::pass_mode::*; use crate::prelude::*; +use rustc_target::abi::call::PassMode as RustcPassMode; + fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> { fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty)) } @@ -12,10 +14,10 @@ pub(crate) fn can_return_to_ssa_var<'tcx>( tcx: TyCtxt<'tcx>, dest_layout: TyAndLayout<'tcx>, ) -> bool { - match get_pass_mode(tcx, dest_layout) { - PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => true, - // FIXME Make it possible to return ByRef to an ssa var. - PassMode::ByRef { size: _ } => false, + match get_arg_abi(tcx, dest_layout).mode { + RustcPassMode::Ignore | RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => true, + // FIXME Make it possible to return Cast and Indirect to an ssa var. + RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => false, } } @@ -27,24 +29,33 @@ pub(super) fn codegen_return_param<'tcx>( start_block: Block, ) -> CPlace<'tcx> { let ret_layout = return_layout(fx); - let ret_pass_mode = get_pass_mode(fx.tcx, ret_layout); - let (ret_place, ret_param) = match ret_pass_mode { - PassMode::NoPass => (CPlace::no_place(ret_layout), Empty), - PassMode::ByVal(_) | PassMode::ByValPair(_, _) => { + let ret_arg_abi = get_arg_abi(fx.tcx, ret_layout); + let (ret_place, ret_param) = match ret_arg_abi.mode { + RustcPassMode::Ignore => (CPlace::no_place(ret_layout), Empty), + RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa), Empty, ) } - PassMode::ByRef { size: Some(_) } => { + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => { let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type); ( CPlace::for_ptr(Pointer::new(ret_param), ret_layout), Single(ret_param), ) } - PassMode::ByRef { size: None } => todo!(), + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), }; #[cfg(not(debug_assertions))] @@ -57,7 +68,7 @@ pub(super) fn codegen_return_param<'tcx>( Some(RETURN_PLACE), None, ret_param, - ret_pass_mode, + &ret_arg_abi, ret_layout.ty, ); @@ -74,36 +85,54 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( ) -> (Inst, T) { let ret_layout = fx.layout_of(fn_sig.output()); - let output_pass_mode = get_pass_mode(fx.tcx, ret_layout); - let return_ptr = match output_pass_mode { - PassMode::NoPass => None, - PassMode::ByRef { size: Some(_) } => match ret_place { + let output_arg_abi = get_arg_abi(fx.tcx, ret_layout); + let return_ptr = match output_arg_abi.mode { + RustcPassMode::Ignore => None, + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => match ret_place { Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)), None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot }, - PassMode::ByRef { size: None } => todo!(), - PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None, + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), + RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => None, }; let (call_inst, meta) = f(fx, return_ptr); - match output_pass_mode { - PassMode::NoPass => {} - PassMode::ByVal(_) => { + match output_arg_abi.mode { + RustcPassMode::Ignore => {} + RustcPassMode::Direct(_) => { if let Some(ret_place) = ret_place { let ret_val = fx.bcx.inst_results(call_inst)[0]; ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout)); } } - PassMode::ByValPair(_, _) => { + RustcPassMode::Pair(_, _) => { if let Some(ret_place) = ret_place { let ret_val_a = fx.bcx.inst_results(call_inst)[0]; let ret_val_b = fx.bcx.inst_results(call_inst)[1]; ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout)); } } - PassMode::ByRef { size: Some(_) } => {} - PassMode::ByRef { size: None } => todo!(), + RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => {} + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), } (call_inst, meta) @@ -111,17 +140,27 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) { - match get_pass_mode(fx.tcx, return_layout(fx)) { - PassMode::NoPass | PassMode::ByRef { size: Some(_) } => { + match get_arg_abi(fx.tcx, return_layout(fx)).mode { + RustcPassMode::Ignore + | RustcPassMode::Cast(_) + | RustcPassMode::Indirect { + attrs: _, + extra_attrs: None, + on_stack: _, + } => { fx.bcx.ins().return_(&[]); } - PassMode::ByRef { size: None } => todo!(), - PassMode::ByVal(_) => { + RustcPassMode::Indirect { + attrs: _, + extra_attrs: Some(_), + on_stack: _, + } => unreachable!("unsized return value"), + RustcPassMode::Direct(_) => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx).load_scalar(fx); fx.bcx.ins().return_(&[ret_val]); } - PassMode::ByValPair(_, _) => { + RustcPassMode::Pair(_, _) => { let place = fx.get_local_place(RETURN_PLACE); let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);