user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replace all uses of PassMode with ArgAbi

Browse source
This commit is contained in:
bjorn3 2021-01-25 15:37:49 +01:00
parent ff3304285a
commit de713a80ca
4 changed files with 232 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/abi/comments.rs
View file

@ -4,6 +4,7 @@
use std::borrow::Cow;
use rustc_middle::mir;
use rustc_target::abi::call::ArgAbi;
use cranelift_codegen::entity::EntityRef;
@ -22,7 +23,7 @@ pub(super) fn add_arg_comment<'tcx>(
local: Option<mir::Local>,
local_field: Option<usize>,
params: EmptySinglePair<Value>,
pass_mode: PassMode,
arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
ty: Ty<'tcx>,
) {
let local = if let Some(local) = local {
@ -42,7 +43,7 @@ pub(super) fn add_arg_comment<'tcx>(
Pair(param_a, param_b) => Cow::Owned(format!("= {:?}, {:?}", param_a, param_b)),
};
let pass_mode = format!("{:?}", pass_mode);
let pass_mode = format!("{:?}", arg_abi.mode);
fx.add_global_comment(format!(
"{kind:5}{local:>3}{local_field:<5} {params:10} {pass_mode:36} {ty:?}",
kind = kind,

61
src/abi/mod.rs
View file

@ -6,9 +6,10 @@ mod pass_mode;
mod returning;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_target::abi::call::PassMode as RustcPassMode;
use rustc_target::spec::abi::Abi;
use cranelift_codegen::ir::{AbiParam, ArgumentPurpose};
use cranelift_codegen::ir::AbiParam;
use self::pass_mode::*;
use crate::prelude::*;
@ -96,7 +97,6 @@ fn clif_sig_from_fn_sig<'tcx>(
tcx: TyCtxt<'tcx>,
triple: &target_lexicon::Triple,
sig: FnSig<'tcx>,
span: Span,
is_vtable_fn: bool,
requires_caller_location: bool,
) -> Signature {
@ -147,54 +147,26 @@ fn clif_sig_from_fn_sig<'tcx>(
.layout_of(ParamEnv::reveal_all().and(tcx.mk_mut_ptr(tcx.mk_unit())))
.unwrap();
}
let pass_mode = get_pass_mode(tcx, layout);
let mut arg_abi = get_arg_abi(tcx, layout);
if abi != Abi::Rust && abi != Abi::RustCall && abi != Abi::RustIntrinsic {
match pass_mode {
PassMode::NoPass | PassMode::ByVal(_) => {}
PassMode::ByRef { size: Some(size) } => {
let purpose = ArgumentPurpose::StructArgument(u32::try_from(size.bytes()).expect("struct too big to pass on stack"));
return EmptySinglePair::Single(AbiParam::special(pointer_ty(tcx), purpose)).into_iter();
}
PassMode::ByValPair(_, _) | PassMode::ByRef { size: None } => {
tcx.sess.span_warn(
span,
&format!(
"Argument of type `{:?}` with pass mode `{:?}` is not yet supported \
for non-rust abi `{}`. Calling this function may result in a crash.",
layout.ty,
pass_mode,
abi,
),
);
}
match arg_abi.mode {
RustcPassMode::Indirect {
ref mut on_stack, ..
} => *on_stack = true,
_ => {}
}
}
pass_mode.get_param_ty(tcx).map(AbiParam::new).into_iter()
arg_abi.get_abi_param(tcx).into_iter()
})
.flatten();
let (mut params, returns): (Vec<_>, Vec<_>) = match get_pass_mode(
let return_arg_abi = get_arg_abi(
tcx,
tcx.layout_of(ParamEnv::reveal_all().and(output)).unwrap(),
) {
PassMode::NoPass => (inputs.collect(), vec![]),
PassMode::ByVal(ret_ty) => (inputs.collect(), vec![AbiParam::new(ret_ty)]),
PassMode::ByValPair(ret_ty_a, ret_ty_b) => (
inputs.collect(),
vec![AbiParam::new(ret_ty_a), AbiParam::new(ret_ty_b)],
),
PassMode::ByRef { size: Some(_) } => {
(
Some(pointer_ty(tcx)) // First param is place to put return val
.into_iter()
.map(|ty| AbiParam::special(ty, ArgumentPurpose::StructReturn))
.chain(inputs)
.collect(),
vec![],
)
}
PassMode::ByRef { size: None } => todo!(),
};
);
let (return_ptr, returns) = return_arg_abi.get_abi_return(tcx);
// Sometimes the first param is an pointer to the place where the return value needs to be stored.
let mut params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
if requires_caller_location {
params.push(AbiParam::new(pointer_ty(tcx)));
@ -226,7 +198,6 @@ pub(crate) fn get_function_name_and_sig<'tcx>(
tcx,
triple,
fn_sig,
tcx.def_span(inst.def_id()),
false,
inst.def.requires_caller_location(tcx),
);
@ -584,7 +555,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
nop_inst,
format!(
"virtual call; self arg pass mode: {:?}",
get_pass_mode(fx.tcx, args[0].layout())
get_arg_abi(fx.tcx, args[0].layout()).mode,
),
);
}
@ -647,7 +618,6 @@ pub(crate) fn codegen_terminator_call<'tcx>(
fx.tcx,
fx.triple(),
fn_sig,
span,
is_virtual_call,
false, // calls through function pointers never pass the caller location
);
@ -723,7 +693,6 @@ pub(crate) fn codegen_drop<'tcx>(
fx.tcx,
fx.triple(),
fn_sig,
span,
true,
false, // `drop_in_place` is never `#[track_caller]`
);

230
src/abi/pass_mode.rs
View file

@ -2,17 +2,10 @@
use crate::prelude::*;
use cranelift_codegen::ir::ArgumentPurpose;
use rustc_target::abi::call::{ArgAbi, ArgAttributes, PassMode as RustcPassMode};
pub(super) use EmptySinglePair::*;
#[derive(Copy, Clone, Debug)]
pub(super) enum PassMode {
NoPass,
ByVal(Type),
ByValPair(Type, Type),
ByRef { size: Option<Size> },
}
#[derive(Copy, Clone, Debug)]
pub(super) enum EmptySinglePair<T> {
Empty,
@ -67,19 +60,126 @@ impl<T: std::fmt::Debug> EmptySinglePair<T> {
}
}
impl PassMode {
pub(super) fn get_param_ty(self, tcx: TyCtxt<'_>) -> EmptySinglePair<Type> {
match self {
PassMode::NoPass => Empty,
PassMode::ByVal(clif_type) => Single(clif_type),
PassMode::ByValPair(a, b) => Pair(a, b),
PassMode::ByRef { size: Some(_) } => Single(pointer_ty(tcx)),
PassMode::ByRef { size: None } => Pair(pointer_ty(tcx), pointer_ty(tcx)),
pub(super) trait ArgAbiExt<'tcx> {
fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair<AbiParam>;
fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>);
}
impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> EmptySinglePair<AbiParam> {
match self.mode {
RustcPassMode::Ignore => EmptySinglePair::Empty,
RustcPassMode::Direct(_) => match &self.layout.abi {
Abi::Scalar(scalar) => {
EmptySinglePair::Single(AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())))
}
Abi::Vector { .. } => {
let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
EmptySinglePair::Single(AbiParam::new(vector_ty))
}
_ => unreachable!("{:?}", self.layout.abi),
},
RustcPassMode::Pair(_, _) => match &self.layout.abi {
Abi::ScalarPair(a, b) => {
let a = scalar_to_clif_type(tcx, a.clone());
let b = scalar_to_clif_type(tcx, b.clone());
EmptySinglePair::Pair(AbiParam::new(a), AbiParam::new(b))
}
_ => unreachable!("{:?}", self.layout.abi),
},
RustcPassMode::Cast(_) => EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx))),
RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack,
} => {
if on_stack {
let size = u32::try_from(self.layout.size.bytes()).unwrap();
EmptySinglePair::Single(AbiParam::special(
pointer_ty(tcx),
ArgumentPurpose::StructArgument(size),
))
} else {
EmptySinglePair::Single(AbiParam::new(pointer_ty(tcx)))
}
}
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack,
} => {
assert!(!on_stack);
EmptySinglePair::Pair(
AbiParam::new(pointer_ty(tcx)),
AbiParam::new(pointer_ty(tcx)),
)
}
}
}
fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>) {
match self.mode {
RustcPassMode::Ignore => (None, vec![]),
RustcPassMode::Direct(_) => match &self.layout.abi {
Abi::Scalar(scalar) => (
None,
vec![AbiParam::new(scalar_to_clif_type(
tcx,
scalar.clone(),
))],
),
// FIXME implement Vector Abi in a cg_llvm compatible way
Abi::Vector { .. } => {
let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
(None, vec![AbiParam::new(vector_ty)])
}
_ => unreachable!("{:?}", self.layout.abi),
},
RustcPassMode::Pair(_, _) => match &self.layout.abi {
Abi::ScalarPair(a, b) => {
let a = scalar_to_clif_type(tcx, a.clone());
let b = scalar_to_clif_type(tcx, b.clone());
(
None,
vec![AbiParam::new(a), AbiParam::new(b)],
)
}
_ => unreachable!("{:?}", self.layout.abi),
},
RustcPassMode::Cast(_) => (
Some(AbiParam::special(
pointer_ty(tcx),
ArgumentPurpose::StructReturn,
)),
vec![],
),
RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack,
} => {
assert!(!on_stack);
(
Some(AbiParam::special(
pointer_ty(tcx),
ArgumentPurpose::StructReturn,
)),
vec![],
)
}
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
}
}
}
pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> PassMode {
pub(super) fn get_arg_abi<'tcx>(
tcx: TyCtxt<'tcx>,
layout: TyAndLayout<'tcx>,
) -> ArgAbi<'tcx, Ty<'tcx>> {
let mut arg_abi = ArgAbi::new(&tcx, layout, |_, _, _| ArgAttributes::new());
if layout.is_zst() {
// WARNING zst arguments must never be passed, as that will break CastKind::ClosureFnPointer
@ -88,7 +188,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>)
match arg_abi.mode {
RustcPassMode::Ignore => {}
RustcPassMode::Direct(_) => match &arg_abi.layout.abi {
Abi::Scalar(_) => {},
Abi::Scalar(_) => {}
// FIXME implement Vector Abi in a cg_llvm compatible way
Abi::Vector { .. } => {
if crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).is_none() {
@ -99,7 +199,7 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>)
};
}
}
_ => unreachable!("{:?}", arg_abi.layout.abi)
_ => unreachable!("{:?}", arg_abi.layout.abi),
},
RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi {
Abi::ScalarPair(a, b) => {
@ -113,54 +213,11 @@ pub(super) fn get_pass_mode<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>)
};
}
}
_ => unreachable!("{:?}", arg_abi.layout.abi)
_ => unreachable!("{:?}", arg_abi.layout.abi),
},
_ => {}
}
match arg_abi.mode {
RustcPassMode::Ignore => PassMode::NoPass,
RustcPassMode::Direct(_) => match &arg_abi.layout.abi {
Abi::Scalar(scalar) => PassMode::ByVal(scalar_to_clif_type(tcx, scalar.clone())),
// FIXME implement Vector Abi in a cg_llvm compatible way
Abi::Vector { .. } => {
let vector_ty = crate::intrinsics::clif_vector_type(tcx, arg_abi.layout).unwrap();
PassMode::ByVal(vector_ty)
}
_ => unreachable!("{:?}", arg_abi.layout.abi)
},
RustcPassMode::Pair(_, _) => match &arg_abi.layout.abi {
Abi::ScalarPair(a, b) => {
let a = scalar_to_clif_type(tcx, a.clone());
let b = scalar_to_clif_type(tcx, b.clone());
PassMode::ByValPair(a, b)
}
_ => unreachable!("{:?}", arg_abi.layout.abi)
},
RustcPassMode::Cast(_) | RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: false,
} => PassMode::ByRef {
size: Some(arg_abi.layout.size),
},
RustcPassMode::Indirect {
attrs: _,
extra_attrs,
on_stack: true,
} => {
assert!(extra_attrs.is_none());
PassMode::ByRef {
size: Some(arg_abi.layout.size)
}
}
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: false,
} => PassMode::ByRef {
size: None,
},
}
arg_abi
}
/// Get a set of values to be passed as function arguments.
@ -168,14 +225,15 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
fx: &mut FunctionCx<'_, 'tcx, impl Module>,
arg: CValue<'tcx>,
) -> EmptySinglePair<Value> {
match get_pass_mode(fx.tcx, arg.layout()) {
PassMode::NoPass => Empty,
PassMode::ByVal(_) => Single(arg.load_scalar(fx)),
PassMode::ByValPair(_, _) => {
let arg_abi = get_arg_abi(fx.tcx, arg.layout());
match arg_abi.mode {
RustcPassMode::Ignore => Empty,
RustcPassMode::Direct(_) => Single(arg.load_scalar(fx)),
RustcPassMode::Pair(_, _) => {
let (a, b) = arg.load_scalar_pair(fx);
Pair(a, b)
}
PassMode::ByRef { size: _ } => match arg.force_stack(fx) {
RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => match arg.force_stack(fx) {
(ptr, None) => Single(ptr.get_addr(fx)),
(ptr, Some(meta)) => Pair(ptr.get_addr(fx), meta),
},
@ -192,14 +250,11 @@ pub(super) fn cvalue_for_param<'tcx>(
arg_ty: Ty<'tcx>,
) -> Option<CValue<'tcx>> {
let layout = fx.layout_of(arg_ty);
let pass_mode = get_pass_mode(fx.tcx, layout);
let arg_abi = get_arg_abi(fx.tcx, layout);
if let PassMode::NoPass = pass_mode {
return None;
}
let clif_types = pass_mode.get_param_ty(fx.tcx);
let block_params = clif_types.map(|t| fx.bcx.append_block_param(start_block, t));
let clif_types = arg_abi.get_abi_param(fx.tcx);
let block_params =
clif_types.map(|abi_param| fx.bcx.append_block_param(start_block, abi_param.value_type));
#[cfg(debug_assertions)]
crate::abi::comments::add_arg_comment(
@ -208,22 +263,31 @@ pub(super) fn cvalue_for_param<'tcx>(
local,
local_field,
block_params,
pass_mode,
&arg_abi,
arg_ty,
);
match pass_mode {
PassMode::NoPass => unreachable!(),
PassMode::ByVal(_) => Some(CValue::by_val(block_params.assert_single(), layout)),
PassMode::ByValPair(_, _) => {
match arg_abi.mode {
RustcPassMode::Ignore => None,
RustcPassMode::Direct(_) => Some(CValue::by_val(block_params.assert_single(), layout)),
RustcPassMode::Pair(_, _) => {
let (a, b) = block_params.assert_pair();
Some(CValue::by_val_pair(a, b, layout))
}
PassMode::ByRef { size: Some(_) } => Some(CValue::by_ref(
RustcPassMode::Cast(_)
| RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => Some(CValue::by_ref(
Pointer::new(block_params.assert_single()),
layout,
)),
PassMode::ByRef { size: None } => {
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => {
let (ptr, meta) = block_params.assert_pair();
Some(CValue::by_ref_unsized(Pointer::new(ptr), meta, layout))
}

95
src/abi/returning.rs
View file

@ -3,6 +3,8 @@
use crate::abi::pass_mode::*;
use crate::prelude::*;
use rustc_target::abi::call::PassMode as RustcPassMode;
fn return_layout<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> TyAndLayout<'tcx> {
fx.layout_of(fx.monomorphize(&fx.mir.local_decls[RETURN_PLACE].ty))
}
@ -12,10 +14,10 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
tcx: TyCtxt<'tcx>,
dest_layout: TyAndLayout<'tcx>,
) -> bool {
match get_pass_mode(tcx, dest_layout) {
PassMode::NoPass | PassMode::ByVal(_) | PassMode::ByValPair(_, _) => true,
// FIXME Make it possible to return ByRef to an ssa var.
PassMode::ByRef { size: _ } => false,
match get_arg_abi(tcx, dest_layout).mode {
RustcPassMode::Ignore | RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => true,
// FIXME Make it possible to return Cast and Indirect to an ssa var.
RustcPassMode::Cast(_) | RustcPassMode::Indirect { .. } => false,
}
}
@ -27,24 +29,33 @@ pub(super) fn codegen_return_param<'tcx>(
start_block: Block,
) -> CPlace<'tcx> {
let ret_layout = return_layout(fx);
let ret_pass_mode = get_pass_mode(fx.tcx, ret_layout);
let (ret_place, ret_param) = match ret_pass_mode {
PassMode::NoPass => (CPlace::no_place(ret_layout), Empty),
PassMode::ByVal(_) | PassMode::ByValPair(_, _) => {
let ret_arg_abi = get_arg_abi(fx.tcx, ret_layout);
let (ret_place, ret_param) = match ret_arg_abi.mode {
RustcPassMode::Ignore => (CPlace::no_place(ret_layout), Empty),
RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => {
let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
(
super::make_local_place(fx, RETURN_PLACE, ret_layout, is_ssa),
Empty,
)
}
PassMode::ByRef { size: Some(_) } => {
RustcPassMode::Cast(_)
| RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {
let ret_param = fx.bcx.append_block_param(start_block, fx.pointer_type);
(
CPlace::for_ptr(Pointer::new(ret_param), ret_layout),
Single(ret_param),
)
}
PassMode::ByRef { size: None } => todo!(),
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
};
#[cfg(not(debug_assertions))]
@ -57,7 +68,7 @@ pub(super) fn codegen_return_param<'tcx>(
Some(RETURN_PLACE),
None,
ret_param,
ret_pass_mode,
&ret_arg_abi,
ret_layout.ty,
);
@ -74,36 +85,54 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
) -> (Inst, T) {
let ret_layout = fx.layout_of(fn_sig.output());
let output_pass_mode = get_pass_mode(fx.tcx, ret_layout);
let return_ptr = match output_pass_mode {
PassMode::NoPass => None,
PassMode::ByRef { size: Some(_) } => match ret_place {
let output_arg_abi = get_arg_abi(fx.tcx, ret_layout);
let return_ptr = match output_arg_abi.mode {
RustcPassMode::Ignore => None,
RustcPassMode::Cast(_)
| RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => match ret_place {
Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)),
None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot
},
PassMode::ByRef { size: None } => todo!(),
PassMode::ByVal(_) | PassMode::ByValPair(_, _) => None,
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
RustcPassMode::Direct(_) | RustcPassMode::Pair(_, _) => None,
};
let (call_inst, meta) = f(fx, return_ptr);
match output_pass_mode {
PassMode::NoPass => {}
PassMode::ByVal(_) => {
match output_arg_abi.mode {
RustcPassMode::Ignore => {}
RustcPassMode::Direct(_) => {
if let Some(ret_place) = ret_place {
let ret_val = fx.bcx.inst_results(call_inst)[0];
ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_layout));
}
}
PassMode::ByValPair(_, _) => {
RustcPassMode::Pair(_, _) => {
if let Some(ret_place) = ret_place {
let ret_val_a = fx.bcx.inst_results(call_inst)[0];
let ret_val_b = fx.bcx.inst_results(call_inst)[1];
ret_place.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_layout));
}
}
PassMode::ByRef { size: Some(_) } => {}
PassMode::ByRef { size: None } => todo!(),
RustcPassMode::Cast(_)
| RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {}
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
}
(call_inst, meta)
@ -111,17 +140,27 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
/// Codegen a return instruction with the right return value(s) if any.
pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) {
match get_pass_mode(fx.tcx, return_layout(fx)) {
PassMode::NoPass | PassMode::ByRef { size: Some(_) } => {
match get_arg_abi(fx.tcx, return_layout(fx)).mode {
RustcPassMode::Ignore
| RustcPassMode::Cast(_)
| RustcPassMode::Indirect {
attrs: _,
extra_attrs: None,
on_stack: _,
} => {
fx.bcx.ins().return_(&[]);
}
PassMode::ByRef { size: None } => todo!(),
PassMode::ByVal(_) => {
RustcPassMode::Indirect {
attrs: _,
extra_attrs: Some(_),
on_stack: _,
} => unreachable!("unsized return value"),
RustcPassMode::Direct(_) => {
let place = fx.get_local_place(RETURN_PLACE);
let ret_val = place.to_cvalue(fx).load_scalar(fx);
fx.bcx.ins().return_(&[ret_val]);
}
PassMode::ByValPair(_, _) => {
RustcPassMode::Pair(_, _) => {
let place = fx.get_local_place(RETURN_PLACE);
let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx);
fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);