From 044a3a65a00e568b66e4bcdf7ee7393105aff6dd Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 4 Jan 2023 17:19:52 +0000 Subject: [PATCH 1/3] Support const and sym operands in inline asm --- src/inline_asm.rs | 166 +++++++++++++++++++++++++++++++++------------- 1 file changed, 121 insertions(+), 45 deletions(-) diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 3fcc84d39295..aa0b51181f40 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -6,12 +6,37 @@ use std::fmt::Write; use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_middle::mir::InlineAsmOperand; +use rustc_middle::ty::SymbolName; use rustc_span::sym; use rustc_target::asm::*; +enum CInlineAsmOperand<'tcx> { + In { + reg: InlineAsmRegOrRegClass, + value: CValue<'tcx>, + }, + Out { + reg: InlineAsmRegOrRegClass, + late: bool, + place: Option>, + }, + InOut { + reg: InlineAsmRegOrRegClass, + _late: bool, + in_value: CValue<'tcx>, + out_place: Option>, + }, + Const { + value: String, + }, + Symbol { + symbol: SymbolName<'tcx>, + }, +} + pub(crate) fn codegen_inline_asm<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - _span: Span, + span: Span, template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperand<'tcx>], options: InlineAsmOptions, @@ -198,6 +223,59 @@ pub(crate) fn codegen_inline_asm<'tcx>( } } + let operands = operands + .into_iter() + .map(|operand| match *operand { + InlineAsmOperand::In { reg, ref value } => { + CInlineAsmOperand::In { reg, value: crate::base::codegen_operand(fx, value) } + } + InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out { + reg, + late, + place: place.map(|place| crate::base::codegen_place(fx, place)), + }, + InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => { + CInlineAsmOperand::InOut { + reg, + _late: late, + in_value: crate::base::codegen_operand(fx, in_value), + out_place: out_place.map(|place| crate::base::codegen_place(fx, place)), + } + } + InlineAsmOperand::Const { ref value } => { + let (const_value, ty) = crate::constant::eval_mir_constant(fx, &*value) + .unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved")); + let value = rustc_codegen_ssa::common::asm_const_to_str( + fx.tcx, + span, + const_value, + fx.layout_of(ty), + ); + CInlineAsmOperand::Const { value } + } + InlineAsmOperand::SymFn { ref value } => { + let literal = fx.monomorphize(value.literal); + if let ty::FnDef(def_id, substs) = *literal.ty().kind() { + let instance = ty::Instance::resolve_for_fn_ptr( + fx.tcx, + ty::ParamEnv::reveal_all(), + def_id, + substs, + ) + .unwrap(); + CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance) } + } else { + span_bug!(span, "invalid type for asm sym (fn)"); + } + } + InlineAsmOperand::SymStatic { def_id } => { + assert!(fx.tcx.is_static(def_id)); + let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); + CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance) } + } + }) + .collect::>(); + let mut inputs = Vec::new(); let mut outputs = Vec::new(); @@ -206,7 +284,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( arch: fx.tcx.sess.asm_arch.unwrap(), enclosing_def_id: fx.instance.def_id(), template, - operands, + operands: &operands, options, registers: Vec::new(), stack_slots_clobber: Vec::new(), @@ -229,36 +307,22 @@ pub(crate) fn codegen_inline_asm<'tcx>( fx.cx.global_asm.push_str(&generated_asm); for (i, operand) in operands.iter().enumerate() { - match *operand { - InlineAsmOperand::In { reg: _, ref value } => { - inputs.push(( - asm_gen.stack_slots_input[i].unwrap(), - crate::base::codegen_operand(fx, value).load_scalar(fx), - )); + match operand { + CInlineAsmOperand::In { reg: _, value } => { + inputs.push((asm_gen.stack_slots_input[i].unwrap(), value.load_scalar(fx))); } - InlineAsmOperand::Out { reg: _, late: _, place } => { + CInlineAsmOperand::Out { reg: _, late: _, place } => { if let Some(place) = place { - outputs.push(( - asm_gen.stack_slots_output[i].unwrap(), - crate::base::codegen_place(fx, place), - )); + outputs.push((asm_gen.stack_slots_output[i].unwrap(), place.clone())); } } - InlineAsmOperand::InOut { reg: _, late: _, ref in_value, out_place } => { - inputs.push(( - asm_gen.stack_slots_input[i].unwrap(), - crate::base::codegen_operand(fx, in_value).load_scalar(fx), - )); + CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => { + inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx))); if let Some(out_place) = out_place { - outputs.push(( - asm_gen.stack_slots_output[i].unwrap(), - crate::base::codegen_place(fx, out_place), - )); + outputs.push((asm_gen.stack_slots_output[i].unwrap(), out_place.clone())); } } - InlineAsmOperand::Const { value: _ } => todo!(), - InlineAsmOperand::SymFn { value: _ } => todo!(), - InlineAsmOperand::SymStatic { def_id: _ } => todo!(), + CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {} } } @@ -280,7 +344,7 @@ struct InlineAssemblyGenerator<'a, 'tcx> { arch: InlineAsmArch, enclosing_def_id: DefId, template: &'a [InlineAsmTemplatePiece], - operands: &'a [InlineAsmOperand<'tcx>], + operands: &'a [CInlineAsmOperand<'tcx>], options: InlineAsmOptions, registers: Vec>, stack_slots_clobber: Vec>, @@ -304,18 +368,20 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Add explicit registers to the allocated set. for (i, operand) in self.operands.iter().enumerate() { match *operand { - InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { + CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { regs[i] = Some(reg); allocated.entry(reg).or_default().0 = true; } - InlineAsmOperand::Out { - reg: InlineAsmRegOrRegClass::Reg(reg), late: true, .. + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(reg), + late: true, + .. } => { regs[i] = Some(reg); allocated.entry(reg).or_default().1 = true; } - InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. } - | InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { + CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. } + | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => { regs[i] = Some(reg); allocated.insert(reg, (true, true)); } @@ -326,12 +392,12 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate out/inout/inlateout registers first because they are more constrained. for (i, operand) in self.operands.iter().enumerate() { match *operand { - InlineAsmOperand::Out { + CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::RegClass(class), late: false, .. } - | InlineAsmOperand::InOut { + | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => { let mut alloc_reg = None; @@ -360,7 +426,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate in/lateout. for (i, operand) in self.operands.iter().enumerate() { match *operand { - InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => { + CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => { let mut alloc_reg = None; for ® in &map[&class] { let mut used = false; @@ -380,7 +446,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { regs[i] = Some(reg); allocated.entry(reg).or_default().0 = true; } - InlineAsmOperand::Out { + CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::RegClass(class), late: true, .. @@ -455,7 +521,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for inout for (i, operand) in self.operands.iter().enumerate() { match *operand { - InlineAsmOperand::InOut { reg, out_place: Some(_), .. } => { + CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => { let slot = new_slot(reg.reg_class()); slots_input[i] = Some(slot); slots_output[i] = Some(slot); @@ -470,8 +536,8 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for input for (i, operand) in self.operands.iter().enumerate() { match *operand { - InlineAsmOperand::In { reg, .. } - | InlineAsmOperand::InOut { reg, out_place: None, .. } => { + CInlineAsmOperand::In { reg, .. } + | CInlineAsmOperand::InOut { reg, out_place: None, .. } => { slots_input[i] = Some(new_slot(reg.reg_class())); } _ => (), @@ -487,7 +553,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for output for (i, operand) in self.operands.iter().enumerate() { match *operand { - InlineAsmOperand::Out { reg, place: Some(_), .. } => { + CInlineAsmOperand::Out { reg, place: Some(_), .. } => { slots_output[i] = Some(new_slot(reg.reg_class())); } _ => (), @@ -549,13 +615,23 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { generated_asm.push_str(s); } InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { - if self.options.contains(InlineAsmOptions::ATT_SYNTAX) { - generated_asm.push('%'); + match self.operands[*operand_idx] { + CInlineAsmOperand::In { .. } + | CInlineAsmOperand::Out { .. } + | CInlineAsmOperand::InOut { .. } => { + if self.options.contains(InlineAsmOptions::ATT_SYNTAX) { + generated_asm.push('%'); + } + self.registers[*operand_idx] + .unwrap() + .emit(&mut generated_asm, self.arch, *modifier) + .unwrap(); + } + CInlineAsmOperand::Const { ref value } => { + generated_asm.push_str(value); + } + CInlineAsmOperand::Symbol { symbol } => generated_asm.push_str(symbol.name), } - self.registers[*operand_idx] - .unwrap() - .emit(&mut generated_asm, self.arch, *modifier) - .unwrap(); } } } From 2e93be3a4c8f59e3856c22152c3771054577b618 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 5 Feb 2023 17:24:02 +0000 Subject: [PATCH 2/3] Add create_wrapper_function helper --- src/allocator.rs | 75 +++++++++--------------------------------------- src/common.rs | 38 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 61 deletions(-) diff --git a/src/allocator.rs b/src/allocator.rs index 8508227179ac..1c73957ca571 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -70,37 +70,13 @@ fn codegen_inner( params: arg_tys.iter().cloned().map(AbiParam::new).collect(), returns: output.into_iter().map(AbiParam::new).collect(), }; - - let caller_name = format!("__rust_{}", method.name); - let callee_name = kind.fn_name(method.name); - - let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap(); - - let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap(); - - let mut ctx = Context::new(); - ctx.func.signature = sig.clone(); - { - let mut func_ctx = FunctionBuilderContext::new(); - let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); - - let block = bcx.create_block(); - bcx.switch_to_block(block); - let args = arg_tys - .into_iter() - .map(|ty| bcx.append_block_param(block, ty)) - .collect::>(); - - let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func); - let call_inst = bcx.ins().call(callee_func_ref, &args); - let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error - - bcx.ins().return_(&results); - bcx.seal_all_blocks(); - bcx.finalize(); - } - module.define_function(func_id, &mut ctx).unwrap(); - unwind_context.add_function(func_id, &ctx, module.isa()); + crate::common::create_wrapper_function( + module, + unwind_context, + sig, + &format!("__rust_{}", method.name), + &kind.fn_name(method.name), + ); } let sig = Signature { @@ -108,36 +84,13 @@ fn codegen_inner( params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)], returns: vec![], }; - - let callee_name = alloc_error_handler_kind.fn_name(sym::oom); - - let func_id = - module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap(); - - let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap(); - - let mut ctx = Context::new(); - ctx.func.signature = sig; - { - let mut func_ctx = FunctionBuilderContext::new(); - let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); - - let block = bcx.create_block(); - bcx.switch_to_block(block); - let args = (&[usize_ty, usize_ty]) - .iter() - .map(|&ty| bcx.append_block_param(block, ty)) - .collect::>(); - - let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func); - bcx.ins().call(callee_func_ref, &args); - - bcx.ins().trap(TrapCode::UnreachableCodeReached); - bcx.seal_all_blocks(); - bcx.finalize(); - } - module.define_function(func_id, &mut ctx).unwrap(); - unwind_context.add_function(func_id, &ctx, module.isa()); + crate::common::create_wrapper_function( + module, + unwind_context, + sig, + "__rust_alloc_error_handler", + &alloc_error_handler_kind.fn_name(sym::oom), + ); let data_id = module.declare_data(OomStrategy::SYMBOL, Linkage::Export, false, false).unwrap(); let mut data_ctx = DataContext::new(); diff --git a/src/common.rs b/src/common.rs index f41af3a9e636..16ae526090b7 100644 --- a/src/common.rs +++ b/src/common.rs @@ -254,6 +254,44 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } } +pub(crate) fn create_wrapper_function( + module: &mut impl Module, + unwind_context: &mut UnwindContext, + sig: Signature, + wrapper_name: &str, + callee_name: &str, +) { + let wrapper_func_id = module.declare_function(wrapper_name, Linkage::Export, &sig).unwrap(); + let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap(); + + let mut ctx = Context::new(); + ctx.func.signature = sig; + { + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + let func = &mut bcx.func.stencil; + let args = func + .signature + .params + .iter() + .map(|param| func.dfg.append_block_param(block, param.value_type)) + .collect::>(); + + let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func); + let call_inst = bcx.ins().call(callee_func_ref, &args); + let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error + + bcx.ins().return_(&results); + bcx.seal_all_blocks(); + bcx.finalize(); + } + module.define_function(wrapper_func_id, &mut ctx).unwrap(); + unwind_context.add_function(wrapper_func_id, &ctx, module.isa()); +} + pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { pub(crate) cx: &'clif mut crate::CodegenCx, pub(crate) module: &'m mut dyn Module, From a2719a285c7baeddb5cb0fd9bdd801b303b74a3e Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 5 Feb 2023 17:39:00 +0000 Subject: [PATCH 3/3] Fix linker error when inline asm sym operand is not exported from local CGU --- src/common.rs | 2 +- src/inline_asm.rs | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/common.rs b/src/common.rs index 16ae526090b7..a8be0d32cc8c 100644 --- a/src/common.rs +++ b/src/common.rs @@ -255,7 +255,7 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } pub(crate) fn create_wrapper_function( - module: &mut impl Module, + module: &mut dyn Module, unwind_context: &mut UnwindContext, sig: Signature, wrapper_name: &str, diff --git a/src/inline_asm.rs b/src/inline_asm.rs index aa0b51181f40..6206fbf7dd57 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -6,7 +6,6 @@ use std::fmt::Write; use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_middle::mir::InlineAsmOperand; -use rustc_middle::ty::SymbolName; use rustc_span::sym; use rustc_target::asm::*; @@ -30,7 +29,7 @@ enum CInlineAsmOperand<'tcx> { value: String, }, Symbol { - symbol: SymbolName<'tcx>, + symbol: String, }, } @@ -263,7 +262,29 @@ pub(crate) fn codegen_inline_asm<'tcx>( substs, ) .unwrap(); - CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance) } + let symbol = fx.tcx.symbol_name(instance); + + // Pass a wrapper rather than the function itself as the function itself may not + // be exported from the main codegen unit and may thus be unreachable from the + // object file created by an external assembler. + let inline_asm_index = fx.cx.inline_asm_index.get(); + fx.cx.inline_asm_index.set(inline_asm_index + 1); + let wrapper_name = format!( + "__inline_asm_{}_wrapper_n{}", + fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"), + inline_asm_index + ); + let sig = + get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance); + create_wrapper_function( + fx.module, + &mut fx.cx.unwind_context, + sig, + &wrapper_name, + symbol.name, + ); + + CInlineAsmOperand::Symbol { symbol: wrapper_name } } else { span_bug!(span, "invalid type for asm sym (fn)"); } @@ -271,7 +292,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( InlineAsmOperand::SymStatic { def_id } => { assert!(fx.tcx.is_static(def_id)); let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); - CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance) } + CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() } } }) .collect::>(); @@ -630,7 +651,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { CInlineAsmOperand::Const { ref value } => { generated_asm.push_str(value); } - CInlineAsmOperand::Symbol { symbol } => generated_asm.push_str(symbol.name), + CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol), } } }