user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rollup merge of #134232 - bjorn3:naked_asm_improvements, r=wesleywiser

Browse source 
Share the naked asm impl between cg_ssa and cg_clif

This was introduced in https://github.com/rust-lang/rust/pull/128004.
This commit is contained in:
Matthias Krüger 2025-04-30 17:27:57 +02:00 • committed by GitHub
commit 555df301f8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 329 additions and 393 deletions
Download patch file Download diff file Expand all files Collapse all files

39
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -8,8 +8,6 @@ use rustc_ast::InlineAsmOptions;
use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_index::IndexVec;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::InlineAsmMacro;
use rustc_middle::ty::TypeVisitableExt;
use rustc_middle::ty::adjustment::PointerCoercion;
use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv};
@ -18,7 +16,6 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
use crate::constant::ConstantCx;
use crate::debuginfo::{FunctionDebugContext, TypeDebugContext};
use crate::enable_verifier;
use crate::inline_asm::codegen_naked_asm;
use crate::prelude::*;
use crate::pretty_clif::CommentWriter;
@ -37,7 +34,7 @@ pub(crate) fn codegen_fn<'tcx>(
cached_func: Function,
module: &mut dyn Module,
instance: Instance<'tcx>,
) -> Option<CodegenedFunction> {
) -> CodegenedFunction {
debug_assert!(!instance.args.has_infer());
let symbol_name = tcx.symbol_name(instance).name.to_string();
@ -54,38 +51,6 @@ pub(crate) fn codegen_fn<'tcx>(
String::from_utf8_lossy(&buf).into_owned()
});
if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) {
assert_eq!(mir.basic_blocks.len(), 1);
assert!(mir.basic_blocks[START_BLOCK].statements.is_empty());
match &mir.basic_blocks[START_BLOCK].terminator().kind {
TerminatorKind::InlineAsm {
asm_macro: InlineAsmMacro::NakedAsm,
template,
operands,
options,
line_spans: _,
targets: _,
unwind: _,
} => {
codegen_naked_asm(
tcx,
cx,
module,
instance,
mir.basic_blocks[START_BLOCK].terminator().source_info.span,
&symbol_name,
template,
operands,
*options,
);
}
_ => unreachable!(),
}
return None;
}
// Declare function
let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance);
let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
@ -166,7 +131,7 @@ pub(crate) fn codegen_fn<'tcx>(
// Verify function
verify_func(tcx, &clif_comments, &func);
Some(CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx })
CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
}
pub(crate) fn compile_fn(

42
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -22,7 +22,10 @@ use rustc_data_structures::sync::{IntoDynSyncSend, par_map};
use rustc_metadata::EncodedMetadata;
use rustc_metadata::fs::copy_to_stdout;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::mono::{
CodegenUnit, Linkage as RLinkage, MonoItem, MonoItemData, Visibility,
};
use rustc_session::Session;
use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType};
@ -30,7 +33,7 @@ use crate::CodegenCx;
use crate::base::CodegenedFunction;
use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
use crate::debuginfo::TypeDebugContext;
use crate::global_asm::GlobalAsmConfig;
use crate::global_asm::{GlobalAsmConfig, GlobalAsmContext};
use crate::prelude::*;
use crate::unwind_module::UnwindModule;
@ -530,19 +533,35 @@ fn codegen_cgu_content(
let mut type_dbg = TypeDebugContext::default();
super::predefine_mono_items(tcx, module, &mono_items);
let mut codegened_functions = vec![];
for (mono_item, _) in mono_items {
for (mono_item, item_data) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
if let Some(codegened_function) = crate::base::codegen_fn(
MonoItem::Fn(instance) => {
if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED)
{
rustc_codegen_ssa::mir::naked_asm::codegen_naked_asm(
&mut GlobalAsmContext { tcx, global_asm: &mut cx.global_asm },
instance,
MonoItemData {
linkage: RLinkage::External,
visibility: if item_data.linkage == RLinkage::Internal {
Visibility::Hidden
} else {
item_data.visibility
},
..item_data
},
);
continue;
}
let codegened_function = crate::base::codegen_fn(
tcx,
&mut cx,
&mut type_dbg,
Function::new(),
module,
inst,
) {
codegened_functions.push(codegened_function);
}
instance,
);
codegened_functions.push(codegened_function);
}
MonoItem::Static(def_id) => {
let data_id = crate::constant::codegen_static(tcx, module, def_id);
@ -551,7 +570,10 @@ fn codegen_cgu_content(
}
}
MonoItem::GlobalAsm(item_id) => {
crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
rustc_codegen_ssa::base::codegen_global_asm(
&mut GlobalAsmContext { tcx, global_asm: &mut cx.global_asm },
item_id,
);
}
}
}

12
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -126,6 +126,11 @@ pub(crate) fn codegen_and_compile_fn<'tcx>(
module: &mut dyn Module,
instance: Instance<'tcx>,
) {
if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) {
tcx.dcx()
.span_fatal(tcx.def_span(instance.def_id()), "Naked asm is not supported in JIT mode");
}
cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler(
tcx.prof.clone(),
)));
@ -135,16 +140,15 @@ pub(crate) fn codegen_and_compile_fn<'tcx>(
crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
if let Some(codegened_func) = crate::base::codegen_fn(
let codegened_func = crate::base::codegen_fn(
tcx,
cx,
&mut TypeDebugContext::default(),
cached_func,
module,
instance,
) {
crate::base::compile_fn(cx, &tcx.prof, cached_context, module, codegened_func);
}
);
crate::base::compile_fn(cx, &tcx.prof, cached_context, module, codegened_func);
});
}

197
compiler/rustc_codegen_cranelift/src/global_asm.rs
View file

@ -7,102 +7,139 @@ use std::process::{Command, Stdio};
use std::sync::Arc;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_hir::{InlineAsmOperand, ItemId};
use rustc_middle::mir::interpret::ErrorHandled;
use rustc_codegen_ssa::traits::{AsmCodegenMethods, GlobalAsmOperandRef};
use rustc_middle::ty::TyCtxt;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers,
};
use rustc_session::config::{OutputFilenames, OutputType};
use rustc_target::asm::InlineAsmArch;
use crate::prelude::*;
pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) {
let item = tcx.hir_item(item_id);
if let rustc_hir::ItemKind::GlobalAsm { asm, .. } = item.kind {
let is_x86 =
matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64);
pub(crate) struct GlobalAsmContext<'a, 'tcx> {
pub tcx: TyCtxt<'tcx>,
pub global_asm: &'a mut String,
}
if is_x86 {
if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
global_asm.push_str("\n.intel_syntax noprefix\n");
} else {
global_asm.push_str("\n.att_syntax\n");
}
impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> {
fn codegen_global_asm(
&mut self,
template: &[InlineAsmTemplatePiece],
operands: &[GlobalAsmOperandRef<'tcx>],
options: InlineAsmOptions,
_line_spans: &[Span],
) {
codegen_global_asm_inner(self.tcx, self.global_asm, template, operands, options);
}
fn mangled_name(&self, instance: Instance<'tcx>) -> String {
let symbol_name = self.tcx.symbol_name(instance).name.to_owned();
if self.tcx.sess.target.is_like_darwin { format!("_{symbol_name}") } else { symbol_name }
}
}
impl<'tcx> LayoutOfHelpers<'tcx> for GlobalAsmContext<'_, 'tcx> {
#[inline]
fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
self.tcx.sess.dcx().span_fatal(span, err.to_string())
} else {
self.tcx
.sess
.dcx()
.span_fatal(span, format!("failed to get layout for `{}`: {}", ty, err))
}
for piece in asm.template {
match *piece {
InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: op_sp } => {
match asm.operands[operand_idx].0 {
InlineAsmOperand::Const { ref anon_const } => {
match tcx.const_eval_poly(anon_const.def_id.to_def_id()) {
Ok(const_value) => {
let ty = tcx
.typeck_body(anon_const.body)
.node_type(anon_const.hir_id);
let string = rustc_codegen_ssa::common::asm_const_to_str(
tcx,
op_sp,
const_value,
FullyMonomorphizedLayoutCx(tcx).layout_of(ty),
);
global_asm.push_str(&string);
}
Err(ErrorHandled::Reported { .. }) => {
// An error has already been reported and compilation is
// guaranteed to fail if execution hits this path.
}
Err(ErrorHandled::TooGeneric(_)) => {
span_bug!(op_sp, "asm const cannot be resolved; too generic");
}
}
}
InlineAsmOperand::SymFn { expr } => {
if cfg!(not(feature = "inline_asm_sym")) {
tcx.dcx().span_err(
item.span,
"asm! and global_asm! sym operands are not yet supported",
);
}
}
}
let ty = tcx.typeck(item_id.owner_id).expr_ty(expr);
let instance = match ty.kind() {
&ty::FnDef(def_id, args) => Instance::new(def_id, args),
_ => span_bug!(op_sp, "asm sym is not a function"),
};
let symbol = tcx.symbol_name(instance);
// FIXME handle the case where the function was made private to the
// current codegen unit
global_asm.push_str(symbol.name);
}
InlineAsmOperand::SymStatic { path: _, def_id } => {
if cfg!(not(feature = "inline_asm_sym")) {
tcx.dcx().span_err(
item.span,
"asm! and global_asm! sym operands are not yet supported",
);
}
impl<'tcx> FnAbiOfHelpers<'tcx> for GlobalAsmContext<'_, 'tcx> {
#[inline]
fn handle_fn_abi_err(
&self,
err: FnAbiError<'tcx>,
span: Span,
fn_abi_request: FnAbiRequest<'tcx>,
) -> ! {
FullyMonomorphizedLayoutCx(self.tcx).handle_fn_abi_err(err, span, fn_abi_request)
}
}
let instance = Instance::mono(tcx, def_id);
let symbol = tcx.symbol_name(instance);
global_asm.push_str(symbol.name);
impl<'tcx> HasTyCtxt<'tcx> for GlobalAsmContext<'_, 'tcx> {
fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
self.tcx
}
}
impl<'tcx> rustc_abi::HasDataLayout for GlobalAsmContext<'_, 'tcx> {
fn data_layout(&self) -> &rustc_abi::TargetDataLayout {
&self.tcx.data_layout
}
}
impl<'tcx> HasTypingEnv<'tcx> for GlobalAsmContext<'_, 'tcx> {
fn typing_env(&self) -> ty::TypingEnv<'tcx> {
ty::TypingEnv::fully_monomorphized()
}
}
fn codegen_global_asm_inner<'tcx>(
tcx: TyCtxt<'tcx>,
global_asm: &mut String,
template: &[InlineAsmTemplatePiece],
operands: &[GlobalAsmOperandRef<'tcx>],
options: InlineAsmOptions,
) {
let is_x86 = matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64);
if is_x86 {
if !options.contains(InlineAsmOptions::ATT_SYNTAX) {
global_asm.push_str("\n.intel_syntax noprefix\n");
} else {
global_asm.push_str("\n.att_syntax\n");
}
}
for piece in template {
match *piece {
InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span } => {
match operands[operand_idx] {
GlobalAsmOperandRef::Const { ref string } => {
global_asm.push_str(string);
}
GlobalAsmOperandRef::SymFn { instance } => {
if cfg!(not(feature = "inline_asm_sym")) {
tcx.dcx().span_err(
span,
"asm! and global_asm! sym operands are not yet supported",
);
}
InlineAsmOperand::In { .. }
| InlineAsmOperand::Out { .. }
| InlineAsmOperand::InOut { .. }
| InlineAsmOperand::SplitInOut { .. }
| InlineAsmOperand::Label { .. } => {
span_bug!(op_sp, "invalid operand type for global_asm!")
let symbol = tcx.symbol_name(instance);
// FIXME handle the case where the function was made private to the
// current codegen unit
global_asm.push_str(symbol.name);
}
GlobalAsmOperandRef::SymStatic { def_id } => {
if cfg!(not(feature = "inline_asm_sym")) {
tcx.dcx().span_err(
span,
"asm! and global_asm! sym operands are not yet supported",
);
}
let instance = Instance::mono(tcx, def_id);
let symbol = tcx.symbol_name(instance);
global_asm.push_str(symbol.name);
}
}
}
}
}
global_asm.push('\n');
if is_x86 {
global_asm.push_str(".att_syntax\n\n");
}
} else {
bug!("Expected GlobalAsm found {:?}", item);
global_asm.push('\n');
if is_x86 {
global_asm.push_str(".att_syntax\n\n");
}
}

193
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -161,7 +161,6 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>(
stack_slots_input: Vec::new(),
stack_slots_output: Vec::new(),
stack_slot_size: Size::from_bytes(0),
is_naked: false,
};
asm_gen.allocate_registers();
asm_gen.allocate_stack_slots();
@ -201,114 +200,6 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>(
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
}
pub(crate) fn codegen_naked_asm<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
module: &mut dyn Module,
instance: Instance<'tcx>,
span: Span,
symbol_name: &str,
template: &[InlineAsmTemplatePiece],
operands: &[InlineAsmOperand<'tcx>],
options: InlineAsmOptions,
) {
// FIXME add .eh_frame unwind info directives
let operands = operands
.iter()
.map(|operand| match *operand {
InlineAsmOperand::In { .. }
| InlineAsmOperand::Out { .. }
| InlineAsmOperand::InOut { .. } => {
span_bug!(span, "invalid operand type for naked asm")
}
InlineAsmOperand::Const { ref value } => {
let cv = instance.instantiate_mir_and_normalize_erasing_regions(
tcx,
ty::TypingEnv::fully_monomorphized(),
ty::EarlyBinder::bind(value.const_),
);
let const_value = cv
.eval(tcx, ty::TypingEnv::fully_monomorphized(), value.span)
.expect("erroneous constant missed by mono item collection");
let value = rustc_codegen_ssa::common::asm_const_to_str(
tcx,
span,
const_value,
FullyMonomorphizedLayoutCx(tcx).layout_of(cv.ty()),
);
CInlineAsmOperand::Const { value }
}
InlineAsmOperand::SymFn { ref value } => {
if cfg!(not(feature = "inline_asm_sym")) {
tcx.dcx()
.span_err(span, "asm! and global_asm! sym operands are not yet supported");
}
let const_ = instance.instantiate_mir_and_normalize_erasing_regions(
tcx,
ty::TypingEnv::fully_monomorphized(),
ty::EarlyBinder::bind(value.const_),
);
if let ty::FnDef(def_id, args) = *const_.ty().kind() {
let instance = ty::Instance::resolve_for_fn_ptr(
tcx,
ty::TypingEnv::fully_monomorphized(),
def_id,
args,
)
.unwrap();
let symbol = tcx.symbol_name(instance);
// Pass a wrapper rather than the function itself as the function itself may not
// be exported from the main codegen unit and may thus be unreachable from the
// object file created by an external assembler.
let wrapper_name = format!(
"__inline_asm_{}_wrapper_n{}",
cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
cx.inline_asm_index
);
cx.inline_asm_index += 1;
let sig =
get_function_sig(tcx, module.target_config().default_call_conv, instance);
create_wrapper_function(module, sig, &wrapper_name, symbol.name);
CInlineAsmOperand::Symbol { symbol: wrapper_name }
} else {
span_bug!(span, "invalid type for asm sym (fn)");
}
}
InlineAsmOperand::SymStatic { def_id } => {
assert!(tcx.is_static(def_id));
let instance = Instance::mono(tcx, def_id);
CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() }
}
InlineAsmOperand::Label { .. } => {
span_bug!(span, "asm! label operands are not yet supported");
}
})
.collect::<Vec<_>>();
let asm_gen = InlineAssemblyGenerator {
tcx,
arch: tcx.sess.asm_arch.unwrap(),
enclosing_def_id: instance.def_id(),
template,
operands: &operands,
options,
registers: Vec::new(),
stack_slots_clobber: Vec::new(),
stack_slots_input: Vec::new(),
stack_slots_output: Vec::new(),
stack_slot_size: Size::from_bytes(0),
is_naked: true,
};
let generated_asm = asm_gen.generate_asm_wrapper(symbol_name);
cx.global_asm.push_str(&generated_asm);
}
struct InlineAssemblyGenerator<'a, 'tcx> {
tcx: TyCtxt<'tcx>,
arch: InlineAsmArch,
@ -321,13 +212,10 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
stack_slots_input: Vec<Option<Size>>,
stack_slots_output: Vec<Option<Size>>,
stack_slot_size: Size,
is_naked: bool,
}
impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
fn allocate_registers(&mut self) {
assert!(!self.is_naked);
let sess = self.tcx.sess;
let map = allocatable_registers(
self.arch,
@ -451,8 +339,6 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
}
fn allocate_stack_slots(&mut self) {
assert!(!self.is_naked);
let mut slot_size = Size::from_bytes(0);
let mut slots_clobber = vec![None; self.operands.len()];
let mut slots_input = vec![None; self.operands.len()];
@ -582,32 +468,31 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
if is_x86 {
generated_asm.push_str(".intel_syntax noprefix\n");
}
if !self.is_naked {
Self::prologue(&mut generated_asm, self.arch);
// Save clobbered registers
if !self.options.contains(InlineAsmOptions::NORETURN) {
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
}
Self::prologue(&mut generated_asm, self.arch);
// Write input registers
// Save clobbered registers
if !self.options.contains(InlineAsmOptions::NORETURN) {
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_input.iter().copied())
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
}
// Write input registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_input.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push_str(".att_syntax\n");
}
@ -701,32 +586,30 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
generated_asm.push_str(".intel_syntax noprefix\n");
}
if !self.is_naked {
if !self.options.contains(InlineAsmOptions::NORETURN) {
// Read output registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_output.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
// Restore clobbered registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
Self::epilogue(&mut generated_asm, self.arch);
} else {
Self::epilogue_noreturn(&mut generated_asm, self.arch);
if !self.options.contains(InlineAsmOptions::NORETURN) {
// Read output registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_output.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
// Restore clobbered registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
Self::epilogue(&mut generated_asm, self.arch);
} else {
Self::epilogue_noreturn(&mut generated_asm, self.arch);
}
if is_x86 {

2
compiler/rustc_codegen_gcc/src/asm.rs
View file

@ -829,7 +829,7 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
impl<'gcc, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
fn codegen_global_asm(
&self,
&mut self,
template: &[InlineAsmTemplatePiece],
operands: &[GlobalAsmOperandRef<'tcx>],
options: InlineAsmOptions,

6
compiler/rustc_codegen_gcc/src/base.rs
View file

@ -206,7 +206,7 @@ pub fn compile_codegen_unit(
let f128_type_supported = target_info.supports_target_dependent_type(CType::Float128);
let u128_type_supported = target_info.supports_target_dependent_type(CType::UInt128t);
// TODO: improve this to avoid passing that many arguments.
let cx = CodegenCx::new(
let mut cx = CodegenCx::new(
&context,
cgu,
tcx,
@ -223,8 +223,8 @@ pub fn compile_codegen_unit(
}
// ... and now that we have everything pre-defined, fill out those definitions.
for &(mono_item, _) in &mono_items {
mono_item.define::<Builder<'_, '_, '_>>(&cx);
for &(mono_item, item_data) in &mono_items {
mono_item.define::<Builder<'_, '_, '_>>(&mut cx, item_data);
}
// If this codegen unit contains the main function, also create the

2
compiler/rustc_codegen_gcc/src/builder.rs
View file

@ -45,7 +45,7 @@ enum ExtremumOperation {
Min,
}
pub struct Builder<'a: 'gcc, 'gcc, 'tcx> {
pub struct Builder<'a, 'gcc, 'tcx> {
pub cx: &'a CodegenCx<'gcc, 'tcx>,
pub block: Block<'gcc>,
pub location: Option<Location<'gcc>>,

2
compiler/rustc_codegen_llvm/src/asm.rs
View file

@ -376,7 +376,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
fn codegen_global_asm(
&self,
&mut self,
template: &[InlineAsmTemplatePiece],
operands: &[GlobalAsmOperandRef<'tcx>],
options: InlineAsmOptions,

6
compiler/rustc_codegen_llvm/src/base.rs
View file

@ -83,15 +83,15 @@ pub(crate) fn compile_codegen_unit(
// Instantiate monomorphizations without filling out definitions yet...
let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str());
{
let cx = CodegenCx::new(tcx, cgu, &llvm_module);
let mut cx = CodegenCx::new(tcx, cgu, &llvm_module);
let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
for &(mono_item, data) in &mono_items {
mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility);
}
// ... and now that we have everything pre-defined, fill out those definitions.
for &(mono_item, _) in &mono_items {
mono_item.define::<Builder<'_, '_, '_>>(&cx);
for &(mono_item, item_data) in &mono_items {
mono_item.define::<Builder<'_, '_, '_>>(&mut cx, item_data);
}
// If this codegen unit contains the main function, also create the

67
compiler/rustc_codegen_ssa/src/base.rs
View file

@ -12,19 +12,21 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
use rustc_data_structures::sync::{IntoDynSyncSend, par_map};
use rustc_data_structures::unord::UnordMap;
use rustc_hir::ItemId;
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
use rustc_hir::lang_items::LangItem;
use rustc_metadata::EncodedMetadata;
use rustc_middle::bug;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, DebuggerVisualizerType};
use rustc_middle::middle::exported_symbols::SymbolExportKind;
use rustc_middle::middle::{exported_symbols, lang_items};
use rustc_middle::mir::BinOp;
use rustc_middle::mir::interpret::ErrorHandled;
use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem, MonoItemPartitions};
use rustc_middle::query::Providers;
use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
use rustc_middle::{bug, span_bug};
use rustc_session::Session;
use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
use rustc_span::{DUMMY_SP, Symbol, sym};
@ -417,6 +419,69 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
mir::codegen_mir::<Bx>(cx, instance);
}
pub fn codegen_global_asm<'tcx, Cx>(cx: &mut Cx, item_id: ItemId)
where
Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> + AsmCodegenMethods<'tcx>,
{
let item = cx.tcx().hir_item(item_id);
if let rustc_hir::ItemKind::GlobalAsm { asm, .. } = item.kind {
let operands: Vec<_> = asm
.operands
.iter()
.map(|(op, op_sp)| match *op {
rustc_hir::InlineAsmOperand::Const { ref anon_const } => {
match cx.tcx().const_eval_poly(anon_const.def_id.to_def_id()) {
Ok(const_value) => {
let ty =
cx.tcx().typeck_body(anon_const.body).node_type(anon_const.hir_id);
let string = common::asm_const_to_str(
cx.tcx(),
*op_sp,
const_value,
cx.layout_of(ty),
);
GlobalAsmOperandRef::Const { string }
}
Err(ErrorHandled::Reported { .. }) => {
// An error has already been reported and
// compilation is guaranteed to fail if execution
// hits this path. So an empty string instead of
// a stringified constant value will suffice.
GlobalAsmOperandRef::Const { string: String::new() }
}
Err(ErrorHandled::TooGeneric(_)) => {
span_bug!(*op_sp, "asm const cannot be resolved; too generic")
}
}
}
rustc_hir::InlineAsmOperand::SymFn { expr } => {
let ty = cx.tcx().typeck(item_id.owner_id).expr_ty(expr);
let instance = match ty.kind() {
&ty::FnDef(def_id, args) => Instance::new(def_id, args),
_ => span_bug!(*op_sp, "asm sym is not a function"),
};
GlobalAsmOperandRef::SymFn { instance }
}
rustc_hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
GlobalAsmOperandRef::SymStatic { def_id }
}
rustc_hir::InlineAsmOperand::In { .. }
| rustc_hir::InlineAsmOperand::Out { .. }
| rustc_hir::InlineAsmOperand::InOut { .. }
| rustc_hir::InlineAsmOperand::SplitInOut { .. }
| rustc_hir::InlineAsmOperand::Label { .. } => {
span_bug!(*op_sp, "invalid operand type for global_asm!")
}
})
.collect();
cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
} else {
span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
}
}
/// Creates the `main` function which will initialize the rust runtime and call
/// users main function.
pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(

7
compiler/rustc_codegen_ssa/src/mir/mod.rs
View file

@ -20,7 +20,7 @@ mod coverageinfo;
pub mod debuginfo;
mod intrinsic;
mod locals;
mod naked_asm;
pub mod naked_asm;
pub mod operand;
pub mod place;
mod rvalue;
@ -178,11 +178,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
debug!("fn_abi: {:?}", fn_abi);
if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) {
crate::mir::naked_asm::codegen_naked_asm::<Bx>(cx, &mir, instance);
return;
}
if tcx.features().ergonomic_clones() {
let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions(
tcx,

48
compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
View file

@ -1,23 +1,33 @@
use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind};
use rustc_attr_parsing::InstructionSetAttr;
use rustc_hir::def_id::DefId;
use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility};
use rustc_middle::mir::{Body, InlineAsmOperand};
use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
use rustc_middle::ty::{Instance, Ty, TyCtxt};
use rustc_middle::mir::mono::{Linkage, MonoItemData, Visibility};
use rustc_middle::mir::{InlineAsmOperand, START_BLOCK};
use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
use rustc_middle::ty::{Instance, Ty, TyCtxt, TypeVisitableExt};
use rustc_middle::{bug, span_bug, ty};
use rustc_span::sym;
use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
use rustc_target::spec::{BinaryFormat, WasmCAbi};
use crate::common;
use crate::traits::{AsmCodegenMethods, BuilderMethods, GlobalAsmOperandRef, MiscCodegenMethods};
use crate::mir::AsmCodegenMethods;
use crate::traits::GlobalAsmOperandRef;
pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
cx: &'a Bx::CodegenCx,
mir: &Body<'tcx>,
pub fn codegen_naked_asm<
'a,
'tcx,
Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
+ FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
+ AsmCodegenMethods<'tcx>,
>(
cx: &'a mut Cx,
instance: Instance<'tcx>,
item_data: MonoItemData,
) {
assert!(!instance.args.has_infer());
let mir = cx.tcx().instance_mir(instance.def);
let rustc_middle::mir::TerminatorKind::InlineAsm {
asm_macro: _,
template,
@ -26,15 +36,14 @@ pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
line_spans,
targets: _,
unwind: _,
} = mir.basic_blocks.iter().next().unwrap().terminator().kind
} = mir.basic_blocks[START_BLOCK].terminator().kind
else {
bug!("#[naked] functions should always terminate with an asm! block")
};
let operands: Vec<_> =
operands.iter().map(|op| inline_to_global_operand::<Bx>(cx, instance, op)).collect();
operands.iter().map(|op| inline_to_global_operand::<Cx>(cx, instance, op)).collect();
let item_data = cx.codegen_unit().items().get(&MonoItem::Fn(instance)).unwrap();
let name = cx.mangled_name(instance);
let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
let (begin, end) = prefix_and_suffix(cx.tcx(), instance, &name, item_data, fn_abi);
@ -47,8 +56,8 @@ pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
cx.codegen_global_asm(&template_vec, &operands, options, line_spans);
}
fn inline_to_global_operand<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
cx: &'a Bx::CodegenCx,
fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>(
cx: &'a Cx,
instance: Instance<'tcx>,
op: &InlineAsmOperand<'tcx>,
) -> GlobalAsmOperandRef<'tcx> {
@ -108,7 +117,7 @@ fn prefix_and_suffix<'tcx>(
tcx: TyCtxt<'tcx>,
instance: Instance<'tcx>,
asm_name: &str,
item_data: &MonoItemData,
item_data: MonoItemData,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
) -> (String, String) {
use std::fmt::Write;
@ -210,8 +219,10 @@ fn prefix_and_suffix<'tcx>(
writeln!(begin, ".pushsection {section},\"ax\", {progbits}").unwrap();
writeln!(begin, ".balign {align_bytes}").unwrap();
write_linkage(&mut begin).unwrap();
if let Visibility::Hidden = item_data.visibility {
writeln!(begin, ".hidden {asm_name}").unwrap();
match item_data.visibility {
Visibility::Default => {}
Visibility::Protected => writeln!(begin, ".protected {asm_name}").unwrap(),
Visibility::Hidden => writeln!(begin, ".hidden {asm_name}").unwrap(),
}
writeln!(begin, ".type {asm_name}, {function}").unwrap();
if !arch_prefix.is_empty() {
@ -231,8 +242,9 @@ fn prefix_and_suffix<'tcx>(
writeln!(begin, ".pushsection {},regular,pure_instructions", section).unwrap();
writeln!(begin, ".balign {align_bytes}").unwrap();
write_linkage(&mut begin).unwrap();
if let Visibility::Hidden = item_data.visibility {
writeln!(begin, ".private_extern {asm_name}").unwrap();
match item_data.visibility {
Visibility::Default | Visibility::Protected => {}
Visibility::Hidden => writeln!(begin, ".private_extern {asm_name}").unwrap(),
}
writeln!(begin, "{asm_name}:").unwrap();

97
compiler/rustc_codegen_ssa/src/mono_item.rs
View file

@ -1,17 +1,18 @@
use rustc_hir as hir;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::interpret::ErrorHandled;
use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility};
use rustc_middle::ty::Instance;
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
use rustc_middle::{span_bug, ty};
use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility};
use rustc_middle::ty::layout::HasTyCtxt;
use tracing::debug;
use crate::base;
use crate::mir::naked_asm;
use crate::traits::*;
use crate::{base, common};
pub trait MonoItemExt<'a, 'tcx> {
fn define<Bx: BuilderMethods<'a, 'tcx>>(&self, cx: &'a Bx::CodegenCx);
fn define<Bx: BuilderMethods<'a, 'tcx>>(
&self,
cx: &'a mut Bx::CodegenCx,
item_data: MonoItemData,
);
fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
&self,
cx: &'a Bx::CodegenCx,
@ -22,7 +23,11 @@ pub trait MonoItemExt<'a, 'tcx> {
}
impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
fn define<Bx: BuilderMethods<'a, 'tcx>>(&self, cx: &'a Bx::CodegenCx) {
fn define<Bx: BuilderMethods<'a, 'tcx>>(
&self,
cx: &'a mut Bx::CodegenCx,
item_data: MonoItemData,
) {
debug!(
"BEGIN IMPLEMENTING '{} ({})' in cgu {}",
self,
@ -35,71 +40,19 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
cx.codegen_static(def_id);
}
MonoItem::GlobalAsm(item_id) => {
let item = cx.tcx().hir_item(item_id);
if let hir::ItemKind::GlobalAsm { asm, .. } = item.kind {
let operands: Vec<_> = asm
.operands
.iter()
.map(|(op, op_sp)| match *op {
hir::InlineAsmOperand::Const { ref anon_const } => {
match cx.tcx().const_eval_poly(anon_const.def_id.to_def_id()) {
Ok(const_value) => {
let ty = cx
.tcx()
.typeck_body(anon_const.body)
.node_type(anon_const.hir_id);
let string = common::asm_const_to_str(
cx.tcx(),
*op_sp,
const_value,
cx.layout_of(ty),
);
GlobalAsmOperandRef::Const { string }
}
Err(ErrorHandled::Reported { .. }) => {
// An error has already been reported and
// compilation is guaranteed to fail if execution
// hits this path. So an empty string instead of
// a stringified constant value will suffice.
GlobalAsmOperandRef::Const { string: String::new() }
}
Err(ErrorHandled::TooGeneric(_)) => {
span_bug!(
*op_sp,
"asm const cannot be resolved; too generic"
)
}
}
}
hir::InlineAsmOperand::SymFn { expr } => {
let ty = cx.tcx().typeck(item_id.owner_id).expr_ty(expr);
let instance = match ty.kind() {
&ty::FnDef(def_id, args) => Instance::new(def_id, args),
_ => span_bug!(*op_sp, "asm sym is not a function"),
};
GlobalAsmOperandRef::SymFn { instance }
}
hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
GlobalAsmOperandRef::SymStatic { def_id }
}
hir::InlineAsmOperand::In { .. }
| hir::InlineAsmOperand::Out { .. }
| hir::InlineAsmOperand::InOut { .. }
| hir::InlineAsmOperand::SplitInOut { .. }
| hir::InlineAsmOperand::Label { .. } => {
span_bug!(*op_sp, "invalid operand type for global_asm!")
}
})
.collect();
cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
} else {
span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
}
base::codegen_global_asm(cx, item_id);
}
MonoItem::Fn(instance) => {
base::codegen_instance::<Bx>(cx, instance);
if cx
.tcx()
.codegen_fn_attrs(instance.def_id())
.flags
.contains(CodegenFnAttrFlags::NAKED)
{
naked_asm::codegen_naked_asm::<Bx::CodegenCx>(cx, instance, item_data);
} else {
base::codegen_instance::<Bx>(cx, instance);
}
}
}

2
compiler/rustc_codegen_ssa/src/traits/asm.rs
View file

@ -62,7 +62,7 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
pub trait AsmCodegenMethods<'tcx> {
fn codegen_global_asm(
&self,
&mut self,
template: &[InlineAsmTemplatePiece],
operands: &[GlobalAsmOperandRef<'tcx>],
options: InlineAsmOptions,