user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #142521 - sayantn:simplify-intrinsics, r=nikic,workingjubilee

Browse source 
Use `LLVMIntrinsicGetDeclaration` to completely remove the hardcoded intrinsics list

Follow-up to rust-lang/rust#142259

This also needs a rustc-perf run, because `Intrinsic::getType` can be expensive

`@rustbot` label A-LLVM A-codegen T-compiler
r? `@workingjubilee`
cc `@nikic`
This commit is contained in:
bors 2025-06-16 03:40:18 +00:00
commit 68ac5abb06
6 changed files with 55 additions and 242 deletions
Download patch file Download diff file Expand all files Collapse all files

15
compiler/rustc_codegen_llvm/src/builder.rs
View file

@ -507,7 +507,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
let res = self.call_intrinsic(&name, &[self.type_ix(width)], &[lhs, rhs]);
let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
(self.extract_value(res, 0), self.extract_value(res, 1))
}
@ -1038,7 +1038,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let size = ty.primitive_size(self.tcx);
let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
Some(self.call_intrinsic(&name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
}
/* Miscellaneous instructions */
@ -1393,7 +1393,8 @@ impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
// Forward to the `get_static` method of `CodegenCx`
let global = self.cx().get_static(def_id);
if self.cx().tcx.is_thread_local_static(def_id) {
let pointer = self.call_intrinsic("llvm.threadlocal.address", &[], &[global]);
let pointer =
self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
// Cast to default address space if globals are in a different addrspace
self.pointercast(pointer, self.type_ptr())
} else {
@ -1590,15 +1591,15 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
pub(crate) fn call_intrinsic(
&mut self,
base_name: &str,
base_name: impl Into<Cow<'static, str>>,
type_params: &[&'ll Type],
args: &[&'ll Value],
) -> &'ll Value {
let (ty, f) = self.cx.get_intrinsic(base_name, type_params);
let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
self.call(ty, None, None, f, args, None, None)
}
fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
let size = size.bytes();
if size == 0 {
return;
@ -1608,7 +1609,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
return;
}
self.call_intrinsic(intrinsic, &[self.type_ptr()], &[self.cx.const_u64(size), ptr]);
self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
}
}
impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {

192
compiler/rustc_codegen_llvm/src/context.rs
View file

@ -1,4 +1,4 @@
use std::borrow::Borrow;
use std::borrow::{Borrow, Cow};
use std::cell::{Cell, RefCell};
use std::ffi::{CStr, c_char, c_uint};
use std::marker::PhantomData;
@ -138,7 +138,7 @@ pub(crate) struct FullCx<'ll, 'tcx> {
pub rust_try_fn: Cell<Option<(&'ll Type, &'ll Value)>>,
intrinsics:
RefCell<FxHashMap<(&'static str, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
RefCell<FxHashMap<(Cow<'static, str>, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
/// A counter that is used for generating local symbol names
local_gen_sym_counter: Cell<usize>,
@ -845,45 +845,16 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
impl<'ll> CodegenCx<'ll, '_> {
pub(crate) fn get_intrinsic(
&self,
base_name: &str,
base_name: Cow<'static, str>,
type_params: &[&'ll Type],
) -> (&'ll Type, &'ll Value) {
if let Some(v) =
self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params)))
{
return *v;
}
self.declare_intrinsic(base_name, type_params)
}
fn insert_intrinsic(
&self,
base_name: &'static str,
type_params: &[&'ll Type],
args: Option<&[&'ll llvm::Type]>,
ret: &'ll llvm::Type,
) -> (&'ll llvm::Type, &'ll llvm::Value) {
let fn_ty = if let Some(args) = args {
self.type_func(args, ret)
} else {
self.type_variadic_func(&[], ret)
};
let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
.expect("Unknown LLVM intrinsic `{base_name}`");
let full_name = if intrinsic.is_overloaded() {
&intrinsic.overloaded_name(self.llmod, type_params)
} else {
base_name
};
let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty);
self.intrinsics
*self
.intrinsics
.borrow_mut()
.insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f));
(fn_ty, f)
.entry((base_name, SmallVec::from_slice(type_params)))
.or_insert_with_key(|(base_name, type_params)| {
self.declare_intrinsic(base_name, type_params)
})
}
fn declare_intrinsic(
@ -891,155 +862,22 @@ impl<'ll> CodegenCx<'ll, '_> {
base_name: &str,
type_params: &[&'ll Type],
) -> (&'ll Type, &'ll Value) {
macro_rules! param {
($index:literal) => {
type_params[$index]
};
($other:expr) => {
$other
};
}
macro_rules! ifn {
($name:expr, fn(...) -> $ret:expr) => (
if base_name == $name {
return self.insert_intrinsic($name, type_params, None, param!($ret));
}
);
($name:expr, fn($($arg:expr),*) -> $ret:expr) => (
if base_name == $name {
return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret));
}
);
}
macro_rules! mk_struct {
($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false))
}
let same_width_vector = |index, element_ty| {
self.type_vector(element_ty, self.vector_length(type_params[index]) as u64)
};
let ptr = self.type_ptr();
let void = self.type_void();
let i1 = self.type_i1();
let t_i32 = self.type_i32();
let t_i64 = self.type_i64();
let t_isize = self.type_isize();
let t_metadata = self.type_metadata();
let t_token = self.type_token();
ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr);
ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32);
ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0);
ifn!("llvm.wasm.trunc.signed", fn(1) -> 0);
ifn!("llvm.fptosi.sat", fn(1) -> 0);
ifn!("llvm.fptoui.sat", fn(1) -> 0);
ifn!("llvm.trap", fn() -> void);
ifn!("llvm.debugtrap", fn() -> void);
ifn!("llvm.frameaddress", fn(t_i32) -> ptr);
ifn!("llvm.powi", fn(0, 1) -> 0);
ifn!("llvm.pow", fn(0, 0) -> 0);
ifn!("llvm.sqrt", fn(0) -> 0);
ifn!("llvm.sin", fn(0) -> 0);
ifn!("llvm.cos", fn(0) -> 0);
ifn!("llvm.exp", fn(0) -> 0);
ifn!("llvm.exp2", fn(0) -> 0);
ifn!("llvm.log", fn(0) -> 0);
ifn!("llvm.log10", fn(0) -> 0);
ifn!("llvm.log2", fn(0) -> 0);
ifn!("llvm.fma", fn(0, 0, 0) -> 0);
ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0);
ifn!("llvm.fabs", fn(0) -> 0);
ifn!("llvm.minnum", fn(0, 0) -> 0);
ifn!("llvm.minimum", fn(0, 0) -> 0);
ifn!("llvm.maxnum", fn(0, 0) -> 0);
ifn!("llvm.maximum", fn(0, 0) -> 0);
ifn!("llvm.floor", fn(0) -> 0);
ifn!("llvm.ceil", fn(0) -> 0);
ifn!("llvm.trunc", fn(0) -> 0);
ifn!("llvm.copysign", fn(0, 0) -> 0);
ifn!("llvm.round", fn(0) -> 0);
ifn!("llvm.rint", fn(0) -> 0);
ifn!("llvm.nearbyint", fn(0) -> 0);
ifn!("llvm.ctpop", fn(0) -> 0);
ifn!("llvm.ctlz", fn(0, i1) -> 0);
ifn!("llvm.cttz", fn(0, i1) -> 0);
ifn!("llvm.bswap", fn(0) -> 0);
ifn!("llvm.bitreverse", fn(0) -> 0);
ifn!("llvm.fshl", fn(0, 0, 0) -> 0);
ifn!("llvm.fshr", fn(0, 0, 0) -> 0);
ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.sadd.sat", fn(0, 0) -> 0);
ifn!("llvm.uadd.sat", fn(0, 0) -> 0);
ifn!("llvm.ssub.sat", fn(0, 0) -> 0);
ifn!("llvm.usub.sat", fn(0, 0) -> 0);
ifn!("llvm.scmp", fn(1, 1) -> 0);
ifn!("llvm.ucmp", fn(1, 1) -> 0);
ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void);
ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void);
ifn!("llvm.is.constant", fn(0) -> i1);
ifn!("llvm.expect", fn(0, 0) -> 0);
ifn!("llvm.eh.typeid.for", fn(0) -> t_i32);
ifn!("llvm.localescape", fn(...) -> void);
ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr);
ifn!("llvm.assume", fn(i1) -> void);
ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void);
// This isn't an "LLVM intrinsic", but LLVM's optimization passes
// recognize it like one (including turning it into `bcmp` sometimes)
// and we use it to implement intrinsics like `raw_eq` and `compare_bytes`
if base_name == "memcmp" {
let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int());
let fn_ty = self
.type_func(&[self.type_ptr(), self.type_ptr(), self.type_isize()], self.type_int());
let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty);
self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f));
return (fn_ty, f);
}
// variadic intrinsics
ifn!("llvm.va_start", fn(0) -> void);
ifn!("llvm.va_end", fn(0) -> void);
ifn!("llvm.va_copy", fn(0, 0) -> void);
let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
.unwrap_or_else(|| bug!("Unknown intrinsic: `{base_name}`"));
let f = intrinsic.get_declaration(self.llmod, &type_params);
if self.sess().instrument_coverage() {
ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void);
ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void);
}
ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1);
ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1});
if self.sess().opts.debuginfo != DebugInfo::None {
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void);
ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void);
}
ifn!("llvm.ptrmask", fn(0, 1) -> 0);
ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);
ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
bug!("Unknown intrinsic: `{base_name}`")
(self.get_type_of_global(f), f)
}
pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value {

47
compiler/rustc_codegen_llvm/src/intrinsic.rs
View file

@ -154,8 +154,6 @@ fn call_simple_intrinsic<'ll, 'tcx>(
sym::roundf64 => ("llvm.round", &[bx.type_f64()]),
sym::roundf128 => ("llvm.round", &[bx.type_f128()]),
sym::ptr_mask => ("llvm.ptrmask", &[bx.type_ptr(), bx.type_isize()]),
_ => return None,
};
Some(bx.call_intrinsic(
@ -181,6 +179,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
let simple = call_simple_intrinsic(self, name, args);
let llval = match name {
_ if simple.is_some() => simple.unwrap(),
sym::ptr_mask => {
let ptr = args[0].immediate();
self.call_intrinsic(
"llvm.ptrmask",
&[self.val_ty(ptr), self.type_isize()],
&[ptr, args[1].immediate()],
)
}
sym::is_val_statically_known => {
if let OperandValue::Immediate(imm) = args[0].val {
self.call_intrinsic(
@ -232,11 +238,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
return Ok(());
}
sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
sym::va_copy => self.call_intrinsic(
"llvm.va_copy",
&[self.type_ptr()],
&[args[0].immediate(), args[1].immediate()],
),
sym::va_copy => {
let dest = args[0].immediate();
self.call_intrinsic(
"llvm.va_copy",
&[self.val_ty(dest)],
&[dest, args[1].immediate()],
)
}
sym::va_arg => {
match result.layout.backend_repr {
BackendRepr::Scalar(scalar) => {
@ -309,15 +318,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
sym::prefetch_write_instruction => (1, 0),
_ => bug!(),
};
let ptr = args[0].immediate();
self.call_intrinsic(
"llvm.prefetch",
&[self.type_ptr()],
&[
args[0].immediate(),
self.const_i32(rw),
args[1].immediate(),
self.const_i32(cache_type),
],
&[self.val_ty(ptr)],
&[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)],
)
}
sym::carrying_mul_add => {
@ -378,7 +383,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
sym::ctlz | sym::cttz => {
let y = self.const_bool(false);
let ret = self.call_intrinsic(
&format!("llvm.{name}"),
format!("llvm.{name}"),
&[llty],
&[args[0].immediate(), y],
);
@ -423,7 +428,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
// always uses `u32`.
let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
self.call_intrinsic(&llvm_name, &[llty], &[val, val, raw_shift])
self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift])
}
sym::saturating_add | sym::saturating_sub => {
let is_add = name == sym::saturating_add;
@ -434,7 +439,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" },
);
self.call_intrinsic(&llvm_name, &[llty], &[lhs, rhs])
self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs])
}
_ => bug!(),
}
@ -637,11 +642,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
self.call_intrinsic("llvm.va_start", &[self.type_ptr()], &[va_list])
self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list])
}
fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
self.call_intrinsic("llvm.va_end", &[self.type_ptr()], &[va_list])
self.call_intrinsic("llvm.va_end", &[self.val_ty(va_list)], &[va_list])
}
}
@ -1018,7 +1023,7 @@ fn codegen_emcc_try<'ll, 'tcx>(
let selector = bx.extract_value(vals, 1);
// Check if the typeid we got is the one for a Rust panic.
let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.type_ptr()], &[tydesc]);
let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.val_ty(tydesc)], &[tydesc]);
let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
@ -2393,7 +2398,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
);
let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
return Ok(bx.call_intrinsic(&llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
}
span_bug!(span, "unknown SIMD intrinsic");

8
compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file

@ -1078,8 +1078,6 @@ unsafe extern "C" {
// Operations on other types
pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type;
pub(crate) fn LLVMTokenTypeInContext(C: &Context) -> &Type;
pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type;
// Operations on all values
pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type;
@ -1198,14 +1196,12 @@ unsafe extern "C" {
// Operations about llvm intrinsics
pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint;
pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero<c_uint>) -> Bool;
pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>(
pub(crate) fn LLVMGetIntrinsicDeclaration<'a>(
Mod: &'a Module,
ID: NonZero<c_uint>,
ParamTypes: *const &'a Type,
ParamCount: size_t,
NameLength: *mut size_t,
) -> *mut c_char;
) -> &'a Value;
// Operations on parameters
pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;

28
compiler/rustc_codegen_llvm/src/llvm/mod.rs
View file

@ -2,9 +2,9 @@
use std::ffi::{CStr, CString};
use std::num::NonZero;
use std::ptr;
use std::str::FromStr;
use std::string::FromUtf8Error;
use std::{ptr, slice};
use libc::c_uint;
use rustc_abi::{Align, Size, WrappingRange};
@ -339,34 +339,14 @@ impl Intrinsic {
NonZero::new(id).map(|id| Self { id })
}
pub(crate) fn is_overloaded(self) -> bool {
unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
}
pub(crate) fn overloaded_name<'ll>(
pub(crate) fn get_declaration<'ll>(
self,
llmod: &'ll Module,
type_params: &[&'ll Type],
) -> String {
let mut len = 0;
let ptr = unsafe {
LLVMIntrinsicCopyOverloadedName2(
llmod,
self.id,
type_params.as_ptr(),
type_params.len(),
&mut len,
)
};
let slice = unsafe { slice::from_raw_parts_mut(ptr.cast(), len) };
let copied = str::from_utf8(slice).expect("Non-UTF8 intrinsic name").to_string();
) -> &'ll Value {
unsafe {
libc::free(ptr.cast());
LLVMGetIntrinsicDeclaration(llmod, self.id, type_params.as_ptr(), type_params.len())
}
copied
}
}

7
compiler/rustc_codegen_llvm/src/type_.rs
View file

@ -58,13 +58,6 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
pub(crate) fn type_void(&self) -> &'ll Type {
unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) }
}
pub(crate) fn type_token(&self) -> &'ll Type {
unsafe { llvm::LLVMTokenTypeInContext(self.llcx()) }
}
pub(crate) fn type_metadata(&self) -> &'ll Type {
unsafe { llvm::LLVMMetadataTypeInContext(self.llcx()) }
}
///x Creates an integer type with the given number of bits, e.g., i24
pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type {