user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #21318 from rust-lang/rustc-pull

Browse source 
minor: Rustc pull update
This commit is contained in:
Laurențiu Nicola 2025-12-22 13:49:55 +00:00 • committed by GitHub
commit b91ed30be3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1036 changed files with 15537 additions and 7614 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock
View file

@ -3613,6 +3613,7 @@ dependencies = [
"gimli 0.31.1",
"itertools",
"libc",
"libloading 0.9.0",
"measureme",
"object 0.37.3",
"rustc-demangle",

25
bootstrap.example.toml
View file

@ -191,6 +191,31 @@
# Currently, this is only supported for the `x86_64-unknown-linux-gnu` target.
#gcc.download-ci-gcc = false
# Provide a directory of prebuilt libgccjit.so dylibs for given (host, target) compilation pairs.
# This is useful when you want to cross-compile `rustc` to another target since GCC is not a
# multi-target compiler.
# You have to use a directory structure that looks like this:
# `<libgccjit-libs-dir>/<host>/<target>/libgccjit.so`.
# For example:
#
# ```
# <libgccjit-libs-dir>
# ├── m68k-unknown-linux-gnu
# │ └── m68k-unknown-linux-gnu
# │ └── libgccjit.so
# └── x86_64-unknown-linux-gnu
# ├── m68k-unknown-linux-gnu
# │ └── libgccjit.so
# └── x86_64-unknown-linux-gnu
# └── libgccjit.so
# ```
# The directory above would allow you to cross-compile rustc from x64 to m68k
#
# Note that this option has priority over `gcc.download-ci-gcc`.
# If you set both, bootstrap will first try to load libgccjit.so from this directory.
# Only if it isn't found, it will try to download it from CI or build it locally.
#gcc.libgccjit-libs-dir = "/path/to/libgccjit-libs-dir"
# =============================================================================
# General build configuration options
# =============================================================================

5
compiler/rustc_abi/src/callconv.rs
View file

@ -60,6 +60,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
/// This is public so that it can be used in unit tests, but
/// should generally only be relevant to the ABI details of
/// specific targets.
#[tracing::instrument(skip(cx), level = "debug")]
pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, Heterogeneous>
where
Ty: TyAbiInterface<'a, C> + Copy,
@ -82,6 +83,10 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
}))
}
BackendRepr::ScalableVector { .. } => {
unreachable!("`homogeneous_aggregate` should not be called for scalable vectors")
}
BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
// Helper for computing `homogeneous_aggregate`, allowing a custom
// starting offset (used below for handling variants).

178
compiler/rustc_abi/src/layout.rs
View file

@ -11,7 +11,7 @@ use tracing::{debug, trace};
use crate::{
AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
Variants, WrappingRange,
TargetDataLayout, Variants, WrappingRange,
};
mod coroutine;
@ -143,58 +143,32 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
})
}
pub fn simd_type<
pub fn scalable_vector_type<FieldIdx, VariantIdx, F>(
&self,
element: F,
count: u64,
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
where
FieldIdx: Idx,
VariantIdx: Idx,
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
>(
{
vector_type_layout(VectorKind::Scalable, self.cx.data_layout(), element, count)
}
pub fn simd_type<FieldIdx, VariantIdx, F>(
&self,
element: F,
count: u64,
repr_packed: bool,
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
let elt = element.as_ref();
if count == 0 {
return Err(LayoutCalculatorError::ZeroLengthSimdType);
} else if count > crate::MAX_SIMD_LANES {
return Err(LayoutCalculatorError::OversizedSimdType {
max_lanes: crate::MAX_SIMD_LANES,
});
}
let BackendRepr::Scalar(e_repr) = elt.backend_repr else {
return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
};
// Compute the size and alignment of the vector
let dl = self.cx.data_layout();
let size =
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
let (repr, align) = if repr_packed && !count.is_power_of_two() {
// Non-power-of-two vectors have padding up to the next power-of-two.
// If we're a packed repr, remove the padding while keeping the alignment as close
// to a vector as possible.
(BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
} else {
(BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
};
let size = size.align_to(align);
Ok(LayoutData {
variants: Variants::Single { index: VariantIdx::new(0) },
fields: FieldsShape::Arbitrary {
offsets: [Size::ZERO].into(),
memory_index: [0].into(),
},
backend_repr: repr,
largest_niche: elt.largest_niche,
uninhabited: false,
size,
align: AbiAlign::new(align),
max_repr_align: None,
unadjusted_abi_align: elt.align.abi,
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
})
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
where
FieldIdx: Idx,
VariantIdx: Idx,
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
{
let kind = if repr_packed { VectorKind::PackedFixed } else { VectorKind::Fixed };
vector_type_layout(kind, self.cx.data_layout(), element, count)
}
/// Compute the layout for a coroutine.
@ -453,6 +427,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
BackendRepr::Scalar(..)
| BackendRepr::ScalarPair(..)
| BackendRepr::SimdVector { .. }
| BackendRepr::ScalableVector { .. }
| BackendRepr::Memory { .. } => repr,
},
};
@ -524,7 +499,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
hide_niches(a);
hide_niches(b);
}
BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
BackendRepr::SimdVector { element, .. }
| BackendRepr::ScalableVector { element, .. } => hide_niches(element),
BackendRepr::Memory { sized: _ } => {}
}
st.largest_niche = None;
@ -738,7 +714,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
},
fields: FieldsShape::Arbitrary {
offsets: [niche_offset].into(),
memory_index: [0].into(),
in_memory_order: [FieldIdx::new(0)].into(),
},
backend_repr: abi,
largest_niche,
@ -1032,8 +1008,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let pair =
LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, tag, prim_scalar);
let pair_offsets = match pair.fields {
FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
assert_eq!(memory_index.raw, [0, 1]);
FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {
assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]);
offsets
}
_ => panic!("encountered a non-arbitrary layout during enum layout"),
@ -1085,7 +1061,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
},
fields: FieldsShape::Arbitrary {
offsets: [Size::ZERO].into(),
memory_index: [0].into(),
in_memory_order: [FieldIdx::new(0)].into(),
},
largest_niche,
uninhabited,
@ -1134,10 +1110,10 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let pack = repr.pack;
let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
let mut max_repr_align = repr.align;
let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
let mut in_memory_order: IndexVec<u32, FieldIdx> = fields.indices().collect();
let optimize_field_order = !repr.inhibit_struct_field_reordering();
let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
let optimizing = &mut inverse_memory_index.raw[..end];
let optimizing = &mut in_memory_order.raw[..end];
let fields_excluding_tail = &fields.raw[..end];
// unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts.
let field_seed = fields_excluding_tail
@ -1272,12 +1248,10 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
// regardless of the status of `-Z randomize-layout`
}
}
// inverse_memory_index holds field indices by increasing memory offset.
// That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
// in_memory_order holds field indices by increasing memory offset.
// That is, if field 5 has offset 0, the first element of in_memory_order is 5.
// We now write field offsets to the corresponding offset slot;
// field 5 with offset 0 puts 0 in offsets[5].
// At the bottom of this function, we invert `inverse_memory_index` to
// produce `memory_index` (see `invert_mapping`).
let mut unsized_field = None::<&F>;
let mut offsets = IndexVec::from_elem(Size::ZERO, fields);
let mut offset = Size::ZERO;
@ -1289,7 +1263,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
align = align.max(prefix_align);
offset = prefix_size.align_to(prefix_align);
}
for &i in &inverse_memory_index {
for &i in &in_memory_order {
let field = &fields[i];
if let Some(unsized_field) = unsized_field {
return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field));
@ -1346,18 +1320,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
debug!("univariant min_size: {:?}", offset);
let min_size = offset;
// As stated above, inverse_memory_index holds field indices by increasing offset.
// This makes it an already-sorted view of the offsets vec.
// To invert it, consider:
// If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
// Field 5 would be the first element, so memory_index is i:
// Note: if we didn't optimize, it's already right.
let memory_index = if optimize_field_order {
inverse_memory_index.invert_bijective_mapping()
} else {
debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
inverse_memory_index.into_iter().map(|it| it.index() as u32).collect()
};
let size = min_size.align_to(align);
// FIXME(oli-obk): deduplicate and harden these checks
if size.bytes() >= dl.obj_size_bound() {
@ -1413,8 +1375,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let pair =
LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, a, b);
let pair_offsets = match pair.fields {
FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
assert_eq!(memory_index.raw, [0, 1]);
FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {
assert_eq!(
in_memory_order.raw,
[FieldIdx::new(0), FieldIdx::new(1)]
);
offsets
}
FieldsShape::Primitive
@ -1458,7 +1423,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
Ok(LayoutData {
variants: Variants::Single { index: VariantIdx::new(0) },
fields: FieldsShape::Arbitrary { offsets, memory_index },
fields: FieldsShape::Arbitrary { offsets, in_memory_order },
backend_repr: abi,
largest_niche,
uninhabited,
@ -1501,3 +1466,70 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
s
}
}
enum VectorKind {
/// `#[rustc_scalable_vector]`
Scalable,
/// `#[repr(simd, packed)]`
PackedFixed,
/// `#[repr(simd)]`
Fixed,
}
fn vector_type_layout<FieldIdx, VariantIdx, F>(
kind: VectorKind,
dl: &TargetDataLayout,
element: F,
count: u64,
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
where
FieldIdx: Idx,
VariantIdx: Idx,
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
{
let elt = element.as_ref();
if count == 0 {
return Err(LayoutCalculatorError::ZeroLengthSimdType);
} else if count > crate::MAX_SIMD_LANES {
return Err(LayoutCalculatorError::OversizedSimdType { max_lanes: crate::MAX_SIMD_LANES });
}
let BackendRepr::Scalar(element) = elt.backend_repr else {
return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
};
// Compute the size and alignment of the vector
let size =
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
let (repr, align) = match kind {
VectorKind::Scalable => {
(BackendRepr::ScalableVector { element, count }, dl.llvmlike_vector_align(size))
}
// Non-power-of-two vectors have padding up to the next power-of-two.
// If we're a packed repr, remove the padding while keeping the alignment as close
// to a vector as possible.
VectorKind::PackedFixed if !count.is_power_of_two() => {
(BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
}
VectorKind::PackedFixed | VectorKind::Fixed => {
(BackendRepr::SimdVector { element, count }, dl.llvmlike_vector_align(size))
}
};
let size = size.align_to(align);
Ok(LayoutData {
variants: Variants::Single { index: VariantIdx::new(0) },
fields: FieldsShape::Arbitrary {
offsets: [Size::ZERO].into(),
in_memory_order: [FieldIdx::new(0)].into(),
},
backend_repr: repr,
largest_niche: elt.largest_niche,
uninhabited: false,
size,
align: AbiAlign::new(align),
max_repr_align: None,
unadjusted_abi_align: elt.align.abi,
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
})
}

46
compiler/rustc_abi/src/layout/coroutine.rs
View file

@ -182,33 +182,29 @@ pub(super) fn layout<
// CoroutineLayout.
debug!("prefix = {:#?}", prefix);
let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields {
FieldsShape::Arbitrary { mut offsets, memory_index } => {
let mut inverse_memory_index = memory_index.invert_bijective_mapping();
FieldsShape::Arbitrary { mut offsets, in_memory_order } => {
// "a" (`0..b_start`) and "b" (`b_start..`) correspond to
// "outer" and "promoted" fields respectively.
let b_start = tag_index.plus(1);
let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index()));
let offsets_a = offsets;
// Disentangle the "a" and "b" components of `inverse_memory_index`
// Disentangle the "a" and "b" components of `in_memory_order`
// by preserving the order but keeping only one disjoint "half" each.
// FIXME(eddyb) build a better abstraction for permutations, if possible.
let inverse_memory_index_b: IndexVec<u32, FieldIdx> = inverse_memory_index
.iter()
.filter_map(|&i| i.index().checked_sub(b_start.index()).map(FieldIdx::new))
.collect();
inverse_memory_index.raw.retain(|&i| i.index() < b_start.index());
let inverse_memory_index_a = inverse_memory_index;
// Since `inverse_memory_index_{a,b}` each only refer to their
// respective fields, they can be safely inverted
let memory_index_a = inverse_memory_index_a.invert_bijective_mapping();
let memory_index_b = inverse_memory_index_b.invert_bijective_mapping();
let mut in_memory_order_a = IndexVec::<u32, FieldIdx>::new();
let mut in_memory_order_b = IndexVec::<u32, FieldIdx>::new();
for i in in_memory_order {
if let Some(j) = i.index().checked_sub(b_start.index()) {
in_memory_order_b.push(FieldIdx::new(j));
} else {
in_memory_order_a.push(i);
}
}
let outer_fields =
FieldsShape::Arbitrary { offsets: offsets_a, memory_index: memory_index_a };
(outer_fields, offsets_b, memory_index_b)
FieldsShape::Arbitrary { offsets: offsets_a, in_memory_order: in_memory_order_a };
(outer_fields, offsets_b, in_memory_order_b.invert_bijective_mapping())
}
_ => unreachable!(),
};
@ -236,7 +232,7 @@ pub(super) fn layout<
)?;
variant.variants = Variants::Single { index };
let FieldsShape::Arbitrary { offsets, memory_index } = variant.fields else {
let FieldsShape::Arbitrary { offsets, in_memory_order } = variant.fields else {
unreachable!();
};
@ -249,8 +245,9 @@ pub(super) fn layout<
// promoted fields were being used, but leave the elements not in the
// subset as `invalid_field_idx`, which we can filter out later to
// obtain a valid (bijective) mapping.
let memory_index = in_memory_order.invert_bijective_mapping();
let invalid_field_idx = promoted_memory_index.len() + memory_index.len();
let mut combined_inverse_memory_index =
let mut combined_in_memory_order =
IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx);
let mut offsets_and_memory_index = iter::zip(offsets, memory_index);
@ -268,19 +265,18 @@ pub(super) fn layout<
(promoted_offsets[field_idx], promoted_memory_index[field_idx])
}
};
combined_inverse_memory_index[memory_index] = i;
combined_in_memory_order[memory_index] = i;
offset
})
.collect();
// Remove the unused slots and invert the mapping to obtain the
// combined `memory_index` (also see previous comment).
combined_inverse_memory_index.raw.retain(|&i| i.index() != invalid_field_idx);
let combined_memory_index = combined_inverse_memory_index.invert_bijective_mapping();
// Remove the unused slots to obtain the combined `in_memory_order`
// (also see previous comment).
combined_in_memory_order.raw.retain(|&i| i.index() != invalid_field_idx);
variant.fields = FieldsShape::Arbitrary {
offsets: combined_offsets,
memory_index: combined_memory_index,
in_memory_order: combined_in_memory_order,
};
size = size.max(variant.size);

6
compiler/rustc_abi/src/layout/simple.rs
View file

@ -16,7 +16,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
variants: Variants::Single { index: VariantIdx::new(0) },
fields: FieldsShape::Arbitrary {
offsets: IndexVec::new(),
memory_index: IndexVec::new(),
in_memory_order: IndexVec::new(),
},
backend_repr: BackendRepr::Memory { sized },
largest_niche: None,
@ -108,7 +108,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
variants: Variants::Single { index: VariantIdx::new(0) },
fields: FieldsShape::Arbitrary {
offsets: [Size::ZERO, b_offset].into(),
memory_index: [0, 1].into(),
in_memory_order: [FieldIdx::new(0), FieldIdx::new(1)].into(),
},
backend_repr: BackendRepr::ScalarPair(a, b),
largest_niche,
@ -133,7 +133,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
Some(fields) => FieldsShape::Union(fields),
None => FieldsShape::Arbitrary {
offsets: IndexVec::new(),
memory_index: IndexVec::new(),
in_memory_order: IndexVec::new(),
},
},
backend_repr: BackendRepr::Memory { sized: true },

10
compiler/rustc_abi/src/layout/ty.rs
View file

@ -155,7 +155,7 @@ impl<'a, Ty> AsRef<LayoutData<FieldIdx, VariantIdx>> for TyAndLayout<'a, Ty> {
/// Trait that needs to be implemented by the higher-level type representation
/// (e.g. `rustc_middle::ty::Ty`), to provide `rustc_target::abi` functionality.
pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug {
pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug + std::fmt::Display {
fn ty_and_layout_for_variant(
this: TyAndLayout<'a, Self>,
cx: &C,
@ -172,6 +172,7 @@ pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug {
fn is_tuple(this: TyAndLayout<'a, Self>) -> bool;
fn is_unit(this: TyAndLayout<'a, Self>) -> bool;
fn is_transparent(this: TyAndLayout<'a, Self>) -> bool;
fn is_scalable_vector(this: TyAndLayout<'a, Self>) -> bool;
/// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
fn is_pass_indirectly_in_non_rustic_abis_flag_set(this: TyAndLayout<'a, Self>) -> bool;
}
@ -271,6 +272,13 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
Ty::is_transparent(self)
}
pub fn is_scalable_vector<C>(self) -> bool
where
Ty: TyAbiInterface<'a, C>,
{
Ty::is_scalable_vector(self)
}
/// If this method returns `true`, then this type should always have a `PassMode` of
/// `Indirect { on_stack: false, .. }` when being used as the argument type of a function with a
/// non-Rustic ABI (this is true for structs annotated with the

112
compiler/rustc_abi/src/lib.rs
View file

@ -96,9 +96,11 @@ bitflags! {
/// If true, the type is always passed indirectly by non-Rustic ABIs.
/// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5;
/// Any of these flags being set prevent field reordering optimisation.
const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
const IS_SCALABLE = 1 << 6;
// Any of these flags being set prevent field reordering optimisation.
const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
| ReprFlags::IS_SIMD.bits()
| ReprFlags::IS_SCALABLE.bits()
| ReprFlags::IS_LINEAR.bits();
const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
}
@ -135,6 +137,19 @@ impl IntegerType {
}
}
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
#[cfg_attr(
feature = "nightly",
derive(Encodable_NoContext, Decodable_NoContext, HashStable_Generic)
)]
pub enum ScalableElt {
/// `N` in `rustc_scalable_vector(N)` - the element count of the scalable vector
ElementCount(u16),
/// `rustc_scalable_vector` w/out `N`, used for tuple types of scalable vectors that only
/// contain other scalable vectors
Container,
}
/// Represents the repr options provided by the user.
#[derive(Copy, Clone, Debug, Eq, PartialEq, Default)]
#[cfg_attr(
@ -146,6 +161,8 @@ pub struct ReprOptions {
pub align: Option<Align>,
pub pack: Option<Align>,
pub flags: ReprFlags,
/// `#[rustc_scalable_vector]`
pub scalable: Option<ScalableElt>,
/// The seed to be used for randomizing a type's layout
///
/// Note: This could technically be a `u128` which would
@ -162,6 +179,11 @@ impl ReprOptions {
self.flags.contains(ReprFlags::IS_SIMD)
}
#[inline]
pub fn scalable(&self) -> bool {
self.flags.contains(ReprFlags::IS_SCALABLE)
}
#[inline]
pub fn c(&self) -> bool {
self.flags.contains(ReprFlags::IS_C)
@ -1614,19 +1636,14 @@ pub enum FieldsShape<FieldIdx: Idx> {
// FIXME(eddyb) use small vector optimization for the common case.
offsets: IndexVec<FieldIdx, Size>,
/// Maps source order field indices to memory order indices,
/// Maps memory order field indices to source order indices,
/// depending on how the fields were reordered (if at all).
/// This is a permutation, with both the source order and the
/// memory order using the same (0..n) index ranges.
///
/// Note that during computation of `memory_index`, sometimes
/// it is easier to operate on the inverse mapping (that is,
/// from memory order to source order), and that is usually
/// named `inverse_memory_index`.
///
// FIXME(eddyb) build a better abstraction for permutations, if possible.
// FIXME(camlorn) also consider small vector optimization here.
memory_index: IndexVec<FieldIdx, u32>,
in_memory_order: IndexVec<u32, FieldIdx>,
},
}
@ -1660,51 +1677,17 @@ impl<FieldIdx: Idx> FieldsShape<FieldIdx> {
}
}
#[inline]
pub fn memory_index(&self, i: usize) -> usize {
match *self {
FieldsShape::Primitive => {
unreachable!("FieldsShape::memory_index: `Primitive`s have no fields")
}
FieldsShape::Union(_) | FieldsShape::Array { .. } => i,
FieldsShape::Arbitrary { ref memory_index, .. } => {
memory_index[FieldIdx::new(i)].try_into().unwrap()
}
}
}
/// Gets source indices of the fields by increasing offsets.
#[inline]
pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator<Item = usize> {
let mut inverse_small = [0u8; 64];
let mut inverse_big = IndexVec::new();
let use_small = self.count() <= inverse_small.len();
// We have to write this logic twice in order to keep the array small.
if let FieldsShape::Arbitrary { ref memory_index, .. } = *self {
if use_small {
for (field_idx, &mem_idx) in memory_index.iter_enumerated() {
inverse_small[mem_idx as usize] = field_idx.index() as u8;
}
} else {
inverse_big = memory_index.invert_bijective_mapping();
}
}
// Primitives don't really have fields in the way that structs do,
// but having this return an empty iterator for them is unhelpful
// since that makes them look kinda like ZSTs, which they're not.
let pseudofield_count = if let FieldsShape::Primitive = self { 1 } else { self.count() };
(0..pseudofield_count).map(move |i| match *self {
(0..pseudofield_count).map(move |i| match self {
FieldsShape::Primitive | FieldsShape::Union(_) | FieldsShape::Array { .. } => i,
FieldsShape::Arbitrary { .. } => {
if use_small {
inverse_small[i] as usize
} else {
inverse_big[i as u32].index()
}
}
FieldsShape::Arbitrary { in_memory_order, .. } => in_memory_order[i as u32].index(),
})
}
}
@ -1736,6 +1719,10 @@ impl AddressSpace {
pub enum BackendRepr {
Scalar(Scalar),
ScalarPair(Scalar, Scalar),
ScalableVector {
element: Scalar,
count: u64,
},
SimdVector {
element: Scalar,
count: u64,
@ -1754,6 +1741,12 @@ impl BackendRepr {
match *self {
BackendRepr::Scalar(_)
| BackendRepr::ScalarPair(..)
// FIXME(rustc_scalable_vector): Scalable vectors are `Sized` while the
// `sized_hierarchy` feature is not yet fully implemented. After `sized_hierarchy` is
// fully implemented, scalable vectors will remain `Sized`, they just won't be
// `const Sized` - whether `is_unsized` continues to return `false` at that point will
// need to be revisited and will depend on what `is_unsized` is used for.
| BackendRepr::ScalableVector { .. }
| BackendRepr::SimdVector { .. } => false,
BackendRepr::Memory { sized } => !sized,
}
@ -1794,7 +1787,9 @@ impl BackendRepr {
BackendRepr::Scalar(s) => Some(s.align(cx).abi),
BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
// The align of a Vector can vary in surprising ways
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
BackendRepr::SimdVector { .. }
| BackendRepr::Memory { .. }
| BackendRepr::ScalableVector { .. } => None,
}
}
@ -1816,7 +1811,9 @@ impl BackendRepr {
Some(size)
}
// The size of a Vector can vary in surprising ways
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
BackendRepr::SimdVector { .. }
| BackendRepr::Memory { .. }
| BackendRepr::ScalableVector { .. } => None,
}
}
@ -1831,6 +1828,9 @@ impl BackendRepr {
BackendRepr::SimdVector { element: element.to_union(), count }
}
BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
BackendRepr::ScalableVector { element, count } => {
BackendRepr::ScalableVector { element: element.to_union(), count }
}
}
}
@ -2071,7 +2071,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
/// Returns `true` if this is an aggregate type (including a ScalarPair!)
pub fn is_aggregate(&self) -> bool {
match self.backend_repr {
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
BackendRepr::Scalar(_)
| BackendRepr::SimdVector { .. }
| BackendRepr::ScalableVector { .. } => false,
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
}
}
@ -2165,6 +2167,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
self.is_sized() && self.size.bytes() == 0 && self.align.bytes() == 1
}
/// Returns `true` if the size of the type is only known at runtime.
pub fn is_runtime_sized(&self) -> bool {
matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
}
/// Returns the elements count of a scalable vector.
pub fn scalable_vector_element_count(&self) -> Option<u64> {
match self.backend_repr {
BackendRepr::ScalableVector { count, .. } => Some(count),
_ => None,
}
}
/// Returns `true` if the type is a ZST and not unsized.
///
/// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@ -2173,6 +2188,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
match self.backend_repr {
BackendRepr::Scalar(_)
| BackendRepr::ScalarPair(..)
| BackendRepr::ScalableVector { .. }
| BackendRepr::SimdVector { .. } => false,
BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
}

1
compiler/rustc_arena/src/lib.rs
View file

@ -10,7 +10,6 @@
// tidy-alphabetical-start
#![allow(clippy::mut_from_ref)] // Arena allocators are one place where this pattern is fine.
#![allow(internal_features)]
#![cfg_attr(bootstrap, feature(maybe_uninit_slice))]
#![cfg_attr(test, feature(test))]
#![deny(unsafe_op_in_unsafe_fn)]
#![doc(test(no_crate_inject, attr(deny(warnings), allow(internal_features))))]

1
compiler/rustc_ast/src/lib.rs
View file

@ -6,7 +6,6 @@
// tidy-alphabetical-start
#![cfg_attr(bootstrap, feature(array_windows))]
#![deny(clippy::manual_let_else)]
#![doc(test(attr(deny(warnings), allow(internal_features))))]
#![feature(associated_type_defaults)]
#![feature(box_patterns)]

2
compiler/rustc_ast_lowering/messages.ftl
View file

@ -56,6 +56,8 @@ ast_lowering_coroutine_too_many_parameters =
ast_lowering_default_field_in_tuple = default fields are not supported in tuple structs
.label = default fields are only supported on structs
ast_lowering_delegation_cycle_in_signature_resolution = encountered a cycle during delegation signature resolution
ast_lowering_delegation_unresolved_callee = failed to resolve delegation callee
ast_lowering_does_not_support_modifiers =
the `{$class_name}` register class does not support template modifiers

68
compiler/rustc_ast_lowering/src/delegation.rs
View file

@ -44,6 +44,7 @@ use hir::{BodyId, HirId};
use rustc_abi::ExternAbi;
use rustc_ast::*;
use rustc_attr_parsing::{AttributeParser, ShouldEmit};
use rustc_data_structures::fx::FxHashSet;
use rustc_errors::ErrorGuaranteed;
use rustc_hir::Target;
use rustc_hir::attrs::{AttributeKind, InlineAttr};
@ -55,6 +56,7 @@ use rustc_span::{DUMMY_SP, Ident, Span, Symbol};
use {rustc_ast as ast, rustc_hir as hir};
use super::{GenericArgsMode, ImplTraitContext, LoweringContext, ParamMode};
use crate::errors::{CycleInDelegationSignatureResolution, UnresolvedDelegationCallee};
use crate::{AllowReturnTypeNotation, ImplTraitPosition, ResolverAstLoweringExt};
pub(crate) struct DelegationResults<'hir> {
@ -119,10 +121,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
&mut self,
delegation: &Delegation,
item_id: NodeId,
is_in_trait_impl: bool,
) -> DelegationResults<'hir> {
let span = self.lower_span(delegation.path.segments.last().unwrap().ident.span);
let sig_id = self.get_delegation_sig_id(item_id, delegation.id, span, is_in_trait_impl);
let sig_id = self.get_delegation_sig_id(
self.resolver.delegation_sig_resolution_nodes[&self.local_def_id(item_id)],
span,
);
match sig_id {
Ok(sig_id) => {
self.add_attributes_if_needed(span, sig_id);
@ -238,24 +244,48 @@ impl<'hir> LoweringContext<'_, 'hir> {
fn get_delegation_sig_id(
&self,
item_id: NodeId,
path_id: NodeId,
mut node_id: NodeId,
span: Span,
is_in_trait_impl: bool,
) -> Result<DefId, ErrorGuaranteed> {
let sig_id = if is_in_trait_impl { item_id } else { path_id };
self.get_resolution_id(sig_id, span)
let mut visited: FxHashSet<NodeId> = Default::default();
loop {
visited.insert(node_id);
let Some(def_id) = self.get_resolution_id(node_id) else {
return Err(self.tcx.dcx().span_delayed_bug(
span,
format!(
"LoweringContext: couldn't resolve node {:?} in delegation item",
node_id
),
));
};
// If def_id is in local crate and it corresponds to another delegation
// it means that we refer to another delegation as a callee, so in order to obtain
// a signature DefId we obtain NodeId of the callee delegation and try to get signature from it.
if let Some(local_id) = def_id.as_local()
&& let Some(next_node_id) =
self.resolver.delegation_sig_resolution_nodes.get(&local_id)
{
node_id = *next_node_id;
if visited.contains(&node_id) {
// We encountered a cycle in the resolution, or delegation callee refers to non-existent
// entity, in this case emit an error.
return Err(match visited.len() {
1 => self.dcx().emit_err(UnresolvedDelegationCallee { span }),
_ => self.dcx().emit_err(CycleInDelegationSignatureResolution { span }),
});
}
} else {
return Ok(def_id);
}
}
}
fn get_resolution_id(&self, node_id: NodeId, span: Span) -> Result<DefId, ErrorGuaranteed> {
let def_id =
self.resolver.get_partial_res(node_id).and_then(|r| r.expect_full_res().opt_def_id());
def_id.ok_or_else(|| {
self.tcx.dcx().span_delayed_bug(
span,
format!("LoweringContext: couldn't resolve node {:?} in delegation item", node_id),
)
})
fn get_resolution_id(&self, node_id: NodeId) -> Option<DefId> {
self.resolver.get_partial_res(node_id).and_then(|r| r.expect_full_res().opt_def_id())
}
fn lower_delegation_generics(&mut self, span: Span) -> &'hir hir::Generics<'hir> {
@ -271,8 +301,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
// Function parameter count, including C variadic `...` if present.
fn param_count(&self, sig_id: DefId) -> (usize, bool /*c_variadic*/) {
if let Some(local_sig_id) = sig_id.as_local() {
// Map may be filled incorrectly due to recursive delegation.
// Error will be emitted later during HIR ty lowering.
match self.resolver.delegation_fn_sigs.get(&local_sig_id) {
Some(sig) => (sig.param_count, sig.c_variadic),
None => (0, false),
@ -489,8 +517,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
delegation.path.segments.iter().rev().skip(1).any(|segment| segment.args.is_some());
let call = if self
.get_resolution_id(delegation.id, span)
.and_then(|def_id| Ok(self.is_method(def_id, span)))
.get_resolution_id(delegation.id)
.map(|def_id| self.is_method(def_id, span))
.unwrap_or_default()
&& delegation.qself.is_none()
&& !has_generic_args

14
compiler/rustc_ast_lowering/src/errors.rs
View file

@ -475,3 +475,17 @@ pub(crate) struct UnionWithDefault {
#[primary_span]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(ast_lowering_delegation_unresolved_callee)]
pub(crate) struct UnresolvedDelegationCallee {
#[primary_span]
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(ast_lowering_delegation_cycle_in_signature_resolution)]
pub(crate) struct CycleInDelegationSignatureResolution {
#[primary_span]
pub span: Span,
}

6
compiler/rustc_ast_lowering/src/item.rs
View file

@ -541,7 +541,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
hir::ItemKind::Macro(ident, macro_def, macro_kinds)
}
ItemKind::Delegation(box delegation) => {
let delegation_results = self.lower_delegation(delegation, id, false);
let delegation_results = self.lower_delegation(delegation, id);
hir::ItemKind::Fn {
sig: delegation_results.sig,
ident: delegation_results.ident,
@ -1026,7 +1026,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
(*ident, generics, kind, ty.is_some())
}
AssocItemKind::Delegation(box delegation) => {
let delegation_results = self.lower_delegation(delegation, i.id, false);
let delegation_results = self.lower_delegation(delegation, i.id);
let item_kind = hir::TraitItemKind::Fn(
delegation_results.sig,
hir::TraitFn::Provided(delegation_results.body_id),
@ -1196,7 +1196,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
)
}
AssocItemKind::Delegation(box delegation) => {
let delegation_results = self.lower_delegation(delegation, i.id, is_in_trait_impl);
let delegation_results = self.lower_delegation(delegation, i.id);
(
delegation.ident,
(

36
compiler/rustc_ast_lowering/src/lib.rs
View file

@ -237,25 +237,27 @@ impl SpanLowerer {
#[extension(trait ResolverAstLoweringExt)]
impl ResolverAstLowering {
fn legacy_const_generic_args(&self, expr: &Expr) -> Option<Vec<usize>> {
if let ExprKind::Path(None, path) = &expr.kind {
// Don't perform legacy const generics rewriting if the path already
// has generic arguments.
if path.segments.last().unwrap().args.is_some() {
return None;
}
let ExprKind::Path(None, path) = &expr.kind else {
return None;
};
if let Res::Def(DefKind::Fn, def_id) = self.partial_res_map.get(&expr.id)?.full_res()? {
// We only support cross-crate argument rewriting. Uses
// within the same crate should be updated to use the new
// const generics style.
if def_id.is_local() {
return None;
}
// Don't perform legacy const generics rewriting if the path already
// has generic arguments.
if path.segments.last().unwrap().args.is_some() {
return None;
}
if let Some(v) = self.legacy_const_generic_args.get(&def_id) {
return v.clone();
}
}
let def_id = self.partial_res_map.get(&expr.id)?.full_res()?.opt_def_id()?;
// We only support cross-crate argument rewriting. Uses
// within the same crate should be updated to use the new
// const generics style.
if def_id.is_local() {
return None;
}
if let Some(v) = self.legacy_const_generic_args.get(&def_id) {
return v.clone();
}
None

2
compiler/rustc_ast_passes/messages.ftl
View file

@ -270,6 +270,8 @@ ast_passes_precise_capturing_duplicated = duplicate `use<...>` precise capturing
ast_passes_precise_capturing_not_allowed_here = `use<...>` precise capturing syntax not allowed in {$loc}
ast_passes_scalable_vector_not_tuple_struct = scalable vectors must be tuple structs
ast_passes_static_without_body =
free static item without body
.suggestion = provide a definition for the static

8
compiler/rustc_ast_passes/src/ast_validation.rs
View file

@ -1319,6 +1319,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
}
ItemKind::Struct(ident, generics, vdata) => {
self.with_tilde_const(Some(TildeConstReason::Struct { span: item.span }), |this| {
// Scalable vectors can only be tuple structs
let is_scalable_vector =
item.attrs.iter().any(|attr| attr.has_name(sym::rustc_scalable_vector));
if is_scalable_vector && !matches!(vdata, VariantData::Tuple(..)) {
this.dcx()
.emit_err(errors::ScalableVectorNotTupleStruct { span: item.span });
}
match vdata {
VariantData::Struct { fields, .. } => {
this.visit_attrs_vis_ident(&item.attrs, &item.vis, ident);

7
compiler/rustc_ast_passes/src/errors.rs
View file

@ -990,3 +990,10 @@ pub(crate) struct AbiX86Interrupt {
pub spans: Vec<Span>,
pub param_count: usize,
}
#[derive(Diagnostic)]
#[diag(ast_passes_scalable_vector_not_tuple_struct)]
pub(crate) struct ScalableVectorNotTupleStruct {
#[primary_span]
pub span: Span,
}

12
compiler/rustc_attr_parsing/messages.ftl
View file

@ -50,11 +50,6 @@ attr_parsing_expects_feature_list =
attr_parsing_expects_features =
`{$name}` expects feature names
attr_parsing_ill_formed_attribute_input = {$num_suggestions ->
[1] attribute must be of the form {$suggestions}
*[other] valid forms for the attribute are {$suggestions}
}
attr_parsing_import_name_type_raw =
import name type can only be used with link kind `raw-dylib`
@ -205,6 +200,9 @@ attr_parsing_rustc_allowed_unstable_pairing =
attr_parsing_rustc_promotable_pairing =
`rustc_promotable` attribute must be paired with either a `rustc_const_unstable` or a `rustc_const_stable` attribute
attr_parsing_rustc_scalable_vector_count_out_of_range = element count in `rustc_scalable_vector` is too large: `{$n}`
.note = the value may not exceed `u16::MAX`
attr_parsing_soft_no_args =
`soft` should not have any arguments
@ -213,10 +211,6 @@ attr_parsing_stability_outside_std = stability attributes may not be used outsid
attr_parsing_suffixed_literal_in_attribute = suffixed literals are not allowed in attributes
.help = instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), use an unsuffixed version (`1`, `1.0`, etc.)
attr_parsing_unknown_meta_item =
unknown meta item '{$item}'
.label = expected one of {$expected}
attr_parsing_unknown_version_literal =
unknown version literal format, assuming it refers to a future version

1
compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
View file

@ -61,7 +61,6 @@ impl<S: Stage> CombineAttributeParser<S> for AllowConstFnUnstableParser {
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);

2
compiler/rustc_attr_parsing/src/attributes/cfg.rs
View file

@ -42,7 +42,7 @@ pub fn parse_cfg<S: Stage>(
args: &ArgParser,
) -> Option<CfgEntry> {
let ArgParser::List(list) = args else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
let Some(single) = list.single() else {

27
compiler/rustc_attr_parsing/src/attributes/cfg_select.rs
View file

@ -28,6 +28,33 @@ pub struct CfgSelectBranches {
pub unreachable: Vec<(CfgSelectPredicate, TokenStream, Span)>,
}
impl CfgSelectBranches {
/// Removes the top-most branch for which `predicate` returns `true`,
/// or the wildcard if none of the reachable branches satisfied the predicate.
pub fn pop_first_match<F>(&mut self, predicate: F) -> Option<(TokenStream, Span)>
where
F: Fn(&CfgEntry) -> bool,
{
for (index, (cfg, _, _)) in self.reachable.iter().enumerate() {
if predicate(cfg) {
let matched = self.reachable.remove(index);
return Some((matched.1, matched.2));
}
}
self.wildcard.take().map(|(_, tts, span)| (tts, span))
}
/// Consume this value and iterate over all the `TokenStream`s that it stores.
pub fn into_iter_tts(self) -> impl Iterator<Item = (TokenStream, Span)> {
let it1 = self.reachable.into_iter().map(|(_, tts, span)| (tts, span));
let it2 = self.wildcard.into_iter().map(|(_, tts, span)| (tts, span));
let it3 = self.unreachable.into_iter().map(|(_, tts, span)| (tts, span));
it1.chain(it2).chain(it3)
}
}
pub fn parse_cfg_select(
p: &mut Parser<'_>,
sess: &Session,

19
compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
View file

@ -25,7 +25,7 @@ impl<S: Stage> SingleAttributeParser<S> for OptimizeParser {
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option<AttributeKind> {
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
@ -57,7 +57,6 @@ impl<S: Stage> NoArgsAttributeParser<S> for ColdParser {
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::ForeignFn),
Allow(Target::Closure),
@ -343,7 +342,7 @@ impl<S: Stage> NoArgsAttributeParser<S> for TrackCallerParser {
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: false })), // `#[track_caller]` is inherited from trait methods
Allow(Target::ForeignFn),
Allow(Target::Closure),
Warn(Target::MacroDef),
@ -478,7 +477,7 @@ fn parse_tf_attribute<S: Stage>(
) -> impl IntoIterator<Item = (Symbol, Span)> {
let mut features = Vec::new();
let ArgParser::List(list) = args else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return features;
};
if list.is_empty() {
@ -601,7 +600,7 @@ impl<S: Stage> SingleAttributeParser<S> for SanitizeParser {
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option<AttributeKind> {
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
@ -691,6 +690,16 @@ impl<S: Stage> SingleAttributeParser<S> for SanitizeParser {
}
}
pub(crate) struct ThreadLocalParser;
impl<S: Stage> NoArgsAttributeParser<S> for ThreadLocalParser {
const PATH: &[Symbol] = &[sym::thread_local];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
const ALLOWED_TARGETS: AllowedTargets =
AllowedTargets::AllowList(&[Allow(Target::Static), Allow(Target::ForeignStatic)]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::ThreadLocal;
}
pub(crate) struct RustcPassIndirectlyInNonRusticAbisParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcPassIndirectlyInNonRusticAbisParser {

2
compiler/rustc_attr_parsing/src/attributes/confusables.rs
View file

@ -13,7 +13,7 @@ impl<S: Stage> AttributeParser<S> for ConfusablesParser {
template!(List: &[r#""name1", "name2", ..."#]),
|this, cx, args| {
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return;
};

2
compiler/rustc_attr_parsing/src/attributes/debugger.rs
View file

@ -21,7 +21,7 @@ impl<S: Stage> CombineAttributeParser<S> for DebuggerViualizerParser {
args: &ArgParser,
) -> impl IntoIterator<Item = Self::Item> {
let Some(l) = args.list() else {
cx.expected_list(args.span().unwrap_or(cx.attr_span));
cx.expected_list(cx.attr_span, args);
return None;
};
let Some(single) = l.single() else {

7
compiler/rustc_attr_parsing/src/attributes/deprecation.rs
View file

@ -110,13 +110,12 @@ impl<S: Stage> SingleAttributeParser<S> for DeprecationParser {
Some(get(cx, name, param.span(), param.args(), &suggestion)?);
}
_ => {
cx.unknown_key(
cx.expected_specific_argument(
param.span(),
param.path().to_string(),
if features.deprecated_suggestion() {
&["since", "note", "suggestion"]
&[sym::since, sym::note, sym::suggestion]
} else {
&["since", "note"]
&[sym::since, sym::note]
},
);
return None;

2
compiler/rustc_attr_parsing/src/attributes/doc.rs
View file

@ -106,7 +106,7 @@ impl DocParser {
}
Some(sym::attr) => {
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return;
};

7
compiler/rustc_attr_parsing/src/attributes/link_attrs.rs
View file

@ -76,7 +76,7 @@ impl<S: Stage> CombineAttributeParser<S> for LinkParser {
return None;
}
_ => {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
}
};
@ -379,7 +379,7 @@ impl LinkParser {
return true;
}
let Some(link_cfg) = item.args().list() else {
cx.expected_list(item.span());
cx.expected_list(item.span(), item.args());
return true;
};
let Some(link_cfg) = link_cfg.single() else {
@ -469,7 +469,6 @@ impl<S: Stage> SingleAttributeParser<S> for LinkSectionParser {
Allow(Target::Static),
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);
@ -587,12 +586,12 @@ impl<S: Stage> SingleAttributeParser<S> for LinkageParser {
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
Allow(Target::Static),
Allow(Target::ForeignStatic),
Allow(Target::ForeignFn),
Warn(Target::Method(MethodKind::Trait { body: false })), // Not inherited
]);
const TEMPLATE: AttributeTemplate = template!(NameValueStr: [

25
compiler/rustc_attr_parsing/src/attributes/macro_attrs.rs
View file

@ -1,9 +1,7 @@
use rustc_errors::DiagArgValue;
use rustc_hir::attrs::MacroUseArgs;
use rustc_session::lint::builtin::INVALID_MACRO_EXPORT_ARGUMENTS;
use super::prelude::*;
use crate::session_diagnostics::IllFormedAttributeInputLint;
pub(crate) struct MacroEscapeParser;
impl<S: Stage> NoArgsAttributeParser<S> for MacroEscapeParser {
@ -101,15 +99,8 @@ impl<S: Stage> AttributeParser<S> for MacroUseParser {
}
}
}
ArgParser::NameValue(_) => {
let suggestions = cx.suggestions();
cx.emit_err(IllFormedAttributeInputLint {
num_suggestions: suggestions.len(),
suggestions: DiagArgValue::StrListSepByAnd(
suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(),
),
span,
});
ArgParser::NameValue(nv) => {
cx.expected_list_or_no_args(nv.args_span());
}
}
},
@ -164,16 +155,8 @@ impl<S: Stage> SingleAttributeParser<S> for MacroExportParser {
}
}
}
ArgParser::NameValue(_) => {
let span = cx.attr_span;
let suggestions = cx.suggestions();
cx.emit_err(IllFormedAttributeInputLint {
num_suggestions: suggestions.len(),
suggestions: DiagArgValue::StrListSepByAnd(
suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(),
),
span,
});
ArgParser::NameValue(nv) => {
cx.expected_list_or_no_args(nv.args_span());
return None;
}
};

1
compiler/rustc_attr_parsing/src/attributes/mod.rs
View file

@ -47,6 +47,7 @@ pub(crate) mod loop_match;
pub(crate) mod macro_attrs;
pub(crate) mod must_use;
pub(crate) mod no_implicit_prelude;
pub(crate) mod no_link;
pub(crate) mod non_exhaustive;
pub(crate) mod path;
pub(crate) mod pin_v2;

14
compiler/rustc_attr_parsing/src/attributes/must_use.rs
View file

@ -1,7 +1,4 @@
use rustc_errors::DiagArgValue;
use super::prelude::*;
use crate::session_diagnostics::IllFormedAttributeInputLint;
pub(crate) struct MustUseParser;
@ -44,15 +41,8 @@ impl<S: Stage> SingleAttributeParser<S> for MustUseParser {
};
Some(value_str)
}
ArgParser::List(_) => {
let suggestions = cx.suggestions();
cx.emit_err(IllFormedAttributeInputLint {
num_suggestions: suggestions.len(),
suggestions: DiagArgValue::StrListSepByAnd(
suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(),
),
span: cx.attr_span,
});
ArgParser::List(list) => {
cx.expected_nv_or_no_args(list.span);
return None;
}
},

14
compiler/rustc_attr_parsing/src/attributes/no_link.rs Normal file
View file

@ -0,0 +1,14 @@
use super::prelude::*;
pub(crate) struct NoLinkParser;
impl<S: Stage> NoArgsAttributeParser<S> for NoLinkParser {
const PATH: &[Symbol] = &[sym::no_link];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Warn;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::ExternCrate),
Warn(Target::Field),
Warn(Target::Arm),
Warn(Target::MacroDef),
]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::NoLink;
}

6
compiler/rustc_attr_parsing/src/attributes/proc_macro_attrs.rs
View file

@ -65,7 +65,7 @@ fn parse_derive_like<S: Stage>(
if args.no_args().is_ok() && !trait_name_mandatory {
return Some((None, ThinVec::new()));
}
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
let mut items = list.mixed();
@ -96,7 +96,7 @@ fn parse_derive_like<S: Stage>(
let mut attributes = ThinVec::new();
if let Some(attrs) = items.next() {
let Some(attr_list) = attrs.meta_item() else {
cx.expected_list(attrs.span());
cx.unexpected_literal(attrs.span());
return None;
};
if !attr_list.path().word_is(sym::attributes) {
@ -104,7 +104,7 @@ fn parse_derive_like<S: Stage>(
return None;
}
let Some(attr_list) = attr_list.args().list() else {
cx.expected_list(attrs.span());
cx.expected_list(attrs.span(), attr_list.args());
return None;
};

7
compiler/rustc_attr_parsing/src/attributes/prototype.rs
View file

@ -27,7 +27,7 @@ impl<S: Stage> SingleAttributeParser<S> for CustomMirParser {
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option<AttributeKind> {
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
@ -46,9 +46,8 @@ impl<S: Stage> SingleAttributeParser<S> for CustomMirParser {
extract_value(cx, sym::dialect, arg, meta_item.span(), &mut dialect, &mut failed);
} else if let Some(arg) = meta_item.word_is(sym::phase) {
extract_value(cx, sym::phase, arg, meta_item.span(), &mut phase, &mut failed);
} else if let Some(word) = meta_item.path().word() {
let word = word.to_string();
cx.unknown_key(meta_item.span(), word, &["dialect", "phase"]);
} else if let Some(..) = meta_item.path().word() {
cx.expected_specific_argument(meta_item.span(), &[sym::dialect, sym::phase]);
failed = true;
} else {
cx.expected_name_value(meta_item.span(), None);

6
compiler/rustc_attr_parsing/src/attributes/repr.rs
View file

@ -33,7 +33,7 @@ impl<S: Stage> CombineAttributeParser<S> for ReprParser {
let mut reprs = Vec::new();
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return reprs;
};
@ -278,7 +278,7 @@ impl AlignParser {
fn parse<S: Stage>(&mut self, cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) {
match args {
ArgParser::NoArgs | ArgParser::NameValue(_) => {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
}
ArgParser::List(list) => {
let Some(align) = list.single() else {
@ -315,7 +315,7 @@ impl<S: Stage> AttributeParser<S> for AlignParser {
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: false })), // `#[align]` is inherited from trait methods
Allow(Target::ForeignFn),
]);

182
compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
View file

@ -1,5 +1,8 @@
use rustc_ast::{LitIntType, LitKind, MetaItemLit};
use super::prelude::*;
use super::util::parse_single_integer;
use crate::session_diagnostics::RustcScalableVectorCountOutOfRange;
pub(crate) struct RustcMainParser;
@ -10,6 +13,36 @@ impl<S: Stage> NoArgsAttributeParser<S> for RustcMainParser {
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcMain;
}
pub(crate) struct RustcNeverReturnsNullPointerParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcNeverReturnsNullPointerParser {
const PATH: &[Symbol] = &[sym::rustc_never_returns_null_ptr];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNeverReturnsNullPointer;
}
pub(crate) struct RustcNoImplicitAutorefsParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcNoImplicitAutorefsParser {
const PATH: &[Symbol] = &[sym::rustc_no_implicit_autorefs];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNoImplicitAutorefs;
}
pub(crate) struct RustcLayoutScalarValidRangeStartParser;
impl<S: Stage> SingleAttributeParser<S> for RustcLayoutScalarValidRangeStartParser {
@ -40,6 +73,129 @@ impl<S: Stage> SingleAttributeParser<S> for RustcLayoutScalarValidRangeEndParser
}
}
pub(crate) struct RustcLegacyConstGenericsParser;
impl<S: Stage> SingleAttributeParser<S> for RustcLegacyConstGenericsParser {
const PATH: &[Symbol] = &[sym::rustc_legacy_const_generics];
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
const TEMPLATE: AttributeTemplate = template!(List: &["N"]);
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option<AttributeKind> {
let ArgParser::List(meta_items) = args else {
cx.expected_list(cx.attr_span, args);
return None;
};
let mut parsed_indexes = ThinVec::new();
let mut errored = false;
for possible_index in meta_items.mixed() {
if let MetaItemOrLitParser::Lit(MetaItemLit {
kind: LitKind::Int(index, LitIntType::Unsuffixed),
..
}) = possible_index
{
parsed_indexes.push((index.0 as usize, possible_index.span()));
} else {
cx.expected_integer_literal(possible_index.span());
errored = true;
}
}
if errored {
return None;
} else if parsed_indexes.is_empty() {
cx.expected_at_least_one_argument(args.span()?);
return None;
}
Some(AttributeKind::RustcLegacyConstGenerics {
fn_indexes: parsed_indexes,
attr_span: cx.attr_span,
})
}
}
pub(crate) struct RustcLintDiagnosticsParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcLintDiagnosticsParser {
const PATH: &[Symbol] = &[sym::rustc_lint_diagnostics];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintDiagnostics;
}
pub(crate) struct RustcLintOptDenyFieldAccessParser;
impl<S: Stage> SingleAttributeParser<S> for RustcLintOptDenyFieldAccessParser {
const PATH: &[Symbol] = &[sym::rustc_lint_opt_deny_field_access];
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Field)]);
const TEMPLATE: AttributeTemplate = template!(Word);
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option<AttributeKind> {
let Some(arg) = args.list().and_then(MetaItemListParser::single) else {
cx.expected_single_argument(cx.attr_span);
return None;
};
let MetaItemOrLitParser::Lit(MetaItemLit { kind: LitKind::Str(lint_message, _), .. }) = arg
else {
cx.expected_string_literal(arg.span(), arg.lit());
return None;
};
Some(AttributeKind::RustcLintOptDenyFieldAccess { lint_message: *lint_message })
}
}
pub(crate) struct RustcLintOptTyParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcLintOptTyParser {
const PATH: &[Symbol] = &[sym::rustc_lint_opt_ty];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintOptTy;
}
pub(crate) struct RustcLintQueryInstabilityParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcLintQueryInstabilityParser {
const PATH: &[Symbol] = &[sym::rustc_lint_query_instability];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintQueryInstability;
}
pub(crate) struct RustcLintUntrackedQueryInformationParser;
impl<S: Stage> NoArgsAttributeParser<S> for RustcLintUntrackedQueryInformationParser {
const PATH: &[Symbol] = &[sym::rustc_lint_untracked_query_information];
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
Allow(Target::Method(MethodKind::Trait { body: false })),
Allow(Target::Method(MethodKind::Trait { body: true })),
Allow(Target::Method(MethodKind::TraitImpl)),
]);
const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintUntrackedQueryInformation;
}
pub(crate) struct RustcObjectLifetimeDefaultParser;
impl<S: Stage> SingleAttributeParser<S> for RustcObjectLifetimeDefaultParser {
@ -76,3 +232,29 @@ impl<S: Stage> SingleAttributeParser<S> for RustcSimdMonomorphizeLaneLimitParser
Some(AttributeKind::RustcSimdMonomorphizeLaneLimit(cx.parse_limit_int(nv)?))
}
}
pub(crate) struct RustcScalableVectorParser;
impl<S: Stage> SingleAttributeParser<S> for RustcScalableVectorParser {
const PATH: &[rustc_span::Symbol] = &[sym::rustc_scalable_vector];
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]);
const TEMPLATE: AttributeTemplate = template!(Word, List: &["count"]);
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option<AttributeKind> {
if args.no_args().is_ok() {
return Some(AttributeKind::RustcScalableVector {
element_count: None,
span: cx.attr_span,
});
}
let n = parse_single_integer(cx, args)?;
let Ok(n) = n.try_into() else {
cx.emit_err(RustcScalableVectorCountOutOfRange { span: cx.attr_span, n });
return None;
};
Some(AttributeKind::RustcScalableVector { element_count: Some(n), span: cx.attr_span })
}
}

26
compiler/rustc_attr_parsing/src/attributes/stability.rs
View file

@ -295,7 +295,7 @@ pub(crate) fn parse_stability<S: Stage>(
let mut since = None;
let ArgParser::List(list) = args else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
@ -315,11 +315,7 @@ pub(crate) fn parse_stability<S: Stage>(
insert_value_into_option_or_error(cx, &param, &mut since, word.unwrap())?
}
_ => {
cx.emit_err(session_diagnostics::UnknownMetaItem {
span: param_span,
item: param.path().to_string(),
expected: &["feature", "since"],
});
cx.expected_specific_argument(param_span, &[sym::feature, sym::since]);
return None;
}
}
@ -371,7 +367,7 @@ pub(crate) fn parse_unstability<S: Stage>(
let mut old_name = None;
let ArgParser::List(list) = args else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
@ -426,11 +422,17 @@ pub(crate) fn parse_unstability<S: Stage>(
insert_value_into_option_or_error(cx, &param, &mut old_name, word.unwrap())?
}
_ => {
cx.emit_err(session_diagnostics::UnknownMetaItem {
span: param.span(),
item: param.path().to_string(),
expected: &["feature", "reason", "issue", "soft", "implied_by", "old_name"],
});
cx.expected_specific_argument(
param.span(),
&[
sym::feature,
sym::reason,
sym::issue,
sym::soft,
sym::implied_by,
sym::old_name,
],
);
return None;
}
}

2
compiler/rustc_attr_parsing/src/attributes/traits.rs
View file

@ -22,7 +22,7 @@ impl<S: Stage> SingleAttributeParser<S> for SkipDuringMethodDispatchParser {
let mut array = false;
let mut boxed_slice = false;
let Some(args) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
if args.is_empty() {

2
compiler/rustc_attr_parsing/src/attributes/util.rs
View file

@ -43,7 +43,7 @@ pub(crate) fn parse_single_integer<S: Stage>(
args: &ArgParser,
) -> Option<u128> {
let Some(list) = args.list() else {
cx.expected_list(cx.attr_span);
cx.expected_list(cx.attr_span, args);
return None;
};
let Some(single) = list.single() else {

190
compiler/rustc_attr_parsing/src/context.rs
View file

@ -23,7 +23,7 @@ use crate::attributes::codegen_attrs::{
ColdParser, CoverageParser, EiiExternItemParser, ExportNameParser, ForceTargetFeatureParser,
NakedParser, NoMangleParser, ObjcClassParser, ObjcSelectorParser, OptimizeParser,
RustcPassIndirectlyInNonRusticAbisParser, SanitizeParser, TargetFeatureParser,
TrackCallerParser, UsedParser,
ThreadLocalParser, TrackCallerParser, UsedParser,
};
use crate::attributes::confusables::ConfusablesParser;
use crate::attributes::crate_level::{
@ -50,6 +50,7 @@ use crate::attributes::macro_attrs::{
};
use crate::attributes::must_use::MustUseParser;
use crate::attributes::no_implicit_prelude::NoImplicitPreludeParser;
use crate::attributes::no_link::NoLinkParser;
use crate::attributes::non_exhaustive::NonExhaustiveParser;
use crate::attributes::path::PathParser as PathAttributeParser;
use crate::attributes::pin_v2::PinV2Parser;
@ -59,8 +60,12 @@ use crate::attributes::proc_macro_attrs::{
use crate::attributes::prototype::CustomMirParser;
use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
use crate::attributes::rustc_internal::{
RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser, RustcMainParser,
RustcObjectLifetimeDefaultParser, RustcSimdMonomorphizeLaneLimitParser,
RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser,
RustcLegacyConstGenericsParser, RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser,
RustcLintOptTyParser, RustcLintQueryInstabilityParser,
RustcLintUntrackedQueryInformationParser, RustcMainParser, RustcNeverReturnsNullPointerParser,
RustcNoImplicitAutorefsParser, RustcObjectLifetimeDefaultParser, RustcScalableVectorParser,
RustcSimdMonomorphizeLaneLimitParser,
};
use crate::attributes::semantics::MayDangleParser;
use crate::attributes::stability::{
@ -77,7 +82,7 @@ use crate::attributes::transparency::TransparencyParser;
use crate::attributes::{AttributeParser as _, Combine, Single, WithoutArgs};
use crate::parser::{ArgParser, RefPathParser};
use crate::session_diagnostics::{
AttributeParseError, AttributeParseErrorReason, ParsedDescription, UnknownMetaItem,
AttributeParseError, AttributeParseErrorReason, ParsedDescription,
};
use crate::target_checking::AllowedTargets;
@ -208,7 +213,10 @@ attribute_parsers!(
Single<RustcForceInlineParser>,
Single<RustcLayoutScalarValidRangeEndParser>,
Single<RustcLayoutScalarValidRangeStartParser>,
Single<RustcLegacyConstGenericsParser>,
Single<RustcLintOptDenyFieldAccessParser>,
Single<RustcObjectLifetimeDefaultParser>,
Single<RustcScalableVectorParser>,
Single<RustcSimdMonomorphizeLaneLimitParser>,
Single<SanitizeParser>,
Single<ShouldPanicParser>,
@ -238,6 +246,7 @@ attribute_parsers!(
Single<WithoutArgs<MayDangleParser>>,
Single<WithoutArgs<NoCoreParser>>,
Single<WithoutArgs<NoImplicitPreludeParser>>,
Single<WithoutArgs<NoLinkParser>>,
Single<WithoutArgs<NoMangleParser>>,
Single<WithoutArgs<NoStdParser>>,
Single<WithoutArgs<NonExhaustiveParser>>,
@ -249,11 +258,18 @@ attribute_parsers!(
Single<WithoutArgs<ProcMacroParser>>,
Single<WithoutArgs<PubTransparentParser>>,
Single<WithoutArgs<RustcCoherenceIsCoreParser>>,
Single<WithoutArgs<RustcLintDiagnosticsParser>>,
Single<WithoutArgs<RustcLintOptTyParser>>,
Single<WithoutArgs<RustcLintQueryInstabilityParser>>,
Single<WithoutArgs<RustcLintUntrackedQueryInformationParser>>,
Single<WithoutArgs<RustcMainParser>>,
Single<WithoutArgs<RustcNeverReturnsNullPointerParser>>,
Single<WithoutArgs<RustcNoImplicitAutorefsParser>>,
Single<WithoutArgs<RustcPassIndirectlyInNonRusticAbisParser>>,
Single<WithoutArgs<RustcShouldNotBeCalledOnConstItems>>,
Single<WithoutArgs<SpecializationTraitParser>>,
Single<WithoutArgs<StdInternalSymbolParser>>,
Single<WithoutArgs<ThreadLocalParser>>,
Single<WithoutArgs<TrackCallerParser>>,
Single<WithoutArgs<TypeConstParser>>,
Single<WithoutArgs<UnsafeSpecializationMarkerParser>>,
@ -426,13 +442,20 @@ impl<'f, 'sess: 'f, S: Stage> SharedContext<'f, 'sess, S> {
}
impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> {
pub(crate) fn unknown_key(
fn emit_parse_error(
&self,
span: Span,
found: String,
options: &[&'static str],
reason: AttributeParseErrorReason<'_>,
) -> ErrorGuaranteed {
self.emit_err(UnknownMetaItem { span, item: found, expected: options })
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason,
suggestions: self.suggestions(),
})
}
/// error that a string literal was expected.
@ -444,133 +467,69 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> {
span: Span,
actual_literal: Option<&MetaItemLit>,
) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
self.emit_parse_error(
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedStringLiteral {
AttributeParseErrorReason::ExpectedStringLiteral {
byte_string: actual_literal.and_then(|i| {
i.kind.is_bytestr().then(|| self.sess().source_map().start_point(i.span))
}),
},
suggestions: self.suggestions(),
})
)
}
pub(crate) fn expected_integer_literal(&self, span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedIntegerLiteral,
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedIntegerLiteral)
}
pub(crate) fn expected_list(&self, span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedList,
suggestions: self.suggestions(),
})
pub(crate) fn expected_list(&self, span: Span, args: &ArgParser) -> ErrorGuaranteed {
let span = match args {
ArgParser::NoArgs => span,
ArgParser::List(list) => list.span,
ArgParser::NameValue(nv) => nv.args_span(),
};
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedList)
}
pub(crate) fn expected_no_args(&self, args_span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span: args_span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedNoArgs,
suggestions: self.suggestions(),
})
pub(crate) fn expected_list_or_no_args(&self, span: Span) -> ErrorGuaranteed {
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedListOrNoArgs)
}
pub(crate) fn expected_nv_or_no_args(&self, span: Span) -> ErrorGuaranteed {
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedNameValueOrNoArgs)
}
pub(crate) fn expected_no_args(&self, span: Span) -> ErrorGuaranteed {
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedNoArgs)
}
/// emit an error that a `name` was expected here
pub(crate) fn expected_identifier(&self, span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedIdentifier,
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedIdentifier)
}
/// emit an error that a `name = value` pair was expected at this span. The symbol can be given for
/// a nicer error message talking about the specific name that was found lacking a value.
pub(crate) fn expected_name_value(&self, span: Span, name: Option<Symbol>) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedNameValue(name),
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedNameValue(name))
}
/// emit an error that a `name = value` pair was found where that name was already seen.
pub(crate) fn duplicate_key(&self, span: Span, key: Symbol) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::DuplicateKey(key),
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::DuplicateKey(key))
}
/// an error that should be emitted when a [`MetaItemOrLitParser`](crate::parser::MetaItemOrLitParser)
/// was expected *not* to be a literal, but instead a meta item.
pub(crate) fn unexpected_literal(&self, span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::UnexpectedLiteral,
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::UnexpectedLiteral)
}
pub(crate) fn expected_single_argument(&self, span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedSingleArgument,
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedSingleArgument)
}
pub(crate) fn expected_at_least_one_argument(&self, span: Span) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedAtLeastOneArgument,
suggestions: self.suggestions(),
})
self.emit_parse_error(span, AttributeParseErrorReason::ExpectedAtLeastOneArgument)
}
/// produces an error along the lines of `expected one of [foo, meow]`
@ -579,19 +538,14 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> {
span: Span,
possibilities: &[Symbol],
) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
self.emit_parse_error(
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedSpecificArgument {
AttributeParseErrorReason::ExpectedSpecificArgument {
possibilities,
strings: false,
list: false,
},
suggestions: self.suggestions(),
})
)
}
/// produces an error along the lines of `expected one of [foo, meow] as an argument`.
@ -601,19 +555,14 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> {
span: Span,
possibilities: &[Symbol],
) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
self.emit_parse_error(
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedSpecificArgument {
AttributeParseErrorReason::ExpectedSpecificArgument {
possibilities,
strings: false,
list: true,
},
suggestions: self.suggestions(),
})
)
}
/// produces an error along the lines of `expected one of ["foo", "meow"]`
@ -622,19 +571,14 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> {
span: Span,
possibilities: &[Symbol],
) -> ErrorGuaranteed {
self.emit_err(AttributeParseError {
self.emit_parse_error(
span,
attr_span: self.attr_span,
template: self.template.clone(),
path: self.attr_path.clone(),
description: self.parsed_description,
reason: AttributeParseErrorReason::ExpectedSpecificArgument {
AttributeParseErrorReason::ExpectedSpecificArgument {
possibilities,
strings: true,
list: false,
},
suggestions: self.suggestions(),
})
)
}
pub(crate) fn warn_empty_attribute(&mut self, span: Span) {

6
compiler/rustc_attr_parsing/src/parser.rs
View file

@ -177,7 +177,7 @@ impl ArgParser {
match self {
Self::NoArgs => Ok(()),
Self::List(args) => Err(args.span),
Self::NameValue(args) => Err(args.eq_span.to(args.value_span)),
Self::NameValue(args) => Err(args.args_span()),
}
}
}
@ -314,6 +314,10 @@ impl NameValueParser {
pub fn value_as_str(&self) -> Option<Symbol> {
self.value_as_lit().kind.str()
}
pub fn args_span(&self) -> Span {
self.eq_span.to(self.value_span)
}
}
fn expr_to_lit(

46
compiler/rustc_attr_parsing/src/session_diagnostics.rs
View file

@ -64,26 +64,6 @@ pub(crate) struct DocAttributeNotAttribute {
pub attribute: Symbol,
}
/// Error code: E0541
pub(crate) struct UnknownMetaItem<'a> {
pub span: Span,
pub item: String,
pub expected: &'a [&'a str],
}
// Manual implementation to be able to format `expected` items correctly.
impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for UnknownMetaItem<'_> {
fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> {
let expected = self.expected.iter().map(|name| format!("`{name}`")).collect::<Vec<_>>();
Diag::new(dcx, level, fluent::attr_parsing_unknown_meta_item)
.with_span(self.span)
.with_code(E0541)
.with_arg("item", self.item)
.with_arg("expected", expected.join(", "))
.with_span_label(self.span, fluent::attr_parsing_label)
}
}
#[derive(Diagnostic)]
#[diag(attr_parsing_missing_since, code = E0542)]
pub(crate) struct MissingSince {
@ -400,15 +380,6 @@ pub(crate) struct UnusedMultiple {
pub name: Symbol,
}
#[derive(Diagnostic)]
#[diag(attr_parsing_ill_formed_attribute_input)]
pub(crate) struct IllFormedAttributeInputLint {
#[primary_span]
pub span: Span,
pub num_suggestions: usize,
pub suggestions: DiagArgValue,
}
#[derive(Diagnostic)]
#[diag(attr_parsing_null_on_export, code = E0648)]
pub(crate) struct NullOnExport {
@ -530,6 +501,15 @@ pub(crate) struct LinkOrdinalOutOfRange {
pub ordinal: u128,
}
#[derive(Diagnostic)]
#[diag(attr_parsing_rustc_scalable_vector_count_out_of_range)]
#[note]
pub(crate) struct RustcScalableVectorCountOutOfRange {
#[primary_span]
pub span: Span,
pub n: u128,
}
pub(crate) enum AttributeParseErrorReason<'a> {
ExpectedNoArgs,
ExpectedStringLiteral {
@ -539,6 +519,8 @@ pub(crate) enum AttributeParseErrorReason<'a> {
ExpectedAtLeastOneArgument,
ExpectedSingleArgument,
ExpectedList,
ExpectedListOrNoArgs,
ExpectedNameValueOrNoArgs,
UnexpectedLiteral,
ExpectedNameValue(Option<Symbol>),
DuplicateKey(Symbol),
@ -611,6 +593,12 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError<'_> {
AttributeParseErrorReason::ExpectedList => {
diag.span_label(self.span, "expected this to be a list");
}
AttributeParseErrorReason::ExpectedListOrNoArgs => {
diag.span_label(self.span, "expected a list or no arguments here");
}
AttributeParseErrorReason::ExpectedNameValueOrNoArgs => {
diag.span_label(self.span, "didn't expect a list here");
}
AttributeParseErrorReason::DuplicateKey(key) => {
diag.span_label(self.span, format!("found `{key}` used as a key more than once"));
diag.code(E0538);

1
compiler/rustc_borrowck/src/lib.rs
View file

@ -2,7 +2,6 @@
// tidy-alphabetical-start
#![allow(internal_features)]
#![deny(clippy::manual_let_else)]
#![feature(assert_matches)]
#![feature(box_patterns)]
#![feature(file_buffered)]

76
compiler/rustc_borrowck/src/region_infer/mod.rs
View file

@ -1275,29 +1275,81 @@ impl<'tcx> RegionInferenceContext<'tcx> {
shorter_fr: RegionVid,
propagated_outlives_requirements: &mut Option<&mut Vec<ClosureOutlivesRequirement<'tcx>>>,
) -> RegionRelationCheckResult {
if let Some(propagated_outlives_requirements) = propagated_outlives_requirements
// Shrink `longer_fr` until we find a non-local region (if we do).
// We'll call it `fr-` -- it's ever so slightly smaller than
if let Some(propagated_outlives_requirements) = propagated_outlives_requirements {
// Shrink `longer_fr` until we find some non-local regions.
// We'll call them `longer_fr-` -- they are ever so slightly smaller than
// `longer_fr`.
&& let Some(fr_minus) = self.universal_region_relations.non_local_lower_bound(longer_fr)
{
debug!("try_propagate_universal_region_error: fr_minus={:?}", fr_minus);
let longer_fr_minus = self.universal_region_relations.non_local_lower_bounds(longer_fr);
debug!("try_propagate_universal_region_error: fr_minus={:?}", longer_fr_minus);
// If we don't find a any non-local regions, we should error out as there is nothing
// to propagate.
if longer_fr_minus.is_empty() {
return RegionRelationCheckResult::Error;
}
let blame_constraint = self
.best_blame_constraint(longer_fr, NllRegionVariableOrigin::FreeRegion, shorter_fr)
.0;
// Grow `shorter_fr` until we find some non-local regions. (We
// always will.) We'll call them `shorter_fr+` -- they're ever
// so slightly larger than `shorter_fr`.
// Grow `shorter_fr` until we find some non-local regions.
// We will always find at least one: `'static`. We'll call
// them `shorter_fr+` -- they're ever so slightly larger
// than `shorter_fr`.
let shorter_fr_plus =
self.universal_region_relations.non_local_upper_bounds(shorter_fr);
debug!("try_propagate_universal_region_error: shorter_fr_plus={:?}", shorter_fr_plus);
for fr in shorter_fr_plus {
// Push the constraint `fr-: shorter_fr+`
// We then create constraints `longer_fr-: shorter_fr+` that may or may not
// be propagated (see below).
let mut constraints = vec![];
for fr_minus in longer_fr_minus {
for shorter_fr_plus in &shorter_fr_plus {
constraints.push((fr_minus, *shorter_fr_plus));
}
}
// We only need to propagate at least one of the constraints for
// soundness. However, we want to avoid arbitrary choices here
// and currently don't support returning OR constraints.
//
// If any of the `shorter_fr+` regions are already outlived by `longer_fr-`,
// we propagate only those.
//
// Consider this example (`'b: 'a` == `a -> b`), where we try to propagate `'d: 'a`:
// a --> b --> d
// \
// \-> c
// Here, `shorter_fr+` of `'a` == `['b, 'c]`.
// Propagating `'d: 'b` is correct and should occur; `'d: 'c` is redundant because of
// `'d: 'b` and could reject valid code.
//
// So we filter the constraints to regions already outlived by `longer_fr-`, but if
// the filter yields an empty set, we fall back to the original one.
let subset: Vec<_> = constraints
.iter()
.filter(|&&(fr_minus, shorter_fr_plus)| {
self.eval_outlives(fr_minus, shorter_fr_plus)
})
.copied()
.collect();
let propagated_constraints = if subset.is_empty() { constraints } else { subset };
debug!(
"try_propagate_universal_region_error: constraints={:?}",
propagated_constraints
);
assert!(
!propagated_constraints.is_empty(),
"Expected at least one constraint to propagate here"
);
for (fr_minus, fr_plus) in propagated_constraints {
// Push the constraint `long_fr-: shorter_fr+`
propagated_outlives_requirements.push(ClosureOutlivesRequirement {
subject: ClosureOutlivesSubject::Region(fr_minus),
outlived_free_region: fr,
outlived_free_region: fr_plus,
blame_span: blame_constraint.cause.span,
category: blame_constraint.category,
});

24
compiler/rustc_borrowck/src/type_check/free_region_relations.rs
View file

@ -94,28 +94,10 @@ impl UniversalRegionRelations<'_> {
/// words, returns the largest (*) known region `fr1` that (a) is
/// outlived by `fr` and (b) is not local.
///
/// (*) If there are multiple competing choices, we pick the "postdominating"
/// one. See `TransitiveRelation::postdom_upper_bound` for details.
pub(crate) fn non_local_lower_bound(&self, fr: RegionVid) -> Option<RegionVid> {
/// (*) If there are multiple competing choices, we return all of them.
pub(crate) fn non_local_lower_bounds(&self, fr: RegionVid) -> Vec<RegionVid> {
debug!("non_local_lower_bound(fr={:?})", fr);
let lower_bounds = self.non_local_bounds(&self.outlives, fr);
// In case we find more than one, reduce to one for
// convenience. This is to prevent us from generating more
// complex constraints, but it will cause spurious errors.
let post_dom = self.outlives.mutual_immediate_postdominator(lower_bounds);
debug!("non_local_bound: post_dom={:?}", post_dom);
post_dom.and_then(|post_dom| {
// If the mutual immediate postdom is not local, then
// there is no non-local result we can return.
if !self.universal_regions.is_local_free_region(post_dom) {
Some(post_dom)
} else {
None
}
})
self.non_local_bounds(&self.outlives, fr)
}
/// Helper for `non_local_upper_bounds` and `non_local_lower_bounds`.

80
compiler/rustc_builtin_macros/src/cfg_select.rs
View file

@ -1,22 +1,65 @@
use rustc_ast::tokenstream::TokenStream;
use rustc_ast::{Expr, ast};
use rustc_attr_parsing as attr;
use rustc_attr_parsing::{
CfgSelectBranches, CfgSelectPredicate, EvalConfigResult, parse_cfg_select,
};
use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacroExpanderResult};
use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacResult, MacroExpanderResult};
use rustc_span::{Ident, Span, sym};
use smallvec::SmallVec;
use crate::errors::{CfgSelectNoMatches, CfgSelectUnreachable};
/// Selects the first arm whose predicate evaluates to true.
fn select_arm(ecx: &ExtCtxt<'_>, branches: CfgSelectBranches) -> Option<(TokenStream, Span)> {
for (cfg, tt, arm_span) in branches.reachable {
if let EvalConfigResult::True = attr::eval_config_entry(&ecx.sess, &cfg) {
return Some((tt, arm_span));
}
}
/// This intermediate structure is used to emit parse errors for the branches that are not chosen.
/// The `MacResult` instance below parses all branches, emitting any errors it encounters, but only
/// keeps the parse result for the selected branch.
struct CfgSelectResult<'cx, 'sess> {
ecx: &'cx mut ExtCtxt<'sess>,
site_span: Span,
selected_tts: TokenStream,
selected_span: Span,
other_branches: CfgSelectBranches,
}
branches.wildcard.map(|(_, tt, span)| (tt, span))
fn tts_to_mac_result<'cx, 'sess>(
ecx: &'cx mut ExtCtxt<'sess>,
site_span: Span,
tts: TokenStream,
span: Span,
) -> Box<dyn MacResult + 'cx> {
match ExpandResult::from_tts(ecx, tts, site_span, span, Ident::with_dummy_span(sym::cfg_select))
{
ExpandResult::Ready(x) => x,
_ => unreachable!("from_tts always returns Ready"),
}
}
macro_rules! forward_to_parser_any_macro {
($method_name:ident, $ret_ty:ty) => {
fn $method_name(self: Box<Self>) -> Option<$ret_ty> {
let CfgSelectResult { ecx, site_span, selected_tts, selected_span, .. } = *self;
for (tts, span) in self.other_branches.into_iter_tts() {
let _ = tts_to_mac_result(ecx, site_span, tts, span).$method_name();
}
tts_to_mac_result(ecx, site_span, selected_tts, selected_span).$method_name()
}
};
}
impl<'cx, 'sess> MacResult for CfgSelectResult<'cx, 'sess> {
forward_to_parser_any_macro!(make_expr, Box<Expr>);
forward_to_parser_any_macro!(make_stmts, SmallVec<[ast::Stmt; 1]>);
forward_to_parser_any_macro!(make_items, SmallVec<[Box<ast::Item>; 1]>);
forward_to_parser_any_macro!(make_impl_items, SmallVec<[Box<ast::AssocItem>; 1]>);
forward_to_parser_any_macro!(make_trait_impl_items, SmallVec<[Box<ast::AssocItem>; 1]>);
forward_to_parser_any_macro!(make_trait_items, SmallVec<[Box<ast::AssocItem>; 1]>);
forward_to_parser_any_macro!(make_foreign_items, SmallVec<[Box<ast::ForeignItem>; 1]>);
forward_to_parser_any_macro!(make_ty, Box<ast::Ty>);
forward_to_parser_any_macro!(make_pat, Box<ast::Pat>);
}
pub(super) fn expand_cfg_select<'cx>(
@ -31,7 +74,7 @@ pub(super) fn expand_cfg_select<'cx>(
Some(ecx.ecfg.features),
ecx.current_expansion.lint_node_id,
) {
Ok(branches) => {
Ok(mut branches) => {
if let Some((underscore, _, _)) = branches.wildcard {
// Warn for every unreachable predicate. We store the fully parsed branch for rustfmt.
for (predicate, _, _) in &branches.unreachable {
@ -44,14 +87,17 @@ pub(super) fn expand_cfg_select<'cx>(
}
}
if let Some((tts, arm_span)) = select_arm(ecx, branches) {
return ExpandResult::from_tts(
if let Some((selected_tts, selected_span)) = branches.pop_first_match(|cfg| {
matches!(attr::eval_config_entry(&ecx.sess, cfg), EvalConfigResult::True)
}) {
let mac = CfgSelectResult {
ecx,
tts,
sp,
arm_span,
Ident::with_dummy_span(sym::cfg_select),
);
selected_tts,
selected_span,
other_branches: branches,
site_span: sp,
};
return ExpandResult::Ready(Box::new(mac));
} else {
// Emit a compiler error when none of the predicates matched.
let guar = ecx.dcx().emit_err(CfgSelectNoMatches { span: sp });

416
compiler/rustc_builtin_macros/src/eii.rs
View file

@ -1,11 +1,12 @@
use rustc_ast::token::{Delimiter, TokenKind};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast::{
DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, Stmt, StmtKind, ast, token, tokenstream,
Attribute, DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, MetaItem, Path, Stmt, StmtKind,
Visibility, ast,
};
use rustc_ast_pretty::pprust::path_to_string;
use rustc_expand::base::{Annotatable, ExtCtxt};
use rustc_span::{Ident, Span, kw, sym};
use rustc_span::{ErrorGuaranteed, Ident, Span, kw, sym};
use thin_vec::{ThinVec, thin_vec};
use crate::errors::{
@ -52,12 +53,12 @@ pub(crate) fn unsafe_eii(
fn eii_(
ecx: &mut ExtCtxt<'_>,
span: Span,
eii_attr_span: Span,
meta_item: &ast::MetaItem,
item: Annotatable,
impl_unsafe: bool,
) -> Vec<Annotatable> {
let span = ecx.with_def_site_ctxt(span);
let eii_attr_span = ecx.with_def_site_ctxt(eii_attr_span);
let (item, stmt) = if let Annotatable::Item(item) = item {
(item, false)
@ -67,7 +68,7 @@ fn eii_(
(item.clone(), true)
} else {
ecx.dcx().emit_err(EiiSharedMacroExpectedFunction {
span,
span: eii_attr_span,
name: path_to_string(&meta_item.path),
});
return vec![item];
@ -77,113 +78,218 @@ fn eii_(
let item = *item;
let ast::Item { attrs, id: _, span: item_span, vis, kind: ItemKind::Fn(mut func), tokens: _ } =
item
else {
let ast::Item { attrs, id: _, span: _, vis, kind: ItemKind::Fn(func), tokens: _ } = item else {
ecx.dcx().emit_err(EiiSharedMacroExpectedFunction {
span,
span: eii_attr_span,
name: path_to_string(&meta_item.path),
});
return vec![Annotatable::Item(Box::new(item))];
};
// Detect when this is the *second* eii attribute on an item.
let mut new_attrs = ThinVec::new();
for i in attrs {
if i.has_name(sym::eii) {
ecx.dcx().emit_err(EiiOnlyOnce {
span: i.span,
first_span: span,
name: path_to_string(&meta_item.path),
});
} else {
new_attrs.push(i);
}
}
let attrs = new_attrs;
let attrs_from_decl =
filter_attrs_for_multiple_eii_attr(ecx, attrs, eii_attr_span, &meta_item.path);
let macro_name = if meta_item.is_word() {
func.ident
} else if let Some([first]) = meta_item.meta_item_list()
&& let Some(m) = first.meta_item()
&& m.path.segments.len() == 1
{
m.path.segments[0].ident
} else {
ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument {
span: meta_item.span,
name: path_to_string(&meta_item.path),
});
let Ok(macro_name) = name_for_impl_macro(ecx, &func, &meta_item) else {
return vec![Annotatable::Item(orig_item)];
};
// span of the declaring item without attributes
let item_span = func.sig.span;
// span of the eii attribute and the item below it, i.e. the full declaration
let decl_span = eii_attr_span.to(item_span);
let foreign_item_name = func.ident;
let mut return_items = Vec::new();
if func.body.is_some() {
let mut default_func = func.clone();
func.body = None;
default_func.eii_impls.push(ast::EiiImpl {
node_id: DUMMY_NODE_ID,
eii_macro_path: ast::Path::from_ident(macro_name),
impl_safety: if impl_unsafe { ast::Safety::Unsafe(span) } else { ast::Safety::Default },
span,
inner_span: macro_name.span,
is_default: true, // important!
});
return_items.push(Box::new(ast::Item {
attrs: ThinVec::new(),
id: ast::DUMMY_NODE_ID,
span,
vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None },
kind: ast::ItemKind::Const(Box::new(ast::ConstItem {
ident: Ident { name: kw::Underscore, span },
defaultness: ast::Defaultness::Final,
generics: ast::Generics::default(),
ty: Box::new(ast::Ty {
id: DUMMY_NODE_ID,
kind: ast::TyKind::Tup(ThinVec::new()),
span,
tokens: None,
}),
rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr {
id: DUMMY_NODE_ID,
kind: ast::ExprKind::Block(
Box::new(ast::Block {
stmts: thin_vec![ast::Stmt {
id: DUMMY_NODE_ID,
kind: ast::StmtKind::Item(Box::new(ast::Item {
attrs: thin_vec![], // FIXME: re-add some original attrs
id: DUMMY_NODE_ID,
span: item_span,
vis: ast::Visibility {
span,
kind: ast::VisibilityKind::Inherited,
tokens: None
},
kind: ItemKind::Fn(default_func),
tokens: None,
})),
span
}],
id: DUMMY_NODE_ID,
rules: ast::BlockCheckMode::Default,
span,
tokens: None,
}),
None,
),
span,
attrs: ThinVec::new(),
tokens: None,
}))),
define_opaque: None,
})),
tokens: None,
}))
return_items.push(Box::new(generate_default_impl(
&func,
impl_unsafe,
macro_name,
eii_attr_span,
item_span,
)))
}
let decl_span = span.to(func.sig.span);
return_items.push(Box::new(generate_foreign_item(
ecx,
eii_attr_span,
item_span,
*func,
vis,
&attrs_from_decl,
)));
return_items.push(Box::new(generate_attribute_macro_to_implement(
ecx,
eii_attr_span,
macro_name,
foreign_item_name,
impl_unsafe,
decl_span,
)));
if stmt {
return_items
.into_iter()
.map(|i| {
Annotatable::Stmt(Box::new(Stmt {
id: DUMMY_NODE_ID,
kind: StmtKind::Item(i),
span: eii_attr_span,
}))
})
.collect()
} else {
return_items.into_iter().map(|i| Annotatable::Item(i)).collect()
}
}
/// Decide on the name of the macro that can be used to implement the EII.
/// This is either an explicitly given name, or the name of the item in the
/// declaration of the EII.
fn name_for_impl_macro(
ecx: &mut ExtCtxt<'_>,
func: &ast::Fn,
meta_item: &MetaItem,
) -> Result<Ident, ErrorGuaranteed> {
if meta_item.is_word() {
Ok(func.ident)
} else if let Some([first]) = meta_item.meta_item_list()
&& let Some(m) = first.meta_item()
&& m.path.segments.len() == 1
{
Ok(m.path.segments[0].ident)
} else {
Err(ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument {
span: meta_item.span,
name: path_to_string(&meta_item.path),
}))
}
}
/// Ensure that in the list of attrs, there's only a single `eii` attribute.
fn filter_attrs_for_multiple_eii_attr(
ecx: &mut ExtCtxt<'_>,
attrs: ThinVec<Attribute>,
eii_attr_span: Span,
eii_attr_path: &Path,
) -> ThinVec<Attribute> {
attrs
.into_iter()
.filter(|i| {
if i.has_name(sym::eii) {
ecx.dcx().emit_err(EiiOnlyOnce {
span: i.span,
first_span: eii_attr_span,
name: path_to_string(eii_attr_path),
});
false
} else {
true
}
})
.collect()
}
fn generate_default_impl(
func: &ast::Fn,
impl_unsafe: bool,
macro_name: Ident,
eii_attr_span: Span,
item_span: Span,
) -> ast::Item {
// FIXME: re-add some original attrs
let attrs = ThinVec::new();
let mut default_func = func.clone();
default_func.eii_impls.push(EiiImpl {
node_id: DUMMY_NODE_ID,
inner_span: macro_name.span,
eii_macro_path: ast::Path::from_ident(macro_name),
impl_safety: if impl_unsafe {
ast::Safety::Unsafe(eii_attr_span)
} else {
ast::Safety::Default
},
span: eii_attr_span,
is_default: true,
});
ast::Item {
attrs: ThinVec::new(),
id: ast::DUMMY_NODE_ID,
span: eii_attr_span,
vis: ast::Visibility {
span: eii_attr_span,
kind: ast::VisibilityKind::Inherited,
tokens: None,
},
kind: ast::ItemKind::Const(Box::new(ast::ConstItem {
ident: Ident { name: kw::Underscore, span: eii_attr_span },
defaultness: ast::Defaultness::Final,
generics: ast::Generics::default(),
ty: Box::new(ast::Ty {
id: DUMMY_NODE_ID,
kind: ast::TyKind::Tup(ThinVec::new()),
span: eii_attr_span,
tokens: None,
}),
rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr {
id: DUMMY_NODE_ID,
kind: ast::ExprKind::Block(
Box::new(ast::Block {
stmts: thin_vec![ast::Stmt {
id: DUMMY_NODE_ID,
kind: ast::StmtKind::Item(Box::new(ast::Item {
attrs,
id: DUMMY_NODE_ID,
span: item_span,
vis: ast::Visibility {
span: eii_attr_span,
kind: ast::VisibilityKind::Inherited,
tokens: None
},
kind: ItemKind::Fn(Box::new(default_func)),
tokens: None,
})),
span: eii_attr_span
}],
id: DUMMY_NODE_ID,
rules: ast::BlockCheckMode::Default,
span: eii_attr_span,
tokens: None,
}),
None,
),
span: eii_attr_span,
attrs: ThinVec::new(),
tokens: None,
}))),
define_opaque: None,
})),
tokens: None,
}
}
/// Generates a foreign item, like
///
/// ```rust, ignore
/// extern "…" { safe fn item(); }
/// ```
fn generate_foreign_item(
ecx: &mut ExtCtxt<'_>,
eii_attr_span: Span,
item_span: Span,
mut func: ast::Fn,
vis: Visibility,
attrs_from_decl: &[Attribute],
) -> ast::Item {
let mut foreign_item_attrs = ThinVec::new();
foreign_item_attrs.extend_from_slice(attrs_from_decl);
// Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled.
// This attribute makes sure that we later know that this foreign item's symbol should not be.
foreign_item_attrs.push(ecx.attr_word(sym::rustc_eii_extern_item, eii_attr_span));
let abi = match func.sig.header.ext {
// extern "X" fn => extern "X" {}
@ -196,85 +302,69 @@ fn eii_(
suffix: None,
symbol_unescaped: sym::Rust,
style: ast::StrStyle::Cooked,
span,
span: eii_attr_span,
}),
};
// ABI has been moved to the extern {} block, so we remove it from the fn item.
func.sig.header.ext = ast::Extern::None;
func.body = None;
// And mark safe functions explicitly as `safe fn`.
if func.sig.header.safety == ast::Safety::Default {
func.sig.header.safety = ast::Safety::Safe(func.sig.span);
}
// extern "…" { safe fn item(); }
let mut extern_item_attrs = attrs.clone();
extern_item_attrs.push(ast::Attribute {
kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr {
item: ast::AttrItem {
unsafety: ast::Safety::Default,
// Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled.
// This attribute makes sure that we later know that this foreign item's symbol should not be.
path: ast::Path::from_ident(Ident::new(sym::rustc_eii_extern_item, span)),
args: ast::AttrArgs::Empty,
tokens: None,
},
tokens: None,
})),
id: ecx.sess.psess.attr_id_generator.mk_attr_id(),
style: ast::AttrStyle::Outer,
span,
});
let extern_block = Box::new(ast::Item {
ast::Item {
attrs: ast::AttrVec::default(),
id: ast::DUMMY_NODE_ID,
span,
vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None },
span: eii_attr_span,
vis: ast::Visibility {
span: eii_attr_span,
kind: ast::VisibilityKind::Inherited,
tokens: None,
},
kind: ast::ItemKind::ForeignMod(ast::ForeignMod {
extern_span: span,
safety: ast::Safety::Unsafe(span),
extern_span: eii_attr_span,
safety: ast::Safety::Unsafe(eii_attr_span),
abi,
items: From::from([Box::new(ast::ForeignItem {
attrs: extern_item_attrs,
attrs: foreign_item_attrs,
id: ast::DUMMY_NODE_ID,
span: item_span,
vis,
kind: ast::ForeignItemKind::Fn(func.clone()),
kind: ast::ForeignItemKind::Fn(Box::new(func.clone())),
tokens: None,
})]),
}),
tokens: None,
});
}
}
let mut macro_attrs = attrs.clone();
macro_attrs.push(
// #[builtin_macro(eii_shared_macro)]
ast::Attribute {
kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr {
item: ast::AttrItem {
unsafety: ast::Safety::Default,
path: ast::Path::from_ident(Ident::new(sym::rustc_builtin_macro, span)),
args: ast::AttrArgs::Delimited(ast::DelimArgs {
dspan: DelimSpan::from_single(span),
delim: Delimiter::Parenthesis,
tokens: TokenStream::new(vec![tokenstream::TokenTree::token_alone(
token::TokenKind::Ident(sym::eii_shared_macro, token::IdentIsRaw::No),
span,
)]),
}),
tokens: None,
},
tokens: None,
})),
id: ecx.sess.psess.attr_id_generator.mk_attr_id(),
style: ast::AttrStyle::Outer,
span,
},
);
/// Generate a stub macro (a bit like in core) that will roughly look like:
///
/// ```rust, ignore, example
/// // Since this a stub macro, the actual code that expands it lives in the compiler.
/// // This attribute tells the compiler that
/// #[builtin_macro(eii_shared_macro)]
/// // the metadata to link this macro to the generated foreign item.
/// #[eii_extern_target(<related_reign_item>)]
/// macro macro_name { () => {} }
/// ```
fn generate_attribute_macro_to_implement(
ecx: &mut ExtCtxt<'_>,
span: Span,
macro_name: Ident,
foreign_item_name: Ident,
impl_unsafe: bool,
decl_span: Span,
) -> ast::Item {
let mut macro_attrs = ThinVec::new();
let macro_def = Box::new(ast::Item {
// #[builtin_macro(eii_shared_macro)]
macro_attrs.push(ecx.attr_nested_word(sym::rustc_builtin_macro, sym::eii_shared_macro, span));
ast::Item {
attrs: macro_attrs,
id: ast::DUMMY_NODE_ID,
span,
@ -305,33 +395,15 @@ fn eii_(
]),
}),
macro_rules: false,
// #[eii_extern_target(func.ident)]
// #[eii_extern_target(foreign_item_ident)]
eii_extern_target: Some(ast::EiiExternTarget {
extern_item_path: ast::Path::from_ident(func.ident),
extern_item_path: ast::Path::from_ident(foreign_item_name),
impl_unsafe,
span: decl_span,
}),
},
),
tokens: None,
});
return_items.push(extern_block);
return_items.push(macro_def);
if stmt {
return_items
.into_iter()
.map(|i| {
Annotatable::Stmt(Box::new(Stmt {
id: DUMMY_NODE_ID,
kind: StmtKind::Item(i),
span,
}))
})
.collect()
} else {
return_items.into_iter().map(|i| Annotatable::Item(i)).collect()
}
}
@ -436,10 +508,10 @@ pub(crate) fn eii_shared_macro(
f.eii_impls.push(EiiImpl {
node_id: DUMMY_NODE_ID,
inner_span: meta_item.path.span,
eii_macro_path: meta_item.path.clone(),
impl_safety: meta_item.unsafety,
span,
inner_span: meta_item.path.span,
is_default,
});

8
compiler/rustc_codegen_cranelift/.github/workflows/main.yml vendored
View file

@ -28,7 +28,7 @@ jobs:
- name: Avoid installing rustc-dev
run: |
sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain
sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain.toml
rustfmt -v
- name: Rustfmt
@ -88,7 +88,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}
- name: Set MinGW as the default toolchain
if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
@ -158,7 +158,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}
- name: Install hyperfine
run: |
@ -207,7 +207,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}
- name: Set MinGW as the default toolchain
if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'

4
compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml vendored
View file

@ -20,7 +20,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }}
key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain.toml', 'Cargo.lock') }}
- name: Test
run: ./scripts/test_bootstrap.sh
@ -40,7 +40,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }}
key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain.toml', 'Cargo.lock') }}
- name: Install ripgrep
run: |

18
compiler/rustc_codegen_cranelift/example/std_example.rs
View file

@ -259,6 +259,9 @@ unsafe fn test_simd() {
test_mm_cvttps_epi32();
test_mm_cvtsi128_si64();
#[cfg(not(jit))]
test_mm_cvtps_ph();
test_mm_extract_epi8();
test_mm_insert_epi16();
test_mm_shuffle_epi8();
@ -558,6 +561,21 @@ unsafe fn test_mm_cvttps_epi32() {
}
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "f16c")]
#[cfg(not(jit))]
unsafe fn test_mm_cvtps_ph() {
const F16_ONE: i16 = 0x3c00;
const F16_TWO: i16 = 0x4000;
const F16_THREE: i16 = 0x4200;
const F16_FOUR: i16 = 0x4400;
let a = _mm_set_ps(1.0, 2.0, 3.0, 4.0);
let r = _mm_cvtps_ph::<_MM_FROUND_CUR_DIRECTION>(a);
let e = _mm_set_epi16(0, 0, 0, 0, F16_ONE, F16_TWO, F16_THREE, F16_FOUR);
assert_eq_m128i(r, e);
}
fn test_checked_mul() {
let u: Option<u8> = u8::from_str_radix("1000", 10).ok();
assert_eq!(u, None);

4
compiler/rustc_codegen_cranelift/rust-toolchain
View file

@ -1,4 +0,0 @@
[toolchain]
channel = "nightly-2025-12-08"
components = ["rust-src", "rustc-dev", "llvm-tools"]
profile = "minimal"

4
compiler/rustc_codegen_cranelift/rust-toolchain.toml Normal file
View file

@ -0,0 +1,4 @@
[toolchain]
channel = "nightly-2025-12-18"
components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"]
profile = "minimal"

5
compiler/rustc_codegen_cranelift/scripts/rustup.sh
View file

@ -22,8 +22,7 @@ case $1 in
"prepare")
echo "=> Installing new nightly"
rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists
sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain
rustup component add rustfmt || true
sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain.toml
echo "=> Uninstalling all old nightlies"
for nightly in $(rustup toolchain list | grep nightly | grep -v "$TOOLCHAIN" | grep -v nightly-x86_64); do
@ -35,7 +34,7 @@ case $1 in
./y.sh prepare
;;
"commit")
git add rust-toolchain
git add rust-toolchain.toml
git commit -m "Rustup to $(rustc -V)"
;;
"push")

29
compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
View file

@ -35,6 +35,7 @@ git checkout -- tests/ui/entry-point/auxiliary/bad_main_functions.rs
rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic
rm tests/ui/simd/dont-invalid-bitcast-x86_64.rs # unimplemented llvm.x86.sse41.round.ps
rm tests/ui/simd/intrinsic/generic-arithmetic-pass.rs # unimplemented simd_funnel_{shl,shr}
rm -r tests/ui/scalable-vectors # scalable vectors are unsupported
# exotic linkages
rm tests/incremental/hashes/function_interfaces.rs
@ -53,23 +54,29 @@ rm tests/ui/sanitizer/kcfi-c-variadic.rs # same
rm tests/ui/c-variadic/same-program-multiple-abis-x86_64.rs # variadics for calling conventions other than C unsupported
rm tests/ui/delegation/fn-header.rs
# inline assembly features
rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly
rm tests/ui/asm/global-asm-mono-sym-fn.rs # same
rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same
rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported
rm tests/ui/asm/label-operand.rs # same
rm tests/ui/asm/may_unwind.rs # asm unwinding not supported
rm tests/ui/asm/aarch64/may_unwind.rs # same
# misc unimplemented things
rm tests/ui/target-feature/missing-plusminus.rs # error not implemented
rm -r tests/run-make/repr128-dwarf # debuginfo test
rm -r tests/run-make/split-debuginfo # same
rm -r tests/run-make/target-specs # i686 not supported by Cranelift
rm -r tests/run-make/mismatching-target-triples # same
rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly
rm tests/ui/asm/global-asm-mono-sym-fn.rs # same
rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same
rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported
rm tests/ui/asm/label-operand.rs # same
rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes
rm -r tests/run-make/used-proc-macro # used(linker) isn't supported yet
rm tests/ui/linking/no-gc-encapsulation-symbols.rs # same
rm tests/ui/attributes/fn-align-dyn.rs # per-function alignment not supported
rm -r tests/ui/explicit-tail-calls # tail calls
rm -r tests/run-make/pointer-auth-link-with-c # pointer auth
rm -r tests/ui/eii # EII not yet implemented
rm -r tests/run-make/forced-unwind-terminate-pof # forced unwinding doesn't take precedence
# requires LTO
rm -r tests/run-make/cdylib
@ -78,6 +85,7 @@ rm -r tests/run-make/lto-*
rm -r tests/run-make/reproducible-build-2
rm -r tests/run-make/no-builtins-lto
rm -r tests/run-make/reachable-extern-fn-available-lto
rm -r tests/run-make/no-builtins-linker-plugin-lto
# coverage instrumentation
rm tests/ui/consts/precise-drop-with-coverage.rs
@ -87,6 +95,7 @@ rm -r tests/ui/instrument-coverage/
# ==================
rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations
rm tests/ui/codegen/init-large-type.rs # same
rm tests/ui/codegen/StackColoring-not-blowup-stack-issue-40883.rs # same
rm tests/ui/statics/const_generics.rs # tests an optimization
rm tests/ui/linking/executable-no-mangle-strip.rs # requires --gc-sections to work for statics
@ -143,6 +152,15 @@ rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path
rm -r tests/run-make/export/extern-opt # something about rustc version mismatches
rm -r tests/run-make/export # same
rm -r tests/ui/compiletest-self-test/compile-flags-incremental.rs # needs compiletest compiled with panic=unwind
rm tests/ui/async-await/in-trait/dont-project-to-specializable-projection.rs # something going wrong with stdlib source remapping
rm tests/ui/consts/miri_unleashed/drop.rs # same
rm tests/ui/error-emitter/multiline-removal-suggestion.rs # same
rm tests/ui/lint/lint-const-item-mutation.rs # same
rm tests/ui/lint/use-redundant/use-redundant-issue-71450.rs # same
rm tests/ui/lint/use-redundant/use-redundant-prelude-rust-2021.rs # same
rm tests/ui/specialization/const_trait_impl.rs # same
rm tests/ui/thir-print/offset_of.rs # same
rm tests/ui/traits/const-traits/const_closure-const_trait_impl-ice-113381.rs # same
# genuine bugs
# ============
@ -157,6 +175,7 @@ rm tests/ui/lint/non-snake-case/lint-non-snake-case-crate.rs # same
rm tests/ui/async-await/async-drop/async-drop-initial.rs # same (rust-lang/rust#140493)
rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables
rm -r tests/run-make-cargo/rustdoc-scrape-examples-paths # FIXME(rust-lang/rust#145580) incr comp bug
rm -r tests/incremental/extern_static/issue-49153.rs # assumes reference to undefined static gets optimized away
rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # really slow with unoptimized libstd
rm tests/ui/process/process-panic-after-fork.rs # same

2
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -190,7 +190,7 @@ fn dep_symbol_lookup_fn(
diag.emit();
}
Linkage::Dynamic => {
dylib_paths.push(src.dylib.as_ref().unwrap().0.clone());
dylib_paths.push(src.dylib.as_ref().unwrap().clone());
}
}
}

14
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -857,19 +857,9 @@ fn call_inline_asm<'tcx>(
fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<types::Type> {
match ty.kind() {
// Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151
// Adapted from https://github.com/rust-lang/rust/blob/df44a57fd29fca899ce473f85ed64efd0708dd7c/compiler/rustc_hir_typeck/src/inline_asm.rs#L180-L183
ty::Adt(adt, args) if fx.tcx.is_lang_item(adt.did(), LangItem::MaybeUninit) => {
let fields = &adt.non_enum_variant().fields;
let ty = fields[FieldIdx::ONE].ty(fx.tcx, args);
let ty::Adt(ty, args) = ty.kind() else {
unreachable!("expected first field of `MaybeUninit` to be an ADT")
};
assert!(
ty.is_manually_drop(),
"expected first field of `MaybeUninit` to be `ManuallyDrop`"
);
let fields = &ty.non_enum_variant().fields;
let ty = fields[FieldIdx::ZERO].ty(fx.tcx, args);
let ty = args.type_at(0);
fx.clif_type(ty)
}
_ => fx.clif_type(ty),

29
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
View file

@ -1313,6 +1313,35 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>(
ret.write_cvalue_transmute(fx, res);
}
"llvm.x86.vcvtps2ph.128" => {
// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_ph
intrinsic_args!(fx, args => (a, _imm8); intrinsic);
let a = a.load_scalar(fx);
let imm8 =
if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[1].node) {
imm8
} else {
fx.tcx
.dcx()
.span_fatal(span, "Index argument for `_mm_cvtps_ph` is not a constant");
};
let imm8 = imm8.to_u32();
codegen_inline_asm_inner(
fx,
&[InlineAsmTemplatePiece::String(format!("vcvtps2ph xmm0, xmm0, {imm8}").into())],
&[CInlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
_late: true,
in_value: a,
out_place: Some(ret),
}],
InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
);
}
_ => {
fx.tcx
.dcx()

4
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -10,7 +10,7 @@
//! function u0:22(i64) -> i8, i8 system_v {
//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE
//! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] }
//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], in_memory_order: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//!
//! ; kind loc.idx param pass mode ty
//! ; ssa _0 (u8, u8) 2b 1 var=(0, 1)
@ -41,7 +41,7 @@
//! ;
//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, copy _2)
//! v2 = stack_load.i64 ss0
//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//! v3, v4 = call fn0(v1, v2) ; v1 = 1
//! v5 -> v3
//! v6 -> v4

3
compiler/rustc_codegen_cranelift/triagebot.toml
View file

@ -2,6 +2,3 @@
# Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust)
[issue-links]
# Prevents mentions in commits to avoid users being spammed
[no-mentions]

12
compiler/rustc_codegen_gcc/build_system/src/build.rs
View file

@ -111,14 +111,20 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
// Symlink libgccjit.so to sysroot.
let lib_path = start_dir.join("sysroot").join("lib");
let rustlib_target_path = lib_path
.join("rustlib")
.join(&config.host_triple)
.join("codegen-backends")
.join("lib")
.join(&config.target_triple);
let libgccjit_path =
PathBuf::from(config.gcc_path.as_ref().expect("libgccjit should be set by this point"))
.join("libgccjit.so");
let libgccjit_in_sysroot_path = lib_path.join("libgccjit.so");
let libgccjit_in_sysroot_path = rustlib_target_path.join("libgccjit.so");
// First remove the file to be able to create the symlink even when the file already exists.
let _ = fs::remove_file(&libgccjit_in_sysroot_path);
create_dir(&lib_path)?;
symlink(libgccjit_path, libgccjit_in_sysroot_path)
create_dir(&rustlib_target_path)?;
symlink(libgccjit_path, &libgccjit_in_sysroot_path)
.map_err(|error| format!("Cannot create symlink for libgccjit.so: {}", error))?;
let library_dir = start_dir.join("sysroot_src").join("library");

2
compiler/rustc_codegen_gcc/rust-toolchain
View file

@ -1,3 +1,3 @@
[toolchain]
channel = "nightly-2025-11-24"
channel = "nightly-2025-12-20"
components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

4
compiler/rustc_codegen_gcc/src/builder.rs
View file

@ -943,6 +943,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
.get_address(self.location)
}
fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> RValue<'gcc> {
todo!()
}
fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
let block = self.llbb();
let function = block.get_function();

123
compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
View file

@ -70,10 +70,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"sve.sm4e" => "__builtin_sve_svsm4e_u32",
"sve.sm4ekey" => "__builtin_sve_svsm4ekey_u32",
"sve.wrffr" => "__builtin_sve_svwrffr",
"tcancel" => "__builtin_arm_tcancel",
"tcommit" => "__builtin_arm_tcommit",
"tstart" => "__builtin_arm_tstart",
"ttest" => "__builtin_arm_ttest",
_ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"),
}
}
@ -1632,6 +1628,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"V6.vabs.f8.128B" => "__builtin_HEXAGON_V6_vabs_f8_128B",
"V6.vabs.hf" => "__builtin_HEXAGON_V6_vabs_hf",
"V6.vabs.hf.128B" => "__builtin_HEXAGON_V6_vabs_hf_128B",
"V6.vabs.qf16.hf" => "__builtin_HEXAGON_V6_vabs_qf16_hf",
"V6.vabs.qf16.hf.128B" => "__builtin_HEXAGON_V6_vabs_qf16_hf_128B",
"V6.vabs.qf16.qf16" => "__builtin_HEXAGON_V6_vabs_qf16_qf16",
"V6.vabs.qf16.qf16.128B" => "__builtin_HEXAGON_V6_vabs_qf16_qf16_128B",
"V6.vabs.qf32.qf32" => "__builtin_HEXAGON_V6_vabs_qf32_qf32",
"V6.vabs.qf32.qf32.128B" => "__builtin_HEXAGON_V6_vabs_qf32_qf32_128B",
"V6.vabs.qf32.sf" => "__builtin_HEXAGON_V6_vabs_qf32_sf",
"V6.vabs.qf32.sf.128B" => "__builtin_HEXAGON_V6_vabs_qf32_sf_128B",
"V6.vabs.sf" => "__builtin_HEXAGON_V6_vabs_sf",
"V6.vabs.sf.128B" => "__builtin_HEXAGON_V6_vabs_sf_128B",
"V6.vabsb" => "__builtin_HEXAGON_V6_vabsb",
@ -1744,6 +1748,8 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"V6.vaddwsat.128B" => "__builtin_HEXAGON_V6_vaddwsat_128B",
"V6.vaddwsat.dv" => "__builtin_HEXAGON_V6_vaddwsat_dv",
"V6.vaddwsat.dv.128B" => "__builtin_HEXAGON_V6_vaddwsat_dv_128B",
"V6.valign4" => "__builtin_HEXAGON_V6_valign4",
"V6.valign4.128B" => "__builtin_HEXAGON_V6_valign4_128B",
"V6.valignb" => "__builtin_HEXAGON_V6_valignb",
"V6.valignb.128B" => "__builtin_HEXAGON_V6_valignb_128B",
"V6.valignbi" => "__builtin_HEXAGON_V6_valignbi",
@ -1862,14 +1868,30 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"V6.vcl0w.128B" => "__builtin_HEXAGON_V6_vcl0w_128B",
"V6.vcombine" => "__builtin_HEXAGON_V6_vcombine",
"V6.vcombine.128B" => "__builtin_HEXAGON_V6_vcombine_128B",
"V6.vconv.bf.qf32" => "__builtin_HEXAGON_V6_vconv_bf_qf32",
"V6.vconv.bf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_bf_qf32_128B",
"V6.vconv.f8.qf16" => "__builtin_HEXAGON_V6_vconv_f8_qf16",
"V6.vconv.f8.qf16.128B" => "__builtin_HEXAGON_V6_vconv_f8_qf16_128B",
"V6.vconv.h.hf" => "__builtin_HEXAGON_V6_vconv_h_hf",
"V6.vconv.h.hf.128B" => "__builtin_HEXAGON_V6_vconv_h_hf_128B",
"V6.vconv.h.hf.rnd" => "__builtin_HEXAGON_V6_vconv_h_hf_rnd",
"V6.vconv.h.hf.rnd.128B" => "__builtin_HEXAGON_V6_vconv_h_hf_rnd_128B",
"V6.vconv.hf.h" => "__builtin_HEXAGON_V6_vconv_hf_h",
"V6.vconv.hf.h.128B" => "__builtin_HEXAGON_V6_vconv_hf_h_128B",
"V6.vconv.hf.qf16" => "__builtin_HEXAGON_V6_vconv_hf_qf16",
"V6.vconv.hf.qf16.128B" => "__builtin_HEXAGON_V6_vconv_hf_qf16_128B",
"V6.vconv.hf.qf32" => "__builtin_HEXAGON_V6_vconv_hf_qf32",
"V6.vconv.hf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_hf_qf32_128B",
"V6.vconv.qf16.f8" => "__builtin_HEXAGON_V6_vconv_qf16_f8",
"V6.vconv.qf16.f8.128B" => "__builtin_HEXAGON_V6_vconv_qf16_f8_128B",
"V6.vconv.qf16.hf" => "__builtin_HEXAGON_V6_vconv_qf16_hf",
"V6.vconv.qf16.hf.128B" => "__builtin_HEXAGON_V6_vconv_qf16_hf_128B",
"V6.vconv.qf16.qf16" => "__builtin_HEXAGON_V6_vconv_qf16_qf16",
"V6.vconv.qf16.qf16.128B" => "__builtin_HEXAGON_V6_vconv_qf16_qf16_128B",
"V6.vconv.qf32.qf32" => "__builtin_HEXAGON_V6_vconv_qf32_qf32",
"V6.vconv.qf32.qf32.128B" => "__builtin_HEXAGON_V6_vconv_qf32_qf32_128B",
"V6.vconv.qf32.sf" => "__builtin_HEXAGON_V6_vconv_qf32_sf",
"V6.vconv.qf32.sf.128B" => "__builtin_HEXAGON_V6_vconv_qf32_sf_128B",
"V6.vconv.sf.qf32" => "__builtin_HEXAGON_V6_vconv_sf_qf32",
"V6.vconv.sf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_sf_qf32_128B",
"V6.vconv.sf.w" => "__builtin_HEXAGON_V6_vconv_sf_w",
@ -1984,6 +2006,22 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"V6.veqh.or.128B" => "__builtin_HEXAGON_V6_veqh_or_128B",
"V6.veqh.xor" => "__builtin_HEXAGON_V6_veqh_xor",
"V6.veqh.xor.128B" => "__builtin_HEXAGON_V6_veqh_xor_128B",
"V6.veqhf" => "__builtin_HEXAGON_V6_veqhf",
"V6.veqhf.128B" => "__builtin_HEXAGON_V6_veqhf_128B",
"V6.veqhf.and" => "__builtin_HEXAGON_V6_veqhf_and",
"V6.veqhf.and.128B" => "__builtin_HEXAGON_V6_veqhf_and_128B",
"V6.veqhf.or" => "__builtin_HEXAGON_V6_veqhf_or",
"V6.veqhf.or.128B" => "__builtin_HEXAGON_V6_veqhf_or_128B",
"V6.veqhf.xor" => "__builtin_HEXAGON_V6_veqhf_xor",
"V6.veqhf.xor.128B" => "__builtin_HEXAGON_V6_veqhf_xor_128B",
"V6.veqsf" => "__builtin_HEXAGON_V6_veqsf",
"V6.veqsf.128B" => "__builtin_HEXAGON_V6_veqsf_128B",
"V6.veqsf.and" => "__builtin_HEXAGON_V6_veqsf_and",
"V6.veqsf.and.128B" => "__builtin_HEXAGON_V6_veqsf_and_128B",
"V6.veqsf.or" => "__builtin_HEXAGON_V6_veqsf_or",
"V6.veqsf.or.128B" => "__builtin_HEXAGON_V6_veqsf_or_128B",
"V6.veqsf.xor" => "__builtin_HEXAGON_V6_veqsf_xor",
"V6.veqsf.xor.128B" => "__builtin_HEXAGON_V6_veqsf_xor_128B",
"V6.veqw" => "__builtin_HEXAGON_V6_veqw",
"V6.veqw.128B" => "__builtin_HEXAGON_V6_veqw_128B",
"V6.veqw.and" => "__builtin_HEXAGON_V6_veqw_and",
@ -2096,6 +2134,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"V6.vgtw.or.128B" => "__builtin_HEXAGON_V6_vgtw_or_128B",
"V6.vgtw.xor" => "__builtin_HEXAGON_V6_vgtw_xor",
"V6.vgtw.xor.128B" => "__builtin_HEXAGON_V6_vgtw_xor_128B",
"V6.vilog2.hf" => "__builtin_HEXAGON_V6_vilog2_hf",
"V6.vilog2.hf.128B" => "__builtin_HEXAGON_V6_vilog2_hf_128B",
"V6.vilog2.qf16" => "__builtin_HEXAGON_V6_vilog2_qf16",
"V6.vilog2.qf16.128B" => "__builtin_HEXAGON_V6_vilog2_qf16_128B",
"V6.vilog2.qf32" => "__builtin_HEXAGON_V6_vilog2_qf32",
"V6.vilog2.qf32.128B" => "__builtin_HEXAGON_V6_vilog2_qf32_128B",
"V6.vilog2.sf" => "__builtin_HEXAGON_V6_vilog2_sf",
"V6.vilog2.sf.128B" => "__builtin_HEXAGON_V6_vilog2_sf_128B",
"V6.vinsertwr" => "__builtin_HEXAGON_V6_vinsertwr",
"V6.vinsertwr.128B" => "__builtin_HEXAGON_V6_vinsertwr_128B",
"V6.vlalignb" => "__builtin_HEXAGON_V6_vlalignb",
@ -2350,6 +2396,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"V6.vnavgub.128B" => "__builtin_HEXAGON_V6_vnavgub_128B",
"V6.vnavgw" => "__builtin_HEXAGON_V6_vnavgw",
"V6.vnavgw.128B" => "__builtin_HEXAGON_V6_vnavgw_128B",
"V6.vneg.qf16.hf" => "__builtin_HEXAGON_V6_vneg_qf16_hf",
"V6.vneg.qf16.hf.128B" => "__builtin_HEXAGON_V6_vneg_qf16_hf_128B",
"V6.vneg.qf16.qf16" => "__builtin_HEXAGON_V6_vneg_qf16_qf16",
"V6.vneg.qf16.qf16.128B" => "__builtin_HEXAGON_V6_vneg_qf16_qf16_128B",
"V6.vneg.qf32.qf32" => "__builtin_HEXAGON_V6_vneg_qf32_qf32",
"V6.vneg.qf32.qf32.128B" => "__builtin_HEXAGON_V6_vneg_qf32_qf32_128B",
"V6.vneg.qf32.sf" => "__builtin_HEXAGON_V6_vneg_qf32_sf",
"V6.vneg.qf32.sf.128B" => "__builtin_HEXAGON_V6_vneg_qf32_sf_128B",
"V6.vnormamth" => "__builtin_HEXAGON_V6_vnormamth",
"V6.vnormamth.128B" => "__builtin_HEXAGON_V6_vnormamth_128B",
"V6.vnormamtw" => "__builtin_HEXAGON_V6_vnormamtw",
@ -2684,6 +2738,24 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"iocsrwr.d" => "__builtin_loongarch_iocsrwr_d",
"iocsrwr.h" => "__builtin_loongarch_iocsrwr_h",
"iocsrwr.w" => "__builtin_loongarch_iocsrwr_w",
"lasx.cast.128" => "__builtin_lasx_cast_128",
"lasx.cast.128.d" => "__builtin_lasx_cast_128_d",
"lasx.cast.128.s" => "__builtin_lasx_cast_128_s",
"lasx.concat.128" => "__builtin_lasx_concat_128",
"lasx.concat.128.d" => "__builtin_lasx_concat_128_d",
"lasx.concat.128.s" => "__builtin_lasx_concat_128_s",
"lasx.extract.128.hi" => "__builtin_lasx_extract_128_hi",
"lasx.extract.128.hi.d" => "__builtin_lasx_extract_128_hi_d",
"lasx.extract.128.hi.s" => "__builtin_lasx_extract_128_hi_s",
"lasx.extract.128.lo" => "__builtin_lasx_extract_128_lo",
"lasx.extract.128.lo.d" => "__builtin_lasx_extract_128_lo_d",
"lasx.extract.128.lo.s" => "__builtin_lasx_extract_128_lo_s",
"lasx.insert.128.hi" => "__builtin_lasx_insert_128_hi",
"lasx.insert.128.hi.d" => "__builtin_lasx_insert_128_hi_d",
"lasx.insert.128.hi.s" => "__builtin_lasx_insert_128_hi_s",
"lasx.insert.128.lo" => "__builtin_lasx_insert_128_lo",
"lasx.insert.128.lo.d" => "__builtin_lasx_insert_128_lo_d",
"lasx.insert.128.lo.s" => "__builtin_lasx_insert_128_lo_s",
"lasx.vext2xv.d.b" => "__builtin_lasx_vext2xv_d_b",
"lasx.vext2xv.d.h" => "__builtin_lasx_vext2xv_d_h",
"lasx.vext2xv.d.w" => "__builtin_lasx_vext2xv_d_w",
@ -4950,8 +5022,20 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"f16x2.to.e5m2x2.rn.relu" => "__nvvm_f16x2_to_e5m2x2_rn_relu",
"f2bf16.rn" => "__nvvm_f2bf16_rn",
"f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
"f2bf16.rn.relu.satfinite" => "__nvvm_f2bf16_rn_relu_satfinite",
"f2bf16.rn.satfinite" => "__nvvm_f2bf16_rn_satfinite",
"f2bf16.rz" => "__nvvm_f2bf16_rz",
"f2bf16.rz.relu" => "__nvvm_f2bf16_rz_relu",
"f2bf16.rz.relu.satfinite" => "__nvvm_f2bf16_rz_relu_satfinite",
"f2bf16.rz.satfinite" => "__nvvm_f2bf16_rz_satfinite",
"f2f16.rn" => "__nvvm_f2f16_rn",
"f2f16.rn.relu" => "__nvvm_f2f16_rn_relu",
"f2f16.rn.relu.satfinite" => "__nvvm_f2f16_rn_relu_satfinite",
"f2f16.rn.satfinite" => "__nvvm_f2f16_rn_satfinite",
"f2f16.rz" => "__nvvm_f2f16_rz",
"f2f16.rz.relu" => "__nvvm_f2f16_rz_relu",
"f2f16.rz.relu.satfinite" => "__nvvm_f2f16_rz_relu_satfinite",
"f2f16.rz.satfinite" => "__nvvm_f2f16_rz_satfinite",
"f2h.rn" => "__nvvm_f2h_rn",
"f2h.rn.ftz" => "__nvvm_f2h_rn_ftz",
"f2i.rm" => "__nvvm_f2i_rm",
@ -5035,20 +5119,28 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"ff.to.ue8m0x2.rz.satfinite" => "__nvvm_ff_to_ue8m0x2_rz_satfinite",
"ff2bf16x2.rn" => "__nvvm_ff2bf16x2_rn",
"ff2bf16x2.rn.relu" => "__nvvm_ff2bf16x2_rn_relu",
"ff2bf16x2.rn.relu.satfinite" => "__nvvm_ff2bf16x2_rn_relu_satfinite",
"ff2bf16x2.rn.satfinite" => "__nvvm_ff2bf16x2_rn_satfinite",
"ff2bf16x2.rs" => "__nvvm_ff2bf16x2_rs",
"ff2bf16x2.rs.relu" => "__nvvm_ff2bf16x2_rs_relu",
"ff2bf16x2.rs.relu.satfinite" => "__nvvm_ff2bf16x2_rs_relu_satfinite",
"ff2bf16x2.rs.satfinite" => "__nvvm_ff2bf16x2_rs_satfinite",
"ff2bf16x2.rz" => "__nvvm_ff2bf16x2_rz",
"ff2bf16x2.rz.relu" => "__nvvm_ff2bf16x2_rz_relu",
"ff2bf16x2.rz.relu.satfinite" => "__nvvm_ff2bf16x2_rz_relu_satfinite",
"ff2bf16x2.rz.satfinite" => "__nvvm_ff2bf16x2_rz_satfinite",
"ff2f16x2.rn" => "__nvvm_ff2f16x2_rn",
"ff2f16x2.rn.relu" => "__nvvm_ff2f16x2_rn_relu",
"ff2f16x2.rn.relu.satfinite" => "__nvvm_ff2f16x2_rn_relu_satfinite",
"ff2f16x2.rn.satfinite" => "__nvvm_ff2f16x2_rn_satfinite",
"ff2f16x2.rs" => "__nvvm_ff2f16x2_rs",
"ff2f16x2.rs.relu" => "__nvvm_ff2f16x2_rs_relu",
"ff2f16x2.rs.relu.satfinite" => "__nvvm_ff2f16x2_rs_relu_satfinite",
"ff2f16x2.rs.satfinite" => "__nvvm_ff2f16x2_rs_satfinite",
"ff2f16x2.rz" => "__nvvm_ff2f16x2_rz",
"ff2f16x2.rz.relu" => "__nvvm_ff2f16x2_rz_relu",
"ff2f16x2.rz.relu.satfinite" => "__nvvm_ff2f16x2_rz_relu_satfinite",
"ff2f16x2.rz.satfinite" => "__nvvm_ff2f16x2_rz_satfinite",
"floor.d" => "__nvvm_floor_d",
"floor.f" => "__nvvm_floor_f",
"floor.ftz.f" => "__nvvm_floor_ftz_f",
@ -5942,6 +6034,8 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"altivec.vupklsb" => "__builtin_altivec_vupklsb",
"altivec.vupklsh" => "__builtin_altivec_vupklsh",
"altivec.vupklsw" => "__builtin_altivec_vupklsw",
"amo.ldat" => "__builtin_amo_ldat",
"amo.lwat" => "__builtin_amo_lwat",
"bcdadd" => "__builtin_ppc_bcdadd",
"bcdadd.p" => "__builtin_ppc_bcdadd_p",
"bcdcopysign" => "__builtin_ppc_bcdcopysign",
@ -6202,6 +6296,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"vsx.xvminsp" => "__builtin_vsx_xvminsp",
"vsx.xvredp" => "__builtin_vsx_xvredp",
"vsx.xvresp" => "__builtin_vsx_xvresp",
"vsx.xvrlw" => "__builtin_vsx_xvrlw",
"vsx.xvrsqrtedp" => "__builtin_vsx_xvrsqrtedp",
"vsx.xvrsqrtesp" => "__builtin_vsx_xvrsqrtesp",
"vsx.xvtdivdp" => "__builtin_vsx_xvtdivdp",
@ -10158,24 +10253,16 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"stui" => "__builtin_ia32_stui",
"subborrow.u32" => "__builtin_ia32_subborrow_u32",
"subborrow.u64" => "__builtin_ia32_subborrow_u64",
"t2rpntlvwz0" => "__builtin_ia32_t2rpntlvwz0",
"t2rpntlvwz0rs" => "__builtin_ia32_t2rpntlvwz0rs",
"t2rpntlvwz0rst1" => "__builtin_ia32_t2rpntlvwz0rst1",
"t2rpntlvwz0t1" => "__builtin_ia32_t2rpntlvwz0t1",
"t2rpntlvwz1" => "__builtin_ia32_t2rpntlvwz1",
"t2rpntlvwz1rs" => "__builtin_ia32_t2rpntlvwz1rs",
"t2rpntlvwz1rst1" => "__builtin_ia32_t2rpntlvwz1rst1",
"t2rpntlvwz1t1" => "__builtin_ia32_t2rpntlvwz1t1",
"tbm.bextri.u32" => "__builtin_ia32_bextri_u32",
"tbm.bextri.u64" => "__builtin_ia32_bextri_u64",
"tcmmimfp16ps" => "__builtin_ia32_tcmmimfp16ps",
"tcmmimfp16ps.internal" => "__builtin_ia32_tcmmimfp16ps_internal",
"tcmmrlfp16ps" => "__builtin_ia32_tcmmrlfp16ps",
"tcmmrlfp16ps.internal" => "__builtin_ia32_tcmmrlfp16ps_internal",
"tconjtcmmimfp16ps" => "__builtin_ia32_tconjtcmmimfp16ps",
"tconjtcmmimfp16ps.internal" => "__builtin_ia32_tconjtcmmimfp16ps_internal",
"tconjtfp16" => "__builtin_ia32_tconjtfp16",
"tconjtfp16.internal" => "__builtin_ia32_tconjtfp16_internal",
"tcvtrowd2ps" => "__builtin_ia32_tcvtrowd2ps",
"tcvtrowd2ps.internal" => "__builtin_ia32_tcvtrowd2ps_internal",
"tcvtrowps2bf16h" => "__builtin_ia32_tcvtrowps2bf16h",
@ -10225,18 +10312,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str {
"tmmultf32ps" => "__builtin_ia32_tmmultf32ps",
"tmmultf32ps.internal" => "__builtin_ia32_tmmultf32ps_internal",
"tpause" => "__builtin_ia32_tpause",
"ttcmmimfp16ps" => "__builtin_ia32_ttcmmimfp16ps",
"ttcmmimfp16ps.internal" => "__builtin_ia32_ttcmmimfp16ps_internal",
"ttcmmrlfp16ps" => "__builtin_ia32_ttcmmrlfp16ps",
"ttcmmrlfp16ps.internal" => "__builtin_ia32_ttcmmrlfp16ps_internal",
"ttdpbf16ps" => "__builtin_ia32_ttdpbf16ps",
"ttdpbf16ps.internal" => "__builtin_ia32_ttdpbf16ps_internal",
"ttdpfp16ps" => "__builtin_ia32_ttdpfp16ps",
"ttdpfp16ps.internal" => "__builtin_ia32_ttdpfp16ps_internal",
"ttmmultf32ps" => "__builtin_ia32_ttmmultf32ps",
"ttmmultf32ps.internal" => "__builtin_ia32_ttmmultf32ps_internal",
"ttransposed" => "__builtin_ia32_ttransposed",
"ttransposed.internal" => "__builtin_ia32_ttransposed_internal",
"umonitor" => "__builtin_ia32_umonitor",
"umwait" => "__builtin_ia32_umwait",
"urdmsr" => "__builtin_ia32_urdmsr",

2
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file

@ -504,7 +504,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
let layout = self.layout_of(tp_ty).layout;
let _use_integer_compare = match layout.backend_repr() {
Scalar(_) | ScalarPair(_, _) => true,
SimdVector { .. } => false,
SimdVector { .. } | ScalableVector { .. } => false,
Memory { .. } => {
// For rusty ABIs, small aggregates are actually passed
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

25
compiler/rustc_codegen_gcc/src/lib.rs
View file

@ -98,7 +98,6 @@ use rustc_middle::ty::TyCtxt;
use rustc_middle::util::Providers;
use rustc_session::Session;
use rustc_session::config::{OptLevel, OutputFilenames};
use rustc_session::filesearch::make_target_lib_path;
use rustc_span::Symbol;
use rustc_target::spec::{Arch, RelocModel};
use tempfile::TempDir;
@ -207,18 +206,38 @@ impl CodegenBackend for GccCodegenBackend {
}
fn init(&self, sess: &Session) {
fn file_path(sysroot_path: &Path, sess: &Session) -> PathBuf {
let rustlib_path =
rustc_target::relative_target_rustlib_path(sysroot_path, &sess.host.llvm_target);
sysroot_path
.join(rustlib_path)
.join("codegen-backends")
.join("lib")
.join(sess.target.llvm_target.as_ref())
.join("libgccjit.so")
}
// We use all_paths() instead of only path() in case the path specified by --sysroot is
// invalid.
// This is the case for instance in Rust for Linux where they specify --sysroot=/dev/null.
for path in sess.opts.sysroot.all_paths() {
let libgccjit_target_lib_file =
make_target_lib_path(path, &sess.target.llvm_target).join("libgccjit.so");
let libgccjit_target_lib_file = file_path(path, sess);
if let Ok(true) = fs::exists(&libgccjit_target_lib_file) {
load_libgccjit_if_needed(&libgccjit_target_lib_file);
break;
}
}
if !gccjit::is_loaded() {
let mut paths = vec![];
for path in sess.opts.sysroot.all_paths() {
let libgccjit_target_lib_file = file_path(path, sess);
paths.push(libgccjit_target_lib_file);
}
panic!("Could not load libgccjit.so. Attempted paths: {:#?}", paths);
}
#[cfg(feature = "master")]
{
let target_cpu = target_cpu(sess);

4
compiler/rustc_codegen_gcc/src/type_of.rs
View file

@ -85,6 +85,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
);
}
BackendRepr::Memory { .. } => {}
BackendRepr::ScalableVector { .. } => todo!(),
}
let name = match *layout.ty.kind() {
@ -179,6 +180,8 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
fn is_gcc_immediate(&self) -> bool {
match self.backend_repr {
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
// FIXME(rustc_scalable_vector): Not yet implemented in rustc_codegen_gcc.
BackendRepr::ScalableVector { .. } => todo!(),
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
}
}
@ -188,6 +191,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
BackendRepr::ScalarPair(..) => true,
BackendRepr::Scalar(_)
| BackendRepr::SimdVector { .. }
| BackendRepr::ScalableVector { .. }
| BackendRepr::Memory { .. } => false,
}
}

9
compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt
View file

@ -88,3 +88,12 @@ tests/ui/test-attrs/test-panic-while-printing.rs
tests/ui/thir-print/offset_of.rs
tests/ui/iterators/rangefrom-overflow-debug.rs
tests/ui/iterators/rangefrom-overflow-overflow-checks.rs
tests/ui/iterators/iter-filter-count-debug-check.rs
tests/ui/eii/codegen_single_crate.rs
tests/ui/eii/codegen_cross_crate.rs
tests/ui/eii/default/local_crate.rs
tests/ui/eii/multiple_impls.rs
tests/ui/eii/default/call_default.rs
tests/ui/eii/same-symbol.rs
tests/ui/eii/privacy1.rs
tests/ui/eii/default/call_impl.rs

3
compiler/rustc_codegen_gcc/triagebot.toml
View file

@ -2,6 +2,3 @@
# Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust)
[issue-links]
# Prevents mentions in commits to avoid users being spammed
[no-mentions]

3
compiler/rustc_codegen_llvm/Cargo.toml
View file

@ -14,6 +14,7 @@ bitflags = "2.4.1"
gimli = "0.31"
itertools = "0.12"
libc = "0.2"
libloading = { version = "0.9.0", optional = true }
measureme = "12.0.1"
object = { version = "0.37.0", default-features = false, features = ["std", "read"] }
rustc-demangle = "0.1.21"
@ -46,7 +47,7 @@ tracing = "0.1"
[features]
# tidy-alphabetical-start
check_only = ["rustc_llvm/check_only"]
llvm_enzyme = []
llvm_enzyme = ["dep:libloading"]
llvm_offload = []
# tidy-alphabetical-end

3
compiler/rustc_codegen_llvm/messages.ftl
View file

@ -1,9 +1,10 @@
codegen_llvm_autodiff_component_unavailable = failed to load our autodiff backend. Did you install it via rustup?
codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z autodiff=Enable
codegen_llvm_autodiff_without_lto = using the autodiff feature requires setting `lto="fat"` in your Cargo.toml
codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
codegen_llvm_fixed_x18_invalid_arch = the `-Zfixed-x18` flag is not supported on the `{$arch}` architecture
codegen_llvm_from_llvm_diag = {$message}

29
compiler/rustc_codegen_llvm/src/back/lto.rs
View file

@ -528,31 +528,34 @@ fn thin_lto(
}
}
fn enable_autodiff_settings(ad: &[config::AutoDiff]) {
#[cfg(feature = "llvm_enzyme")]
pub(crate) fn enable_autodiff_settings(ad: &[config::AutoDiff]) {
let mut enzyme = llvm::EnzymeWrapper::get_instance();
for val in ad {
// We intentionally don't use a wildcard, to not forget handling anything new.
match val {
config::AutoDiff::PrintPerf => {
llvm::set_print_perf(true);
enzyme.set_print_perf(true);
}
config::AutoDiff::PrintAA => {
llvm::set_print_activity(true);
enzyme.set_print_activity(true);
}
config::AutoDiff::PrintTA => {
llvm::set_print_type(true);
enzyme.set_print_type(true);
}
config::AutoDiff::PrintTAFn(fun) => {
llvm::set_print_type(true); // Enable general type printing
llvm::set_print_type_fun(&fun); // Set specific function to analyze
enzyme.set_print_type(true); // Enable general type printing
enzyme.set_print_type_fun(&fun); // Set specific function to analyze
}
config::AutoDiff::Inline => {
llvm::set_inline(true);
enzyme.set_inline(true);
}
config::AutoDiff::LooseTypes => {
llvm::set_loose_types(true);
enzyme.set_loose_types(true);
}
config::AutoDiff::PrintSteps => {
llvm::set_print(true);
enzyme.set_print(true);
}
// We handle this in the PassWrapper.cpp
config::AutoDiff::PrintPasses => {}
@ -571,9 +574,9 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) {
}
}
// This helps with handling enums for now.
llvm::set_strict_aliasing(false);
enzyme.set_strict_aliasing(false);
// FIXME(ZuseZ4): Test this, since it was added a long time ago.
llvm::set_rust_rules(true);
enzyme.set_rust_rules(true);
}
pub(crate) fn run_pass_manager(
@ -607,10 +610,6 @@ pub(crate) fn run_pass_manager(
if enable_ad { write::AutodiffStage::DuringAD } else { write::AutodiffStage::PostAD }
};
if enable_ad {
enable_autodiff_settings(&config.autodiff);
}
unsafe {
write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage);
}

12
compiler/rustc_codegen_llvm/src/back/write.rs
View file

@ -707,8 +707,7 @@ pub(crate) unsafe fn llvm_optimize(
if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Enable) {
let cx =
SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size);
// For now we only support up to 10 kernels named kernel_0 ... kernel_9, a follow-up PR is
// introducing a proper offload intrinsic to solve this limitation.
for func in cx.get_functions() {
let offload_kernel = "offload-kernel";
if attributes::has_string_attr(func, offload_kernel) {
@ -730,6 +729,13 @@ pub(crate) unsafe fn llvm_optimize(
let llvm_plugins = config.llvm_plugins.join(",");
let enzyme_fn = if consider_ad {
let wrapper = llvm::EnzymeWrapper::get_instance();
wrapper.registerEnzymeAndPassPipeline
} else {
std::ptr::null()
};
let result = unsafe {
llvm::LLVMRustOptimize(
module.module_llvm.llmod(),
@ -749,7 +755,7 @@ pub(crate) unsafe fn llvm_optimize(
vectorize_loop,
config.no_builtins,
config.emit_lifetime_markers,
run_enzyme,
enzyme_fn,
print_before_enzyme,
print_after_enzyme,
print_passes,

16
compiler/rustc_codegen_llvm/src/base.rs
View file

@ -23,13 +23,14 @@ use rustc_middle::dep_graph;
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, SanitizerFnAttrs};
use rustc_middle::mir::mono::Visibility;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::DebugInfo;
use rustc_session::config::{DebugInfo, Offload};
use rustc_span::Symbol;
use rustc_target::spec::SanitizerSet;
use super::ModuleLlvm;
use crate::attributes;
use crate::builder::Builder;
use crate::builder::gpu_offload::OffloadGlobals;
use crate::context::CodegenCx;
use crate::llvm::{self, Value};
@ -85,6 +86,19 @@ pub(crate) fn compile_codegen_unit(
let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str());
{
let mut cx = CodegenCx::new(tcx, cgu, &llvm_module);
// Declare and store globals shared by all offload kernels
//
// These globals are left in the LLVM-IR host module so all kernels can access them.
// They are necessary for correct offload execution. We do this here to simplify the
// `offload` intrinsic, avoiding the need for tracking whether it's the first
// intrinsic call or not.
if cx.sess().opts.unstable_opts.offload.contains(&Offload::Enable)
&& !cx.sess().target.is_like_gpu
{
cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx)));
}
let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
for &(mono_item, data) in &mono_items {
mono_item.predefine::<Builder<'_, '_, '_>>(

19
compiler/rustc_codegen_llvm/src/builder.rs
View file

@ -613,6 +613,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
}
}
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
let mut bx = Builder::with_cx(self.cx);
bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
let llvm_ty = match element_ty.kind() {
ty::Bool => bx.type_i1(),
ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
_ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
};
unsafe {
let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
alloca
}
}
fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
unsafe {
let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);

188
compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs
View file

@ -2,17 +2,76 @@ use std::ffi::CString;
use llvm::Linkage::*;
use rustc_abi::Align;
use rustc_codegen_ssa::traits::BaseTypeCodegenMethods;
use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods};
use rustc_middle::ty::offload_meta::OffloadMetadata;
use crate::builder::SBuilder;
use crate::builder::Builder;
use crate::common::CodegenCx;
use crate::llvm::AttributePlace::Function;
use crate::llvm::{self, BasicBlock, Linkage, Type, Value};
use crate::llvm::{self, Linkage, Type, Value};
use crate::{SimpleCx, attributes};
// LLVM kernel-independent globals required for offloading
pub(crate) struct OffloadGlobals<'ll> {
pub launcher_fn: &'ll llvm::Value,
pub launcher_ty: &'ll llvm::Type,
pub bin_desc: &'ll llvm::Type,
pub kernel_args_ty: &'ll llvm::Type,
pub offload_entry_ty: &'ll llvm::Type,
pub begin_mapper: &'ll llvm::Value,
pub end_mapper: &'ll llvm::Value,
pub mapper_fn_ty: &'ll llvm::Type,
pub ident_t_global: &'ll llvm::Value,
pub register_lib: &'ll llvm::Value,
pub unregister_lib: &'ll llvm::Value,
pub init_rtls: &'ll llvm::Value,
}
impl<'ll> OffloadGlobals<'ll> {
pub(crate) fn declare(cx: &CodegenCx<'ll, '_>) -> Self {
let (launcher_fn, launcher_ty) = generate_launcher(cx);
let kernel_args_ty = KernelArgsTy::new_decl(cx);
let offload_entry_ty = TgtOffloadEntry::new_decl(cx);
let (begin_mapper, _, end_mapper, mapper_fn_ty) = gen_tgt_data_mappers(cx);
let ident_t_global = generate_at_one(cx);
let tptr = cx.type_ptr();
let ti32 = cx.type_i32();
let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr];
let bin_desc = cx.type_named_struct("struct.__tgt_bin_desc");
cx.set_struct_body(bin_desc, &tgt_bin_desc_ty, false);
let register_lib = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty);
let unregister_lib = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty);
let init_ty = cx.type_func(&[], cx.type_void());
let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty);
OffloadGlobals {
launcher_fn,
launcher_ty,
bin_desc,
kernel_args_ty,
offload_entry_ty,
begin_mapper,
end_mapper,
mapper_fn_ty,
ident_t_global,
register_lib,
unregister_lib,
init_rtls,
}
}
}
// ; Function Attrs: nounwind
// declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) #2
fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm::Type) {
fn generate_launcher<'ll>(cx: &CodegenCx<'ll, '_>) -> (&'ll llvm::Value, &'ll llvm::Type) {
let tptr = cx.type_ptr();
let ti64 = cx.type_i64();
let ti32 = cx.type_i32();
@ -30,7 +89,7 @@ fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm
// @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8
// FIXME(offload): @0 should include the file name (e.g. lib.rs) in which the function to be
// offloaded was defined.
fn generate_at_one<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Value {
pub(crate) fn generate_at_one<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Value {
let unknown_txt = ";unknown;unknown;0;0;;";
let c_entry_name = CString::new(unknown_txt).unwrap();
let c_val = c_entry_name.as_bytes_with_nul();
@ -68,7 +127,7 @@ pub(crate) struct TgtOffloadEntry {
}
impl TgtOffloadEntry {
pub(crate) fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Type {
pub(crate) fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Type {
let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry");
let tptr = cx.type_ptr();
let ti64 = cx.type_i64();
@ -82,7 +141,7 @@ impl TgtOffloadEntry {
}
fn new<'ll>(
cx: &'ll SimpleCx<'_>,
cx: &CodegenCx<'ll, '_>,
region_id: &'ll Value,
llglobal: &'ll Value,
) -> [&'ll Value; 9] {
@ -126,7 +185,7 @@ impl KernelArgsTy {
const OFFLOAD_VERSION: u64 = 3;
const FLAGS: u64 = 0;
const TRIPCOUNT: u64 = 0;
fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll Type {
fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Type {
let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments");
let tptr = cx.type_ptr();
let ti64 = cx.type_i64();
@ -140,8 +199,8 @@ impl KernelArgsTy {
kernel_arguments_ty
}
fn new<'ll>(
cx: &'ll SimpleCx<'_>,
fn new<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
num_args: u64,
memtransfer_types: &'ll Value,
geps: [&'ll Value; 3],
@ -171,7 +230,8 @@ impl KernelArgsTy {
}
// Contains LLVM values needed to manage offloading for a single kernel.
pub(crate) struct OffloadKernelData<'ll> {
#[derive(Copy, Clone)]
pub(crate) struct OffloadKernelGlobals<'ll> {
pub offload_sizes: &'ll llvm::Value,
pub memtransfer_types: &'ll llvm::Value,
pub region_id: &'ll llvm::Value,
@ -179,7 +239,7 @@ pub(crate) struct OffloadKernelData<'ll> {
}
fn gen_tgt_data_mappers<'ll>(
cx: &'ll SimpleCx<'_>,
cx: &CodegenCx<'ll, '_>,
) -> (&'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Type) {
let tptr = cx.type_ptr();
let ti64 = cx.type_i64();
@ -241,12 +301,18 @@ pub(crate) fn add_global<'ll>(
// mapped to/from the gpu. It also returns a region_id with the name of this kernel, to be
// concatenated into the list of region_ids.
pub(crate) fn gen_define_handling<'ll>(
cx: &SimpleCx<'ll>,
offload_entry_ty: &'ll llvm::Type,
cx: &CodegenCx<'ll, '_>,
metadata: &[OffloadMetadata],
types: &[&Type],
symbol: &str,
) -> OffloadKernelData<'ll> {
types: &[&'ll Type],
symbol: String,
offload_globals: &OffloadGlobals<'ll>,
) -> OffloadKernelGlobals<'ll> {
if let Some(entry) = cx.offload_kernel_cache.borrow().get(&symbol) {
return *entry;
}
let offload_entry_ty = offload_globals.offload_entry_ty;
// It seems like non-pointer values are automatically mapped. So here, we focus on pointer (or
// reference) types.
let ptr_meta = types.iter().zip(metadata).filter_map(|(&x, meta)| match cx.type_kind(x) {
@ -272,9 +338,9 @@ pub(crate) fn gen_define_handling<'ll>(
let name = format!(".{symbol}.region_id");
let initializer = cx.get_const_i8(0);
let region_id = add_unnamed_global(&cx, &name, initializer, WeakAnyLinkage);
let region_id = add_global(&cx, &name, initializer, WeakAnyLinkage);
let c_entry_name = CString::new(symbol).unwrap();
let c_entry_name = CString::new(symbol.clone()).unwrap();
let c_val = c_entry_name.as_bytes_with_nul();
let offload_entry_name = format!(".offloading.entry_name.{symbol}");
@ -298,11 +364,16 @@ pub(crate) fn gen_define_handling<'ll>(
let c_section_name = CString::new("llvm_offload_entries").unwrap();
llvm::set_section(offload_entry, &c_section_name);
OffloadKernelData { offload_sizes, memtransfer_types, region_id, offload_entry }
let result =
OffloadKernelGlobals { offload_sizes, memtransfer_types, region_id, offload_entry };
cx.offload_kernel_cache.borrow_mut().insert(symbol, result);
result
}
fn declare_offload_fn<'ll>(
cx: &'ll SimpleCx<'_>,
cx: &CodegenCx<'ll, '_>,
name: &str,
ty: &'ll llvm::Type,
) -> &'ll llvm::Value {
@ -335,28 +406,28 @@ fn declare_offload_fn<'ll>(
// 4. set insert point after kernel call.
// 5. generate all the GEPS and stores, to be used in 6)
// 6. generate __tgt_target_data_end calls to move data from the GPU
pub(crate) fn gen_call_handling<'ll>(
cx: &SimpleCx<'ll>,
bb: &BasicBlock,
offload_data: &OffloadKernelData<'ll>,
pub(crate) fn gen_call_handling<'ll, 'tcx>(
builder: &mut Builder<'_, 'll, 'tcx>,
offload_data: &OffloadKernelGlobals<'ll>,
args: &[&'ll Value],
types: &[&Type],
metadata: &[OffloadMetadata],
offload_globals: &OffloadGlobals<'ll>,
) {
let OffloadKernelData { offload_sizes, offload_entry, memtransfer_types, region_id } =
let cx = builder.cx;
let OffloadKernelGlobals { offload_sizes, offload_entry, memtransfer_types, region_id } =
offload_data;
let (tgt_decl, tgt_target_kernel_ty) = generate_launcher(&cx);
let tgt_decl = offload_globals.launcher_fn;
let tgt_target_kernel_ty = offload_globals.launcher_ty;
// %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr }
let tptr = cx.type_ptr();
let ti32 = cx.type_i32();
let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr];
let tgt_bin_desc = cx.type_named_struct("struct.__tgt_bin_desc");
cx.set_struct_body(tgt_bin_desc, &tgt_bin_desc_ty, false);
let tgt_bin_desc = offload_globals.bin_desc;
let tgt_kernel_decl = KernelArgsTy::new_decl(&cx);
let (begin_mapper_decl, _, end_mapper_decl, fn_ty) = gen_tgt_data_mappers(&cx);
let mut builder = SBuilder::build(cx, bb);
let tgt_kernel_decl = offload_globals.kernel_args_ty;
let begin_mapper_decl = offload_globals.begin_mapper;
let end_mapper_decl = offload_globals.end_mapper;
let fn_ty = offload_globals.mapper_fn_ty;
let num_args = types.len() as u64;
let ip = unsafe { llvm::LLVMRustGetInsertPoint(&builder.llbuilder) };
@ -378,9 +449,8 @@ pub(crate) fn gen_call_handling<'ll>(
// Step 0)
// %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr }
// %6 = alloca %struct.__tgt_bin_desc, align 8
let llfn = unsafe { llvm::LLVMGetBasicBlockParent(bb) };
unsafe {
llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, llfn);
llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, builder.llfn());
}
let tgt_bin_desc_alloca = builder.direct_alloca(tgt_bin_desc, Align::EIGHT, "EmptyDesc");
@ -413,16 +483,16 @@ pub(crate) fn gen_call_handling<'ll>(
}
let mapper_fn_ty = cx.type_func(&[cx.type_ptr()], cx.type_void());
let register_lib_decl = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty);
let unregister_lib_decl = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty);
let register_lib_decl = offload_globals.register_lib;
let unregister_lib_decl = offload_globals.unregister_lib;
let init_ty = cx.type_func(&[], cx.type_void());
let init_rtls_decl = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty);
let init_rtls_decl = offload_globals.init_rtls;
// FIXME(offload): Later we want to add them to the wrapper code, rather than our main function.
// call void @__tgt_register_lib(ptr noundef %6)
builder.call(mapper_fn_ty, register_lib_decl, &[tgt_bin_desc_alloca], None);
builder.call(mapper_fn_ty, None, None, register_lib_decl, &[tgt_bin_desc_alloca], None, None);
// call void @__tgt_init_all_rtls()
builder.call(init_ty, init_rtls_decl, &[], None);
builder.call(init_ty, None, None, init_rtls_decl, &[], None, None);
for i in 0..num_args {
let idx = cx.get_const_i32(i);
@ -437,15 +507,15 @@ pub(crate) fn gen_call_handling<'ll>(
// For now we have a very simplistic indexing scheme into our
// offload_{baseptrs,ptrs,sizes}. We will probably improve this along with our gpu frontend pr.
fn get_geps<'a, 'll>(
builder: &mut SBuilder<'a, 'll>,
cx: &'ll SimpleCx<'ll>,
fn get_geps<'ll, 'tcx>(
builder: &mut Builder<'_, 'll, 'tcx>,
ty: &'ll Type,
ty2: &'ll Type,
a1: &'ll Value,
a2: &'ll Value,
a4: &'ll Value,
) -> [&'ll Value; 3] {
let cx = builder.cx;
let i32_0 = cx.get_const_i32(0);
let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]);
@ -454,9 +524,8 @@ pub(crate) fn gen_call_handling<'ll>(
[gep1, gep2, gep3]
}
fn generate_mapper_call<'a, 'll>(
builder: &mut SBuilder<'a, 'll>,
cx: &'ll SimpleCx<'ll>,
fn generate_mapper_call<'ll, 'tcx>(
builder: &mut Builder<'_, 'll, 'tcx>,
geps: [&'ll Value; 3],
o_type: &'ll Value,
fn_to_call: &'ll Value,
@ -464,20 +533,20 @@ pub(crate) fn gen_call_handling<'ll>(
num_args: u64,
s_ident_t: &'ll Value,
) {
let cx = builder.cx;
let nullptr = cx.const_null(cx.type_ptr());
let i64_max = cx.get_const_i64(u64::MAX);
let num_args = cx.get_const_i32(num_args);
let args =
vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr];
builder.call(fn_ty, fn_to_call, &args, None);
builder.call(fn_ty, None, None, fn_to_call, &args, None, None);
}
// Step 2)
let s_ident_t = generate_at_one(&cx);
let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4);
let s_ident_t = offload_globals.ident_t_global;
let geps = get_geps(builder, ty, ty2, a1, a2, a4);
generate_mapper_call(
&mut builder,
&cx,
builder,
geps,
memtransfer_types,
begin_mapper_decl,
@ -504,14 +573,13 @@ pub(crate) fn gen_call_handling<'ll>(
region_id,
a5,
];
builder.call(tgt_target_kernel_ty, tgt_decl, &args, None);
builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None);
// %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args)
// Step 4)
let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4);
let geps = get_geps(builder, ty, ty2, a1, a2, a4);
generate_mapper_call(
&mut builder,
&cx,
builder,
geps,
memtransfer_types,
end_mapper_decl,
@ -520,7 +588,5 @@ pub(crate) fn gen_call_handling<'ll>(
s_ident_t,
);
builder.call(mapper_fn_ty, unregister_lib_decl, &[tgt_bin_desc_alloca], None);
drop(builder);
builder.call(mapper_fn_ty, None, None, unregister_lib_decl, &[tgt_bin_desc_alloca], None, None);
}

9
compiler/rustc_codegen_llvm/src/context.rs
View file

@ -35,6 +35,7 @@ use smallvec::SmallVec;
use crate::abi::to_llvm_calling_convention;
use crate::back::write::to_llvm_code_model;
use crate::builder::gpu_offload::{OffloadGlobals, OffloadKernelGlobals};
use crate::callee::get_fn;
use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value};
@ -156,6 +157,12 @@ pub(crate) struct FullCx<'ll, 'tcx> {
/// Cache of Objective-C selector references
pub objc_selrefs: RefCell<FxHashMap<Symbol, &'ll Value>>,
/// Globals shared by the offloading runtime
pub offload_globals: RefCell<Option<OffloadGlobals<'ll>>>,
/// Cache of kernel-specific globals
pub offload_kernel_cache: RefCell<FxHashMap<String, OffloadKernelGlobals<'ll>>>,
}
fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode {
@ -639,6 +646,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
objc_class_t: Cell::new(None),
objc_classrefs: Default::default(),
objc_selrefs: Default::default(),
offload_globals: Default::default(),
offload_kernel_cache: Default::default(),
},
PhantomData,
)

16
compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
View file

@ -7,7 +7,7 @@ use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods};
use rustc_data_structures::fx::FxIndexMap;
use rustc_index::IndexVec;
use rustc_middle::ty::TyCtxt;
use rustc_span::{RemapPathScopeComponents, SourceFile, StableSourceFileId};
use rustc_span::{FileName, RemapPathScopeComponents, SourceFile, StableSourceFileId};
use tracing::debug;
use crate::common::CodegenCx;
@ -125,7 +125,19 @@ impl GlobalFileTable {
for file in all_files {
raw_file_table.entry(file.stable_id).or_insert_with(|| {
file.name.display(RemapPathScopeComponents::COVERAGE).to_string_lossy().into_owned()
// Prefer using the embeddable filename as this filename is going to
// end-up in the coverage artifacts (see rust-lang/rust#150020).
if let FileName::Real(real) = &file.name {
let (_work_dir, abs_name) =
real.embeddable_name(RemapPathScopeComponents::COVERAGE);
abs_name.to_string_lossy().into_owned()
} else {
file.name
.display(RemapPathScopeComponents::COVERAGE)
.to_string_lossy()
.into_owned()
}
});
}

31
compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs
View file

@ -1,8 +1,37 @@
use std::ptr;
use libc::c_uint;
use rustc_abi::Align;
use crate::llvm::debuginfo::DIBuilder;
use crate::llvm::{self, ToLlvmBool};
use crate::llvm::{self, Module, ToLlvmBool};
/// Owning pointer to a `DIBuilder<'ll>` that will dispose of the builder
/// when dropped. Use `.as_ref()` to get the underlying `&DIBuilder`
/// needed for debuginfo FFI calls.
pub(crate) struct DIBuilderBox<'ll> {
raw: ptr::NonNull<DIBuilder<'ll>>,
}
impl<'ll> DIBuilderBox<'ll> {
pub(crate) fn new(llmod: &'ll Module) -> Self {
let raw = unsafe { llvm::LLVMCreateDIBuilder(llmod) };
let raw = ptr::NonNull::new(raw).unwrap();
Self { raw }
}
pub(crate) fn as_ref(&self) -> &DIBuilder<'ll> {
// SAFETY: This is an owning pointer, so `&DIBuilder` is valid
// for as long as `&self` is.
unsafe { self.raw.as_ref() }
}
}
impl<'ll> Drop for DIBuilderBox<'ll> {
fn drop(&mut self) {
unsafe { llvm::LLVMDisposeDIBuilder(self.raw) };
}
}
/// Extension trait for defining safe wrappers and helper methods on
/// `&DIBuilder<'ll>`, without requiring it to be defined in the same crate.

3
compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
View file

@ -38,8 +38,9 @@ use self::namespace::mangled_name_of_instance;
use self::utils::{DIB, create_DIArray, is_node_local_to_unit};
use crate::builder::Builder;
use crate::common::{AsCCharPtr, CodegenCx};
use crate::debuginfo::di_builder::DIBuilderBox;
use crate::llvm::debuginfo::{
DIArray, DIBuilderBox, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope,
DIArray, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope,
DITemplateTypeParameter, DIType, DIVariable,
};
use crate::llvm::{self, Value};

5
compiler/rustc_codegen_llvm/src/errors.rs
View file

@ -32,6 +32,11 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
}
}
#[cfg(feature = "llvm_enzyme")]
#[derive(Diagnostic)]
#[diag(codegen_llvm_autodiff_component_unavailable)]
pub(crate) struct AutoDiffComponentUnavailable;
#[derive(Diagnostic)]
#[diag(codegen_llvm_autodiff_without_lto)]
pub(crate) struct AutoDiffWithoutLto;

59
compiler/rustc_codegen_llvm/src/intrinsic.rs
View file

@ -26,7 +26,7 @@ use tracing::debug;
use crate::abi::FnAbiLlvmExt;
use crate::builder::Builder;
use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
use crate::builder::gpu_offload::TgtOffloadEntry;
use crate::builder::gpu_offload::{gen_call_handling, gen_define_handling};
use crate::context::CodegenCx;
use crate::errors::{
AutoDiffWithoutEnable, AutoDiffWithoutLto, OffloadWithoutEnable, OffloadWithoutFatLTO,
@ -480,6 +480,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
let use_integer_compare = match layout.backend_repr() {
Scalar(_) | ScalarPair(_, _) => true,
SimdVector { .. } => false,
ScalableVector { .. } => {
tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
span,
name: sym::raw_eq,
ty: tp_ty,
});
return Ok(());
}
Memory { .. } => {
// For rusty ABIs, small aggregates are actually passed
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
@ -1287,8 +1295,6 @@ fn codegen_offload<'ll, 'tcx>(
let args = get_args_from_tuple(bx, args[1], fn_target);
let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE);
let offload_entry_ty = TgtOffloadEntry::new_decl(&cx);
let sig = tcx.fn_sig(fn_target.def_id()).skip_binder().skip_binder();
let inputs = sig.inputs();
@ -1296,17 +1302,16 @@ fn codegen_offload<'ll, 'tcx>(
let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::<Vec<_>>();
let offload_data = crate::builder::gpu_offload::gen_define_handling(
cx,
offload_entry_ty,
&metadata,
&types,
&target_symbol,
);
// FIXME(Sa4dUs): pass the original builder once we separate kernel launch logic from globals
let bb = unsafe { llvm::LLVMGetInsertBlock(bx.llbuilder) };
crate::builder::gpu_offload::gen_call_handling(cx, bb, &offload_data, &args, &types, &metadata);
let offload_globals_ref = cx.offload_globals.borrow();
let offload_globals = match offload_globals_ref.as_ref() {
Some(globals) => globals,
None => {
// Offload is not initialized, cannot continue
return;
}
};
let offload_data = gen_define_handling(&cx, &metadata, &types, target_symbol, offload_globals);
gen_call_handling(bx, &offload_data, &args, &types, &metadata, offload_globals);
}
fn get_args_from_tuple<'ll, 'tcx>(
@ -1679,11 +1684,27 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
m_len == v_len,
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
);
let in_elem_bitwidth = require_int_or_uint_ty!(
m_elem_ty.kind(),
InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
);
let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
let m_i1s = if args[1].layout.ty.is_scalable_vector() {
match m_elem_ty.kind() {
ty::Bool => {}
_ => return_error!(InvalidMonomorphization::MaskWrongElementType {
span,
name,
ty: m_elem_ty
}),
};
let i1 = bx.type_i1();
let i1xn = bx.type_scalable_vector(i1, m_len as u64);
bx.trunc(args[0].immediate(), i1xn)
} else {
let in_elem_bitwidth = require_int_or_uint_ty!(
m_elem_ty.kind(),
InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
);
vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
};
return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
}

15
compiler/rustc_codegen_llvm/src/lib.rs
View file

@ -5,7 +5,6 @@
//! This API is completely unstable and subject to change.
// tidy-alphabetical-start
#![cfg_attr(bootstrap, feature(slice_as_array))]
#![feature(assert_matches)]
#![feature(extern_types)]
#![feature(file_buffered)]
@ -13,6 +12,7 @@
#![feature(impl_trait_in_assoc_type)]
#![feature(iter_intersperse)]
#![feature(macro_derive)]
#![feature(once_cell_try)]
#![feature(trim_prefix_suffix)]
#![feature(try_blocks)]
// tidy-alphabetical-end
@ -240,6 +240,19 @@ impl CodegenBackend for LlvmCodegenBackend {
fn init(&self, sess: &Session) {
llvm_util::init(sess); // Make sure llvm is inited
#[cfg(feature = "llvm_enzyme")]
{
use rustc_session::config::AutoDiff;
use crate::back::lto::enable_autodiff_settings;
if sess.opts.unstable_opts.autodiff.contains(&AutoDiff::Enable) {
if let Err(_) = llvm::EnzymeWrapper::get_or_init(&sess.opts.sysroot) {
sess.dcx().emit_fatal(crate::errors::AutoDiffComponentUnavailable);
}
enable_autodiff_settings(&sess.opts.unstable_opts.autodiff);
}
}
}
fn provide(&self, providers: &mut Providers) {

40
compiler/rustc_codegen_llvm/src/llvm/conversions.rs
View file

@ -1,6 +1,6 @@
//! Conversions from backend-independent data types to/from LLVM FFI types.
use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate};
use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, TypeKind};
use rustc_middle::ty::AtomicOrdering;
use rustc_session::config::DebugInfo;
use rustc_target::spec::SymbolVisibility;
@ -9,10 +9,22 @@ use crate::llvm;
/// Helper trait for converting backend-independent types to LLVM-specific
/// types, for FFI purposes.
///
/// FIXME(#147327): These trait/method names were chosen to avoid churn in
/// existing code, but are not great and could probably be made clearer.
pub(crate) trait FromGeneric<T> {
fn from_generic(other: T) -> Self;
}
/// Helper trait for converting LLVM-specific types to backend-independent
/// types, for FFI purposes.
///
/// FIXME(#147327): These trait/method names were chosen to avoid churn in
/// existing code, but are not great and could probably be made clearer.
pub(crate) trait ToGeneric<T> {
fn to_generic(&self) -> T;
}
impl FromGeneric<SymbolVisibility> for llvm::Visibility {
fn from_generic(visibility: SymbolVisibility) -> Self {
match visibility {
@ -113,3 +125,29 @@ impl FromGeneric<DebugInfo> for llvm::debuginfo::DebugEmissionKind {
}
}
}
impl ToGeneric<TypeKind> for llvm::TypeKind {
fn to_generic(&self) -> TypeKind {
match self {
Self::Void => TypeKind::Void,
Self::Half => TypeKind::Half,
Self::Float => TypeKind::Float,
Self::Double => TypeKind::Double,
Self::X86_FP80 => TypeKind::X86_FP80,
Self::FP128 => TypeKind::FP128,
Self::PPC_FP128 => TypeKind::PPC_FP128,
Self::Label => TypeKind::Label,
Self::Integer => TypeKind::Integer,
Self::Function => TypeKind::Function,
Self::Struct => TypeKind::Struct,
Self::Array => TypeKind::Array,
Self::Pointer => TypeKind::Pointer,
Self::Vector => TypeKind::Vector,
Self::Metadata => TypeKind::Metadata,
Self::Token => TypeKind::Token,
Self::ScalableVector => TypeKind::ScalableVector,
Self::BFloat => TypeKind::BFloat,
Self::X86_AMX => TypeKind::X86_AMX,
}
}
}

646
compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
View file

@ -91,102 +91,361 @@ pub(crate) use self::Enzyme_AD::*;
#[cfg(feature = "llvm_enzyme")]
pub(crate) mod Enzyme_AD {
use std::ffi::{CString, c_char};
use std::ffi::{c_char, c_void};
use std::sync::{Mutex, MutexGuard, OnceLock};
use libc::c_void;
use rustc_middle::bug;
use rustc_session::config::{Sysroot, host_tuple};
use rustc_session::filesearch;
use super::{CConcreteType, CTypeTreeRef, Context};
use crate::llvm::{EnzymeTypeTree, LLVMRustVersionMajor};
unsafe extern "C" {
pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8);
pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char);
type EnzymeSetCLBoolFn = unsafe extern "C" fn(*mut c_void, u8);
type EnzymeSetCLStringFn = unsafe extern "C" fn(*mut c_void, *const c_char);
type EnzymeNewTypeTreeFn = unsafe extern "C" fn() -> CTypeTreeRef;
type EnzymeNewTypeTreeCTFn = unsafe extern "C" fn(CConcreteType, &Context) -> CTypeTreeRef;
type EnzymeNewTypeTreeTRFn = unsafe extern "C" fn(CTypeTreeRef) -> CTypeTreeRef;
type EnzymeFreeTypeTreeFn = unsafe extern "C" fn(CTypeTreeRef);
type EnzymeMergeTypeTreeFn = unsafe extern "C" fn(CTypeTreeRef, CTypeTreeRef) -> bool;
type EnzymeTypeTreeOnlyEqFn = unsafe extern "C" fn(CTypeTreeRef, i64);
type EnzymeTypeTreeData0EqFn = unsafe extern "C" fn(CTypeTreeRef);
type EnzymeTypeTreeShiftIndiciesEqFn =
unsafe extern "C" fn(CTypeTreeRef, *const c_char, i64, i64, u64);
type EnzymeTypeTreeInsertEqFn =
unsafe extern "C" fn(CTypeTreeRef, *const i64, usize, CConcreteType, &Context);
type EnzymeTypeTreeToStringFn = unsafe extern "C" fn(CTypeTreeRef) -> *const c_char;
type EnzymeTypeTreeToStringFreeFn = unsafe extern "C" fn(*const c_char);
#[allow(non_snake_case)]
pub(crate) struct EnzymeWrapper {
EnzymeNewTypeTree: EnzymeNewTypeTreeFn,
EnzymeNewTypeTreeCT: EnzymeNewTypeTreeCTFn,
EnzymeNewTypeTreeTR: EnzymeNewTypeTreeTRFn,
EnzymeFreeTypeTree: EnzymeFreeTypeTreeFn,
EnzymeMergeTypeTree: EnzymeMergeTypeTreeFn,
EnzymeTypeTreeOnlyEq: EnzymeTypeTreeOnlyEqFn,
EnzymeTypeTreeData0Eq: EnzymeTypeTreeData0EqFn,
EnzymeTypeTreeShiftIndiciesEq: EnzymeTypeTreeShiftIndiciesEqFn,
EnzymeTypeTreeInsertEq: EnzymeTypeTreeInsertEqFn,
EnzymeTypeTreeToString: EnzymeTypeTreeToStringFn,
EnzymeTypeTreeToStringFree: EnzymeTypeTreeToStringFreeFn,
EnzymePrintPerf: *mut c_void,
EnzymePrintActivity: *mut c_void,
EnzymePrintType: *mut c_void,
EnzymeFunctionToAnalyze: *mut c_void,
EnzymePrint: *mut c_void,
EnzymeStrictAliasing: *mut c_void,
EnzymeInline: *mut c_void,
EnzymeMaxTypeDepth: *mut c_void,
RustTypeRules: *mut c_void,
looseTypeAnalysis: *mut c_void,
EnzymeSetCLBool: EnzymeSetCLBoolFn,
EnzymeSetCLString: EnzymeSetCLStringFn,
pub registerEnzymeAndPassPipeline: *const c_void,
lib: libloading::Library,
}
// TypeTree functions
unsafe extern "C" {
pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef;
pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef;
pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef;
pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef);
pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool;
pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64);
pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef);
pub(crate) fn EnzymeTypeTreeShiftIndiciesEq(
arg1: CTypeTreeRef,
unsafe impl Sync for EnzymeWrapper {}
unsafe impl Send for EnzymeWrapper {}
fn load_ptr_by_symbol_mut_void(
lib: &libloading::Library,
bytes: &[u8],
) -> Result<*mut c_void, Box<dyn std::error::Error>> {
unsafe {
let s: libloading::Symbol<'_, *mut c_void> = lib.get(bytes)?;
// libloading = 0.9.0: try_as_raw_ptr always succeeds and returns Some
let s = s.try_as_raw_ptr().unwrap();
Ok(s)
}
}
// e.g.
// load_ptrs_by_symbols_mut_void(ABC, XYZ);
// =>
// let ABC = load_ptr_mut_void(&lib, b"ABC")?;
// let XYZ = load_ptr_mut_void(&lib, b"XYZ")?;
macro_rules! load_ptrs_by_symbols_mut_void {
($lib:expr, $($name:ident),* $(,)?) => {
$(
#[allow(non_snake_case)]
let $name = load_ptr_by_symbol_mut_void(&$lib, stringify!($name).as_bytes())?;
)*
};
}
// e.g.
// load_ptrs_by_symbols_fn(ABC: ABCFn, XYZ: XYZFn);
// =>
// let ABC: libloading::Symbol<'_, ABCFn> = unsafe { lib.get(b"ABC")? };
// let XYZ: libloading::Symbol<'_, XYZFn> = unsafe { lib.get(b"XYZ")? };
macro_rules! load_ptrs_by_symbols_fn {
($lib:expr, $($name:ident : $ty:ty),* $(,)?) => {
$(
#[allow(non_snake_case)]
let $name: $ty = *unsafe { $lib.get::<$ty>(stringify!($name).as_bytes())? };
)*
};
}
static ENZYME_INSTANCE: OnceLock<Mutex<EnzymeWrapper>> = OnceLock::new();
impl EnzymeWrapper {
/// Initialize EnzymeWrapper with the given sysroot if not already initialized.
/// Safe to call multiple times - subsequent calls are no-ops due to OnceLock.
pub(crate) fn get_or_init(
sysroot: &rustc_session::config::Sysroot,
) -> Result<MutexGuard<'static, Self>, Box<dyn std::error::Error>> {
let mtx: &'static Mutex<EnzymeWrapper> = ENZYME_INSTANCE.get_or_try_init(|| {
let w = Self::call_dynamic(sysroot)?;
Ok::<_, Box<dyn std::error::Error>>(Mutex::new(w))
})?;
Ok(mtx.lock().unwrap())
}
/// Get the EnzymeWrapper instance. Panics if not initialized.
pub(crate) fn get_instance() -> MutexGuard<'static, Self> {
ENZYME_INSTANCE
.get()
.expect("EnzymeWrapper not initialized. Call get_or_init with sysroot first.")
.lock()
.unwrap()
}
pub(crate) fn new_type_tree(&self) -> CTypeTreeRef {
unsafe { (self.EnzymeNewTypeTree)() }
}
pub(crate) fn new_type_tree_ct(
&self,
t: CConcreteType,
ctx: &Context,
) -> *mut EnzymeTypeTree {
unsafe { (self.EnzymeNewTypeTreeCT)(t, ctx) }
}
pub(crate) fn new_type_tree_tr(&self, tree: CTypeTreeRef) -> CTypeTreeRef {
unsafe { (self.EnzymeNewTypeTreeTR)(tree) }
}
pub(crate) fn free_type_tree(&self, tree: CTypeTreeRef) {
unsafe { (self.EnzymeFreeTypeTree)(tree) }
}
pub(crate) fn merge_type_tree(&self, tree1: CTypeTreeRef, tree2: CTypeTreeRef) -> bool {
unsafe { (self.EnzymeMergeTypeTree)(tree1, tree2) }
}
pub(crate) fn tree_only_eq(&self, tree: CTypeTreeRef, num: i64) {
unsafe { (self.EnzymeTypeTreeOnlyEq)(tree, num) }
}
pub(crate) fn tree_data0_eq(&self, tree: CTypeTreeRef) {
unsafe { (self.EnzymeTypeTreeData0Eq)(tree) }
}
pub(crate) fn shift_indicies_eq(
&self,
tree: CTypeTreeRef,
data_layout: *const c_char,
offset: i64,
max_size: i64,
add_offset: u64,
);
pub(crate) fn EnzymeTypeTreeInsertEq(
CTT: CTypeTreeRef,
) {
unsafe {
(self.EnzymeTypeTreeShiftIndiciesEq)(
tree,
data_layout,
offset,
max_size,
add_offset,
)
}
}
pub(crate) fn tree_insert_eq(
&self,
tree: CTypeTreeRef,
indices: *const i64,
len: usize,
ct: CConcreteType,
ctx: &Context,
);
pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char;
pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char);
}
) {
unsafe { (self.EnzymeTypeTreeInsertEq)(tree, indices, len, ct, ctx) }
}
unsafe extern "C" {
static mut EnzymePrintPerf: c_void;
static mut EnzymePrintActivity: c_void;
static mut EnzymePrintType: c_void;
static mut EnzymeFunctionToAnalyze: c_void;
static mut EnzymePrint: c_void;
static mut EnzymeStrictAliasing: c_void;
static mut looseTypeAnalysis: c_void;
static mut EnzymeInline: c_void;
static mut RustTypeRules: c_void;
}
pub(crate) fn set_print_perf(print: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintPerf), print as u8);
pub(crate) fn tree_to_string(&self, tree: *mut EnzymeTypeTree) -> *const c_char {
unsafe { (self.EnzymeTypeTreeToString)(tree) }
}
}
pub(crate) fn set_print_activity(print: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintActivity), print as u8);
pub(crate) fn tree_to_string_free(&self, ch: *const c_char) {
unsafe { (self.EnzymeTypeTreeToStringFree)(ch) }
}
}
pub(crate) fn set_print_type(print: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintType), print as u8);
pub(crate) fn get_max_type_depth(&self) -> usize {
unsafe { std::ptr::read::<u32>(self.EnzymeMaxTypeDepth as *const u32) as usize }
}
}
pub(crate) fn set_print_type_fun(fun_name: &str) {
let c_fun_name = CString::new(fun_name).unwrap();
unsafe {
EnzymeSetCLString(
std::ptr::addr_of_mut!(EnzymeFunctionToAnalyze),
c_fun_name.as_ptr() as *const c_char,
pub(crate) fn set_print_perf(&mut self, print: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.EnzymePrintPerf, print as u8);
}
}
pub(crate) fn set_print_activity(&mut self, print: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.EnzymePrintActivity, print as u8);
}
}
pub(crate) fn set_print_type(&mut self, print: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.EnzymePrintType, print as u8);
}
}
pub(crate) fn set_print_type_fun(&mut self, fun_name: &str) {
let c_fun_name = std::ffi::CString::new(fun_name)
.unwrap_or_else(|err| bug!("failed to set_print_type_fun: {err}"));
unsafe {
(self.EnzymeSetCLString)(
self.EnzymeFunctionToAnalyze,
c_fun_name.as_ptr() as *const c_char,
);
}
}
pub(crate) fn set_print(&mut self, print: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.EnzymePrint, print as u8);
}
}
pub(crate) fn set_strict_aliasing(&mut self, strict: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.EnzymeStrictAliasing, strict as u8);
}
}
pub(crate) fn set_loose_types(&mut self, loose: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.looseTypeAnalysis, loose as u8);
}
}
pub(crate) fn set_inline(&mut self, val: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.EnzymeInline, val as u8);
}
}
pub(crate) fn set_rust_rules(&mut self, val: bool) {
unsafe {
(self.EnzymeSetCLBool)(self.RustTypeRules, val as u8);
}
}
#[allow(non_snake_case)]
fn call_dynamic(
sysroot: &rustc_session::config::Sysroot,
) -> Result<Self, Box<dyn std::error::Error>> {
let enzyme_path = Self::get_enzyme_path(sysroot)?;
let lib = unsafe { libloading::Library::new(enzyme_path)? };
load_ptrs_by_symbols_fn!(
lib,
EnzymeNewTypeTree: EnzymeNewTypeTreeFn,
EnzymeNewTypeTreeCT: EnzymeNewTypeTreeCTFn,
EnzymeNewTypeTreeTR: EnzymeNewTypeTreeTRFn,
EnzymeFreeTypeTree: EnzymeFreeTypeTreeFn,
EnzymeMergeTypeTree: EnzymeMergeTypeTreeFn,
EnzymeTypeTreeOnlyEq: EnzymeTypeTreeOnlyEqFn,
EnzymeTypeTreeData0Eq: EnzymeTypeTreeData0EqFn,
EnzymeTypeTreeShiftIndiciesEq: EnzymeTypeTreeShiftIndiciesEqFn,
EnzymeTypeTreeInsertEq: EnzymeTypeTreeInsertEqFn,
EnzymeTypeTreeToString: EnzymeTypeTreeToStringFn,
EnzymeTypeTreeToStringFree: EnzymeTypeTreeToStringFreeFn,
EnzymeSetCLBool: EnzymeSetCLBoolFn,
EnzymeSetCLString: EnzymeSetCLStringFn,
);
load_ptrs_by_symbols_mut_void!(
lib,
registerEnzymeAndPassPipeline,
EnzymePrintPerf,
EnzymePrintActivity,
EnzymePrintType,
EnzymeFunctionToAnalyze,
EnzymePrint,
EnzymeStrictAliasing,
EnzymeInline,
EnzymeMaxTypeDepth,
RustTypeRules,
looseTypeAnalysis,
);
Ok(Self {
EnzymeNewTypeTree,
EnzymeNewTypeTreeCT,
EnzymeNewTypeTreeTR,
EnzymeFreeTypeTree,
EnzymeMergeTypeTree,
EnzymeTypeTreeOnlyEq,
EnzymeTypeTreeData0Eq,
EnzymeTypeTreeShiftIndiciesEq,
EnzymeTypeTreeInsertEq,
EnzymeTypeTreeToString,
EnzymeTypeTreeToStringFree,
EnzymePrintPerf,
EnzymePrintActivity,
EnzymePrintType,
EnzymeFunctionToAnalyze,
EnzymePrint,
EnzymeStrictAliasing,
EnzymeInline,
EnzymeMaxTypeDepth,
RustTypeRules,
looseTypeAnalysis,
EnzymeSetCLBool,
EnzymeSetCLString,
registerEnzymeAndPassPipeline,
lib,
})
}
}
pub(crate) fn set_print(print: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrint), print as u8);
}
}
pub(crate) fn set_strict_aliasing(strict: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymeStrictAliasing), strict as u8);
}
}
pub(crate) fn set_loose_types(loose: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(looseTypeAnalysis), loose as u8);
}
}
pub(crate) fn set_inline(val: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymeInline), val as u8);
}
}
pub(crate) fn set_rust_rules(val: bool) {
unsafe {
EnzymeSetCLBool(std::ptr::addr_of_mut!(RustTypeRules), val as u8);
fn get_enzyme_path(sysroot: &Sysroot) -> Result<String, String> {
let llvm_version_major = unsafe { LLVMRustVersionMajor() };
let path_buf = sysroot
.all_paths()
.map(|sysroot_path| {
filesearch::make_target_lib_path(sysroot_path, host_tuple())
.join("lib")
.with_file_name(format!("libEnzyme-{llvm_version_major}"))
.with_extension(std::env::consts::DLL_EXTENSION)
})
.find(|f| f.exists())
.ok_or_else(|| {
let candidates = sysroot
.all_paths()
.map(|p| p.join("lib").display().to_string())
.collect::<Vec<String>>()
.join("\n* ");
format!(
"failed to find a `libEnzyme-{llvm_version_major}` folder \
in the sysroot candidates:\n* {candidates}"
)
})?;
Ok(path_buf
.to_str()
.ok_or_else(|| format!("invalid UTF-8 in path: {}", path_buf.display()))?
.to_string())
}
}
}
@ -198,111 +457,156 @@ pub(crate) use self::Fallback_AD::*;
pub(crate) mod Fallback_AD {
#![allow(unused_variables)]
use std::ffi::c_void;
use std::sync::{Mutex, MutexGuard};
use libc::c_char;
use rustc_codegen_ssa::back::write::CodegenContext;
use rustc_codegen_ssa::traits::WriteBackendMethods;
use super::{CConcreteType, CTypeTreeRef, Context};
use super::{CConcreteType, CTypeTreeRef, Context, EnzymeTypeTree};
// TypeTree function fallbacks
pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef {
unimplemented!()
pub(crate) struct EnzymeWrapper {
pub registerEnzymeAndPassPipeline: *const c_void,
}
pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef {
unimplemented!()
}
impl EnzymeWrapper {
pub(crate) fn get_or_init(
_sysroot: &rustc_session::config::Sysroot,
) -> Result<MutexGuard<'static, Self>, Box<dyn std::error::Error>> {
unimplemented!("Enzyme not available: build with llvm_enzyme feature")
}
pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef {
unimplemented!()
}
pub(crate) fn init<'a, B: WriteBackendMethods>(
_cgcx: &'a CodegenContext<B>,
) -> &'static Mutex<Self> {
unimplemented!("Enzyme not available: build with llvm_enzyme feature")
}
pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) {
unimplemented!()
}
pub(crate) fn get_instance() -> MutexGuard<'static, Self> {
unimplemented!("Enzyme not available: build with llvm_enzyme feature")
}
pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool {
unimplemented!()
}
pub(crate) fn new_type_tree(&self) -> CTypeTreeRef {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) {
unimplemented!()
}
pub(crate) fn new_type_tree_ct(
&self,
t: CConcreteType,
ctx: &Context,
) -> *mut EnzymeTypeTree {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) {
unimplemented!()
}
pub(crate) fn new_type_tree_tr(&self, tree: CTypeTreeRef) -> CTypeTreeRef {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq(
arg1: CTypeTreeRef,
data_layout: *const c_char,
offset: i64,
max_size: i64,
add_offset: u64,
) {
unimplemented!()
}
pub(crate) fn free_type_tree(&self, tree: CTypeTreeRef) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeInsertEq(
CTT: CTypeTreeRef,
indices: *const i64,
len: usize,
ct: CConcreteType,
ctx: &Context,
) {
unimplemented!()
}
pub(crate) fn merge_type_tree(&self, tree1: CTypeTreeRef, tree2: CTypeTreeRef) -> bool {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char {
unimplemented!()
}
pub(crate) fn tree_only_eq(&self, tree: CTypeTreeRef, num: i64) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) {
unimplemented!()
}
pub(crate) fn tree_data0_eq(&self, tree: CTypeTreeRef) {
unimplemented!()
}
pub(crate) fn set_inline(val: bool) {
unimplemented!()
}
pub(crate) fn set_print_perf(print: bool) {
unimplemented!()
}
pub(crate) fn set_print_activity(print: bool) {
unimplemented!()
}
pub(crate) fn set_print_type(print: bool) {
unimplemented!()
}
pub(crate) fn set_print_type_fun(fun_name: &str) {
unimplemented!()
}
pub(crate) fn set_print(print: bool) {
unimplemented!()
}
pub(crate) fn set_strict_aliasing(strict: bool) {
unimplemented!()
}
pub(crate) fn set_loose_types(loose: bool) {
unimplemented!()
}
pub(crate) fn set_rust_rules(val: bool) {
unimplemented!()
pub(crate) fn shift_indicies_eq(
&self,
tree: CTypeTreeRef,
data_layout: *const c_char,
offset: i64,
max_size: i64,
add_offset: u64,
) {
unimplemented!()
}
pub(crate) fn tree_insert_eq(
&self,
tree: CTypeTreeRef,
indices: *const i64,
len: usize,
ct: CConcreteType,
ctx: &Context,
) {
unimplemented!()
}
pub(crate) fn tree_to_string(&self, tree: *mut EnzymeTypeTree) -> *const c_char {
unimplemented!()
}
pub(crate) fn tree_to_string_free(&self, ch: *const c_char) {
unimplemented!()
}
pub(crate) fn get_max_type_depth(&self) -> usize {
unimplemented!()
}
pub(crate) fn set_inline(&mut self, val: bool) {
unimplemented!()
}
pub(crate) fn set_print_perf(&mut self, print: bool) {
unimplemented!()
}
pub(crate) fn set_print_activity(&mut self, print: bool) {
unimplemented!()
}
pub(crate) fn set_print_type(&mut self, print: bool) {
unimplemented!()
}
pub(crate) fn set_print_type_fun(&mut self, fun_name: &str) {
unimplemented!()
}
pub(crate) fn set_print(&mut self, print: bool) {
unimplemented!()
}
pub(crate) fn set_strict_aliasing(&mut self, strict: bool) {
unimplemented!()
}
pub(crate) fn set_loose_types(&mut self, loose: bool) {
unimplemented!()
}
pub(crate) fn set_rust_rules(&mut self, val: bool) {
unimplemented!()
}
}
}
impl TypeTree {
pub(crate) fn new() -> TypeTree {
let inner = unsafe { EnzymeNewTypeTree() };
let wrapper = EnzymeWrapper::get_instance();
let inner = wrapper.new_type_tree();
TypeTree { inner }
}
pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree {
let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) };
let wrapper = EnzymeWrapper::get_instance();
let inner = wrapper.new_type_tree_ct(t, ctx);
TypeTree { inner }
}
pub(crate) fn merge(self, other: Self) -> Self {
unsafe {
EnzymeMergeTypeTree(self.inner, other.inner);
}
let wrapper = EnzymeWrapper::get_instance();
wrapper.merge_type_tree(self.inner, other.inner);
drop(other);
self
}
@ -316,37 +620,36 @@ impl TypeTree {
add_offset: usize,
) -> Self {
let layout = std::ffi::CString::new(layout).unwrap();
unsafe {
EnzymeTypeTreeShiftIndiciesEq(
self.inner,
layout.as_ptr(),
offset as i64,
max_size as i64,
add_offset as u64,
);
}
let wrapper = EnzymeWrapper::get_instance();
wrapper.shift_indicies_eq(
self.inner,
layout.as_ptr(),
offset as i64,
max_size as i64,
add_offset as u64,
);
self
}
pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) {
unsafe {
EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx);
}
let wrapper = EnzymeWrapper::get_instance();
wrapper.tree_insert_eq(self.inner, indices.as_ptr(), indices.len(), ct, ctx);
}
}
impl Clone for TypeTree {
fn clone(&self) -> Self {
let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) };
let wrapper = EnzymeWrapper::get_instance();
let inner = wrapper.new_type_tree_tr(self.inner);
TypeTree { inner }
}
}
impl std::fmt::Display for TypeTree {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let ptr = unsafe { EnzymeTypeTreeToString(self.inner) };
let wrapper = EnzymeWrapper::get_instance();
let ptr = wrapper.tree_to_string(self.inner);
let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) };
match cstr.to_str() {
Ok(x) => write!(f, "{}", x)?,
@ -354,9 +657,7 @@ impl std::fmt::Display for TypeTree {
}
// delete C string pointer
unsafe {
EnzymeTypeTreeToStringFree(ptr);
}
wrapper.tree_to_string_free(ptr);
Ok(())
}
@ -370,6 +671,7 @@ impl std::fmt::Debug for TypeTree {
impl Drop for TypeTree {
fn drop(&mut self) {
unsafe { EnzymeFreeTypeTree(self.inner) }
let wrapper = EnzymeWrapper::get_instance();
wrapper.free_type_tree(self.inner)
}
}

60
compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file

@ -363,33 +363,6 @@ pub(crate) enum TypeKind {
X86_AMX = 19,
}
impl TypeKind {
pub(crate) fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind {
use rustc_codegen_ssa::common::TypeKind as Common;
match self {
Self::Void => Common::Void,
Self::Half => Common::Half,
Self::Float => Common::Float,
Self::Double => Common::Double,
Self::X86_FP80 => Common::X86_FP80,
Self::FP128 => Common::FP128,
Self::PPC_FP128 => Common::PPC_FP128,
Self::Label => Common::Label,
Self::Integer => Common::Integer,
Self::Function => Common::Function,
Self::Struct => Common::Struct,
Self::Array => Common::Array,
Self::Pointer => Common::Pointer,
Self::Vector => Common::Vector,
Self::Metadata => Common::Metadata,
Self::Token => Common::Token,
Self::ScalableVector => Common::ScalableVector,
Self::BFloat => Common::BFloat,
Self::X86_AMX => Common::X86_AMX,
}
}
}
/// LLVMAtomicRmwBinOp
#[derive(Copy, Clone)]
#[repr(C)]
@ -738,12 +711,9 @@ unsafe extern "C" {
pub(crate) type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void);
pub(crate) mod debuginfo {
use std::ptr;
use bitflags::bitflags;
use super::{InvariantOpaque, Metadata};
use crate::llvm::{self, Module};
/// Opaque target type for references to an LLVM debuginfo builder.
///
@ -756,33 +726,6 @@ pub(crate) mod debuginfo {
#[repr(C)]
pub(crate) struct DIBuilder<'ll>(InvariantOpaque<'ll>);
/// Owning pointer to a `DIBuilder<'ll>` that will dispose of the builder
/// when dropped. Use `.as_ref()` to get the underlying `&DIBuilder`
/// needed for debuginfo FFI calls.
pub(crate) struct DIBuilderBox<'ll> {
raw: ptr::NonNull<DIBuilder<'ll>>,
}
impl<'ll> DIBuilderBox<'ll> {
pub(crate) fn new(llmod: &'ll Module) -> Self {
let raw = unsafe { llvm::LLVMCreateDIBuilder(llmod) };
let raw = ptr::NonNull::new(raw).unwrap();
Self { raw }
}
pub(crate) fn as_ref(&self) -> &DIBuilder<'ll> {
// SAFETY: This is an owning pointer, so `&DIBuilder` is valid
// for as long as `&self` is.
unsafe { self.raw.as_ref() }
}
}
impl<'ll> Drop for DIBuilderBox<'ll> {
fn drop(&mut self) {
unsafe { llvm::LLVMDisposeDIBuilder(self.raw) };
}
}
pub(crate) type DIDescriptor = Metadata;
pub(crate) type DILocation = Metadata;
pub(crate) type DIScope = DIDescriptor;
@ -998,6 +941,7 @@ unsafe extern "C" {
// Operations on array, pointer, and vector types (sequence types)
pub(crate) safe fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
pub(crate) fn LLVMScalableVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type;
pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint;
@ -2411,7 +2355,7 @@ unsafe extern "C" {
LoopVectorize: bool,
DisableSimplifyLibCalls: bool,
EmitLifetimeMarkers: bool,
RunEnzyme: bool,
RunEnzyme: *const c_void,
PrintBeforeEnzyme: bool,
PrintAfterEnzyme: bool,
PrintPasses: bool,

26
compiler/rustc_codegen_llvm/src/llvm_util.rs
View file

@ -266,6 +266,10 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea
"leoncasa" => Some(LLVMFeature::new("hasleoncasa")),
s => Some(LLVMFeature::new(s)),
},
Arch::Wasm32 | Arch::Wasm64 => match s {
"gc" if major < 22 => None,
s => Some(LLVMFeature::new(s)),
},
Arch::X86 | Arch::X86_64 => {
match s {
"sse4.2" => Some(LLVMFeature::with_dependencies(
@ -360,25 +364,25 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
let target_abi = &sess.target.options.abi;
let target_pointer_width = sess.target.pointer_width;
let version = get_version();
let lt_20_1_1 = version < (20, 1, 1);
let lt_21_0_0 = version < (21, 0, 0);
let (major, _, _) = version;
cfg.has_reliable_f16 = match (target_arch, target_os) {
// LLVM crash without neon <https://github.com/llvm/llvm-project/issues/129394> (fixed in llvm20)
// LLVM crash without neon <https://github.com/llvm/llvm-project/issues/129394> (fixed in LLVM 20.1.1)
(Arch::AArch64, _)
if !cfg.target_features.iter().any(|f| f.as_str() == "neon") && lt_20_1_1 =>
if !cfg.target_features.iter().any(|f| f.as_str() == "neon")
&& version < (20, 1, 1) =>
{
false
}
// Unsupported <https://github.com/llvm/llvm-project/issues/94434>
(Arch::Arm64EC, _) => false,
// Selection failure <https://github.com/llvm/llvm-project/issues/50374> (fixed in llvm21)
(Arch::S390x, _) if lt_21_0_0 => false,
(Arch::S390x, _) if major < 21 => false,
// MinGW ABI bugs <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054>
(Arch::X86_64, Os::Windows) if *target_env == Env::Gnu && *target_abi != Abi::Llvm => false,
// Infinite recursion <https://github.com/llvm/llvm-project/issues/97981>
(Arch::CSky, _) => false,
(Arch::Hexagon, _) if lt_21_0_0 => false, // (fixed in llvm21)
(Arch::Hexagon, _) if major < 21 => false, // (fixed in llvm21)
(Arch::PowerPC | Arch::PowerPC64, _) => false,
(Arch::Sparc | Arch::Sparc64, _) => false,
(Arch::Wasm32 | Arch::Wasm64, _) => false,
@ -389,15 +393,15 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
};
cfg.has_reliable_f128 = match (target_arch, target_os) {
// Unsupported https://github.com/llvm/llvm-project/issues/121122
(Arch::AmdGpu, _) => false,
// Unsupported <https://github.com/llvm/llvm-project/issues/94434>
(Arch::Arm64EC, _) => false,
// Selection bug <https://github.com/llvm/llvm-project/issues/96432> (fixed in llvm20)
(Arch::Mips64 | Arch::Mips64r6, _) if lt_20_1_1 => false,
// Selection bug <https://github.com/llvm/llvm-project/issues/96432> (fixed in LLVM 20.1.0)
(Arch::Mips64 | Arch::Mips64r6, _) if version < (20, 1, 0) => false,
// Selection bug <https://github.com/llvm/llvm-project/issues/95471>. This issue is closed
// but basic math still does not work.
(Arch::Nvptx64, _) => false,
// Unsupported https://github.com/llvm/llvm-project/issues/121122
(Arch::AmdGpu, _) => false,
// ABI bugs <https://github.com/rust-lang/rust/issues/125109> et al. (full
// list at <https://github.com/rust-lang/rust/issues/116909>)
(Arch::PowerPC | Arch::PowerPC64, _) => false,
@ -405,7 +409,7 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
(Arch::Sparc, _) => false,
// Stack alignment bug <https://github.com/llvm/llvm-project/issues/77401>. NB: tests may
// not fail if our compiler-builtins is linked. (fixed in llvm21)
(Arch::X86, _) if lt_21_0_0 => false,
(Arch::X86, _) if major < 21 => false,
// MinGW ABI bugs <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054>
(Arch::X86_64, Os::Windows) if *target_env == Env::Gnu && *target_abi != Abi::Llvm => false,
// There are no known problems on other platforms, so the only requirement is that symbols

6
compiler/rustc_codegen_llvm/src/type_.rs
View file

@ -15,7 +15,7 @@ use rustc_target::callconv::{CastTarget, FnAbi};
use crate::abi::{FnAbiLlvmExt, LlvmType};
use crate::common;
use crate::context::{CodegenCx, GenericCx, SCx};
use crate::llvm::{self, FALSE, Metadata, TRUE, ToLlvmBool, Type, Value};
use crate::llvm::{self, FALSE, Metadata, TRUE, ToGeneric, ToLlvmBool, Type, Value};
use crate::type_of::LayoutLlvmExt;
impl PartialEq for Type {
@ -68,6 +68,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
}
pub(crate) fn type_scalable_vector(&self, ty: &'ll Type, count: u64) -> &'ll Type {
unsafe { llvm::LLVMScalableVectorType(ty, count as c_uint) }
}
pub(crate) fn add_func(&self, name: &str, ty: &'ll Type) -> &'ll Value {
let name = SmallCStr::new(name);
unsafe { llvm::LLVMAddFunction(self.llmod(), name.as_ptr(), ty) }

14
compiler/rustc_codegen_llvm/src/type_of.rs
View file

@ -24,6 +24,15 @@ fn uncached_llvm_type<'a, 'tcx>(
let element = layout.scalar_llvm_type_at(cx, element);
return cx.type_vector(element, count);
}
BackendRepr::ScalableVector { ref element, count } => {
let element = if element.is_bool() {
cx.type_i1()
} else {
layout.scalar_llvm_type_at(cx, *element)
};
return cx.type_scalable_vector(element, count);
}
BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {}
}
@ -176,7 +185,9 @@ pub(crate) trait LayoutLlvmExt<'tcx> {
impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
fn is_llvm_immediate(&self) -> bool {
match self.backend_repr {
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
BackendRepr::Scalar(_)
| BackendRepr::SimdVector { .. }
| BackendRepr::ScalableVector { .. } => true,
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
}
}
@ -186,6 +197,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
BackendRepr::ScalarPair(..) => true,
BackendRepr::Scalar(_)
| BackendRepr::SimdVector { .. }
| BackendRepr::ScalableVector { .. }
| BackendRepr::Memory { .. } => false,
}
}

11
compiler/rustc_codegen_llvm/src/typetree.rs
View file

@ -2,6 +2,7 @@ use rustc_ast::expand::typetree::FncTree;
#[cfg(feature = "llvm_enzyme")]
use {
crate::attributes,
crate::llvm::EnzymeWrapper,
rustc_ast::expand::typetree::TypeTree as RustTypeTree,
std::ffi::{CString, c_char, c_uint},
};
@ -77,7 +78,8 @@ pub(crate) fn add_tt<'ll>(
for (i, input) in inputs.iter().enumerate() {
unsafe {
let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx);
let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner);
let enzyme_wrapper = EnzymeWrapper::get_instance();
let c_str = enzyme_wrapper.tree_to_string(enzyme_tt.inner);
let c_str = std::ffi::CStr::from_ptr(c_str);
let attr = llvm::LLVMCreateStringAttribute(
@ -89,13 +91,14 @@ pub(crate) fn add_tt<'ll>(
);
attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]);
llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr());
enzyme_wrapper.tree_to_string_free(c_str.as_ptr());
}
}
unsafe {
let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx);
let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner);
let enzyme_wrapper = EnzymeWrapper::get_instance();
let c_str = enzyme_wrapper.tree_to_string(enzyme_tt.inner);
let c_str = std::ffi::CStr::from_ptr(c_str);
let ret_attr = llvm::LLVMCreateStringAttribute(
@ -107,7 +110,7 @@ pub(crate) fn add_tt<'ll>(
);
attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]);
llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr());
enzyme_wrapper.tree_to_string_free(c_str.as_ptr());
}
}

138
compiler/rustc_codegen_llvm/src/va_arg.rs
View file

@ -7,7 +7,7 @@ use rustc_codegen_ssa::traits::{
};
use rustc_middle::ty::Ty;
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
use rustc_target::spec::{Abi, Arch};
use rustc_target::spec::{Abi, Arch, Env};
use crate::builder::Builder;
use crate::llvm::{Type, Value};
@ -549,7 +549,7 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
registers_for_primitive(scalar1.primitive());
registers_for_primitive(scalar2.primitive());
}
BackendRepr::SimdVector { .. } => {
BackendRepr::SimdVector { .. } | BackendRepr::ScalableVector { .. } => {
// Because no instance of VaArgSafe uses a non-scalar `BackendRepr`.
unreachable!(
"No x86-64 SysV va_arg implementation for {:?}",
@ -689,7 +689,9 @@ fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
}
}
// The Previous match on `BackendRepr` means control flow already escaped.
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => unreachable!(),
BackendRepr::SimdVector { .. }
| BackendRepr::ScalableVector { .. }
| BackendRepr::Memory { .. } => unreachable!(),
};
// AMD64-ABI 3.5.7p5: Step 5. Set:
@ -780,6 +782,129 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
mem_addr
}
fn emit_hexagon_va_arg_musl<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
list: OperandRef<'tcx, &'ll Value>,
target_ty: Ty<'tcx>,
) -> &'ll Value {
// Implementation of va_arg for Hexagon musl target.
// Based on LLVM's HexagonBuiltinVaList implementation.
//
// struct __va_list_tag {
// void *__current_saved_reg_area_pointer;
// void *__saved_reg_area_end_pointer;
// void *__overflow_area_pointer;
// };
//
// All variadic arguments are passed on the stack, but the musl implementation
// uses a register save area for compatibility.
let va_list_addr = list.immediate();
let layout = bx.cx.layout_of(target_ty);
let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi;
let ptr_size = bx.tcx().data_layout.pointer_size().bytes();
// Check if argument fits in register save area
let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg");
let from_overflow = bx.append_sibling_block("va_arg.from_overflow");
let end = bx.append_sibling_block("va_arg.end");
// Load the three pointers from va_list
let current_ptr_addr = va_list_addr;
let end_ptr_addr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(ptr_size));
let overflow_ptr_addr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(2 * ptr_size));
let current_ptr = bx.load(bx.type_ptr(), current_ptr_addr, ptr_align_abi);
let end_ptr = bx.load(bx.type_ptr(), end_ptr_addr, ptr_align_abi);
let overflow_ptr = bx.load(bx.type_ptr(), overflow_ptr_addr, ptr_align_abi);
// Align current pointer based on argument type size (following LLVM's implementation)
// Arguments <= 32 bits (4 bytes) use 4-byte alignment, > 32 bits use 8-byte alignment
let type_size_bits = bx.cx.size_of(target_ty).bits();
let arg_align = if type_size_bits > 32 {
Align::from_bytes(8).unwrap()
} else {
Align::from_bytes(4).unwrap()
};
let aligned_current = round_pointer_up_to_alignment(bx, current_ptr, arg_align, bx.type_ptr());
// Calculate next pointer position (following LLVM's logic)
// Arguments <= 32 bits take 4 bytes, > 32 bits take 8 bytes
let arg_size = if type_size_bits > 32 { 8 } else { 4 };
let next_ptr = bx.inbounds_ptradd(aligned_current, bx.const_usize(arg_size));
// Check if argument fits in register save area
let fits_in_regs = bx.icmp(IntPredicate::IntULE, next_ptr, end_ptr);
bx.cond_br(fits_in_regs, maybe_reg, from_overflow);
// Load from register save area
bx.switch_to_block(maybe_reg);
let reg_value_addr = aligned_current;
// Update current pointer
bx.store(next_ptr, current_ptr_addr, ptr_align_abi);
bx.br(end);
// Load from overflow area (stack)
bx.switch_to_block(from_overflow);
// Align overflow pointer using the same alignment rules
let aligned_overflow =
round_pointer_up_to_alignment(bx, overflow_ptr, arg_align, bx.type_ptr());
let overflow_value_addr = aligned_overflow;
// Update overflow pointer - use the same size calculation
let next_overflow = bx.inbounds_ptradd(aligned_overflow, bx.const_usize(arg_size));
bx.store(next_overflow, overflow_ptr_addr, ptr_align_abi);
// IMPORTANT: Also update the current saved register area pointer to match
// This synchronizes the pointers when switching to overflow area
bx.store(next_overflow, current_ptr_addr, ptr_align_abi);
bx.br(end);
// Return the value
bx.switch_to_block(end);
let value_addr =
bx.phi(bx.type_ptr(), &[reg_value_addr, overflow_value_addr], &[maybe_reg, from_overflow]);
bx.load(layout.llvm_type(bx), value_addr, layout.align.abi)
}
fn emit_hexagon_va_arg_bare_metal<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
list: OperandRef<'tcx, &'ll Value>,
target_ty: Ty<'tcx>,
) -> &'ll Value {
// Implementation of va_arg for Hexagon bare-metal (non-musl) targets.
// Based on LLVM's EmitVAArgForHexagon implementation.
//
// va_list is a simple pointer (char *)
let va_list_addr = list.immediate();
let layout = bx.cx.layout_of(target_ty);
let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi;
// Load current pointer from va_list
let current_ptr = bx.load(bx.type_ptr(), va_list_addr, ptr_align_abi);
// Handle address alignment for types with alignment > 4 bytes
let ty_align = layout.align.abi;
let aligned_ptr = if ty_align.bytes() > 4 {
// Ensure alignment is a power of 2
debug_assert!(ty_align.bytes().is_power_of_two(), "Alignment is not power of 2!");
round_pointer_up_to_alignment(bx, current_ptr, ty_align, bx.type_ptr())
} else {
current_ptr
};
// Calculate offset: round up type size to 4-byte boundary (minimum stack slot size)
let type_size = layout.size.bytes();
let offset = type_size.next_multiple_of(4); // align to 4 bytes
// Update va_list to point to next argument
let next_ptr = bx.inbounds_ptradd(aligned_ptr, bx.const_usize(offset));
bx.store(next_ptr, va_list_addr, ptr_align_abi);
// Load and return the argument value
bx.load(layout.llvm_type(bx), aligned_ptr, layout.align.abi)
}
fn emit_xtensa_va_arg<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
list: OperandRef<'tcx, &'ll Value>,
@ -964,6 +1089,13 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
// This includes `target.is_like_darwin`, which on x86_64 targets is like sysv64.
Arch::X86_64 => emit_x86_64_sysv64_va_arg(bx, addr, target_ty),
Arch::Xtensa => emit_xtensa_va_arg(bx, addr, target_ty),
Arch::Hexagon => {
if target.env == Env::Musl {
emit_hexagon_va_arg_musl(bx, addr, target_ty)
} else {
emit_hexagon_va_arg_bare_metal(bx, addr, target_ty)
}
}
// For all other architecture/OS combinations fall back to using
// the LLVM va_arg instruction.
// https://llvm.org/docs/LangRef.html#va-arg-instruction

1
compiler/rustc_codegen_ssa/messages.ftl
View file

@ -129,6 +129,7 @@ codegen_ssa_invalid_monomorphization_mask_wrong_element_type = invalid monomorph
codegen_ssa_invalid_monomorphization_mismatched_lengths = invalid monomorphization of `{$name}` intrinsic: mismatched lengths: mask length `{$m_len}` != other vector length `{$v_len}`
codegen_ssa_invalid_monomorphization_non_scalable_type = invalid monomorphization of `{$name}` intrinsic: expected non-scalable type, found scalable type `{$ty}`
codegen_ssa_invalid_monomorphization_return_element = invalid monomorphization of `{$name}` intrinsic: expected return element type `{$in_elem}` (element of input `{$in_ty}`), found `{$ret_ty}` with element type `{$out_ty}`
codegen_ssa_invalid_monomorphization_return_integer_type = invalid monomorphization of `{$name}` intrinsic: expected return type with integer elements, found `{$ret_ty}` with non-integer `{$out_ty}`

4
compiler/rustc_codegen_ssa/src/back/link.rs
View file

@ -2792,11 +2792,9 @@ fn add_upstream_rust_crates(
// We must always link crates `compiler_builtins` and `profiler_builtins` statically.
// Even if they were already included into a dylib
// (e.g. `libstd` when `-C prefer-dynamic` is used).
// FIXME: `dependency_formats` can report `profiler_builtins` as `NotLinked` for some
// reason, it shouldn't do that because `profiler_builtins` should indeed be linked.
let linkage = data[cnum];
let link_static_crate = linkage == Linkage::Static
|| (linkage == Linkage::IncludedFromDylib || linkage == Linkage::NotLinked)
|| linkage == Linkage::IncludedFromDylib
&& (codegen_results.crate_info.compiler_builtins == Some(cnum)
|| codegen_results.crate_info.profiler_runtime == Some(cnum));

4
compiler/rustc_codegen_ssa/src/codegen_attrs.rs
View file

@ -350,6 +350,9 @@ fn process_builtin_attrs(
codegen_fn_attrs.flags |= CodegenFnAttrFlags::EXTERNALLY_IMPLEMENTABLE_ITEM;
}
}
AttributeKind::ThreadLocal => {
codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL
}
_ => {}
}
}
@ -366,7 +369,6 @@ fn process_builtin_attrs(
sym::rustc_allocator_zeroed => {
codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED
}
sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL,
sym::instruction_set => {
codegen_fn_attrs.instruction_set = parse_instruction_set_attr(tcx, attr)
}

8
compiler/rustc_codegen_ssa/src/errors.rs
View file

@ -1094,6 +1094,14 @@ pub enum InvalidMonomorphization<'tcx> {
expected_element: Ty<'tcx>,
vector_type: Ty<'tcx>,
},
#[diag(codegen_ssa_invalid_monomorphization_non_scalable_type, code = E0511)]
NonScalableType {
#[primary_span]
span: Span,
name: Symbol,
ty: Ty<'tcx>,
},
}
pub enum ExpectedPointerMutability {

45
compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
View file

@ -2,7 +2,7 @@ use std::collections::hash_map::Entry;
use std::marker::PhantomData;
use std::ops::Range;
use rustc_abi::{BackendRepr, FieldIdx, FieldsShape, Size, VariantIdx};
use rustc_abi::{BackendRepr, FieldIdx, FieldsShape, ScalableElt, Size, VariantIdx};
use rustc_data_structures::fx::FxHashMap;
use rustc_index::IndexVec;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
@ -408,6 +408,49 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
return;
}
// Don't spill `<vscale x N x i1>` for `N != 16`:
//
// SVE predicates are only one bit for each byte in an SVE vector (which makes
// sense, the predicate only needs to keep track of whether a lane is
// enabled/disabled). i.e. a `<vscale x 16 x i8>` vector has a `<vscale x 16 x i1>`
// predicate type. `<vscale x 16 x i1>` corresponds to two bytes of storage,
// multiplied by the `vscale`, with one bit for each of the sixteen lanes.
//
// For a vector with fewer elements, such as `svint32_t`/`<vscale x 4 x i32>`,
// while only a `<vscale x 4 x i1>` predicate type would be strictly necessary,
// relevant intrinsics still take a `svbool_t`/`<vscale x 16 x i1>` - this is
// because a `<vscale x 4 x i1>` is only half of a byte (for `vscale=1`), and with
// memory being byte-addressable, it's unclear how to store that.
//
// Due to this, LLVM ultimately decided not to support stores of `<vscale x N x i1>`
// for `N != 16`. As for `vscale=1` and `N` fewer than sixteen, partial bytes would
// need to be stored (except for `N=8`, but that also isn't supported). `N` can
// never be greater than sixteen as that ends up larger than the 128-bit increment
// size.
//
// Internally, with an intrinsic operating on a `svint32_t`/`<vscale x 4 x i32>`
// (for example), the intrinsic takes the `svbool_t`/`<vscale x 16 x i1>` predicate
// and casts it to a `svbool4_t`/`<vscale x 4 x i1>`. Therefore, it's important that
// the `<vscale x 4 x i32>` never spills because that'll cause errors during
// instruction selection. Spilling to the stack to create debuginfo for these
// intermediate values must be avoided and won't degrade the debugging experience
// anyway.
if operand.layout.ty.is_scalable_vector()
&& bx.sess().target.arch == rustc_target::spec::Arch::AArch64
&& let ty::Adt(adt, args) = &operand.layout.ty.kind()
&& let Some(marker_type_field) =
adt.non_enum_variant().fields.get(FieldIdx::from_u32(0))
{
let marker_type = marker_type_field.ty(bx.tcx(), args);
// i.e. `<vscale x N x i1>` when `N != 16`
if let ty::Slice(element_ty) = marker_type.kind()
&& element_ty.is_bool()
&& adt.repr().scalable != Some(ScalableElt::ElementCount(16))
{
return;
}
}
Self::spill_operand_to_stack(*operand, name, bx)
}

8
compiler/rustc_codegen_ssa/src/mir/operand.rs
View file

@ -405,7 +405,9 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
imm
}
}
BackendRepr::ScalarPair(_, _) | BackendRepr::Memory { .. } => bug!(),
BackendRepr::ScalarPair(_, _)
| BackendRepr::Memory { .. }
| BackendRepr::ScalableVector { .. } => bug!(),
})
};
@ -692,7 +694,9 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
BackendRepr::ScalarPair(a, b) => {
OperandValueBuilder::Pair(Either::Right(a), Either::Right(b))
}
BackendRepr::SimdVector { .. } => OperandValueBuilder::Vector(Either::Right(())),
BackendRepr::SimdVector { .. } | BackendRepr::ScalableVector { .. } => {
OperandValueBuilder::Vector(Either::Right(()))
}
BackendRepr::Memory { .. } => {
bug!("Cannot use non-ZST Memory-ABI type in operand builder: {layout:?}");
}

18
compiler/rustc_codegen_ssa/src/mir/place.rs
View file

@ -109,7 +109,11 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
bx: &mut Bx,
layout: TyAndLayout<'tcx>,
) -> Self {
Self::alloca_size(bx, layout.size, layout)
if layout.is_runtime_sized() {
Self::alloca_runtime_sized(bx, layout)
} else {
Self::alloca_size(bx, layout.size, layout)
}
}
pub fn alloca_size<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
@ -146,6 +150,18 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
bug!("unexpected layout `{:#?}` in PlaceRef::len", self.layout)
}
}
fn alloca_runtime_sized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
bx: &mut Bx,
layout: TyAndLayout<'tcx>,
) -> Self {
let (element_count, ty) = layout.ty.scalable_vector_element_count_and_type(bx.tcx());
PlaceValue::new_sized(
bx.scalable_alloca(element_count as u64, layout.align.abi, ty),
layout.align.abi,
)
.with_type(layout)
}
}
impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {

1
compiler/rustc_codegen_ssa/src/traits/builder.rs
View file

@ -235,6 +235,7 @@ pub trait BuilderMethods<'a, 'tcx>:
fn to_immediate_scalar(&mut self, val: Self::Value, scalar: Scalar) -> Self::Value;
fn alloca(&mut self, size: Size, align: Align) -> Self::Value;
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value;
fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value;
fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value;

50
compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file

@ -6,14 +6,17 @@ mod simd;
use std::assert_matches::assert_matches;
use rustc_abi::{FieldIdx, HasDataLayout, Size, VariantIdx};
use rustc_abi::{FIRST_VARIANT, FieldIdx, HasDataLayout, Size, VariantIdx};
use rustc_apfloat::ieee::{Double, Half, Quad, Single};
use rustc_hir::def_id::CRATE_DEF_ID;
use rustc_infer::infer::TyCtxtInferExt;
use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
use rustc_middle::ty::layout::TyAndLayout;
use rustc_middle::ty::{FloatTy, Ty, TyCtxt};
use rustc_middle::ty::{FloatTy, PolyExistentialPredicate, Ty, TyCtxt, TypeFoldable};
use rustc_middle::{bug, span_bug, ty};
use rustc_span::{Symbol, sym};
use rustc_trait_selection::traits::{Obligation, ObligationCause, ObligationCtxt};
use tracing::trace;
use super::memory::MemoryKind;
@ -219,6 +222,49 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.write_scalar(Scalar::from_target_usize(offset, self), dest)?;
}
sym::vtable_for => {
let tp_ty = instance.args.type_at(0);
let result_ty = instance.args.type_at(1);
ensure_monomorphic_enough(tcx, tp_ty)?;
ensure_monomorphic_enough(tcx, result_ty)?;
let ty::Dynamic(preds, _) = result_ty.kind() else {
span_bug!(
self.find_closest_untracked_caller_location(),
"Invalid type provided to vtable_for::<T, U>. U must be dyn Trait, got {result_ty}."
);
};
let (infcx, param_env) =
self.tcx.infer_ctxt().build_with_typing_env(self.typing_env);
let ocx = ObligationCtxt::new(&infcx);
ocx.register_obligations(preds.iter().map(|pred: PolyExistentialPredicate<'_>| {
let pred = pred.with_self_ty(tcx, tp_ty);
// Lifetimes can only be 'static because of the bound on T
let pred = pred.fold_with(&mut ty::BottomUpFolder {
tcx,
ty_op: |ty| ty,
lt_op: |lt| {
if lt == tcx.lifetimes.re_erased { tcx.lifetimes.re_static } else { lt }
},
ct_op: |ct| ct,
});
Obligation::new(tcx, ObligationCause::dummy(), param_env, pred)
}));
let type_impls_trait = ocx.evaluate_obligations_error_on_ambiguity().is_empty();
// Since `assumed_wf_tys=[]` the choice of LocalDefId is irrelevant, so using the "default"
let regions_are_valid = ocx.resolve_regions(CRATE_DEF_ID, param_env, []).is_empty();
if regions_are_valid && type_impls_trait {
let vtable_ptr = self.get_vtable_ptr(tp_ty, preds)?;
// Writing a non-null pointer into an `Option<NonNull>` will automatically make it `Some`.
self.write_pointer(vtable_ptr, dest)?;
} else {
// Write `None`
self.write_discriminant(FIRST_VARIANT, dest)?;
}
}
sym::variant_count => {
let tp_ty = instance.args.type_at(0);
let ty = match tp_ty.kind() {

7
compiler/rustc_const_eval/src/interpret/step.rs
View file

@ -561,8 +561,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
)?;
// Sanity-check that `eval_fn_call` either pushed a new frame or
// did a jump to another block.
if self.frame_idx() == old_stack && self.frame().loc == old_loc {
// did a jump to another block. We disable the sanity check for functions that
// can't return, since Miri sometimes does have to keep the location the same
// for those (which is fine since execution will continue on a different thread).
if target.is_some() && self.frame_idx() == old_stack && self.frame().loc == old_loc
{
span_bug!(terminator.source_info.span, "evaluating this call made no progress");
}
}

Some files were not shown because too many files have changed in this diff Show more