user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge ref '4fa824bb78' from rust-lang/rust

Browse source 
Pull recent changes from https://github.com/rust-lang/rust via Josh.

Upstream ref: 4fa824bb78
Filtered ref: b8d3c3cc8b2048bd34d7611095d36d82259331af
Upstream diff: 9f32ccf35f...4fa824bb78

This merge was created using https://github.com/rust-lang/josh-sync.
This commit is contained in:
The rustc-josh-sync Cronjob Bot 2025-10-06 04:19:25 +00:00
commit cbee1ab175
1658 changed files with 29284 additions and 13550 deletions
Download patch file Download diff file Expand all files Collapse all files

5
CONTRIBUTING.md
View file

@ -31,8 +31,8 @@ bootstrapping, the compiler architecture, source code representation, and more.
## [Getting help](https://rustc-dev-guide.rust-lang.org/getting-started.html#asking-questions)
There are many ways you can get help when you're stuck. Rust has many platforms for this:
[internals], [rust-zulip], and [rust-discord]. It is recommended to ask for help on
There are many ways you can get help when you're stuck. Rust has two platforms for this:
[internals] and [rust-zulip]. It is recommended to ask for help on
the [rust-zulip], but any of these platforms are great ways to seek help and even
find a mentor! You can learn more about asking questions and getting help in the
[Asking Questions](https://rustc-dev-guide.rust-lang.org/getting-started.html#asking-questions) chapter of the [rustc-dev-guide].
@ -47,5 +47,4 @@ refer to [this section][contributing-bug-reports] and [open an issue][issue temp
[contributing-bug-reports]: https://rustc-dev-guide.rust-lang.org/contributing.html#bug-reports
[issue template]: https://github.com/rust-lang/rust/issues/new/choose
[internals]: https://internals.rust-lang.org
[rust-discord]: http://discord.gg/rust-lang
[rust-zulip]: https://rust-lang.zulipchat.com

20
Cargo.lock
View file

@ -128,9 +128,9 @@ dependencies = [
[[package]]
name = "anstyle-svg"
version = "0.1.10"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc03a770ef506fe1396c0e476120ac0e6523cf14b74218dd5f18cd6833326fa9"
checksum = "26b9ec8c976eada1b0f9747a3d7cc4eae3bef10613e443746e7487f26c872fde"
dependencies = [
"anstyle",
"anstyle-lossy",
@ -334,8 +334,10 @@ dependencies = [
"anyhow",
"build_helper",
"curl",
"hex",
"indexmap",
"serde",
"sha2",
"toml 0.8.23",
]
@ -1332,6 +1334,12 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959"
[[package]]
name = "flate2"
version = "1.1.2"
@ -3556,7 +3564,7 @@ dependencies = [
"ar_archive_writer",
"bitflags",
"bstr",
"cc",
"find-msvc-tools",
"itertools",
"libc",
"object 0.37.3",
@ -4760,7 +4768,7 @@ dependencies = [
name = "rustc_windows_rc"
version = "0.0.0"
dependencies = [
"cc",
"find-msvc-tools",
]
[[package]]
@ -5233,9 +5241,9 @@ dependencies = [
[[package]]
name = "stringdex"
version = "0.0.1-alpha9"
version = "0.0.1-alpha10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7081029913fd7d591c0112182aba8c98ae886b4f12edb208130496cd17dc3c15"
checksum = "0fa846a7d509d1828a4f90962dc09810e161abcada7fc6a921e92c168d0811d7"
dependencies = [
"stacker",
]

10
bootstrap.example.toml
View file

@ -476,9 +476,6 @@
# when the stage 0 compiler is actually built from in-tree sources.
#build.compiletest-allow-stage0 = false
# Whether to use the precompiled stage0 libtest with compiletest.
#build.compiletest-use-stage0-libtest = true
# Default value for the `--extra-checks` flag of tidy.
#
# See `./x test tidy --help` for details.
@ -768,8 +765,7 @@
# make this default to false.
#rust.lld = false in all cases, except on `x86_64-unknown-linux-gnu` as described above, where it is true
# Indicates whether LLD will be used to link Rust crates during bootstrap on
# supported platforms.
# Indicates if we should override the linker used to link Rust crates during bootstrap to be LLD.
# If set to `true` or `"external"`, a global `lld` binary that has to be in $PATH
# will be used.
# If set to `"self-contained"`, rust-lld from the snapshot compiler will be used.
@ -777,7 +773,7 @@
# On MSVC, LLD will not be used if we're cross linking.
#
# Explicitly setting the linker for a target will override this option when targeting MSVC.
#rust.use-lld = false
#rust.bootstrap-override-lld = false
# Indicates whether some LLVM tools, like llvm-objdump, will be made available in the
# sysroot.
@ -950,7 +946,7 @@
# Linker to be used to bootstrap Rust code. Note that the
# default value is platform specific, and if not specified it may also depend on
# what platform is crossing to what platform.
# Setting this will override the `use-lld` option for Rust code when targeting MSVC.
# Setting this will override the `bootstrap-override-lld` option for Rust code when targeting MSVC.
#linker = "cc" (path)
# Should rustc and the standard library be built with split debuginfo? Default

22
compiler/rustc_abi/src/callconv/reg.rs
View file

@ -42,22 +42,22 @@ impl Reg {
let dl = cx.data_layout();
match self.kind {
RegKind::Integer => match self.size.bits() {
1 => dl.i1_align.abi,
2..=8 => dl.i8_align.abi,
9..=16 => dl.i16_align.abi,
17..=32 => dl.i32_align.abi,
33..=64 => dl.i64_align.abi,
65..=128 => dl.i128_align.abi,
1 => dl.i1_align,
2..=8 => dl.i8_align,
9..=16 => dl.i16_align,
17..=32 => dl.i32_align,
33..=64 => dl.i64_align,
65..=128 => dl.i128_align,
_ => panic!("unsupported integer: {self:?}"),
},
RegKind::Float => match self.size.bits() {
16 => dl.f16_align.abi,
32 => dl.f32_align.abi,
64 => dl.f64_align.abi,
128 => dl.f128_align.abi,
16 => dl.f16_align,
32 => dl.f32_align,
64 => dl.f64_align,
128 => dl.f128_align,
_ => panic!("unsupported float: {self:?}"),
},
RegKind::Vector => dl.llvmlike_vector_align(self.size).abi,
RegKind::Vector => dl.llvmlike_vector_align(self.size),
}
}
}

71
compiler/rustc_abi/src/layout.rs
View file

@ -174,11 +174,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
// Non-power-of-two vectors have padding up to the next power-of-two.
// If we're a packed repr, remove the padding while keeping the alignment as close
// to a vector as possible.
(BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) })
(BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
} else {
(BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
};
let size = size.align_to(align.abi);
let size = size.align_to(align);
Ok(LayoutData {
variants: Variants::Single { index: VariantIdx::new(0) },
@ -190,7 +190,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
largest_niche: elt.largest_niche,
uninhabited: false,
size,
align,
align: AbiAlign::new(align),
max_repr_align: None,
unadjusted_abi_align: elt.align.abi,
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
@ -388,7 +388,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
return Err(LayoutCalculatorError::UnexpectedUnsized(*field));
}
align = align.max(field.align);
align = align.max(field.align.abi);
max_repr_align = max_repr_align.max(field.max_repr_align);
size = cmp::max(size, field.size);
@ -423,13 +423,13 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
}
if let Some(pack) = repr.pack {
align = align.min(AbiAlign::new(pack));
align = align.min(pack);
}
// The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
// See documentation on `LayoutData::unadjusted_abi_align`.
let unadjusted_abi_align = align.abi;
let unadjusted_abi_align = align;
if let Some(repr_align) = repr.align {
align = align.max(AbiAlign::new(repr_align));
align = align.max(repr_align);
}
// `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.
let align = align;
@ -441,14 +441,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
Ok(Some((repr, _))) => match repr {
// Mismatched alignment (e.g. union is #[repr(packed)]): disable opt
BackendRepr::Scalar(_) | BackendRepr::ScalarPair(_, _)
if repr.scalar_align(dl).unwrap() != align.abi =>
if repr.scalar_align(dl).unwrap() != align =>
{
BackendRepr::Memory { sized: true }
}
// Vectors require at least element alignment, else disable the opt
BackendRepr::SimdVector { element, count: _ }
if element.align(dl).abi > align.abi =>
{
BackendRepr::SimdVector { element, count: _ } if element.align(dl).abi > align => {
BackendRepr::Memory { sized: true }
}
// the alignment tests passed and we can use this
@ -474,8 +472,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
backend_repr,
largest_niche: None,
uninhabited: false,
align,
size: size.align_to(align.abi),
align: AbiAlign::new(align),
size: size.align_to(align),
max_repr_align,
unadjusted_abi_align,
randomization_seed: combined_seed,
@ -611,7 +609,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let mut align = dl.aggregate_align;
let mut max_repr_align = repr.align;
let mut unadjusted_abi_align = align.abi;
let mut unadjusted_abi_align = align;
let mut variant_layouts = variants
.iter_enumerated()
@ -619,7 +617,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let mut st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?;
st.variants = Variants::Single { index: j };
align = align.max(st.align);
align = align.max(st.align.abi);
max_repr_align = max_repr_align.max(st.max_repr_align);
unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);
@ -646,7 +644,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let (niche_start, niche_scalar) = niche.reserve(dl, count)?;
let niche_offset = niche.offset;
let niche_size = niche.value.size(dl);
let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
let size = variant_layouts[largest_variant_index].size.align_to(align);
let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
if i == largest_variant_index {
@ -699,7 +697,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
.iter_enumerated()
.all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO);
let same_size = size == variant_layouts[largest_variant_index].size;
let same_align = align == variant_layouts[largest_variant_index].align;
let same_align = align == variant_layouts[largest_variant_index].align.abi;
let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited());
let abi = if same_size && same_align && others_zst {
@ -746,7 +744,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
largest_niche,
uninhabited,
size,
align,
align: AbiAlign::new(align),
max_repr_align,
unadjusted_abi_align,
randomization_seed: combined_seed,
@ -818,7 +816,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
let mut align = dl.aggregate_align;
let mut max_repr_align = repr.align;
let mut unadjusted_abi_align = align.abi;
let mut unadjusted_abi_align = align;
let mut size = Size::ZERO;
@ -860,7 +858,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
}
}
size = cmp::max(size, st.size);
align = align.max(st.align);
align = align.max(st.align.abi);
max_repr_align = max_repr_align.max(st.max_repr_align);
unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);
Ok(st)
@ -868,7 +866,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
.collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
// Align the maximum variant size to the largest alignment.
size = size.align_to(align.abi);
size = size.align_to(align);
// FIXME(oli-obk): deduplicate and harden these checks
if size.bytes() >= dl.obj_size_bound() {
@ -1042,7 +1040,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
};
if pair_offsets[FieldIdx::new(0)] == Size::ZERO
&& pair_offsets[FieldIdx::new(1)] == *offset
&& align == pair.align
&& align == pair.align.abi
&& size == pair.size
{
// We can use `ScalarPair` only when it matches our
@ -1066,7 +1064,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
// Also need to bump up the size and alignment, so that the entire value fits
// in here.
variant.size = cmp::max(variant.size, size);
variant.align.abi = cmp::max(variant.align.abi, align.abi);
variant.align.abi = cmp::max(variant.align.abi, align);
}
}
}
@ -1092,7 +1090,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
largest_niche,
uninhabited,
backend_repr: abi,
align,
align: AbiAlign::new(align),
size,
max_repr_align,
unadjusted_abi_align,
@ -1169,7 +1167,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
// To allow unsizing `&Foo<Type>` -> `&Foo<dyn Trait>`, the layout of the struct must
// not depend on the layout of the tail.
let max_field_align =
fields_excluding_tail.iter().map(|f| f.align.abi.bytes()).max().unwrap_or(1);
fields_excluding_tail.iter().map(|f| f.align.bytes()).max().unwrap_or(1);
let largest_niche_size = fields_excluding_tail
.iter()
.filter_map(|f| f.largest_niche)
@ -1189,7 +1187,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
} else {
// Returns `log2(effective-align)`. The calculation assumes that size is an
// integer multiple of align, except for ZSTs.
let align = layout.align.abi.bytes();
let align = layout.align.bytes();
let size = layout.size.bytes();
let niche_size = layout.largest_niche.map(|n| n.available(dl)).unwrap_or(0);
// Group [u8; 4] with align-4 or [u8; 6] with align-2 fields.
@ -1288,7 +1286,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
let prefix_align =
if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
align = align.max(AbiAlign::new(prefix_align));
align = align.max(prefix_align);
offset = prefix_size.align_to(prefix_align);
}
for &i in &inverse_memory_index {
@ -1312,7 +1310,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
field.align
};
offset = offset.align_to(field_align.abi);
align = align.max(field_align);
align = align.max(field_align.abi);
max_repr_align = max_repr_align.max(field.max_repr_align);
debug!("univariant offset: {:?} field: {:#?}", offset, field);
@ -1339,9 +1337,9 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
// The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
// See documentation on `LayoutData::unadjusted_abi_align`.
let unadjusted_abi_align = align.abi;
let unadjusted_abi_align = align;
if let Some(repr_align) = repr.align {
align = align.max(AbiAlign::new(repr_align));
align = align.max(repr_align);
}
// `align` must not be modified after this point, or `unadjusted_abi_align` could be inaccurate.
let align = align;
@ -1360,7 +1358,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
inverse_memory_index.into_iter().map(|it| it.index() as u32).collect()
};
let size = min_size.align_to(align.abi);
let size = min_size.align_to(align);
// FIXME(oli-obk): deduplicate and harden these checks
if size.bytes() >= dl.obj_size_bound() {
return Err(LayoutCalculatorError::SizeOverflow);
@ -1383,8 +1381,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
layout_of_single_non_zst_field = Some(field);
// Field fills the struct and it has a scalar or scalar pair ABI.
if offsets[i].bytes() == 0 && align.abi == field.align.abi && size == field.size
{
if offsets[i].bytes() == 0 && align == field.align.abi && size == field.size {
match field.backend_repr {
// For plain scalars, or vectors of them, we can't unpack
// newtypes for `#[repr(C)]`, as that affects C ABIs.
@ -1428,7 +1425,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
};
if offsets[i] == pair_offsets[FieldIdx::new(0)]
&& offsets[j] == pair_offsets[FieldIdx::new(1)]
&& align == pair.align
&& align == pair.align.abi
&& size == pair.size
{
// We can use `ScalarPair` only when it matches our
@ -1450,7 +1447,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
Some(l) => l.unadjusted_abi_align,
None => {
// `repr(transparent)` with all ZST fields.
align.abi
align
}
}
} else {
@ -1465,7 +1462,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
backend_repr: abi,
largest_niche,
uninhabited,
align,
align: AbiAlign::new(align),
size,
max_repr_align,
unadjusted_abi_align,
@ -1488,7 +1485,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
for i in layout.fields.index_by_increasing_offset() {
let offset = layout.fields.offset(i);
let f = &fields[FieldIdx::new(i)];
write!(s, "[o{}a{}s{}", offset.bytes(), f.align.abi.bytes(), f.size.bytes()).unwrap();
write!(s, "[o{}a{}s{}", offset.bytes(), f.align.bytes(), f.size.bytes()).unwrap();
if let Some(n) = f.largest_niche {
write!(
s,

27
compiler/rustc_abi/src/layout/simple.rs
View file

@ -4,7 +4,8 @@ use rustc_hashes::Hash64;
use rustc_index::{Idx, IndexVec};
use crate::{
BackendRepr, FieldsShape, HasDataLayout, LayoutData, Niche, Primitive, Scalar, Size, Variants,
AbiAlign, BackendRepr, FieldsShape, HasDataLayout, LayoutData, Niche, Primitive, Scalar, Size,
Variants,
};
/// "Simple" layout constructors that cannot fail.
@ -20,10 +21,10 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
backend_repr: BackendRepr::Memory { sized },
largest_niche: None,
uninhabited: false,
align: dl.i8_align,
align: AbiAlign::new(dl.i8_align),
size: Size::ZERO,
max_repr_align: None,
unadjusted_abi_align: dl.i8_align.abi,
unadjusted_abi_align: dl.i8_align,
randomization_seed: Hash64::new(0),
}
}
@ -37,10 +38,10 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
backend_repr: BackendRepr::Memory { sized: true },
largest_niche: None,
uninhabited: true,
align: dl.i8_align,
align: AbiAlign::new(dl.i8_align),
size: Size::ZERO,
max_repr_align: None,
unadjusted_abi_align: dl.i8_align.abi,
unadjusted_abi_align: dl.i8_align,
randomization_seed: Hash64::ZERO,
}
}
@ -89,10 +90,10 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
pub fn scalar_pair<C: HasDataLayout>(cx: &C, a: Scalar, b: Scalar) -> Self {
let dl = cx.data_layout();
let b_align = b.align(dl);
let align = a.align(dl).max(b_align).max(dl.aggregate_align);
let b_offset = a.size(dl).align_to(b_align.abi);
let size = (b_offset + b.size(dl)).align_to(align.abi);
let b_align = b.align(dl).abi;
let align = a.align(dl).abi.max(b_align).max(dl.aggregate_align);
let b_offset = a.size(dl).align_to(b_align);
let size = (b_offset + b.size(dl)).align_to(align);
// HACK(nox): We iter on `b` and then `a` because `max_by_key`
// returns the last maximum.
@ -112,10 +113,10 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
backend_repr: BackendRepr::ScalarPair(a, b),
largest_niche,
uninhabited: false,
align,
align: AbiAlign::new(align),
size,
max_repr_align: None,
unadjusted_abi_align: align.abi,
unadjusted_abi_align: align,
randomization_seed: Hash64::new(combined_seed),
}
}
@ -138,10 +139,10 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
backend_repr: BackendRepr::Memory { sized: true },
largest_niche: None,
uninhabited: true,
align: dl.i8_align,
align: AbiAlign::new(dl.i8_align),
size: Size::ZERO,
max_repr_align: None,
unadjusted_abi_align: dl.i8_align.abi,
unadjusted_abi_align: dl.i8_align,
randomization_seed: Hash64::ZERO,
}
}

87
compiler/rustc_abi/src/lib.rs
View file

@ -229,7 +229,7 @@ pub struct PointerSpec {
/// The size of the bitwise representation of the pointer.
pointer_size: Size,
/// The alignment of pointers for this address space
pointer_align: AbiAlign,
pointer_align: Align,
/// The size of the value a pointer can be offset by in this address space.
pointer_offset: Size,
/// Pointers into this address space contain extra metadata
@ -242,20 +242,20 @@ pub struct PointerSpec {
#[derive(Debug, PartialEq, Eq)]
pub struct TargetDataLayout {
pub endian: Endian,
pub i1_align: AbiAlign,
pub i8_align: AbiAlign,
pub i16_align: AbiAlign,
pub i32_align: AbiAlign,
pub i64_align: AbiAlign,
pub i128_align: AbiAlign,
pub f16_align: AbiAlign,
pub f32_align: AbiAlign,
pub f64_align: AbiAlign,
pub f128_align: AbiAlign,
pub aggregate_align: AbiAlign,
pub i1_align: Align,
pub i8_align: Align,
pub i16_align: Align,
pub i32_align: Align,
pub i64_align: Align,
pub i128_align: Align,
pub f16_align: Align,
pub f32_align: Align,
pub f64_align: Align,
pub f128_align: Align,
pub aggregate_align: Align,
/// Alignments for vector types.
pub vector_align: Vec<(Size, AbiAlign)>,
pub vector_align: Vec<(Size, Align)>,
pub default_address_space: AddressSpace,
pub default_address_space_pointer_spec: PointerSpec,
@ -282,25 +282,25 @@ impl Default for TargetDataLayout {
let align = |bits| Align::from_bits(bits).unwrap();
TargetDataLayout {
endian: Endian::Big,
i1_align: AbiAlign::new(align(8)),
i8_align: AbiAlign::new(align(8)),
i16_align: AbiAlign::new(align(16)),
i32_align: AbiAlign::new(align(32)),
i64_align: AbiAlign::new(align(32)),
i128_align: AbiAlign::new(align(32)),
f16_align: AbiAlign::new(align(16)),
f32_align: AbiAlign::new(align(32)),
f64_align: AbiAlign::new(align(64)),
f128_align: AbiAlign::new(align(128)),
aggregate_align: AbiAlign { abi: align(8) },
i1_align: align(8),
i8_align: align(8),
i16_align: align(16),
i32_align: align(32),
i64_align: align(32),
i128_align: align(32),
f16_align: align(16),
f32_align: align(32),
f64_align: align(64),
f128_align: align(128),
aggregate_align: align(8),
vector_align: vec![
(Size::from_bits(64), AbiAlign::new(align(64))),
(Size::from_bits(128), AbiAlign::new(align(128))),
(Size::from_bits(64), align(64)),
(Size::from_bits(128), align(128)),
],
default_address_space: AddressSpace::ZERO,
default_address_space_pointer_spec: PointerSpec {
pointer_size: Size::from_bits(64),
pointer_align: AbiAlign::new(align(64)),
pointer_align: align(64),
pointer_offset: Size::from_bits(64),
_is_fat: false,
},
@ -360,7 +360,7 @@ impl TargetDataLayout {
.map_err(|err| TargetDataLayoutErrors::InvalidAlignment { cause, err })
};
let abi = parse_bits(s, "alignment", cause)?;
Ok(AbiAlign::new(align_from_bits(abi)?))
Ok(align_from_bits(abi)?)
};
// Parse an alignment sequence, possibly in the form `<align>[:<preferred_alignment>]`,
@ -596,7 +596,7 @@ impl TargetDataLayout {
/// psABI-mandated alignment for a vector type, if any
#[inline]
fn cabi_vector_align(&self, vec_size: Size) -> Option<AbiAlign> {
fn cabi_vector_align(&self, vec_size: Size) -> Option<Align> {
self.vector_align
.iter()
.find(|(size, _align)| *size == vec_size)
@ -605,10 +605,9 @@ impl TargetDataLayout {
/// an alignment resembling the one LLVM would pick for a vector
#[inline]
pub fn llvmlike_vector_align(&self, vec_size: Size) -> AbiAlign {
self.cabi_vector_align(vec_size).unwrap_or(AbiAlign::new(
Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap(),
))
pub fn llvmlike_vector_align(&self, vec_size: Size) -> Align {
self.cabi_vector_align(vec_size)
.unwrap_or(Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap())
}
/// Get the pointer size in the default data address space.
@ -654,21 +653,19 @@ impl TargetDataLayout {
/// Get the pointer alignment in the default data address space.
#[inline]
pub fn pointer_align(&self) -> AbiAlign {
self.default_address_space_pointer_spec.pointer_align
AbiAlign::new(self.default_address_space_pointer_spec.pointer_align)
}
/// Get the pointer alignment in a specific address space.
#[inline]
pub fn pointer_align_in(&self, c: AddressSpace) -> AbiAlign {
if c == self.default_address_space {
return self.default_address_space_pointer_spec.pointer_align;
}
if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) {
AbiAlign::new(if c == self.default_address_space {
self.default_address_space_pointer_spec.pointer_align
} else if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) {
e.1.pointer_align
} else {
panic!("Use of unknown address space {c:?}");
}
})
}
}
@ -1185,13 +1182,13 @@ impl Integer {
use Integer::*;
let dl = cx.data_layout();
match self {
AbiAlign::new(match self {
I8 => dl.i8_align,
I16 => dl.i16_align,
I32 => dl.i32_align,
I64 => dl.i64_align,
I128 => dl.i128_align,
}
})
}
/// Returns the largest signed value that can be represented by this Integer.
@ -1311,12 +1308,12 @@ impl Float {
use Float::*;
let dl = cx.data_layout();
match self {
AbiAlign::new(match self {
F16 => dl.f16_align,
F32 => dl.f32_align,
F64 => dl.f64_align,
F128 => dl.f128_align,
}
})
}
}
@ -2159,7 +2156,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
/// Returns `true` if the type is sized and a 1-ZST (meaning it has size 0 and alignment 1).
pub fn is_1zst(&self) -> bool {
self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
self.is_sized() && self.size.bytes() == 0 && self.align.bytes() == 1
}
/// Returns `true` if the type is a ZST and not unsized.

3
compiler/rustc_ast/src/ast.rs
View file

@ -114,8 +114,7 @@ impl PartialEq<Symbol> for Path {
impl PartialEq<&[Symbol]> for Path {
#[inline]
fn eq(&self, names: &&[Symbol]) -> bool {
self.segments.len() == names.len()
&& self.segments.iter().zip(names.iter()).all(|(s1, s2)| s1 == s2)
self.segments.iter().eq(*names)
}
}

17
compiler/rustc_ast/src/expand/autodiff_attrs.rs
View file

@ -6,6 +6,7 @@
use std::fmt::{self, Display, Formatter};
use std::str::FromStr;
use crate::expand::typetree::TypeTree;
use crate::expand::{Decodable, Encodable, HashStable_Generic};
use crate::{Ty, TyKind};
@ -84,6 +85,8 @@ pub struct AutoDiffItem {
/// The name of the function being generated
pub target: String,
pub attrs: AutoDiffAttrs,
pub inputs: Vec<TypeTree>,
pub output: TypeTree,
}
#[derive(Clone, Eq, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@ -275,14 +278,22 @@ impl AutoDiffAttrs {
!matches!(self.mode, DiffMode::Error | DiffMode::Source)
}
pub fn into_item(self, source: String, target: String) -> AutoDiffItem {
AutoDiffItem { source, target, attrs: self }
pub fn into_item(
self,
source: String,
target: String,
inputs: Vec<TypeTree>,
output: TypeTree,
) -> AutoDiffItem {
AutoDiffItem { source, target, inputs, output, attrs: self }
}
}
impl fmt::Display for AutoDiffItem {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Differentiating {} -> {}", self.source, self.target)?;
write!(f, " with attributes: {:?}", self.attrs)
write!(f, " with attributes: {:?}", self.attrs)?;
write!(f, " with inputs: {:?}", self.inputs)?;
write!(f, " with output: {:?}", self.output)
}
}

1
compiler/rustc_ast/src/expand/typetree.rs
View file

@ -31,6 +31,7 @@ pub enum Kind {
Half,
Float,
Double,
F128,
Unknown,
}

1
compiler/rustc_ast/src/lib.rs
View file

@ -15,6 +15,7 @@
#![feature(associated_type_defaults)]
#![feature(box_patterns)]
#![feature(if_let_guard)]
#![feature(iter_order_by)]
#![feature(macro_metavar_expr)]
#![feature(rustdoc_internals)]
#![recursion_limit = "256"]

4
compiler/rustc_ast/src/tokenstream.rs
View file

@ -48,9 +48,7 @@ impl TokenTree {
match (self, other) {
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
(TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
delim == delim2
&& tts.len() == tts2.len()
&& tts.iter().zip(tts2.iter()).all(|(a, b)| a.eq_unspanned(b))
delim == delim2 && tts.iter().eq_by(tts2.iter(), |a, b| a.eq_unspanned(b))
}
_ => false,
}

20
compiler/rustc_ast_lowering/src/format.rs
View file

@ -487,26 +487,6 @@ fn expand_format_args<'hir>(
// Generate:
// []
(vec![], ctx.arena.alloc(ctx.expr(macsp, hir::ExprKind::Array(&[]))))
} else if argmap.len() == 1 && arguments.len() == 1 {
// Only one argument, so we don't need to make the `args` tuple.
//
// Generate:
// super let args = [<core::fmt::Argument>::new_display(&arg)];
let args = ctx.arena.alloc_from_iter(argmap.iter().map(
|(&(arg_index, ty), &placeholder_span)| {
let arg = &arguments[arg_index];
let placeholder_span =
placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt());
let arg = ctx.lower_expr(&arg.expr);
let ref_arg = ctx.arena.alloc(ctx.expr_ref(arg.span.with_ctxt(macsp.ctxt()), arg));
make_argument(ctx, placeholder_span, ref_arg, ty)
},
));
let args = ctx.arena.alloc(ctx.expr(macsp, hir::ExprKind::Array(args)));
let args_ident = Ident::new(sym::args, macsp);
let (args_pat, args_hir_id) = ctx.pat_ident(macsp, args_ident);
let let_statement = ctx.stmt_super_let_pat(macsp, args_pat, Some(args));
(vec![let_statement], ctx.arena.alloc(ctx.expr_ident_mut(macsp, args_ident, args_hir_id)))
} else {
// Generate:
// super let args = (&arg0, &arg1, &…);

8
compiler/rustc_ast_passes/src/feature_gate.rs
View file

@ -183,7 +183,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
gate_doc!(
"experimental" {
cfg => doc_cfg
cfg_hide => doc_cfg_hide
auto_cfg => doc_cfg
masked => doc_masked
notable_trait => doc_notable_trait
}
@ -622,11 +622,7 @@ fn maybe_stage_features(sess: &Session, features: &Features, krate: &ast::Crate)
}
fn check_incompatible_features(sess: &Session, features: &Features) {
let enabled_lang_features =
features.enabled_lang_features().iter().map(|feat| (feat.gate_name, feat.attr_sp));
let enabled_lib_features =
features.enabled_lib_features().iter().map(|feat| (feat.gate_name, feat.attr_sp));
let enabled_features = enabled_lang_features.chain(enabled_lib_features);
let enabled_features = features.enabled_features_iter_stable_order();
for (f1, f2) in rustc_feature::INCOMPATIBLE_FEATURES
.iter()

165
compiler/rustc_attr_parsing/messages.ftl
View file

@ -1,3 +1,9 @@
attr_parsing_as_needed_compatibility =
linking modifier `as-needed` is only compatible with `dylib` and `framework` linking kinds
attr_parsing_bundle_needs_static =
linking modifier `bundle` is only compatible with `static` linking kind
attr_parsing_cfg_predicate_identifier =
`cfg` predicate key must be an identifier
@ -8,18 +14,22 @@ attr_parsing_deprecated_item_suggestion =
attr_parsing_empty_attribute =
unused attribute
.suggestion = remove this attribute
.suggestion = {$valid_without_list ->
[true] remove these parentheses
*[other] remove this attribute
}
.note = {$valid_without_list ->
[true] using `{$attr_path}` with an empty list is equivalent to not using a list at all
*[other] using `{$attr_path}` with an empty list has no effect
}
attr_parsing_invalid_target = `#[{$name}]` attribute cannot be used on {$target}
.help = `#[{$name}]` can {$only}be applied to {$applied}
.suggestion = remove the attribute
attr_parsing_invalid_target_lint = `#[{$name}]` attribute cannot be used on {$target}
.warn = {-attr_parsing_previously_accepted}
.help = `#[{$name}]` can {$only}be applied to {$applied}
.suggestion = remove the attribute
attr_parsing_empty_confusables =
expected at least one confusable name
attr_parsing_empty_link_name =
link name must not be empty
.label = empty link name
attr_parsing_expected_one_cfg_pattern =
expected 1 cfg-pattern
@ -40,6 +50,15 @@ attr_parsing_ill_formed_attribute_input = {$num_suggestions ->
*[other] valid forms for the attribute are {$suggestions}
}
attr_parsing_import_name_type_raw =
import name type can only be used with link kind `raw-dylib`
attr_parsing_import_name_type_x86 =
import name type is only supported on x86
attr_parsing_incompatible_wasm_link =
`wasm_import_module` is incompatible with other arguments in `#[link]` attributes
attr_parsing_incorrect_repr_format_align_one_arg =
incorrect `repr(align)` attribute format: `align` takes exactly one argument in parentheses
@ -59,6 +78,11 @@ attr_parsing_incorrect_repr_format_packed_one_or_zero_arg =
attr_parsing_invalid_alignment_value =
invalid alignment value: {$error_part}
attr_parsing_invalid_attr_unsafe = `{$name}` is not an unsafe attribute
.label = this is not an unsafe attribute
.suggestion = remove the `unsafe(...)`
.note = extraneous unsafe is not allowed in attributes
attr_parsing_invalid_issue_string =
`issue` must be a non-zero numeric string or "none"
.must_not_be_zero = `issue` must not be "0", use "none" instead
@ -67,6 +91,13 @@ attr_parsing_invalid_issue_string =
.pos_overflow = number too large to fit in target type
.neg_overflow = number too small to fit in target type
attr_parsing_invalid_link_modifier =
invalid linking modifier syntax, expected '+' or '-' prefix before one of: bundle, verbatim, whole-archive, as-needed
attr_parsing_invalid_meta_item = expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found {$descr}
.remove_neg_sugg = negative numbers are not literals, try removing the `-` sign
.quote_ident_sugg = surround the identifier with quotation marks to make it into a string literal
attr_parsing_invalid_predicate =
invalid predicate `{$predicate}`
@ -92,9 +123,36 @@ attr_parsing_invalid_style = {$is_used_as_inner ->
}
.note = This attribute does not have an `!`, which means it is applied to this {$target}
attr_parsing_invalid_target = `#[{$name}]` attribute cannot be used on {$target}
.help = `#[{$name}]` can {$only}be applied to {$applied}
.suggestion = remove the attribute
attr_parsing_invalid_target_lint = `#[{$name}]` attribute cannot be used on {$target}
.warn = {-attr_parsing_previously_accepted}
.help = `#[{$name}]` can {$only}be applied to {$applied}
.suggestion = remove the attribute
attr_parsing_limit_invalid =
`limit` must be a non-negative integer
.label = {$error_str}
attr_parsing_link_arg_unstable =
link kind `link-arg` is unstable
attr_parsing_link_cfg_unstable =
link cfg is unstable
attr_parsing_link_framework_apple =
link kind `framework` is only supported on Apple targets
attr_parsing_link_ordinal_out_of_range = ordinal value in `link_ordinal` is too large: `{$ordinal}`
.note = the value may not exceed `u16::MAX`
attr_parsing_link_requires_name =
`#[link]` attribute requires a `name = "string"` argument
.label = missing `name` argument
attr_parsing_meta_bad_delim = wrong meta list delimiters
attr_parsing_meta_bad_delim_suggestion = the delimiters should be `(` and `)`
attr_parsing_missing_feature =
missing 'feature'
@ -107,6 +165,9 @@ attr_parsing_missing_note =
attr_parsing_missing_since =
missing 'since'
attr_parsing_multiple_modifiers =
multiple `{$modifier}` modifiers in a single `modifiers` argument
attr_parsing_multiple_stability_levels =
multiple stability levels
@ -130,6 +191,15 @@ attr_parsing_objc_class_expected_string_literal = `objc::class!` expected a stri
attr_parsing_objc_selector_expected_string_literal = `objc::selector!` expected a string literal
attr_parsing_raw_dylib_elf_unstable =
link kind `raw-dylib` is unstable on ELF platforms
attr_parsing_raw_dylib_no_nul =
link name must not contain NUL characters if link kind is `raw-dylib`
attr_parsing_raw_dylib_only_windows =
link kind `raw-dylib` is only supported on Windows targets
attr_parsing_repr_ident =
meta item in `repr` must be an identifier
@ -144,6 +214,9 @@ attr_parsing_soft_no_args =
attr_parsing_stability_outside_std = stability attributes may not be used outside of the standard library
attr_parsing_suffixed_literal_in_attribute = suffixed literals are not allowed in attributes
.help = instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), use an unsuffixed version (`1`, `1.0`, etc.)
attr_parsing_unknown_meta_item =
unknown meta item '{$item}'
.label = expected one of {$expected}
@ -156,6 +229,10 @@ attr_parsing_unrecognized_repr_hint =
.help = valid reprs are `Rust` (default), `C`, `align`, `packed`, `transparent`, `simd`, `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`
.note = for more information, visit <https://doc.rust-lang.org/reference/type-layout.html?highlight=repr#representations>
attr_parsing_unsafe_attr_outside_unsafe = unsafe attribute used without unsafe
.label = usage of unsafe attribute
attr_parsing_unsafe_attr_outside_unsafe_suggestion = wrap the attribute in `unsafe(...)`
attr_parsing_unstable_cfg_target_compact =
compact `cfg(target(..))` is experimental and subject to change
@ -185,77 +262,5 @@ attr_parsing_unused_multiple =
-attr_parsing_previously_accepted =
this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
attr_parsing_meta_bad_delim = wrong meta list delimiters
attr_parsing_meta_bad_delim_suggestion = the delimiters should be `(` and `)`
attr_parsing_unsafe_attr_outside_unsafe = unsafe attribute used without unsafe
.label = usage of unsafe attribute
attr_parsing_unsafe_attr_outside_unsafe_suggestion = wrap the attribute in `unsafe(...)`
attr_parsing_invalid_attr_unsafe = `{$name}` is not an unsafe attribute
.label = this is not an unsafe attribute
.suggestion = remove the `unsafe(...)`
.note = extraneous unsafe is not allowed in attributes
attr_parsing_invalid_meta_item = expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found {$descr}
.remove_neg_sugg = negative numbers are not literals, try removing the `-` sign
.quote_ident_sugg = surround the identifier with quotation marks to make it into a string literal
attr_parsing_suffixed_literal_in_attribute = suffixed literals are not allowed in attributes
.help = instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), use an unsuffixed version (`1`, `1.0`, etc.)
attr_parsing_as_needed_compatibility =
linking modifier `as-needed` is only compatible with `dylib` and `framework` linking kinds
attr_parsing_bundle_needs_static =
linking modifier `bundle` is only compatible with `static` linking kind
attr_parsing_empty_link_name =
link name must not be empty
.label = empty link name
attr_parsing_import_name_type_raw =
import name type can only be used with link kind `raw-dylib`
attr_parsing_import_name_type_x86 =
import name type is only supported on x86
attr_parsing_incompatible_wasm_link =
`wasm_import_module` is incompatible with other arguments in `#[link]` attributes
attr_parsing_invalid_link_modifier =
invalid linking modifier syntax, expected '+' or '-' prefix before one of: bundle, verbatim, whole-archive, as-needed
attr_parsing_link_arg_unstable =
link kind `link-arg` is unstable
attr_parsing_link_cfg_unstable =
link cfg is unstable
attr_parsing_link_framework_apple =
link kind `framework` is only supported on Apple targets
attr_parsing_link_requires_name =
`#[link]` attribute requires a `name = "string"` argument
.label = missing `name` argument
attr_parsing_multiple_modifiers =
multiple `{$modifier}` modifiers in a single `modifiers` argument
attr_parsing_multiple_renamings =
multiple renamings were specified for library `{$lib_name}`
attr_parsing_raw_dylib_no_nul =
link name must not contain NUL characters if link kind is `raw-dylib`
attr_parsing_raw_dylib_elf_unstable =
link kind `raw-dylib` is unstable on ELF platforms
attr_parsing_raw_dylib_only_windows =
link kind `raw-dylib` is only supported on Windows targets
attr_parsing_whole_archive_needs_static =
linking modifier `whole-archive` is only compatible with `static` linking kind
attr_parsing_limit_invalid =
`limit` must be a non-negative integer
.label = {$error_str}

36
compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
View file

@ -370,6 +370,7 @@ impl<S: Stage> NoArgsAttributeParser<S> for NoMangleParser {
pub(crate) struct UsedParser {
first_compiler: Option<Span>,
first_linker: Option<Span>,
first_default: Option<Span>,
}
// A custom `AttributeParser` is used rather than a Simple attribute parser because
@ -382,7 +383,7 @@ impl<S: Stage> AttributeParser<S> for UsedParser {
template!(Word, List: &["compiler", "linker"]),
|group: &mut Self, cx, args| {
let used_by = match args {
ArgParser::NoArgs => UsedBy::Linker,
ArgParser::NoArgs => UsedBy::Default,
ArgParser::List(list) => {
let Some(l) = list.single() else {
cx.expected_single_argument(list.span);
@ -423,12 +424,29 @@ impl<S: Stage> AttributeParser<S> for UsedParser {
ArgParser::NameValue(_) => return,
};
let attr_span = cx.attr_span;
// `#[used]` is interpreted as `#[used(linker)]` (though depending on target OS the
// circumstances are more complicated). While we're checking `used_by`, also report
// these cross-`UsedBy` duplicates to warn.
let target = match used_by {
UsedBy::Compiler => &mut group.first_compiler,
UsedBy::Linker => &mut group.first_linker,
UsedBy::Linker => {
if let Some(prev) = group.first_default {
cx.warn_unused_duplicate(prev, attr_span);
return;
}
&mut group.first_linker
}
UsedBy::Default => {
if let Some(prev) = group.first_linker {
cx.warn_unused_duplicate(prev, attr_span);
return;
}
&mut group.first_default
}
};
let attr_span = cx.attr_span;
if let Some(prev) = *target {
cx.warn_unused_duplicate(prev, attr_span);
} else {
@ -440,11 +458,13 @@ impl<S: Stage> AttributeParser<S> for UsedParser {
AllowedTargets::AllowList(&[Allow(Target::Static), Warn(Target::MacroCall)]);
fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option<AttributeKind> {
// Ratcheting behaviour, if both `linker` and `compiler` are specified, use `linker`
Some(match (self.first_compiler, self.first_linker) {
(_, Some(span)) => AttributeKind::Used { used_by: UsedBy::Linker, span },
(Some(span), _) => AttributeKind::Used { used_by: UsedBy::Compiler, span },
(None, None) => return None,
// If a specific form of `used` is specified, it takes precedence over generic `#[used]`.
// If both `linker` and `compiler` are specified, use `linker`.
Some(match (self.first_compiler, self.first_linker, self.first_default) {
(_, Some(span), _) => AttributeKind::Used { used_by: UsedBy::Linker, span },
(Some(span), _, _) => AttributeKind::Used { used_by: UsedBy::Compiler, span },
(_, _, Some(span)) => AttributeKind::Used { used_by: UsedBy::Default, span },
(None, None, None) => return None,
})
}
}

36
compiler/rustc_attr_parsing/src/attributes/crate_level.rs
View file

@ -1,40 +1,4 @@
use std::num::IntErrorKind;
use rustc_hir::limit::Limit;
use super::prelude::*;
use crate::session_diagnostics::LimitInvalid;
impl<S: Stage> AcceptContext<'_, '_, S> {
fn parse_limit_int(&self, nv: &NameValueParser) -> Option<Limit> {
let Some(limit) = nv.value_as_str() else {
self.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
return None;
};
let error_str = match limit.as_str().parse() {
Ok(i) => return Some(Limit::new(i)),
Err(e) => match e.kind() {
IntErrorKind::PosOverflow => "`limit` is too large",
IntErrorKind::Empty => "`limit` must be a non-negative integer",
IntErrorKind::InvalidDigit => "not a valid integer",
IntErrorKind::NegOverflow => {
panic!(
"`limit` should never negatively overflow since we're parsing into a usize and we'd get Empty instead"
)
}
IntErrorKind::Zero => {
panic!("zero is a valid `limit` so should have returned Ok() when parsing")
}
kind => panic!("unimplemented IntErrorKind variant: {:?}", kind),
},
};
self.emit_err(LimitInvalid { span: self.attr_span, value_span: nv.value_span, error_str });
None
}
}
pub(crate) struct CrateNameParser;

60
compiler/rustc_attr_parsing/src/attributes/debugger.rs Normal file
View file

@ -0,0 +1,60 @@
use rustc_hir::attrs::{DebugVisualizer, DebuggerVisualizerType};
use super::prelude::*;
pub(crate) struct DebuggerViualizerParser;
impl<S: Stage> CombineAttributeParser<S> for DebuggerViualizerParser {
const PATH: &[Symbol] = &[sym::debugger_visualizer];
const ALLOWED_TARGETS: AllowedTargets =
AllowedTargets::AllowList(&[Allow(Target::Mod), Allow(Target::Crate)]);
const TEMPLATE: AttributeTemplate = template!(
List: &[r#"natvis_file = "...", gdb_script_file = "...""#],
"https://doc.rust-lang.org/reference/attributes/debugger.html#the-debugger_visualizer-attribute"
);
type Item = DebugVisualizer;
const CONVERT: ConvertFn<Self::Item> = |v, _| AttributeKind::DebuggerVisualizer(v);
fn extend<'c>(
cx: &'c mut AcceptContext<'_, '_, S>,
args: &'c ArgParser<'_>,
) -> impl IntoIterator<Item = Self::Item> + 'c {
let Some(l) = args.list() else {
cx.expected_list(args.span().unwrap_or(cx.attr_span));
return None;
};
let Some(single) = l.single() else {
cx.expected_single_argument(l.span);
return None;
};
let Some(mi) = single.meta_item() else {
cx.expected_name_value(single.span(), None);
return None;
};
let path = mi.path().word_sym();
let visualizer_type = match path {
Some(sym::natvis_file) => DebuggerVisualizerType::Natvis,
Some(sym::gdb_script_file) => DebuggerVisualizerType::GdbPrettyPrinter,
_ => {
cx.expected_specific_argument(
mi.path().span(),
&[sym::natvis_file, sym::gdb_script_file],
);
return None;
}
};
let Some(path) = mi.args().name_value() else {
cx.expected_name_value(single.span(), path);
return None;
};
let Some(path) = path.value_as_str() else {
cx.expected_string_literal(path.value_span, Some(path.value_as_lit()));
return None;
};
Some(DebugVisualizer { span: mi.span(), visualizer_type, path })
}
}

6
compiler/rustc_attr_parsing/src/attributes/inline.rs
View file

@ -72,7 +72,11 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
const PATH: &'static [Symbol] = &[sym::rustc_force_inline];
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]);
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[
Allow(Target::Fn),
Allow(Target::Method(MethodKind::Inherent)),
]);
const TEMPLATE: AttributeTemplate = template!(Word, List: &["reason"], NameValueStr: "reason");
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option<AttributeKind> {

29
compiler/rustc_attr_parsing/src/attributes/link_attrs.rs
View file

@ -65,10 +65,22 @@ impl<S: Stage> CombineAttributeParser<S> for LinkParser {
cx: &'c mut AcceptContext<'_, '_, S>,
args: &'c ArgParser<'_>,
) -> impl IntoIterator<Item = Self::Item> + 'c {
let mut result = None;
let Some(items) = args.list() else {
cx.expected_list(cx.attr_span);
return result;
let items = match args {
ArgParser::List(list) => list,
// This is an edgecase added because making this a hard error would break too many crates
// Specifically `#[link = "dl"]` is accepted with a FCW
// For more information, see https://github.com/rust-lang/rust/pull/143193
ArgParser::NameValue(nv) if nv.value_as_str().is_some_and(|v| v == sym::dl) => {
let suggestions = <Self as CombineAttributeParser<S>>::TEMPLATE
.suggestions(cx.attr_style, "link");
let span = cx.attr_span;
cx.emit_lint(AttributeLintKind::IllFormedAttributeInput { suggestions }, span);
return None;
}
_ => {
cx.expected_list(cx.attr_span);
return None;
}
};
let sess = cx.sess();
@ -113,7 +125,7 @@ impl<S: Stage> CombineAttributeParser<S> for LinkParser {
}
};
if !cont {
return result;
return None;
}
}
@ -202,7 +214,7 @@ impl<S: Stage> CombineAttributeParser<S> for LinkParser {
}
let Some((name, name_span)) = name else {
cx.emit_err(LinkRequiresName { span: cx.attr_span });
return result;
return None;
};
// Do this outside of the loop so that `import_name_type` can be specified before `kind`.
@ -218,15 +230,14 @@ impl<S: Stage> CombineAttributeParser<S> for LinkParser {
cx.emit_err(RawDylibNoNul { span: name_span });
}
result = Some(LinkEntry {
Some(LinkEntry {
span: cx.attr_span,
kind: kind.unwrap_or(NativeLibKind::Unspecified),
name,
cfg,
verbatim,
import_name_type,
});
result
})
}
}

1
compiler/rustc_attr_parsing/src/attributes/mod.rs
View file

@ -36,6 +36,7 @@ pub(crate) mod cfg_old;
pub(crate) mod codegen_attrs;
pub(crate) mod confusables;
pub(crate) mod crate_level;
pub(crate) mod debugger;
pub(crate) mod deprecation;
pub(crate) mod dummy;
pub(crate) mod inline;

18
compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs
View file

@ -49,3 +49,21 @@ impl<S: Stage> SingleAttributeParser<S> for RustcObjectLifetimeDefaultParser {
Some(AttributeKind::RustcObjectLifetimeDefault)
}
}
pub(crate) struct RustcSimdMonomorphizeLaneLimitParser;
impl<S: Stage> SingleAttributeParser<S> for RustcSimdMonomorphizeLaneLimitParser {
const PATH: &[Symbol] = &[sym::rustc_simd_monomorphize_lane_limit];
const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost;
const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Error;
const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]);
const TEMPLATE: AttributeTemplate = template!(NameValueStr: "N");
fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option<AttributeKind> {
let ArgParser::NameValue(nv) = args else {
cx.expected_name_value(cx.attr_span, None);
return None;
};
Some(AttributeKind::RustcSimdMonomorphizeLaneLimit(cx.parse_limit_int(nv)?))
}
}

37
compiler/rustc_attr_parsing/src/attributes/util.rs
View file

@ -1,11 +1,15 @@
use std::num::IntErrorKind;
use rustc_ast::LitKind;
use rustc_ast::attr::AttributeExt;
use rustc_feature::is_builtin_attr_name;
use rustc_hir::RustcVersion;
use rustc_hir::limit::Limit;
use rustc_span::{Symbol, sym};
use crate::context::{AcceptContext, Stage};
use crate::parser::ArgParser;
use crate::parser::{ArgParser, NameValueParser};
use crate::session_diagnostics::LimitInvalid;
/// Parse a rustc version number written inside string literal in an attribute,
/// like appears in `since = "1.0.0"`. Suffixes like "-dev" and "-nightly" are
@ -85,3 +89,34 @@ pub(crate) fn parse_single_integer<S: Stage>(
};
Some(num.0)
}
impl<S: Stage> AcceptContext<'_, '_, S> {
pub(crate) fn parse_limit_int(&self, nv: &NameValueParser) -> Option<Limit> {
let Some(limit) = nv.value_as_str() else {
self.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
return None;
};
let error_str = match limit.as_str().parse() {
Ok(i) => return Some(Limit::new(i)),
Err(e) => match e.kind() {
IntErrorKind::PosOverflow => "`limit` is too large",
IntErrorKind::Empty => "`limit` must be a non-negative integer",
IntErrorKind::InvalidDigit => "not a valid integer",
IntErrorKind::NegOverflow => {
panic!(
"`limit` should never negatively overflow since we're parsing into a usize and we'd get Empty instead"
)
}
IntErrorKind::Zero => {
panic!("zero is a valid `limit` so should have returned Ok() when parsing")
}
kind => panic!("unimplemented IntErrorKind variant: {:?}", kind),
},
};
self.emit_err(LimitInvalid { span: self.attr_span, value_span: nv.value_span, error_str });
None
}
}

12
compiler/rustc_attr_parsing/src/context.rs
View file

@ -28,6 +28,7 @@ use crate::attributes::crate_level::{
CrateNameParser, MoveSizeLimitParser, NoCoreParser, NoStdParser, PatternComplexityLimitParser,
RecursionLimitParser, RustcCoherenceIsCoreParser, TypeLengthLimitParser,
};
use crate::attributes::debugger::DebuggerViualizerParser;
use crate::attributes::deprecation::DeprecationParser;
use crate::attributes::dummy::DummyParser;
use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
@ -53,7 +54,7 @@ use crate::attributes::prototype::CustomMirParser;
use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser};
use crate::attributes::rustc_internal::{
RustcLayoutScalarValidRangeEnd, RustcLayoutScalarValidRangeStart,
RustcObjectLifetimeDefaultParser,
RustcObjectLifetimeDefaultParser, RustcSimdMonomorphizeLaneLimitParser,
};
use crate::attributes::semantics::MayDangleParser;
use crate::attributes::stability::{
@ -163,6 +164,7 @@ attribute_parsers!(
// tidy-alphabetical-start
Combine<AllowConstFnUnstableParser>,
Combine<AllowInternalUnstableParser>,
Combine<DebuggerViualizerParser>,
Combine<ForceTargetFeatureParser>,
Combine<LinkParser>,
Combine<ReprParser>,
@ -198,6 +200,7 @@ attribute_parsers!(
Single<RustcLayoutScalarValidRangeEnd>,
Single<RustcLayoutScalarValidRangeStart>,
Single<RustcObjectLifetimeDefaultParser>,
Single<RustcSimdMonomorphizeLaneLimitParser>,
Single<SanitizeParser>,
Single<ShouldPanicParser>,
Single<SkipDuringMethodDispatchParser>,
@ -594,7 +597,12 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> {
}
pub(crate) fn warn_empty_attribute(&mut self, span: Span) {
self.emit_lint(AttributeLintKind::EmptyAttribute { first_span: span }, span);
let attr_path = self.attr_path.clone();
let valid_without_list = self.template.word;
self.emit_lint(
AttributeLintKind::EmptyAttribute { first_span: span, attr_path, valid_without_list },
span,
);
}
}

18
compiler/rustc_attr_parsing/src/lints.rs
View file

@ -43,12 +43,18 @@ pub fn emit_attribute_lint<L: LintEmitter>(lint: &AttributeLint<L::Id>, lint_emi
),
},
),
AttributeLintKind::EmptyAttribute { first_span } => lint_emitter.emit_node_span_lint(
rustc_session::lint::builtin::UNUSED_ATTRIBUTES,
*id,
*first_span,
session_diagnostics::EmptyAttributeList { attr_span: *first_span },
),
AttributeLintKind::EmptyAttribute { first_span, attr_path, valid_without_list } => {
lint_emitter.emit_node_span_lint(
rustc_session::lint::builtin::UNUSED_ATTRIBUTES,
*id,
*first_span,
session_diagnostics::EmptyAttributeList {
attr_span: *first_span,
attr_path: attr_path.clone(),
valid_without_list: *valid_without_list,
},
)
}
AttributeLintKind::InvalidTarget { name, target, applied, only } => lint_emitter
.emit_node_span_lint(
// This check is here because `deprecated` had its own lint group and removing this would be a breaking change

2
compiler/rustc_attr_parsing/src/parser.rs
View file

@ -49,7 +49,7 @@ impl<'a> PathParser<'a> {
}
pub fn segments_is(&self, segments: &[Symbol]) -> bool {
self.len() == segments.len() && self.segments().zip(segments).all(|(a, b)| a.name == *b)
self.segments().map(|segment| &segment.name).eq(segments)
}
pub fn word(&self) -> Option<Ident> {

3
compiler/rustc_attr_parsing/src/session_diagnostics.rs
View file

@ -503,9 +503,12 @@ pub(crate) struct EmptyConfusables {
#[derive(LintDiagnostic)]
#[diag(attr_parsing_empty_attribute)]
#[note]
pub(crate) struct EmptyAttributeList {
#[suggestion(code = "", applicability = "machine-applicable")]
pub attr_span: Span,
pub attr_path: AttrPath,
pub valid_without_list: bool,
}
#[derive(LintDiagnostic)]

12
compiler/rustc_attr_parsing/src/validate_attr.rs
View file

@ -207,10 +207,9 @@ pub fn check_attribute_safety(
}
}
// - Normal builtin attribute, or any non-builtin attribute
// - All non-builtin attributes are currently considered safe; writing `#[unsafe(..)]` is
// not permitted on non-builtin attributes or normal builtin attributes
(Some(AttributeSafety::Normal) | None, Safety::Unsafe(unsafe_span)) => {
// - Normal builtin attribute
// - Writing `#[unsafe(..)]` is not permitted on normal builtin attributes
(Some(AttributeSafety::Normal), Safety::Unsafe(unsafe_span)) => {
psess.dcx().emit_err(errors::InvalidAttrUnsafe {
span: unsafe_span,
name: attr_item.path.clone(),
@ -224,9 +223,8 @@ pub fn check_attribute_safety(
}
// - Non-builtin attribute
// - No explicit `#[unsafe(..)]` written.
(None, Safety::Default) => {
// OK
(None, Safety::Unsafe(_) | Safety::Default) => {
// OK (not checked here)
}
(

4
compiler/rustc_borrowck/src/borrowck_errors.rs
View file

@ -426,7 +426,7 @@ impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, 'infcx, 'tcx> {
}
pub(crate) fn path_does_not_live_long_enough(&self, span: Span, path: &str) -> Diag<'infcx> {
struct_span_code_err!(self.dcx(), span, E0597, "{} does not live long enough", path,)
struct_span_code_err!(self.dcx(), span, E0597, "{} does not live long enough", path)
}
pub(crate) fn cannot_return_reference_to_local(
@ -480,7 +480,7 @@ impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, 'infcx, 'tcx> {
}
pub(crate) fn temporary_value_borrowed_for_too_long(&self, span: Span) -> Diag<'infcx> {
struct_span_code_err!(self.dcx(), span, E0716, "temporary value dropped while borrowed",)
struct_span_code_err!(self.dcx(), span, E0716, "temporary value dropped while borrowed")
}
}

9
compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
View file

@ -2992,6 +2992,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
self.buffer_error(err);
}
#[tracing::instrument(level = "debug", skip(self, explanation))]
fn report_local_value_does_not_live_long_enough(
&self,
location: Location,
@ -3001,13 +3002,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
borrow_spans: UseSpans<'tcx>,
explanation: BorrowExplanation<'tcx>,
) -> Diag<'infcx> {
debug!(
"report_local_value_does_not_live_long_enough(\
{:?}, {:?}, {:?}, {:?}, {:?}\
)",
location, name, borrow, drop_span, borrow_spans
);
let borrow_span = borrow_spans.var_or_use_path_span();
if let BorrowExplanation::MustBeValidFor {
category,
@ -3974,7 +3968,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
}
ProjectionElem::ConstantIndex { .. }
| ProjectionElem::Subslice { .. }
| ProjectionElem::Subtype(_)
| ProjectionElem::Index(_)
| ProjectionElem::UnwrapUnsafeBinder(_) => kind,
},

20
compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
View file

@ -416,6 +416,26 @@ impl<'tcx> BorrowExplanation<'tcx> {
{
self.add_object_lifetime_default_note(tcx, err, unsize_ty);
}
let mut preds = path
.iter()
.filter_map(|constraint| match constraint.category {
ConstraintCategory::Predicate(pred) if !pred.is_dummy() => Some(pred),
_ => None,
})
.collect::<Vec<Span>>();
preds.sort();
preds.dedup();
if !preds.is_empty() {
let s = if preds.len() == 1 { "" } else { "s" };
err.span_note(
preds,
format!(
"requirement{s} that the value outlives `{region_name}` introduced here"
),
);
}
self.add_lifetime_bound_suggestion_to_diagnostic(err, &category, span, region_name);
}
_ => {}

7
compiler/rustc_borrowck/src/diagnostics/mod.rs
View file

@ -402,7 +402,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
ProjectionElem::Downcast(..) if opt.including_downcast => return None,
ProjectionElem::Downcast(..) => (),
ProjectionElem::OpaqueCast(..) => (),
ProjectionElem::Subtype(..) => (),
ProjectionElem::UnwrapUnsafeBinder(_) => (),
ProjectionElem::Field(field, _ty) => {
// FIXME(project-rfc_2229#36): print capture precisely here.
@ -484,9 +483,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
PlaceRef { local, projection: proj_base }.ty(self.body, self.infcx.tcx)
}
ProjectionElem::Downcast(..) => place.ty(self.body, self.infcx.tcx),
ProjectionElem::Subtype(ty)
| ProjectionElem::OpaqueCast(ty)
| ProjectionElem::UnwrapUnsafeBinder(ty) => PlaceTy::from_ty(*ty),
ProjectionElem::OpaqueCast(ty) | ProjectionElem::UnwrapUnsafeBinder(ty) => {
PlaceTy::from_ty(*ty)
}
ProjectionElem::Field(_, field_type) => PlaceTy::from_ty(*field_type),
},
};

1
compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
View file

@ -192,7 +192,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
[
..,
ProjectionElem::Index(_)
| ProjectionElem::Subtype(_)
| ProjectionElem::ConstantIndex { .. }
| ProjectionElem::OpaqueCast { .. }
| ProjectionElem::Subslice { .. }

10
compiler/rustc_borrowck/src/lib.rs
View file

@ -119,7 +119,7 @@ pub fn provide(providers: &mut Providers) {
fn mir_borrowck(
tcx: TyCtxt<'_>,
def: LocalDefId,
) -> Result<&ConcreteOpaqueTypes<'_>, ErrorGuaranteed> {
) -> Result<&DefinitionSiteHiddenTypes<'_>, ErrorGuaranteed> {
assert!(!tcx.is_typeck_child(def.to_def_id()));
let (input_body, _) = tcx.mir_promoted(def);
debug!("run query mir_borrowck: {}", tcx.def_path_str(def));
@ -130,7 +130,7 @@ fn mir_borrowck(
Err(guar)
} else if input_body.should_skip() {
debug!("Skipping borrowck because of injected body");
let opaque_types = ConcreteOpaqueTypes(Default::default());
let opaque_types = DefinitionSiteHiddenTypes(Default::default());
Ok(tcx.arena.alloc(opaque_types))
} else {
let mut root_cx = BorrowCheckRootCtxt::new(tcx, def, None);
@ -278,7 +278,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
) -> Ty<'tcx> {
fold_regions(tcx, self.inner, |r, depth| match r.kind() {
ty::ReBound(debruijn, br) => {
ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), br) => {
debug_assert_eq!(debruijn, depth);
map(ty::RegionVid::from_usize(br.var.index()))
}
@ -1989,10 +1989,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
},
// `OpaqueCast`: only transmutes the type, so no moves there.
// `Downcast` : only changes information about a `Place` without moving.
// `Subtype` : only transmutes the type, so no moves.
// So it's safe to skip these.
ProjectionElem::OpaqueCast(_)
| ProjectionElem::Subtype(_)
| ProjectionElem::Downcast(_, _)
| ProjectionElem::UnwrapUnsafeBinder(_) => (),
}
@ -2218,7 +2216,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
for (place_base, elem) in place.iter_projections().rev() {
match elem {
ProjectionElem::Index(_/*operand*/) |
ProjectionElem::Subtype(_) |
ProjectionElem::OpaqueCast(_) |
ProjectionElem::ConstantIndex { .. } |
// assigning to P[i] requires P to be valid.
@ -2610,7 +2607,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
| ProjectionElem::Index(..)
| ProjectionElem::ConstantIndex { .. }
| ProjectionElem::Subslice { .. }
| ProjectionElem::Subtype(..)
| ProjectionElem::OpaqueCast { .. }
| ProjectionElem::Downcast(..)
| ProjectionElem::UnwrapUnsafeBinder(_) => {

2
compiler/rustc_borrowck/src/places_conflict.rs
View file

@ -249,7 +249,6 @@ fn place_components_conflict<'tcx>(
| (ProjectionElem::ConstantIndex { .. }, _, _)
| (ProjectionElem::Subslice { .. }, _, _)
| (ProjectionElem::OpaqueCast { .. }, _, _)
| (ProjectionElem::Subtype(_), _, _)
| (ProjectionElem::Downcast { .. }, _, _)
| (ProjectionElem::UnwrapUnsafeBinder(_), _, _) => {
// Recursive case. This can still be disjoint on a
@ -510,7 +509,6 @@ fn place_projection_conflict<'tcx>(
| ProjectionElem::Field(..)
| ProjectionElem::Index(..)
| ProjectionElem::ConstantIndex { .. }
| ProjectionElem::Subtype(_)
| ProjectionElem::OpaqueCast { .. }
| ProjectionElem::Subslice { .. }
| ProjectionElem::Downcast(..),

3
compiler/rustc_borrowck/src/prefixes.rs
View file

@ -77,9 +77,6 @@ impl<'tcx> Iterator for Prefixes<'tcx> {
| ProjectionElem::Index(_) => {
cursor = cursor_base;
}
ProjectionElem::Subtype(..) => {
panic!("Subtype projection is not allowed before borrow check")
}
ProjectionElem::Deref => {
match self.kind {
PrefixSet::Shallow => {

11
compiler/rustc_borrowck/src/region_infer/mod.rs
View file

@ -1382,10 +1382,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
}
/// The constraints we get from equating the hidden type of each use of an opaque
/// with its final concrete type may end up getting preferred over other, potentially
/// with its final hidden type may end up getting preferred over other, potentially
/// longer constraint paths.
///
/// Given that we compute the final concrete type by relying on this existing constraint
/// Given that we compute the final hidden type by relying on this existing constraint
/// path, this can easily end up hiding the actual reason for why we require these regions
/// to be equal.
///
@ -1736,9 +1736,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
// `BoringNoLocation` constraints can point to user-written code, but are less
// specific, and are not used for relations that would make sense to blame.
ConstraintCategory::BoringNoLocation => 6,
// Do not blame internal constraints.
ConstraintCategory::OutlivesUnnameablePlaceholder(_) => 7,
ConstraintCategory::Internal => 8,
// Do not blame internal constraints if we can avoid it. Never blame
// the `'region: 'static` constraints introduced by placeholder outlives.
ConstraintCategory::Internal => 7,
ConstraintCategory::OutlivesUnnameablePlaceholder(_) => 8,
};
debug!("constraint {constraint:?} category: {category:?}, interest: {interest:?}");

76
compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs
View file

@ -8,7 +8,7 @@ use rustc_infer::infer::outlives::env::RegionBoundPairs;
use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, OpaqueTypeStorageEntries};
use rustc_infer::traits::ObligationCause;
use rustc_macros::extension;
use rustc_middle::mir::{Body, ConcreteOpaqueTypes, ConstraintCategory};
use rustc_middle::mir::{Body, ConstraintCategory, DefinitionSiteHiddenTypes};
use rustc_middle::ty::{
self, DefiningScopeKind, EarlyBinder, FallibleTypeFolder, GenericArg, GenericArgsRef,
OpaqueHiddenType, OpaqueTypeKey, Region, RegionVid, Ty, TyCtxt, TypeFoldable,
@ -129,9 +129,9 @@ fn nll_var_to_universal_region<'tcx>(
/// Collect all defining uses of opaque types inside of this typeck root. This
/// expects the hidden type to be mapped to the definition parameters of the opaque
/// and errors if we end up with distinct hidden types.
fn add_concrete_opaque_type<'tcx>(
fn add_hidden_type<'tcx>(
tcx: TyCtxt<'tcx>,
concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>,
def_id: LocalDefId,
hidden_ty: OpaqueHiddenType<'tcx>,
) {
@ -139,7 +139,7 @@ fn add_concrete_opaque_type<'tcx>(
// back to the opaque type definition. E.g. we may have `OpaqueType<X, Y>` mapped to
// `(X, Y)` and `OpaqueType<Y, X>` mapped to `(Y, X)`, and those are the same, but we
// only know that once we convert the generic parameters to those of the opaque type.
if let Some(prev) = concrete_opaque_types.0.get_mut(&def_id) {
if let Some(prev) = hidden_types.0.get_mut(&def_id) {
if prev.ty != hidden_ty.ty {
let guar = hidden_ty.ty.error_reported().err().unwrap_or_else(|| {
let (Ok(e) | Err(e)) = prev.build_mismatch_error(&hidden_ty, tcx).map(|d| d.emit());
@ -151,15 +151,15 @@ fn add_concrete_opaque_type<'tcx>(
// FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
prev.span = prev.span.substitute_dummy(hidden_ty.span);
} else {
concrete_opaque_types.0.insert(def_id, hidden_ty);
hidden_types.0.insert(def_id, hidden_ty);
}
}
fn get_concrete_opaque_type<'tcx>(
concrete_opaque_types: &ConcreteOpaqueTypes<'tcx>,
fn get_hidden_type<'tcx>(
hidden_types: &DefinitionSiteHiddenTypes<'tcx>,
def_id: LocalDefId,
) -> Option<EarlyBinder<'tcx, OpaqueHiddenType<'tcx>>> {
concrete_opaque_types.0.get(&def_id).map(|ty| EarlyBinder::bind(*ty))
hidden_types.0.get(&def_id).map(|ty| EarlyBinder::bind(*ty))
}
#[derive(Debug)]
@ -173,22 +173,22 @@ struct DefiningUse<'tcx> {
}
/// This computes the actual hidden types of the opaque types and maps them to their
/// definition sites. Outside of registering the computed concrete types this function
/// definition sites. Outside of registering the computed hidden types this function
/// does not mutate the current borrowck state.
///
/// While it may fail to infer the hidden type and return errors, we always apply
/// the computed concrete hidden type to all opaque type uses to check whether they
/// the computed hidden type to all opaque type uses to check whether they
/// are correct. This is necessary to support non-defining uses of opaques in their
/// defining scope.
///
/// It also means that this whole function is not really soundness critical as we
/// recheck all uses of the opaques regardless.
pub(crate) fn compute_concrete_opaque_types<'tcx>(
pub(crate) fn compute_definition_site_hidden_types<'tcx>(
infcx: &BorrowckInferCtxt<'tcx>,
universal_region_relations: &Frozen<UniversalRegionRelations<'tcx>>,
constraints: &MirTypeckRegionConstraints<'tcx>,
location_map: Rc<DenseLocationMap>,
concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>,
opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)],
) -> Vec<DeferredOpaqueTypeError<'tcx>> {
let mut errors = Vec::new();
@ -201,8 +201,7 @@ pub(crate) fn compute_concrete_opaque_types<'tcx>(
// We start by checking each use of an opaque type during type check and
// check whether the generic arguments of the opaque type are fully
// universal, if so, it's a defining use.
let defining_uses =
collect_defining_uses(&mut rcx, concrete_opaque_types, opaque_types, &mut errors);
let defining_uses = collect_defining_uses(&mut rcx, hidden_types, opaque_types, &mut errors);
// We now compute and apply member constraints for all regions in the hidden
// types of each defining use. This mutates the region values of the `rcx` which
@ -210,11 +209,11 @@ pub(crate) fn compute_concrete_opaque_types<'tcx>(
apply_member_constraints(&mut rcx, &defining_uses);
// After applying member constraints, we now check whether all member regions ended
// up equal to one of their choice regions and compute the actual concrete type of
// up equal to one of their choice regions and compute the actual hidden type of
// the opaque type definition. This is stored in the `root_cx`.
compute_concrete_types_from_defining_uses(
compute_definition_site_hidden_types_from_defining_uses(
&rcx,
concrete_opaque_types,
hidden_types,
&defining_uses,
&mut errors,
);
@ -224,7 +223,7 @@ pub(crate) fn compute_concrete_opaque_types<'tcx>(
#[instrument(level = "debug", skip_all, ret)]
fn collect_defining_uses<'tcx>(
rcx: &mut RegionCtxt<'_, 'tcx>,
concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>,
opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)],
errors: &mut Vec<DeferredOpaqueTypeError<'tcx>>,
) -> Vec<DefiningUse<'tcx>> {
@ -244,9 +243,9 @@ fn collect_defining_uses<'tcx>(
// with `TypingMode::Borrowck`.
if infcx.tcx.use_typing_mode_borrowck() {
match err {
NonDefiningUseReason::Tainted(guar) => add_concrete_opaque_type(
NonDefiningUseReason::Tainted(guar) => add_hidden_type(
infcx.tcx,
concrete_opaque_types,
hidden_types,
opaque_type_key.def_id,
OpaqueHiddenType::new_error(infcx.tcx, guar),
),
@ -277,9 +276,9 @@ fn collect_defining_uses<'tcx>(
defining_uses
}
fn compute_concrete_types_from_defining_uses<'tcx>(
fn compute_definition_site_hidden_types_from_defining_uses<'tcx>(
rcx: &RegionCtxt<'_, 'tcx>,
concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>,
defining_uses: &[DefiningUse<'tcx>],
errors: &mut Vec<DeferredOpaqueTypeError<'tcx>>,
) {
@ -358,9 +357,9 @@ fn compute_concrete_types_from_defining_uses<'tcx>(
},
));
}
add_concrete_opaque_type(
add_hidden_type(
tcx,
concrete_opaque_types,
hidden_types,
opaque_type_key.def_id,
OpaqueHiddenType { span: hidden_type.span, ty },
);
@ -489,20 +488,20 @@ impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for ToArgRegionsFolder<'_, 'tcx> {
///
/// It does this by equating the hidden type of each use with the instantiated final
/// hidden type of the opaque.
pub(crate) fn apply_computed_concrete_opaque_types<'tcx>(
pub(crate) fn apply_definition_site_hidden_types<'tcx>(
infcx: &BorrowckInferCtxt<'tcx>,
body: &Body<'tcx>,
universal_regions: &UniversalRegions<'tcx>,
region_bound_pairs: &RegionBoundPairs<'tcx>,
known_type_outlives_obligations: &[ty::PolyTypeOutlivesPredicate<'tcx>],
constraints: &mut MirTypeckRegionConstraints<'tcx>,
concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>,
opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)],
) -> Vec<DeferredOpaqueTypeError<'tcx>> {
let tcx = infcx.tcx;
let mut errors = Vec::new();
for &(key, hidden_type) in opaque_types {
let Some(expected) = get_concrete_opaque_type(concrete_opaque_types, key.def_id) else {
let Some(expected) = get_hidden_type(hidden_types, key.def_id) else {
if !tcx.use_typing_mode_borrowck() {
if let ty::Alias(ty::Opaque, alias_ty) = hidden_type.ty.kind()
&& alias_ty.def_id == key.def_id.to_def_id()
@ -521,12 +520,7 @@ pub(crate) fn apply_computed_concrete_opaque_types<'tcx>(
hidden_type.span,
"non-defining use in the defining scope with no defining uses",
);
add_concrete_opaque_type(
tcx,
concrete_opaque_types,
key.def_id,
OpaqueHiddenType::new_error(tcx, guar),
);
add_hidden_type(tcx, hidden_types, key.def_id, OpaqueHiddenType::new_error(tcx, guar));
continue;
};
@ -566,18 +560,13 @@ pub(crate) fn apply_computed_concrete_opaque_types<'tcx>(
"equating opaque types",
),
) {
add_concrete_opaque_type(
tcx,
concrete_opaque_types,
key.def_id,
OpaqueHiddenType::new_error(tcx, guar),
);
add_hidden_type(tcx, hidden_types, key.def_id, OpaqueHiddenType::new_error(tcx, guar));
}
}
errors
}
/// In theory `apply_concrete_opaque_types` could introduce new uses of opaque types.
/// In theory `apply_definition_site_hidden_types` could introduce new uses of opaque types.
/// We do not check these new uses so this could be unsound.
///
/// We detect any new uses and simply delay a bug if they occur. If this results in
@ -682,13 +671,6 @@ impl<'tcx> InferCtxt<'tcx> {
///
/// (*) C1 and C2 were introduced in the comments on
/// `register_member_constraints`. Read that comment for more context.
///
/// # Parameters
///
/// - `def_id`, the `impl Trait` type
/// - `args`, the args used to instantiate this opaque type
/// - `instantiated_ty`, the inferred type C1 -- fully resolved, lifted version of
/// `opaque_defn.concrete_ty`
#[instrument(level = "debug", skip(self))]
fn infer_opaque_definition_from_instantiation(
&self,

22
compiler/rustc_borrowck/src/root_cx.rs
View file

@ -12,12 +12,12 @@ use smallvec::SmallVec;
use crate::consumers::BorrowckConsumer;
use crate::nll::compute_closure_requirements_modulo_opaques;
use crate::region_infer::opaque_types::{
apply_computed_concrete_opaque_types, clone_and_resolve_opaque_types,
compute_concrete_opaque_types, detect_opaque_types_added_while_handling_opaque_types,
apply_definition_site_hidden_types, clone_and_resolve_opaque_types,
compute_definition_site_hidden_types, detect_opaque_types_added_while_handling_opaque_types,
};
use crate::type_check::{Locations, constraint_conversion};
use crate::{
ClosureRegionRequirements, CollectRegionConstraintsResult, ConcreteOpaqueTypes,
ClosureRegionRequirements, CollectRegionConstraintsResult, DefinitionSiteHiddenTypes,
PropagatedBorrowCheckResults, borrowck_check_region_constraints,
borrowck_collect_region_constraints,
};
@ -27,7 +27,7 @@ use crate::{
pub(super) struct BorrowCheckRootCtxt<'tcx> {
pub tcx: TyCtxt<'tcx>,
root_def_id: LocalDefId,
concrete_opaque_types: ConcreteOpaqueTypes<'tcx>,
hidden_types: DefinitionSiteHiddenTypes<'tcx>,
/// The region constraints computed by [borrowck_collect_region_constraints]. This uses
/// an [FxIndexMap] to guarantee that iterating over it visits nested bodies before
/// their parents.
@ -49,7 +49,7 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
BorrowCheckRootCtxt {
tcx,
root_def_id,
concrete_opaque_types: Default::default(),
hidden_types: Default::default(),
collect_region_constraints_results: Default::default(),
propagated_borrowck_results: Default::default(),
tainted_by_errors: None,
@ -72,11 +72,11 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
&self.propagated_borrowck_results[&nested_body_def_id].used_mut_upvars
}
pub(super) fn finalize(self) -> Result<&'tcx ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> {
pub(super) fn finalize(self) -> Result<&'tcx DefinitionSiteHiddenTypes<'tcx>, ErrorGuaranteed> {
if let Some(guar) = self.tainted_by_errors {
Err(guar)
} else {
Ok(self.tcx.arena.alloc(self.concrete_opaque_types))
Ok(self.tcx.arena.alloc(self.hidden_types))
}
}
@ -88,12 +88,12 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
&input.universal_region_relations,
&mut input.constraints,
);
input.deferred_opaque_type_errors = compute_concrete_opaque_types(
input.deferred_opaque_type_errors = compute_definition_site_hidden_types(
&input.infcx,
&input.universal_region_relations,
&input.constraints,
Rc::clone(&input.location_map),
&mut self.concrete_opaque_types,
&mut self.hidden_types,
&opaque_types,
);
per_body_info.push((num_entries, opaque_types));
@ -103,14 +103,14 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> {
self.collect_region_constraints_results.values_mut().zip(per_body_info)
{
if input.deferred_opaque_type_errors.is_empty() {
input.deferred_opaque_type_errors = apply_computed_concrete_opaque_types(
input.deferred_opaque_type_errors = apply_definition_site_hidden_types(
&input.infcx,
&input.body_owned,
&input.universal_region_relations.universal_regions,
&input.region_bound_pairs,
&input.known_type_outlives_obligations,
&mut input.constraints,
&mut self.concrete_opaque_types,
&mut self.hidden_types,
&opaque_types,
);
}

22
compiler/rustc_borrowck/src/type_check/mod.rs
View file

@ -505,6 +505,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
let mut constraints = Default::default();
let mut liveness_constraints =
LivenessValues::without_specific_points(Rc::new(DenseLocationMap::new(promoted_body)));
let mut deferred_closure_requirements = Default::default();
// Don't try to add borrow_region facts for the promoted MIR as they refer
// to the wrong locations.
@ -512,6 +513,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
mem::swap(this.polonius_facts, polonius_facts);
mem::swap(&mut this.constraints.outlives_constraints, &mut constraints);
mem::swap(&mut this.constraints.liveness_constraints, &mut liveness_constraints);
mem::swap(this.deferred_closure_requirements, &mut deferred_closure_requirements);
};
swap_constraints(self);
@ -536,6 +538,17 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
}
self.constraints.outlives_constraints.push(constraint)
}
// If there are nested bodies in promoteds, we also need to update their
// location to something in the actual body, not the promoted.
//
// We don't update the constraint categories of the resulting constraints
// as returns in nested bodies are a proper return, even if that nested body
// is in a promoted.
for (closure_def_id, args, _locations) in deferred_closure_requirements {
self.deferred_closure_requirements.push((closure_def_id, args, locations));
}
// If the region is live at least one location in the promoted MIR,
// then add a liveness constraint to the main MIR for this region
// at the location provided as an argument to this method
@ -1545,6 +1558,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
),
}
}
CastKind::Subtype => {
bug!("CastKind::Subtype shouldn't exist in borrowck")
}
}
}
@ -1869,9 +1885,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
)
.unwrap();
}
ProjectionElem::Subtype(_) => {
bug!("ProjectionElem::Subtype shouldn't exist in borrowck")
}
}
}
}
@ -2399,9 +2412,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
| ProjectionElem::UnwrapUnsafeBinder(_) => {
// other field access
}
ProjectionElem::Subtype(_) => {
bug!("ProjectionElem::Subtype shouldn't exist in borrowck")
}
}
}
}

18
compiler/rustc_builtin_macros/messages.ftl
View file

@ -135,6 +135,15 @@ builtin_macros_concat_missing_literal = expected a literal
builtin_macros_default_arg = `#[default]` attribute does not accept a value
.suggestion = try using `#[default]`
builtin_macros_derive_from_usage_note = `#[derive(From)]` can only be used on structs with exactly one field
builtin_macros_derive_from_wrong_field_count = `#[derive(From)]` used on a struct with {$multiple_fields ->
[true] multiple fields
*[false] no fields
}
builtin_macros_derive_from_wrong_target = `#[derive(From)]` used on {$kind}
builtin_macros_derive_macro_call = `derive` cannot be used on items with type macros
builtin_macros_derive_path_args_list = traits in `#[derive(...)]` don't accept arguments
@ -229,15 +238,6 @@ builtin_macros_format_unused_args = multiple unused formatting arguments
builtin_macros_format_use_positional = consider using a positional formatting argument instead
builtin_macros_derive_from_wrong_target = `#[derive(From)]` used on {$kind}
builtin_macros_derive_from_wrong_field_count = `#[derive(From)]` used on a struct with {$multiple_fields ->
[true] multiple fields
*[false] no fields
}
builtin_macros_derive_from_usage_note = `#[derive(From)]` can only be used on structs with exactly one field
builtin_macros_incomplete_include = include macro expected single expression in source
builtin_macros_multiple_default_attrs = multiple `#[default]` attributes

3
compiler/rustc_builtin_macros/src/autodiff.rs
View file

@ -377,8 +377,7 @@ mod llvm_enzyme {
(ast::AttrKind::Normal(a), ast::AttrKind::Normal(b)) => {
let a = &a.item.path;
let b = &b.item.path;
a.segments.len() == b.segments.len()
&& a.segments.iter().zip(b.segments.iter()).all(|(a, b)| a.ident == b.ident)
a.segments.iter().eq_by(&b.segments, |a, b| a.ident == b.ident)
}
_ => false,
}

15
compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs
View file

@ -356,21 +356,14 @@ fn contains_maybe_sized_bound(bounds: &[GenericBound]) -> bool {
bounds.iter().any(is_maybe_sized_bound)
}
fn path_segment_is_exact_match(path_segments: &[ast::PathSegment], syms: &[Symbol]) -> bool {
path_segments.iter().zip(syms).all(|(segment, &symbol)| segment.ident.name == symbol)
}
fn is_sized_marker(path: &ast::Path) -> bool {
const CORE_UNSIZE: [Symbol; 3] = [sym::core, sym::marker, sym::Sized];
const STD_UNSIZE: [Symbol; 3] = [sym::std, sym::marker, sym::Sized];
if path.segments.len() == 4 && path.is_global() {
path_segment_is_exact_match(&path.segments[1..], &CORE_UNSIZE)
|| path_segment_is_exact_match(&path.segments[1..], &STD_UNSIZE)
} else if path.segments.len() == 3 {
path_segment_is_exact_match(&path.segments, &CORE_UNSIZE)
|| path_segment_is_exact_match(&path.segments, &STD_UNSIZE)
let segments = || path.segments.iter().map(|segment| segment.ident.name);
if path.is_global() {
segments().skip(1).eq(CORE_UNSIZE) || segments().skip(1).eq(STD_UNSIZE)
} else {
*path == sym::Sized
segments().eq(CORE_UNSIZE) || segments().eq(STD_UNSIZE) || *path == sym::Sized
}
}

2
compiler/rustc_builtin_macros/src/format.rs
View file

@ -768,7 +768,7 @@ fn report_missing_placeholders(
if !found_foreign && invalid_refs.is_empty() {
// Show example if user didn't use any format specifiers
let show_example = used.iter().all(|used| !used);
let show_example = !used.contains(&true);
if !show_example {
if unused.len() > 1 {

16
compiler/rustc_builtin_macros/src/global_allocator.rs
View file

@ -85,7 +85,7 @@ impl AllocFnFactory<'_, '_> {
body,
define_opaque: None,
}));
let item = self.cx.item(self.span, self.attrs(), kind);
let item = self.cx.item(self.span, self.attrs(method), kind);
self.cx.stmt_item(self.ty_span, item)
}
@ -100,8 +100,18 @@ impl AllocFnFactory<'_, '_> {
self.cx.expr_call(self.ty_span, method, args)
}
fn attrs(&self) -> AttrVec {
thin_vec![self.cx.attr_word(sym::rustc_std_internal_symbol, self.span)]
fn attrs(&self, method: &AllocatorMethod) -> AttrVec {
let alloc_attr = match method.name {
sym::alloc => sym::rustc_allocator,
sym::dealloc => sym::rustc_deallocator,
sym::realloc => sym::rustc_reallocator,
sym::alloc_zeroed => sym::rustc_allocator_zeroed,
_ => unreachable!("Unknown allocator method!"),
};
thin_vec![
self.cx.attr_word(sym::rustc_std_internal_symbol, self.span),
self.cx.attr_word(alloc_attr, self.span)
]
}
fn arg_ty(&self, input: &AllocatorMethodInput, args: &mut ThinVec<Param>) -> Box<Expr> {

1
compiler/rustc_builtin_macros/src/lib.rs
View file

@ -11,6 +11,7 @@
#![feature(box_patterns)]
#![feature(decl_macro)]
#![feature(if_let_guard)]
#![feature(iter_order_by)]
#![feature(proc_macro_internals)]
#![feature(proc_macro_quote)]
#![feature(rustdoc_internals)]

10
compiler/rustc_builtin_macros/src/test_harness.rs
View file

@ -63,8 +63,8 @@ pub fn inject(
if sess.is_test_crate() {
let panic_strategy = match (panic_strategy, sess.opts.unstable_opts.panic_abort_tests) {
(PanicStrategy::Abort, true) => PanicStrategy::Abort,
(PanicStrategy::Abort, false) => {
(PanicStrategy::Abort | PanicStrategy::ImmediateAbort, true) => panic_strategy,
(PanicStrategy::Abort | PanicStrategy::ImmediateAbort, false) => {
if panic_strategy == platform_panic_strategy {
// Silently allow compiling with panic=abort on these platforms,
// but with old behavior (abort if a test fails).
@ -287,10 +287,8 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> Box<ast::Item> {
let ecx = &cx.ext_cx;
let test_ident = Ident::new(sym::test, sp);
let runner_name = match cx.panic_strategy {
PanicStrategy::Unwind => "test_main_static",
PanicStrategy::Abort => "test_main_static_abort",
};
let runner_name =
if cx.panic_strategy.unwinds() { "test_main_static" } else { "test_main_static_abort" };
// test::test_main_static(...)
let mut test_runner = cx.test_runner.clone().unwrap_or_else(|| {

2
compiler/rustc_codegen_cranelift/src/abi/comments.rs
View file

@ -89,7 +89,7 @@ pub(super) fn add_local_place_comments<'tcx>(
format!("{:?}", local),
format!("{:?}", ty),
size.bytes(),
align.abi.bytes(),
align.bytes(),
if extra.is_empty() { "" } else { " " },
extra,
));

2
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -233,7 +233,7 @@ pub(super) fn from_casted_value<'tcx>(
// It may also be smaller for example when the type is a wrapper around an integer with a
// larger alignment than the integer.
std::cmp::max(abi_param_size, layout_size),
u32::try_from(layout.align.abi.bytes()).unwrap(),
u32::try_from(layout.align.bytes()).unwrap(),
);
let mut block_params_iter = block_params.iter().copied();
for (offset, _) in abi_params {

6
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -789,7 +789,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
let operand = codegen_operand(fx, operand);
crate::unsize::coerce_unsized_into(fx, operand, lval);
}
Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => {
Rvalue::Cast(CastKind::Transmute | CastKind::Subtype, ref operand, _to_ty) => {
let operand = codegen_operand(fx, operand);
lval.write_cvalue_transmute(fx, operand);
}
@ -846,7 +846,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
let layout = fx.layout_of(fx.monomorphize(ty));
let val = match null_op {
NullOp::SizeOf => layout.size.bytes(),
NullOp::AlignOf => layout.align.abi.bytes(),
NullOp::AlignOf => layout.align.bytes(),
NullOp::OffsetOf(fields) => fx
.tcx
.offset_of_subfield(
@ -996,7 +996,7 @@ pub(crate) fn codegen_place<'tcx>(
cplace = cplace.place_deref(fx);
}
PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"),
PlaceElem::Subtype(ty) | PlaceElem::UnwrapUnsafeBinder(ty) => {
PlaceElem::UnwrapUnsafeBinder(ty) => {
cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty));
}
PlaceElem::Field(field, _ty) => {

9
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -439,7 +439,10 @@ pub(crate) struct FullyMonomorphizedLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>);
impl<'tcx> LayoutOfHelpers<'tcx> for FullyMonomorphizedLayoutCx<'tcx> {
#[inline]
fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
if let LayoutError::SizeOverflow(_)
| LayoutError::InvalidSimd { .. }
| LayoutError::ReferencesError(_) = err
{
self.0.sess.dcx().span_fatal(span, err.to_string())
} else {
self.0
@ -458,7 +461,9 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for FullyMonomorphizedLayoutCx<'tcx> {
span: Span,
fn_abi_request: FnAbiRequest<'tcx>,
) -> ! {
if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err {
if let FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::InvalidSimd { .. }) =
err
{
self.0.sess.dcx().emit_fatal(Spanned { span, node: err })
} else {
match fn_abi_request {

2
compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
View file

@ -304,7 +304,7 @@ impl DebugContext {
entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line));
entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(static_layout.align.abi.bytes()));
entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(static_layout.align.bytes()));
let mut expr = Expression::new();
expr.op_addr(address_for_data(data_id));

6
compiler/rustc_codegen_cranelift/src/debuginfo/types.rs
View file

@ -166,7 +166,7 @@ impl DebugContext {
let tuple_entry = self.dwarf.unit.get_mut(tuple_type_id);
tuple_entry.set(gimli::DW_AT_name, AttributeValue::StringRef(self.dwarf.strings.add(name)));
tuple_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
tuple_entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(layout.align.abi.bytes()));
tuple_entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(layout.align.bytes()));
for (i, (ty, dw_ty)) in components.into_iter().enumerate() {
let member_id = self.dwarf.unit.add(tuple_type_id, gimli::DW_TAG_member);
@ -178,9 +178,7 @@ impl DebugContext {
member_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
member_entry.set(
gimli::DW_AT_alignment,
AttributeValue::Udata(
FullyMonomorphizedLayoutCx(tcx).layout_of(ty).align.abi.bytes(),
),
AttributeValue::Udata(FullyMonomorphizedLayoutCx(tcx).layout_of(ty).align.bytes()),
);
member_entry.set(
gimli::DW_AT_data_member_location,

5
compiler/rustc_codegen_cranelift/src/global_asm.rs
View file

@ -42,7 +42,10 @@ impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> {
impl<'tcx> LayoutOfHelpers<'tcx> for GlobalAsmContext<'_, 'tcx> {
#[inline]
fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
if let LayoutError::SizeOverflow(_)
| LayoutError::InvalidSimd { .. }
| LayoutError::ReferencesError(_) = err
{
self.tcx.sess.dcx().span_fatal(span, err.to_string())
} else {
self.tcx

4
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -165,6 +165,10 @@ impl CodegenBackend for CraneliftCodegenBackend {
""
}
fn name(&self) -> &'static str {
"cranelift"
}
fn init(&self, sess: &Session) {
use rustc_session::config::{InstrumentCoverage, Lto};
match sess.lto() {

6
compiler/rustc_codegen_cranelift/src/unsize.rs
View file

@ -167,7 +167,7 @@ pub(crate) fn size_and_align_of<'tcx>(
if layout.is_sized() {
return (
fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64),
fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64),
fx.bcx.ins().iconst(fx.pointer_type, layout.align.bytes() as i64),
);
}
@ -186,7 +186,7 @@ pub(crate) fn size_and_align_of<'tcx>(
// times the unit size.
(
fx.bcx.ins().imul_imm(info.unwrap(), unit.size.bytes() as i64),
fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
fx.bcx.ins().iconst(fx.pointer_type, unit.align.bytes() as i64),
)
}
ty::Foreign(_) => {
@ -224,7 +224,7 @@ pub(crate) fn size_and_align_of<'tcx>(
let unsized_offset_unadjusted = layout.fields.offset(i).bytes();
let unsized_offset_unadjusted =
fx.bcx.ins().iconst(fx.pointer_type, unsized_offset_unadjusted as i64);
let sized_align = layout.align.abi.bytes();
let sized_align = layout.align.bytes();
let sized_align = fx.bcx.ins().iconst(fx.pointer_type, sized_align as i64);
// Recurse to get the size of the dynamically sized field (must be

8
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -383,7 +383,7 @@ impl<'tcx> CPlace<'tcx> {
let stack_slot = fx.create_stack_slot(
u32::try_from(layout.size.bytes()).unwrap(),
u32::try_from(layout.align.abi.bytes()).unwrap(),
u32::try_from(layout.align.bytes()).unwrap(),
);
CPlace { inner: CPlaceInner::Addr(stack_slot, None), layout }
}
@ -641,8 +641,8 @@ impl<'tcx> CPlace<'tcx> {
let size = dst_layout.size.bytes();
// `emit_small_memory_copy` uses `u8` for alignments, just use the maximum
// alignment that fits in a `u8` if the actual alignment is larger.
let src_align = src_layout.align.abi.bytes().try_into().unwrap_or(128);
let dst_align = dst_layout.align.abi.bytes().try_into().unwrap_or(128);
let src_align = src_layout.align.bytes().try_into().unwrap_or(128);
let dst_align = dst_layout.align.bytes().try_into().unwrap_or(128);
fx.bcx.emit_small_memory_copy(
fx.target_config,
to_addr,
@ -660,7 +660,7 @@ impl<'tcx> CPlace<'tcx> {
}
}
/// Used for `ProjectionElem::Subtype`, `ty` has to be monomorphized before
/// Used for `ProjectionElem::UnwrapUnsafeBinder`, `ty` has to be monomorphized before
/// passed on.
pub(crate) fn place_transmute_type(
self,

16
compiler/rustc_codegen_gcc/src/asm.rs
View file

@ -698,8 +698,12 @@ fn reg_class_to_gcc(reg_class: InlineAsmRegClass) -> &'static str {
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
| InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
InlineAsmRegClass::PowerPC(
PowerPCInlineAsmRegClass::cr
| PowerPCInlineAsmRegClass::ctr
| PowerPCInlineAsmRegClass::lr
| PowerPCInlineAsmRegClass::xer,
) => {
unreachable!("clobber-only")
}
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
@ -777,8 +781,12 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => {
cx.type_vector(cx.type_i32(), 4)
}
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
| InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
InlineAsmRegClass::PowerPC(
PowerPCInlineAsmRegClass::cr
| PowerPCInlineAsmRegClass::ctr
| PowerPCInlineAsmRegClass::lr
| PowerPCInlineAsmRegClass::xer,
) => {
unreachable!("clobber-only")
}
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),

4
compiler/rustc_codegen_gcc/src/base.rs
View file

@ -15,9 +15,9 @@ use rustc_middle::mir::mono::Visibility;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::DebugInfo;
use rustc_span::Symbol;
use rustc_target::spec::RelocModel;
#[cfg(feature = "master")]
use rustc_target::spec::SymbolVisibility;
use rustc_target::spec::{PanicStrategy, RelocModel};
use crate::builder::Builder;
use crate::context::CodegenCx;
@ -101,7 +101,7 @@ pub fn compile_codegen_unit(
// Instantiate monomorphizations without filling out definitions yet...
let context = new_context(tcx);
if tcx.sess.panic_strategy() == PanicStrategy::Unwind {
if tcx.sess.panic_strategy().unwinds() {
context.add_command_line_option("-fexceptions");
context.add_driver_option("-fexceptions");
}

1
compiler/rustc_codegen_gcc/src/builder.rs
View file

@ -1383,6 +1383,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
_src_align: Align,
size: RValue<'gcc>,
flags: MemFlags,
_tt: Option<rustc_ast::expand::typetree::FncTree>, // Autodiff TypeTrees are LLVM-only, ignored in GCC backend
) {
assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
let size = self.intcast(size, self.type_size_t(), false);

15
compiler/rustc_codegen_gcc/src/context.rs
View file

@ -147,7 +147,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
let layout = tcx
.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(rust_type))
.unwrap();
let align = layout.align.abi.bytes();
let align = layout.align.bytes();
// For types with size 1, the alignment can be 1 and only 1
// So, we can skip the call to ``get_aligned`.
// In the future, we can add a GCC API to query the type align,
@ -186,9 +186,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
(i128_type, u128_type)
} else {
/*let layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.i128)).unwrap();
let i128_align = layout.align.abi.bytes();
let i128_align = layout.align.bytes();
let layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.u128)).unwrap();
let u128_align = layout.align.abi.bytes();*/
let u128_align = layout.align.bytes();*/
// TODO(antoyo): re-enable the alignment when libgccjit fixed the issue in
// gcc_jit_context_new_array_constructor (it should not use reinterpret_cast).
@ -529,7 +529,10 @@ impl<'gcc, 'tcx> HasX86AbiOpt for CodegenCx<'gcc, 'tcx> {
impl<'gcc, 'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> {
#[inline]
fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
if let LayoutError::SizeOverflow(_)
| LayoutError::InvalidSimd { .. }
| LayoutError::ReferencesError(_) = err
{
self.tcx.dcx().emit_fatal(respan(span, err.into_diagnostic()))
} else {
self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err })
@ -545,7 +548,9 @@ impl<'gcc, 'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> {
span: Span,
fn_abi_request: FnAbiRequest<'tcx>,
) -> ! {
if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err {
if let FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::InvalidSimd { .. }) =
err
{
self.tcx.dcx().emit_fatal(respan(span, err))
} else {
match fn_abi_request {

13
compiler/rustc_codegen_gcc/src/debuginfo.rs
View file

@ -29,13 +29,24 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
_variable_alloca: Self::Value,
_direct_offset: Size,
_indirect_offsets: &[Size],
_fragment: Option<Range<Size>>,
_fragment: &Option<Range<Size>>,
) {
// FIXME(tempdragon): Not sure if this is correct, probably wrong but still keep it here.
#[cfg(feature = "master")]
_variable_alloca.set_location(_dbg_loc);
}
fn dbg_var_value(
&mut self,
_dbg_var: Self::DIVariable,
_dbg_loc: Self::DILocation,
_value: Self::Value,
_direct_offset: Size,
_indirect_offsets: &[Size],
_fragment: &Option<Range<Size>>,
) {
}
fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) {
// TODO(antoyo): insert reference to gdb debug scripts section global.
}

4
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file

@ -29,7 +29,6 @@ use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::{self, Instance, Ty};
use rustc_span::{Span, Symbol, sym};
use rustc_target::callconv::{ArgAbi, PassMode};
use rustc_target::spec::PanicStrategy;
#[cfg(feature = "master")]
use crate::abi::FnAbiGccExt;
@ -771,6 +770,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
scratch_align,
bx.const_usize(self.layout.size.bytes()),
MemFlags::empty(),
None,
);
bx.lifetime_end(scratch, scratch_size);
@ -1334,7 +1334,7 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>(
_catch_func: RValue<'gcc>,
dest: PlaceRef<'tcx, RValue<'gcc>>,
) {
if bx.sess().panic_strategy() == PanicStrategy::Abort {
if !bx.sess().panic_strategy().unwinds() {
bx.call(bx.type_void(), None, None, try_func, &[data], None, None);
// Return 0 unconditionally from the intrinsic call;
// we can never unwind.

4
compiler/rustc_codegen_gcc/src/lib.rs
View file

@ -184,6 +184,10 @@ impl CodegenBackend for GccCodegenBackend {
crate::DEFAULT_LOCALE_RESOURCE
}
fn name(&self) -> &'static str {
"gcc"
}
fn init(&self, _sess: &Session) {
#[cfg(feature = "master")]
{

87
compiler/rustc_codegen_llvm/src/abi.rs
View file

@ -1,4 +1,3 @@
use std::borrow::Borrow;
use std::cmp;
use libc::c_uint;
@ -13,7 +12,7 @@ use rustc_codegen_ssa::traits::*;
use rustc_middle::ty::Ty;
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::{bug, ty};
use rustc_session::config;
use rustc_session::{Session, config};
use rustc_target::callconv::{
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode,
};
@ -246,6 +245,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
scratch_align,
bx.const_usize(copy_bytes),
MemFlags::empty(),
None,
);
bx.lifetime_end(llscratch, scratch_size);
}
@ -399,7 +399,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
}
fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv {
llvm::CallConv::from_conv(self.conv, cx.tcx.sess.target.arch.borrow())
to_llvm_calling_convention(cx.tcx.sess, self.conv)
}
fn apply_attrs_llfn(
@ -538,7 +538,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
// If the declaration has an associated instance, compute extra attributes based on that.
if let Some(instance) = instance {
llfn_attrs_from_instance(cx, llfn, instance);
llfn_attrs_from_instance(
cx,
cx.tcx,
llfn,
&cx.tcx.codegen_instance_attrs(instance.def),
Some(instance),
);
}
}
@ -656,43 +662,44 @@ impl AbiBuilderMethods for Builder<'_, '_, '_> {
}
}
impl llvm::CallConv {
pub(crate) fn from_conv(conv: CanonAbi, arch: &str) -> Self {
match conv {
CanonAbi::C | CanonAbi::Rust => llvm::CCallConv,
CanonAbi::RustCold => llvm::PreserveMost,
// Functions with this calling convention can only be called from assembly, but it is
// possible to declare an `extern "custom"` block, so the backend still needs a calling
// convention for declaring foreign functions.
CanonAbi::Custom => llvm::CCallConv,
CanonAbi::GpuKernel => {
if arch == "amdgpu" {
llvm::AmdgpuKernel
} else if arch == "nvptx64" {
llvm::PtxKernel
} else {
panic!("Architecture {arch} does not support GpuKernel calling convention");
}
/// Determines the appropriate [`llvm::CallConv`] to use for a given function
/// ABI, for the current target.
pub(crate) fn to_llvm_calling_convention(sess: &Session, abi: CanonAbi) -> llvm::CallConv {
match abi {
CanonAbi::C | CanonAbi::Rust => llvm::CCallConv,
CanonAbi::RustCold => llvm::PreserveMost,
// Functions with this calling convention can only be called from assembly, but it is
// possible to declare an `extern "custom"` block, so the backend still needs a calling
// convention for declaring foreign functions.
CanonAbi::Custom => llvm::CCallConv,
CanonAbi::GpuKernel => {
let arch = sess.target.arch.as_ref();
if arch == "amdgpu" {
llvm::AmdgpuKernel
} else if arch == "nvptx64" {
llvm::PtxKernel
} else {
panic!("Architecture {arch} does not support GpuKernel calling convention");
}
CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind {
InterruptKind::Avr => llvm::AvrInterrupt,
InterruptKind::AvrNonBlocking => llvm::AvrNonBlockingInterrupt,
InterruptKind::Msp430 => llvm::Msp430Intr,
InterruptKind::RiscvMachine | InterruptKind::RiscvSupervisor => llvm::CCallConv,
InterruptKind::X86 => llvm::X86_Intr,
},
CanonAbi::Arm(arm_call) => match arm_call {
ArmCall::Aapcs => llvm::ArmAapcsCallConv,
ArmCall::CCmseNonSecureCall | ArmCall::CCmseNonSecureEntry => llvm::CCallConv,
},
CanonAbi::X86(x86_call) => match x86_call {
X86Call::Fastcall => llvm::X86FastcallCallConv,
X86Call::Stdcall => llvm::X86StdcallCallConv,
X86Call::SysV64 => llvm::X86_64_SysV,
X86Call::Thiscall => llvm::X86_ThisCall,
X86Call::Vectorcall => llvm::X86_VectorCall,
X86Call::Win64 => llvm::X86_64_Win64,
},
}
CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind {
InterruptKind::Avr => llvm::AvrInterrupt,
InterruptKind::AvrNonBlocking => llvm::AvrNonBlockingInterrupt,
InterruptKind::Msp430 => llvm::Msp430Intr,
InterruptKind::RiscvMachine | InterruptKind::RiscvSupervisor => llvm::CCallConv,
InterruptKind::X86 => llvm::X86_Intr,
},
CanonAbi::Arm(arm_call) => match arm_call {
ArmCall::Aapcs => llvm::ArmAapcsCallConv,
ArmCall::CCmseNonSecureCall | ArmCall::CCmseNonSecureEntry => llvm::CCallConv,
},
CanonAbi::X86(x86_call) => match x86_call {
X86Call::Fastcall => llvm::X86FastcallCallConv,
X86Call::Stdcall => llvm::X86StdcallCallConv,
X86Call::SysV64 => llvm::X86_64_SysV,
X86Call::Thiscall => llvm::X86_ThisCall,
X86Call::Vectorcall => llvm::X86_VectorCall,
X86Call::Win64 => llvm::X86_64_Win64,
},
}
}

51
compiler/rustc_codegen_llvm/src/allocator.rs
View file

@ -5,15 +5,17 @@ use rustc_ast::expand::allocator::{
};
use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _;
use rustc_middle::bug;
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
use rustc_middle::ty::TyCtxt;
use rustc_session::config::{DebugInfo, OomStrategy};
use rustc_span::sym;
use rustc_symbol_mangling::mangle_internal_symbol;
use smallvec::SmallVec;
use crate::attributes::llfn_attrs_from_instance;
use crate::builder::SBuilder;
use crate::declare::declare_simple_fn;
use crate::llvm::{self, FALSE, TRUE, Type, Value};
use crate::{SimpleCx, attributes, debuginfo, llvm_util};
use crate::llvm::{self, FALSE, FromGeneric, TRUE, Type, Value};
use crate::{SimpleCx, attributes, debuginfo};
pub(crate) unsafe fn codegen(
tcx: TyCtxt<'_>,
@ -58,7 +60,26 @@ pub(crate) unsafe fn codegen(
let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name));
let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name));
create_wrapper_function(tcx, &cx, &from_name, Some(&to_name), &args, output, false);
let alloc_attr_flag = match method.name {
sym::alloc => CodegenFnAttrFlags::ALLOCATOR,
sym::dealloc => CodegenFnAttrFlags::DEALLOCATOR,
sym::realloc => CodegenFnAttrFlags::REALLOCATOR,
sym::alloc_zeroed => CodegenFnAttrFlags::ALLOCATOR_ZEROED,
_ => unreachable!("Unknown allocator method!"),
};
let mut attrs = CodegenFnAttrs::new();
attrs.flags |= alloc_attr_flag;
create_wrapper_function(
tcx,
&cx,
&from_name,
Some(&to_name),
&args,
output,
false,
&attrs,
);
}
}
@ -71,6 +92,7 @@ pub(crate) unsafe fn codegen(
&[usize, usize], // size, align
None,
true,
&CodegenFnAttrs::new(),
);
unsafe {
@ -92,6 +114,7 @@ pub(crate) unsafe fn codegen(
&[],
None,
false,
&CodegenFnAttrs::new(),
);
}
@ -138,6 +161,7 @@ fn create_wrapper_function(
args: &[&Type],
output: Option<&Type>,
no_return: bool,
attrs: &CodegenFnAttrs,
) {
let ty = cx.type_func(args, output.unwrap_or_else(|| cx.type_void()));
let llfn = declare_simple_fn(
@ -149,18 +173,7 @@ fn create_wrapper_function(
ty,
);
let mut attrs = SmallVec::<[_; 2]>::new();
let target_cpu = llvm_util::target_cpu(tcx.sess);
let target_cpu_attr = llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu);
let tune_cpu_attr = llvm_util::tune_cpu(tcx.sess)
.map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu));
attrs.push(target_cpu_attr);
attrs.extend(tune_cpu_attr);
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs);
llfn_attrs_from_instance(cx, tcx, llfn, attrs, None);
let no_return = if no_return {
// -> ! DIFlagNoReturn
@ -171,12 +184,6 @@ fn create_wrapper_function(
None
};
if tcx.sess.must_emit_unwind_tables() {
let uwtable =
attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind);
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]);
}
let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) };
let mut bx = SBuilder::build(&cx, llbb);

22
compiler/rustc_codegen_llvm/src/asm.rs
View file

@ -340,8 +340,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
} else if options.contains(InlineAsmOptions::NOMEM) {
attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
} else {
// LLVM doesn't have an attribute to represent ReadOnly + SideEffect
} else if options.contains(InlineAsmOptions::READONLY) {
attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx));
}
attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
@ -538,9 +538,7 @@ pub(crate) fn inline_asm_call<'ll>(
bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
)
}));
let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) };
let md = bx.get_metadata_value(md);
llvm::LLVMSetMetadata(call, kind, md);
bx.cx.set_metadata_node(call, kind, &srcloc);
Some(call)
}
@ -662,7 +660,12 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
PowerPC(
PowerPCInlineAsmRegClass::cr
| PowerPCInlineAsmRegClass::ctr
| PowerPCInlineAsmRegClass::lr
| PowerPCInlineAsmRegClass::xer,
) => {
unreachable!("clobber-only")
}
RiscV(RiscVInlineAsmRegClass::reg) => "r",
@ -830,7 +833,12 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => {
PowerPC(
PowerPCInlineAsmRegClass::cr
| PowerPCInlineAsmRegClass::ctr
| PowerPCInlineAsmRegClass::lr
| PowerPCInlineAsmRegClass::xer,
) => {
unreachable!("clobber-only")
}
RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),

196
compiler/rustc_codegen_llvm/src/attributes.rs
View file

@ -1,20 +1,21 @@
//! Set and unset common attributes on LLVM values.
use rustc_codegen_ssa::traits::*;
use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr};
use rustc_hir::def_id::DefId;
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry};
use rustc_middle::middle::codegen_fn_attrs::{
CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry,
};
use rustc_middle::ty::{self, TyCtxt};
use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet};
use rustc_symbol_mangling::mangle_internal_symbol;
use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector};
use smallvec::SmallVec;
use crate::context::CodegenCx;
use crate::context::SimpleCx;
use crate::errors::SanitizerMemtagRequiresMte;
use crate::llvm::AttributePlace::Function;
use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects};
use crate::value::Value;
use crate::{attributes, llvm_util};
use crate::{Session, attributes, llvm_util};
pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) {
if !attrs.is_empty() {
@ -30,18 +31,19 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[
/// Get LLVM attribute for the provided inline heuristic.
pub(crate) fn inline_attr<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
cx: &SimpleCx<'ll>,
tcx: TyCtxt<'tcx>,
instance: ty::Instance<'tcx>,
) -> Option<&'ll Attribute> {
// `optnone` requires `noinline`
let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id());
let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id());
let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) {
(_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never,
(InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint,
(InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint,
(inline, _) => inline,
};
if !cx.tcx.sess.opts.unstable_opts.inline_llvm {
if !tcx.sess.opts.unstable_opts.inline_llvm {
// disable LLVM inlining
return Some(AttributeKind::NoInline.create_attr(cx.llcx));
}
@ -51,7 +53,7 @@ pub(crate) fn inline_attr<'ll, 'tcx>(
Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
}
InlineAttr::Never => {
if cx.sess().target.arch != "amdgpu" {
if tcx.sess.target.arch != "amdgpu" {
Some(AttributeKind::NoInline.create_attr(cx.llcx))
} else {
None
@ -63,12 +65,13 @@ pub(crate) fn inline_attr<'ll, 'tcx>(
#[inline]
fn patchable_function_entry_attrs<'ll>(
cx: &CodegenCx<'ll, '_>,
cx: &SimpleCx<'ll>,
sess: &Session,
attr: Option<PatchableFunctionEntry>,
) -> SmallVec<[&'ll Attribute; 2]> {
let mut attrs = SmallVec::new();
let patchable_spec = attr.unwrap_or_else(|| {
PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry)
PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry)
});
let entry = patchable_spec.entry();
let prefix = patchable_spec.prefix();
@ -91,12 +94,13 @@ fn patchable_function_entry_attrs<'ll>(
/// Get LLVM sanitize attributes.
#[inline]
pub(crate) fn sanitize_attrs<'ll>(
cx: &CodegenCx<'ll, '_>,
pub(crate) fn sanitize_attrs<'ll, 'tcx>(
cx: &SimpleCx<'ll>,
tcx: TyCtxt<'tcx>,
no_sanitize: SanitizerSet,
) -> SmallVec<[&'ll Attribute; 4]> {
let mut attrs = SmallVec::new();
let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize;
let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize;
if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) {
attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx));
}
@ -114,11 +118,11 @@ pub(crate) fn sanitize_attrs<'ll>(
}
if enabled.contains(SanitizerSet::MEMTAG) {
// Check to make sure the mte target feature is actually enabled.
let features = cx.tcx.global_backend_features(());
let features = tcx.global_backend_features(());
let mte_feature =
features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..]));
if let None | Some("-mte") = mte_feature {
cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte);
tcx.dcx().emit_err(SanitizerMemtagRequiresMte);
}
attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx));
@ -139,9 +143,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option<bool>)
llvm::CreateUWTableAttr(llcx, async_unwind)
}
pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
let mut fp = cx.sess().target.frame_pointer;
let opts = &cx.sess().opts;
pub(crate) fn frame_pointer_type_attr<'ll>(
cx: &SimpleCx<'ll>,
sess: &Session,
) -> Option<&'ll Attribute> {
let mut fp = sess.target.frame_pointer;
let opts = &sess.opts;
// "mcount" function relies on stack pointer.
// See <https://sourceware.org/binutils/docs/gprof/Implementation.html>.
if opts.unstable_opts.instrument_mcount {
@ -156,8 +163,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'
Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value))
}
fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
let function_return_attr = match cx.sess().opts.unstable_opts.function_return {
fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> {
let function_return_attr = match sess.opts.unstable_opts.function_return {
FunctionReturn::Keep => return None,
FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern,
};
@ -167,17 +174,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute>
/// Tell LLVM what instrument function to insert.
#[inline]
fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> {
fn instrument_function_attr<'ll>(
cx: &SimpleCx<'ll>,
sess: &Session,
) -> SmallVec<[&'ll Attribute; 4]> {
let mut attrs = SmallVec::new();
if cx.sess().opts.unstable_opts.instrument_mcount {
if sess.opts.unstable_opts.instrument_mcount {
// Similar to `clang -pg` behavior. Handled by the
// `post-inline-ee-instrument` LLVM pass.
// The function name varies on platforms.
// See test/CodeGen/mcount.c in clang.
let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic {
let mcount_name = match &sess.target.llvm_mcount_intrinsic {
Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(),
None => cx.sess().target.mcount.as_ref(),
None => sess.target.mcount.as_ref(),
};
attrs.push(llvm::CreateAttrStringValue(
@ -186,7 +196,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr
mcount_name,
));
}
if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray {
if let Some(options) = &sess.opts.unstable_opts.instrument_xray {
// XRay instrumentation is similar to __cyg_profile_func_{enter,exit}.
// Function prologue and epilogue are instrumented with NOP sleds,
// a runtime library later replaces them with detours into tracing code.
@ -217,20 +227,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr
attrs
}
fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
if !cx.sess().opts.unstable_opts.no_jump_tables {
fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> {
if !sess.opts.unstable_opts.no_jump_tables {
return None;
}
Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true"))
}
fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> {
// Currently stack probes seem somewhat incompatible with the address
// sanitizer and thread sanitizer. With asan we're already protected from
// stack overflow anyway so we don't really need stack probes regardless.
if cx
.sess()
if tcx
.sess
.opts
.unstable_opts
.sanitizer
@ -240,22 +250,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
}
// probestack doesn't play nice either with `-C profile-generate`.
if cx.sess().opts.cg.profile_generate.enabled() {
if tcx.sess.opts.cg.profile_generate.enabled() {
return None;
}
let attr_value = match cx.sess().target.stack_probes {
let attr_value = match tcx.sess.target.stack_probes {
StackProbeType::None => return None,
// Request LLVM to generate the probes inline. If the given LLVM version does not support
// this, no probe is generated at all (even if the attribute is specified).
StackProbeType::Inline => "inline-asm",
// Flag our internal `__rust_probestack` function as the stack probe symbol.
// This is defined in the `compiler-builtins` crate for each architecture.
StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"),
StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"),
// Pick from the two above based on the LLVM version.
StackProbeType::InlineOrCall { min_llvm_version_for_inline } => {
if llvm_util::get_version() < min_llvm_version_for_inline {
&mangle_internal_symbol(cx.tcx, "__rust_probestack")
&mangle_internal_symbol(tcx, "__rust_probestack")
} else {
"inline-asm"
}
@ -264,8 +274,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value))
}
fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
let sspattr = match cx.sess().stack_protector() {
fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> {
let sspattr = match sess.stack_protector() {
StackProtector::None => return None,
StackProtector::All => AttributeKind::StackProtectReq,
StackProtector::Strong => AttributeKind::StackProtectStrong,
@ -275,33 +285,34 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
Some(sspattr.create_attr(cx.llcx))
}
fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
if cx.sess().target.arch != "s390x" {
fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> {
if sess.target.arch != "s390x" {
return None;
}
let requested_features = cx.sess().opts.cg.target_feature.split(',');
let requested_features = sess.opts.cg.target_feature.split(',');
let found_positive = requested_features.clone().any(|r| r == "+backchain");
if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None }
}
pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute {
let target_cpu = llvm_util::target_cpu(cx.tcx.sess);
pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute {
let target_cpu = llvm_util::target_cpu(sess);
llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu)
}
pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
llvm_util::tune_cpu(cx.tcx.sess)
pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> {
llvm_util::tune_cpu(sess)
.map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu))
}
/// Get the `target-features` LLVM attribute.
pub(crate) fn target_features_attr<'ll>(
cx: &CodegenCx<'ll, '_>,
pub(crate) fn target_features_attr<'ll, 'tcx>(
cx: &SimpleCx<'ll>,
tcx: TyCtxt<'tcx>,
function_features: Vec<String>,
) -> Option<&'ll Attribute> {
let global_features = cx.tcx.global_backend_features(()).iter().map(String::as_str);
let global_features = tcx.global_backend_features(()).iter().map(String::as_str);
let function_features = function_features.iter().map(String::as_str);
let target_features =
global_features.chain(function_features).intersperse(",").collect::<String>();
@ -311,22 +322,22 @@ pub(crate) fn target_features_attr<'ll>(
/// Get the `NonLazyBind` LLVM attribute,
/// if the codegen options allow skipping the PLT.
pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
pub(crate) fn non_lazy_bind_attr<'ll>(
cx: &SimpleCx<'ll>,
sess: &Session,
) -> Option<&'ll Attribute> {
// Don't generate calls through PLT if it's not necessary
if !cx.sess().needs_plt() {
Some(AttributeKind::NonLazyBind.create_attr(cx.llcx))
} else {
None
}
if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None }
}
/// Get the default optimizations attrs for a function.
#[inline]
pub(crate) fn default_optimisation_attrs<'ll>(
cx: &CodegenCx<'ll, '_>,
cx: &SimpleCx<'ll>,
sess: &Session,
) -> SmallVec<[&'ll Attribute; 2]> {
let mut attrs = SmallVec::new();
match cx.sess().opts.optimize {
match sess.opts.optimize {
OptLevel::Size => {
attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx));
}
@ -347,17 +358,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute {
/// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`)
/// attributes.
pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
cx: &SimpleCx<'ll>,
tcx: TyCtxt<'tcx>,
llfn: &'ll Value,
instance: ty::Instance<'tcx>,
codegen_fn_attrs: &CodegenFnAttrs,
instance: Option<ty::Instance<'tcx>>,
) {
let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def);
let sess = tcx.sess;
let mut to_add = SmallVec::<[_; 16]>::new();
match codegen_fn_attrs.optimize {
OptimizeAttr::Default => {
to_add.extend(default_optimisation_attrs(cx));
to_add.extend(default_optimisation_attrs(cx, sess));
}
OptimizeAttr::DoNotOptimize => {
to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx));
@ -369,21 +381,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
OptimizeAttr::Speed => {}
}
if cx.sess().must_emit_unwind_tables() {
to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind));
if sess.must_emit_unwind_tables() {
to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind));
}
if cx.sess().opts.unstable_opts.profile_sample_use.is_some() {
if sess.opts.unstable_opts.profile_sample_use.is_some() {
to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile"));
}
// FIXME: none of these functions interact with source level attributes.
to_add.extend(frame_pointer_type_attr(cx));
to_add.extend(function_return_attr(cx));
to_add.extend(instrument_function_attr(cx));
to_add.extend(nojumptables_attr(cx));
to_add.extend(probestack_attr(cx));
to_add.extend(stackprotector_attr(cx));
to_add.extend(frame_pointer_type_attr(cx, sess));
to_add.extend(function_return_attr(cx, sess));
to_add.extend(instrument_function_attr(cx, sess));
to_add.extend(nojumptables_attr(cx, sess));
to_add.extend(probestack_attr(cx, tcx));
to_add.extend(stackprotector_attr(cx, sess));
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) {
to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins"));
@ -404,16 +416,19 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
// not used.
} else {
// Do not set sanitizer attributes for naked functions.
to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize));
to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize));
// For non-naked functions, set branch protection attributes on aarch64.
if let Some(BranchProtection { bti, pac_ret }) =
cx.sess().opts.unstable_opts.branch_protection
if let Some(BranchProtection { bti, pac_ret, gcs }) =
sess.opts.unstable_opts.branch_protection
{
assert!(cx.sess().target.arch == "aarch64");
assert!(sess.target.arch == "aarch64");
if bti {
to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement"));
}
if gcs {
to_add.push(llvm::CreateAttrString(cx.llcx, "guarded-control-stack"));
}
if let Some(PacRet { leaf, pc, key }) = pac_ret {
if pc {
to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr"));
@ -435,14 +450,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
|| codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED)
{
to_add.push(create_alloc_family_attr(cx.llcx));
if let Some(zv) =
cx.tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant)
if let Some(instance) = instance
&& let Some(zv) =
tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant)
&& let Some(name) = zv.value_str()
{
to_add.push(llvm::CreateAttrStringValue(
cx.llcx,
"alloc-variant-zeroed",
&mangle_internal_symbol(cx.tcx, name.as_str()),
&mangle_internal_symbol(tcx, name.as_str()),
));
}
// apply to argument place instead of function
@ -487,18 +503,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
if let Some(align) = codegen_fn_attrs.alignment {
llvm::set_alignment(llfn, align);
}
if let Some(backchain) = backchain_attr(cx) {
if let Some(backchain) = backchain_attr(cx, sess) {
to_add.push(backchain);
}
to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry));
to_add.extend(patchable_function_entry_attrs(
cx,
sess,
codegen_fn_attrs.patchable_function_entry,
));
// Always annotate functions with the target-cpu they are compiled for.
// Without this, ThinLTO won't inline Rust functions into Clang generated
// functions (because Clang annotates functions this way too).
to_add.push(target_cpu_attr(cx));
to_add.push(target_cpu_attr(cx, sess));
// tune-cpu is only conveyed through the attribute for our purpose.
// The target doesn't care; the subtarget reads our attribute.
to_add.extend(tune_cpu_attr(cx));
to_add.extend(tune_cpu_attr(cx, sess));
let function_features =
codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::<Vec<&str>>();
@ -506,7 +526,9 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
// Apply function attributes as per usual if there are no user defined
// target features otherwise this will get applied at the callsite.
if function_features.is_empty() {
if let Some(inline_attr) = inline_attr(cx, instance) {
if let Some(instance) = instance
&& let Some(inline_attr) = inline_attr(cx, tcx, instance)
{
to_add.push(inline_attr);
}
}
@ -514,7 +536,7 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
let function_features = function_features
.iter()
// Convert to LLVMFeatures and filter out unavailable ones
.flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat))
.flat_map(|feat| llvm_util::to_llvm_features(sess, feat))
// Convert LLVMFeatures & dependencies to +<feats>s
.flat_map(|feat| feat.into_iter().map(|f| format!("+{f}")))
.chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
@ -523,20 +545,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
}))
.collect::<Vec<String>>();
if cx.tcx.sess.target.is_like_wasm {
if sess.target.is_like_wasm {
// If this function is an import from the environment but the wasm
// import has a specific module/name, apply them here.
if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) {
if let Some(instance) = instance
&& let Some(module) = wasm_import_module(tcx, instance.def_id())
{
to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module));
let name =
codegen_fn_attrs.symbol_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id()));
codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id()));
let name = name.as_str();
to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name));
}
}
to_add.extend(target_features_attr(cx, function_features));
to_add.extend(target_features_attr(cx, tcx, function_features));
attributes::apply_to_llfn(llfn, Function, &to_add);
}

2
compiler/rustc_codegen_llvm/src/back/lto.rs
View file

@ -563,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) {
config::AutoDiff::Enable => {}
// We handle this below
config::AutoDiff::NoPostopt => {}
// Disables TypeTree generation
config::AutoDiff::NoTT => {}
}
}
// This helps with handling enums for now.

4
compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
View file

@ -95,8 +95,6 @@ impl Drop for OwnedTargetMachine {
// SAFETY: constructing ensures we have a valid pointer created by
// llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no
// double free or use after free.
unsafe {
llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_ptr());
}
unsafe { llvm::LLVMDisposeTargetMachine(self.tm_unique) };
}
}

14
compiler/rustc_codegen_llvm/src/back/write.rs
View file

@ -44,7 +44,7 @@ use crate::errors::{
};
use crate::llvm::diagnostic::OptimizationDiagnosticKind::*;
use crate::llvm::{self, DiagnosticInfo};
use crate::type_::Type;
use crate::type_::llvm_type_ptr;
use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util};
pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! {
@ -204,6 +204,9 @@ pub(crate) fn target_machine_factory(
optlvl: config::OptLevel,
target_features: &[String],
) -> TargetMachineFactoryFn<LlvmCodegenBackend> {
// Self-profile timer for creating a _factory_.
let _prof_timer = sess.prof.generic_activity("target_machine_factory");
let reloc_model = to_llvm_relocation_model(sess.relocation_model());
let (opt_level, _) = to_llvm_opt_settings(optlvl);
@ -259,6 +262,9 @@ pub(crate) fn target_machine_factory(
.into_string()
.unwrap_or_default();
let command_line_args = quote_command_line_args(&sess.expanded_args);
// Self-profile counter for the number of bytes produced by command-line quoting.
// Values are summed, so the summary result is cumulative across all TM factories.
sess.prof.artifact_size("quoted_command_line_args", "-", command_line_args.len() as u64);
let debuginfo_compression = sess.opts.debuginfo_compression.to_string();
match sess.opts.debuginfo_compression {
@ -281,7 +287,11 @@ pub(crate) fn target_machine_factory(
let use_wasm_eh = wants_wasm_eh(sess);
let prof = SelfProfilerRef::clone(&sess.prof);
Arc::new(move |config: TargetMachineFactoryConfig| {
// Self-profile timer for invoking a factory to create a target machine.
let _prof_timer = prof.generic_activity("target_machine_factory_inner");
let path_to_cstring_helper = |path: Option<PathBuf>| -> CString {
let path = path.unwrap_or_default();
let path = path_mapping
@ -1150,7 +1160,7 @@ fn create_msvc_imps(
// underscores added in front).
let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" };
let ptr_ty = Type::ptr_llcx(llcx);
let ptr_ty = llvm_type_ptr(llcx);
let globals = base::iter_globals(llmod)
.filter(|&val| {
llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val)

2
compiler/rustc_codegen_llvm/src/base.rs
View file

@ -105,7 +105,7 @@ pub(crate) fn compile_codegen_unit(
if let Some(entry) =
maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit)
{
let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty());
let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty());
attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
}

74
compiler/rustc_codegen_llvm/src/builder.rs
View file

@ -1,11 +1,12 @@
use std::borrow::{Borrow, Cow};
use std::iter;
use std::ops::Deref;
use std::{iter, ptr};
use rustc_ast::expand::typetree::FncTree;
pub(crate) mod autodiff;
pub(crate) mod gpu_offload;
use libc::{c_char, c_uint, size_t};
use libc::{c_char, c_uint};
use rustc_abi as abi;
use rustc_abi::{Align, Size, WrappingRange};
use rustc_codegen_ssa::MemFlags;
@ -35,7 +36,8 @@ use crate::attributes;
use crate::common::Funclet;
use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
use crate::llvm::{
self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, GEPNoWrapFlags, Metadata, TRUE, ToLlvmBool,
self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
ToLlvmBool,
};
use crate::type_::Type;
use crate::type_of::LayoutLlvmExt;
@ -395,10 +397,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
md.push(weight(is_cold));
}
unsafe {
let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
self.cx.set_metadata(switch, llvm::MD_prof, md_node);
}
self.cx.set_metadata_node(switch, llvm::MD_prof, &md);
}
fn invoke(
@ -800,22 +799,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
return;
}
unsafe {
let llty = self.cx.val_ty(load);
let md = [
llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
];
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
self.set_metadata(load, llvm::MD_range, md);
}
let llty = self.cx.val_ty(load);
let md = [
llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
];
self.set_metadata_node(load, llvm::MD_range, &md);
}
fn nonnull_metadata(&mut self, load: &'ll Value) {
unsafe {
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
self.set_metadata(load, llvm::MD_nonnull, md);
}
self.set_metadata_node(load, llvm::MD_nonnull, &[]);
}
fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
@ -864,8 +857,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
//
// [1]: https://llvm.org/docs/LangRef.html#store-instruction
let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
self.set_metadata(store, llvm::MD_nontemporal, md);
self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
}
}
store
@ -1107,11 +1099,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
src_align: Align,
size: &'ll Value,
flags: MemFlags,
tt: Option<FncTree>,
) {
assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
let size = self.intcast(size, self.type_isize(), false);
let is_volatile = flags.contains(MemFlags::VOLATILE);
unsafe {
let memcpy = unsafe {
llvm::LLVMRustBuildMemCpy(
self.llbuilder,
dst,
@ -1120,7 +1113,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
src_align.bytes() as c_uint,
size,
is_volatile,
);
)
};
// TypeTree metadata for memcpy is especially important: when Enzyme encounters
// a memcpy during autodiff, it needs to know the structure of the data being
// copied to properly track derivatives. For example, copying an array of floats
// vs. copying a struct with mixed types requires different derivative handling.
// The TypeTree tells Enzyme exactly what memory layout to expect.
if let Some(tt) = tt {
crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt);
}
}
@ -1370,10 +1372,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
}
fn set_invariant_load(&mut self, load: &'ll Value) {
unsafe {
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
self.set_metadata(load, llvm::MD_invariant_load, md);
}
self.set_metadata_node(load, llvm::MD_invariant_load, &[]);
}
fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
@ -1433,7 +1432,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
// If there is an inline attribute and a target feature that matches
// we will add the attribute to the callsite otherwise we'll omit
// this and not add the attribute to prevent soundness issues.
&& let Some(inlining_rule) = attributes::inline_attr(&self.cx, instance)
&& let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance)
&& self.cx.tcx.is_target_feature_call_safe(
&fn_call_attrs.target_features,
&fn_defn_attrs.target_features,
@ -1517,25 +1516,16 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
}
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
fn align_metadata(&mut self, load: &'ll Value, align: Align) {
unsafe {
let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
self.set_metadata(load, llvm::MD_align, md);
}
let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
self.set_metadata_node(load, llvm::MD_align, &md);
}
fn noundef_metadata(&mut self, load: &'ll Value) {
unsafe {
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
self.set_metadata(load, llvm::MD_noundef, md);
}
self.set_metadata_node(load, llvm::MD_noundef, &[]);
}
pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
unsafe {
let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
self.set_metadata(inst, llvm::MD_unpredictable, md);
}
self.set_metadata_node(inst, llvm::MD_unpredictable, &[]);
}
}
impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {

15
compiler/rustc_codegen_llvm/src/builder/autodiff.rs
View file

@ -1,6 +1,7 @@
use std::ptr;
use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode};
use rustc_ast::expand::typetree::FncTree;
use rustc_codegen_ssa::common::TypeKind;
use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods};
use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv};
@ -294,6 +295,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>(
fn_args: &[&'ll Value],
attrs: AutoDiffAttrs,
dest: PlaceRef<'tcx, &'ll Value>,
fnc_tree: FncTree,
) {
// We have to pick the name depending on whether we want forward or reverse mode autodiff.
let mut ad_name: String = match attrs.mode {
@ -370,7 +372,18 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>(
fn_args,
);
if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() {
crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree);
}
let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None);
builder.store_to_place(call, dest.val);
let fn_ret_ty = builder.cx.val_ty(call);
if fn_ret_ty != builder.cx.type_void() && fn_ret_ty != builder.cx.type_struct(&[], false) {
// If we return void or an empty struct, then our caller (due to how we generated it)
// does not expect a return value. As such, we have no pointer (or place) into which
// we could store our value, and would store into an undef, which would cause UB.
// As such, we just ignore the return value in those cases.
builder.store_to_place(call, dest.val);
}
}

15
compiler/rustc_codegen_llvm/src/consts.rs
View file

@ -494,16 +494,7 @@ impl<'ll> CodegenCx<'ll, '_> {
let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len());
let alloc = self.create_metadata(bytes);
let data = [section, alloc];
let meta =
unsafe { llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()) };
let val = self.get_metadata_value(meta);
unsafe {
llvm::LLVMAddNamedMetadataOperand(
self.llmod,
c"wasm.custom_sections".as_ptr(),
val,
)
};
self.module_add_named_metadata_node(self.llmod(), c"wasm.custom_sections", &data);
}
} else {
base::set_link_section(g, attrs);
@ -564,7 +555,7 @@ impl<'ll> CodegenCx<'ll, '_> {
let g = self.define_global(&sym, llty).unwrap_or_else(|| {
bug!("symbol `{}` is already defined", sym);
});
set_global_alignment(self, g, self.tcx.data_layout.i8_align.abi);
set_global_alignment(self, g, self.tcx.data_layout.i8_align);
llvm::set_initializer(g, llval);
llvm::set_linkage(g, llvm::Linkage::PrivateLinkage);
llvm::set_section(g, c"__TEXT,__cstring,cstring_literals");
@ -680,7 +671,7 @@ impl<'ll> CodegenCx<'ll, '_> {
let methname_g = self.define_global(&methname_sym, methname_llty).unwrap_or_else(|| {
bug!("symbol `{}` is already defined", methname_sym);
});
set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align.abi);
set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align);
llvm::set_initializer(methname_g, methname_llval);
llvm::set_linkage(methname_g, llvm::Linkage::PrivateLinkage);
llvm::set_section(

113
compiler/rustc_codegen_llvm/src/context.rs
View file

@ -31,10 +31,11 @@ use rustc_symbol_mangling::mangle_internal_symbol;
use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel};
use smallvec::SmallVec;
use crate::abi::to_llvm_calling_convention;
use crate::back::write::to_llvm_code_model;
use crate::callee::get_fn;
use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
use crate::llvm::Metadata;
use crate::llvm::{Metadata, MetadataKindId, Module};
use crate::type_::Type;
use crate::value::Value;
use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util};
@ -370,7 +371,8 @@ pub(crate) unsafe fn create_module<'ll>(
);
}
if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection {
if let Some(BranchProtection { bti, pac_ret, gcs }) = sess.opts.unstable_opts.branch_protection
{
if sess.target.arch == "aarch64" {
llvm::add_module_flag_u32(
llmod,
@ -403,6 +405,12 @@ pub(crate) unsafe fn create_module<'ll>(
"sign-return-address-with-bkey",
u32::from(pac_opts.key == PAuthKey::B),
);
llvm::add_module_flag_u32(
llmod,
llvm::ModuleFlagMergeBehavior::Min,
"guarded-control-stack",
gcs.into(),
);
} else {
bug!(
"branch-protection used on non-AArch64 target; \
@ -488,14 +496,7 @@ pub(crate) unsafe fn create_module<'ll>(
format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"));
let name_metadata = cx.create_metadata(rustc_producer.as_bytes());
unsafe {
llvm::LLVMAddNamedMetadataOperand(
llmod,
c"llvm.ident".as_ptr(),
&cx.get_metadata_value(llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)),
);
}
cx.module_add_named_metadata_node(llmod, c"llvm.ident", &[name_metadata]);
// Emit RISC-V specific target-abi metadata
// to workaround lld as the LTO plugin not
@ -740,7 +741,7 @@ impl<'ll> SimpleCx<'ll> {
llcx: &'ll llvm::Context,
pointer_size: Size,
) -> Self {
let isize_ty = llvm::Type::ix_llcx(llcx, pointer_size.bits());
let isize_ty = llvm::LLVMIntTypeInContext(llcx, pointer_size.bits() as c_uint);
Self(SCx { llmod, llcx, isize_ty }, PhantomData)
}
}
@ -869,7 +870,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
} else {
let fty = self.type_variadic_func(&[], self.type_i32());
let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty);
let target_cpu = attributes::target_cpu_attr(self);
let target_cpu = attributes::target_cpu_attr(self, self.sess());
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]);
llfn
}
@ -884,15 +885,15 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
}
fn set_frame_pointer_type(&self, llfn: &'ll Value) {
if let Some(attr) = attributes::frame_pointer_type_attr(self) {
if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) {
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]);
}
}
fn apply_target_cpu_attr(&self, llfn: &'ll Value) {
let mut attrs = SmallVec::<[_; 2]>::new();
attrs.push(attributes::target_cpu_attr(self));
attrs.extend(attributes::tune_cpu_attr(self));
attrs.push(attributes::target_cpu_attr(self, self.sess()));
attrs.extend(attributes::tune_cpu_attr(self, self.sess()));
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs);
}
@ -901,17 +902,14 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
if self.get_declared_value(entry_name).is_none() {
let llfn = self.declare_entry_fn(
entry_name,
llvm::CallConv::from_conv(
self.sess().target.entry_abi,
self.sess().target.arch.borrow(),
),
to_llvm_calling_convention(self.sess(), self.sess().target.entry_abi),
llvm::UnnamedAddr::Global,
fn_type,
);
attributes::apply_to_llfn(
llfn,
llvm::AttributePlace::Function,
attributes::target_features_attr(self, vec![]).as_slice(),
attributes::target_features_attr(self, self.tcx, vec![]).as_slice(),
);
Some(llfn)
} else {
@ -995,15 +993,75 @@ impl CodegenCx<'_, '_> {
}
impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
/// Wrapper for `LLVMMDNodeInContext2`, i.e. `llvm::MDNode::get`.
pub(crate) fn md_node_in_context(&self, md_list: &[&'ll Metadata]) -> &'ll Metadata {
unsafe { llvm::LLVMMDNodeInContext2(self.llcx(), md_list.as_ptr(), md_list.len()) }
}
/// A wrapper for [`llvm::LLVMSetMetadata`], but it takes `Metadata` as a parameter instead of `Value`.
pub(crate) fn set_metadata<'a>(
&self,
val: &'a Value,
kind_id: impl Into<llvm::MetadataKindId>,
kind_id: MetadataKindId,
md: &'ll Metadata,
) {
let node = self.get_metadata_value(md);
llvm::LLVMSetMetadata(val, kind_id.into(), node);
llvm::LLVMSetMetadata(val, kind_id, node);
}
/// Helper method for the sequence of calls:
/// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata)
/// - `LLVMMetadataAsValue` (to adapt that node to an `llvm::Value`)
/// - `LLVMSetMetadata` (to set that node as metadata of `kind_id` for `instruction`)
pub(crate) fn set_metadata_node(
&self,
instruction: &'ll Value,
kind_id: MetadataKindId,
md_list: &[&'ll Metadata],
) {
let md = self.md_node_in_context(md_list);
self.set_metadata(instruction, kind_id, md);
}
/// Helper method for the sequence of calls:
/// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata)
/// - `LLVMMetadataAsValue` (to adapt that node to an `llvm::Value`)
/// - `LLVMAddNamedMetadataOperand` (to set that node as metadata of `kind_name` for `module`)
pub(crate) fn module_add_named_metadata_node(
&self,
module: &'ll Module,
kind_name: &CStr,
md_list: &[&'ll Metadata],
) {
let md = self.md_node_in_context(md_list);
let md_as_val = self.get_metadata_value(md);
unsafe { llvm::LLVMAddNamedMetadataOperand(module, kind_name.as_ptr(), md_as_val) };
}
/// Helper method for the sequence of calls:
/// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata)
/// - `LLVMRustGlobalAddMetadata` (to set that node as metadata of `kind_id` for `global`)
pub(crate) fn global_add_metadata_node(
&self,
global: &'ll Value,
kind_id: MetadataKindId,
md_list: &[&'ll Metadata],
) {
let md = self.md_node_in_context(md_list);
unsafe { llvm::LLVMRustGlobalAddMetadata(global, kind_id, md) };
}
/// Helper method for the sequence of calls:
/// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata)
/// - `LLVMGlobalSetMetadata` (to set that node as metadata of `kind_id` for `global`)
pub(crate) fn global_set_metadata_node(
&self,
global: &'ll Value,
kind_id: MetadataKindId,
md_list: &[&'ll Metadata],
) {
let md = self.md_node_in_context(md_list);
unsafe { llvm::LLVMGlobalSetMetadata(global, kind_id, md) };
}
}
@ -1037,7 +1095,10 @@ impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> {
impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
#[inline]
fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
if let LayoutError::SizeOverflow(_)
| LayoutError::ReferencesError(_)
| LayoutError::InvalidSimd { .. } = err
{
self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() })
} else {
self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err })
@ -1054,7 +1115,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
fn_abi_request: FnAbiRequest<'tcx>,
) -> ! {
match err {
FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => {
FnAbiError::Layout(
LayoutError::SizeOverflow(_)
| LayoutError::Cycle(_)
| LayoutError::InvalidSimd { .. },
) => {
self.tcx.dcx().emit_fatal(Spanned { span, node: err });
}
_ => match fn_abi_request {

45
compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
View file

@ -1,5 +1,3 @@
use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId};
/// Must match the layout of `LLVMRustCounterKind`.
#[derive(Copy, Clone, Debug)]
#[repr(C)]
@ -26,30 +24,12 @@ pub(crate) enum CounterKind {
pub(crate) struct Counter {
// Important: The layout (order and types of fields) must match its C++ counterpart.
pub(crate) kind: CounterKind,
id: u32,
pub(crate) id: u32,
}
impl Counter {
/// A `Counter` of kind `Zero`. For this counter kind, the `id` is not used.
pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 };
/// Constructs a new `Counter` of kind `CounterValueReference`.
pub(crate) fn counter_value_reference(counter_id: CounterId) -> Self {
Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() }
}
/// Constructs a new `Counter` of kind `Expression`.
pub(crate) fn expression(expression_id: ExpressionId) -> Self {
Self { kind: CounterKind::Expression, id: expression_id.as_u32() }
}
pub(crate) fn from_term(term: CovTerm) -> Self {
match term {
CovTerm::Zero => Self::ZERO,
CovTerm::Counter(id) => Self::counter_value_reference(id),
CovTerm::Expression(id) => Self::expression(id),
}
}
}
/// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`.
@ -94,29 +74,6 @@ pub(crate) struct CoverageSpan {
pub(crate) end_col: u32,
}
/// Holds tables of the various region types in one struct.
///
/// Don't pass this struct across FFI; pass the individual region tables as
/// pointer/length pairs instead.
///
/// Each field name has a `_regions` suffix for improved readability after
/// exhaustive destructing, which ensures that all region types are handled.
#[derive(Clone, Debug, Default)]
pub(crate) struct Regions {
pub(crate) code_regions: Vec<CodeRegion>,
pub(crate) expansion_regions: Vec<ExpansionRegion>,
pub(crate) branch_regions: Vec<BranchRegion>,
}
impl Regions {
/// Returns true if none of this structure's tables contain any regions.
pub(crate) fn has_no_regions(&self) -> bool {
let Self { code_regions, expansion_regions, branch_regions } = self;
code_regions.is_empty() && expansion_regions.is_empty() && branch_regions.is_empty()
}
}
/// Must match the layout of `LLVMRustCoverageCodeRegion`.
#[derive(Clone, Debug)]
#[repr(C)]

42
compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs
View file

@ -2,26 +2,25 @@
use std::ffi::CString;
use crate::common::AsCCharPtr;
use crate::coverageinfo::ffi;
use crate::llvm;
pub(crate) fn covmap_var_name() -> CString {
CString::new(llvm::build_byte_buffer(|s| unsafe {
CString::new(llvm::build_byte_buffer(|s| {
llvm::LLVMRustCoverageWriteCovmapVarNameToString(s);
}))
.expect("covmap variable name should not contain NUL")
}
pub(crate) fn covmap_section_name(llmod: &llvm::Module) -> CString {
CString::new(llvm::build_byte_buffer(|s| unsafe {
CString::new(llvm::build_byte_buffer(|s| {
llvm::LLVMRustCoverageWriteCovmapSectionNameToString(llmod, s);
}))
.expect("covmap section name should not contain NUL")
}
pub(crate) fn covfun_section_name(llmod: &llvm::Module) -> CString {
CString::new(llvm::build_byte_buffer(|s| unsafe {
CString::new(llvm::build_byte_buffer(|s| {
llvm::LLVMRustCoverageWriteCovfunSectionNameToString(llmod, s);
}))
.expect("covfun section name should not contain NUL")
@ -34,7 +33,7 @@ pub(crate) fn create_pgo_func_name_var<'ll>(
unsafe {
llvm::LLVMRustCoverageCreatePGOFuncNameVar(
llfn,
mangled_fn_name.as_c_char_ptr(),
mangled_fn_name.as_ptr(),
mangled_fn_name.len(),
)
}
@ -44,7 +43,7 @@ pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef<str>]) -> Vec<u8
let (pointers, lengths) = filenames
.into_iter()
.map(AsRef::as_ref)
.map(|s: &str| (s.as_c_char_ptr(), s.len()))
.map(|s: &str| (s.as_ptr(), s.len()))
.unzip::<_, _, Vec<_>, Vec<_>>();
llvm::build_byte_buffer(|buffer| unsafe {
@ -58,12 +57,35 @@ pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef<str>]) -> Vec<u8
})
}
/// Holds tables of the various region types in one struct.
///
/// Don't pass this struct across FFI; pass the individual region tables as
/// pointer/length pairs instead.
///
/// Each field name has a `_regions` suffix for improved readability after
/// exhaustive destructing, which ensures that all region types are handled.
#[derive(Clone, Debug, Default)]
pub(crate) struct Regions {
pub(crate) code_regions: Vec<ffi::CodeRegion>,
pub(crate) expansion_regions: Vec<ffi::ExpansionRegion>,
pub(crate) branch_regions: Vec<ffi::BranchRegion>,
}
impl Regions {
/// Returns true if none of this structure's tables contain any regions.
pub(crate) fn has_no_regions(&self) -> bool {
let Self { code_regions, expansion_regions, branch_regions } = self;
code_regions.is_empty() && expansion_regions.is_empty() && branch_regions.is_empty()
}
}
pub(crate) fn write_function_mappings_to_buffer(
virtual_file_mapping: &[u32],
expressions: &[ffi::CounterExpression],
regions: &ffi::Regions,
regions: &Regions,
) -> Vec<u8> {
let ffi::Regions { code_regions, expansion_regions, branch_regions } = regions;
let Regions { code_regions, expansion_regions, branch_regions } = regions;
// SAFETY:
// - All types are FFI-compatible and have matching representations in Rust/C++.
@ -89,12 +111,12 @@ pub(crate) fn write_function_mappings_to_buffer(
/// Hashes some bytes into a 64-bit hash, via LLVM's `IndexedInstrProf::ComputeHash`,
/// as required for parts of the LLVM coverage mapping format.
pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 {
unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_c_char_ptr(), bytes.len()) }
unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_ptr(), bytes.len()) }
}
/// Returns LLVM's `coverage::CovMapVersion::CurrentVersion` (CoverageMapping.h)
/// as a raw numeric value. For historical reasons, the numeric value is 1 less
/// than the number in the version's name, so `Version7` is actually `6u32`.
pub(crate) fn mapping_version() -> u32 {
unsafe { llvm::LLVMRustCoverageMappingVersion() }
llvm::LLVMRustCoverageMappingVersion()
}

27
compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs
View file

@ -10,8 +10,8 @@ use std::sync::Arc;
use rustc_abi::Align;
use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods as _, ConstCodegenMethods};
use rustc_middle::mir::coverage::{
BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping,
MappingKind, Op,
BasicCoverageBlock, CounterId, CovTerm, CoverageIdsInfo, Expression, ExpressionId,
FunctionCoverageInfo, Mapping, MappingKind, Op,
};
use rustc_middle::ty::{Instance, TyCtxt};
use rustc_span::{SourceFile, Span};
@ -36,7 +36,7 @@ pub(crate) struct CovfunRecord<'tcx> {
virtual_file_mapping: VirtualFileMapping,
expressions: Vec<ffi::CounterExpression>,
regions: ffi::Regions,
regions: llvm_cov::Regions,
}
impl<'tcx> CovfunRecord<'tcx> {
@ -64,7 +64,7 @@ pub(crate) fn prepare_covfun_record<'tcx>(
is_used,
virtual_file_mapping: VirtualFileMapping::default(),
expressions,
regions: ffi::Regions::default(),
regions: llvm_cov::Regions::default(),
};
fill_region_tables(tcx, fn_cov_info, ids_info, &mut covfun);
@ -77,10 +77,21 @@ pub(crate) fn prepare_covfun_record<'tcx>(
Some(covfun)
}
pub(crate) fn counter_for_term(term: CovTerm) -> ffi::Counter {
use ffi::Counter;
match term {
CovTerm::Zero => Counter::ZERO,
CovTerm::Counter(id) => {
Counter { kind: ffi::CounterKind::CounterValueReference, id: CounterId::as_u32(id) }
}
CovTerm::Expression(id) => {
Counter { kind: ffi::CounterKind::Expression, id: ExpressionId::as_u32(id) }
}
}
}
/// Convert the function's coverage-counter expressions into a form suitable for FFI.
fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec<ffi::CounterExpression> {
let counter_for_term = ffi::Counter::from_term;
// We know that LLVM will optimize out any unused expressions before
// producing the final coverage map, so there's no need to do the same
// thing on the Rust side unless we're confident we can do much better.
@ -113,7 +124,7 @@ fn fill_region_tables<'tcx>(
} else {
CovTerm::Zero
};
ffi::Counter::from_term(term)
counter_for_term(term)
};
// Currently a function's mappings must all be in the same file, so use the
@ -136,7 +147,7 @@ fn fill_region_tables<'tcx>(
if discard_all { None } else { spans::make_coords(source_map, &source_file, span) }
};
let ffi::Regions {
let llvm_cov::Regions {
code_regions,
expansion_regions: _, // FIXME(Zalathar): Fill out support for expansion regions
branch_regions,

3
compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs
View file

@ -35,3 +35,6 @@ declare_constant!(DW_OP_plus_uconst: u64);
/// Double-checked by a static assertion in `RustWrapper.cpp`.
#[allow(non_upper_case_globals)]
pub(crate) const DW_OP_LLVM_fragment: u64 = 0x1000;
// It describes the actual value of a source variable which might not exist in registers or in memory.
#[allow(non_upper_case_globals)]
pub(crate) const DW_OP_stack_value: u64 = 0x9f;

2
compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
View file

@ -2,9 +2,9 @@
use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive;
use rustc_codegen_ssa::traits::*;
use rustc_hir::attrs::DebuggerVisualizerType;
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_middle::bug;
use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType;
use rustc_session::config::{CrateType, DebugInfo};
use crate::builder::Builder;

36
compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
View file

@ -37,11 +37,11 @@ use crate::common::{AsCCharPtr, CodegenCx};
use crate::debuginfo::dwarf_const;
use crate::debuginfo::metadata::type_map::build_type_with_children;
use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind};
use crate::llvm;
use crate::llvm::debuginfo::{
DIBasicType, DIBuilder, DICompositeType, DIDescriptor, DIFile, DIFlags, DILexicalBlock,
DIScope, DIType, DebugEmissionKind, DebugNameTableKind,
};
use crate::llvm::{self, FromGeneric};
use crate::value::Value;
impl PartialEq for llvm::Metadata {
@ -117,7 +117,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>(
.try_to_target_usize(cx.tcx)
.expect("expected monomorphic const in codegen") as c_longlong;
let subrange = unsafe { llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) };
let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) };
let subscripts = &[subrange];
let di_node = unsafe {
@ -477,8 +477,8 @@ pub(crate) fn spanned_type_di_node<'ll, 'tcx>(
ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id),
ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id),
ty::Adt(def, ..) => match def.adt_kind() {
AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id),
AdtKind::Union => build_union_type_di_node(cx, unique_type_id),
AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id, span),
AdtKind::Union => build_union_type_di_node(cx, unique_type_id, span),
AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span),
},
ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id),
@ -1043,7 +1043,7 @@ fn create_member_type<'ll, 'tcx>(
file_metadata,
line_number,
layout.size.bits(),
layout.align.abi.bits() as u32,
layout.align.bits() as u32,
offset.bits(),
flags,
type_di_node,
@ -1076,6 +1076,7 @@ fn visibility_di_flags<'ll, 'tcx>(
fn build_struct_type_di_node<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
unique_type_id: UniqueTypeId<'tcx>,
span: Span,
) -> DINodeCreationResult<'ll> {
let struct_type = unique_type_id.expect_ty();
let ty::Adt(adt_def, _) = struct_type.kind() else {
@ -1083,7 +1084,7 @@ fn build_struct_type_di_node<'ll, 'tcx>(
};
assert!(adt_def.is_struct());
let containing_scope = get_namespace_for_item(cx, adt_def.did());
let struct_type_and_layout = cx.layout_of(struct_type);
let struct_type_and_layout = cx.spanned_layout_of(struct_type, span);
let variant_def = adt_def.non_enum_variant();
let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
Some(file_metadata_from_def_id(cx, Some(adt_def.did())))
@ -1276,6 +1277,7 @@ fn build_closure_env_di_node<'ll, 'tcx>(
fn build_union_type_di_node<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
unique_type_id: UniqueTypeId<'tcx>,
span: Span,
) -> DINodeCreationResult<'ll> {
let union_type = unique_type_id.expect_ty();
let (union_def_id, variant_def) = match union_type.kind() {
@ -1283,7 +1285,7 @@ fn build_union_type_di_node<'ll, 'tcx>(
_ => bug!("build_union_type_di_node on a non-ADT"),
};
let containing_scope = get_namespace_for_item(cx, union_def_id);
let union_ty_and_layout = cx.layout_of(union_type);
let union_ty_and_layout = cx.spanned_layout_of(union_type, span);
let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false);
let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers {
Some(file_metadata_from_def_id(cx, Some(union_def_id)))
@ -1605,21 +1607,11 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>(
let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref);
let typeid = cx.create_metadata(trait_ref_typeid.as_bytes());
unsafe {
let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid];
llvm::LLVMRustGlobalAddMetadata(
vtable,
llvm::MD_type as c_uint,
llvm::LLVMMDNodeInContext2(cx.llcx, v.as_ptr(), v.len()),
);
let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64));
let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1);
llvm::LLVMGlobalSetMetadata(
vtable,
llvm::MetadataType::MD_vcall_visibility as c_uint,
vcall_visibility_metadata,
);
}
let type_ = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid];
cx.global_add_metadata_node(vtable, llvm::MD_type, &type_);
let vcall_visibility = [llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64))];
cx.global_set_metadata_node(vtable, llvm::MD_vcall_visibility, &vcall_visibility);
}
/// Creates debug information for the given vtable, which is for the

4
compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
View file

@ -289,7 +289,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>(
file_metadata,
line_number,
enum_type_and_layout.size.bits(),
enum_type_and_layout.align.abi.bits() as u32,
enum_type_and_layout.align.bits() as u32,
DIFlags::FlagZero,
tag_member_di_node,
create_DIArray(DIB(cx), &[]),
@ -449,7 +449,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>(
file_di_node,
line_number,
enum_type_and_layout.size.bits(),
enum_type_and_layout.align.abi.bits() as u32,
enum_type_and_layout.align.bits() as u32,
Size::ZERO.bits(),
discr,
DIFlags::FlagZero,

124
compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
View file

@ -52,15 +52,6 @@ mod utils;
use self::create_scope_map::compute_mir_scopes;
pub(crate) use self::metadata::build_global_var_di_node;
// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were
// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp`
// to decide which C++ API to call. Instead, we should just have two separate
// FFI functions and choose the correct one on the Rust side.
#[allow(non_upper_case_globals)]
const DW_TAG_auto_variable: c_uint = 0x100;
#[allow(non_upper_case_globals)]
const DW_TAG_arg_variable: c_uint = 0x101;
/// A context object for maintaining all state needed by the debuginfo module.
pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> {
llmod: &'ll llvm::Module,
@ -165,16 +156,66 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
variable_alloca: Self::Value,
direct_offset: Size,
indirect_offsets: &[Size],
fragment: Option<Range<Size>>,
fragment: &Option<Range<Size>>,
) {
use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst};
// Convert the direct and indirect offsets and fragment byte range to address ops.
let mut addr_ops = SmallVec::<[u64; 8]>::new();
if direct_offset.bytes() > 0 {
addr_ops.push(DW_OP_plus_uconst);
addr_ops.push(direct_offset.bytes());
}
for &offset in indirect_offsets {
addr_ops.push(DW_OP_deref);
if offset.bytes() > 0 {
addr_ops.push(DW_OP_plus_uconst);
addr_ops.push(offset.bytes());
}
}
if let Some(fragment) = fragment {
// `DW_OP_LLVM_fragment` takes as arguments the fragment's
// offset and size, both of them in bits.
addr_ops.push(DW_OP_LLVM_fragment);
addr_ops.push(fragment.start.bits());
addr_ops.push((fragment.end - fragment.start).bits());
}
let di_builder = DIB(self.cx());
let addr_expr = unsafe {
llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len())
};
unsafe {
llvm::LLVMDIBuilderInsertDeclareRecordAtEnd(
di_builder,
variable_alloca,
dbg_var,
addr_expr,
dbg_loc,
self.llbb(),
)
};
}
fn dbg_var_value(
&mut self,
dbg_var: &'ll DIVariable,
dbg_loc: &'ll DILocation,
value: Self::Value,
direct_offset: Size,
indirect_offsets: &[Size],
fragment: &Option<Range<Size>>,
) {
use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst, DW_OP_stack_value};
// Convert the direct and indirect offsets and fragment byte range to address ops.
let mut addr_ops = SmallVec::<[u64; 8]>::new();
if direct_offset.bytes() > 0 {
addr_ops.push(DW_OP_plus_uconst);
addr_ops.push(direct_offset.bytes() as u64);
addr_ops.push(DW_OP_stack_value);
}
for &offset in indirect_offsets {
addr_ops.push(DW_OP_deref);
@ -191,14 +232,16 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
addr_ops.push((fragment.end - fragment.start).bits() as u64);
}
let di_builder = DIB(self.cx());
let addr_expr = unsafe {
llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len())
};
unsafe {
// FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`.
llvm::LLVMRustDIBuilderInsertDeclareAtEnd(
DIB(self.cx()),
variable_alloca,
llvm::LLVMDIBuilderInsertDbgValueRecordAtEnd(
di_builder,
value,
dbg_var,
addr_ops.as_ptr(),
addr_ops.len() as c_uint,
addr_expr,
dbg_loc,
self.llbb(),
);
@ -630,28 +673,39 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
let type_metadata = spanned_type_di_node(self, variable_type, span);
let (argument_index, dwarf_tag) = match variable_kind {
ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable),
LocalVariable => (0, DW_TAG_auto_variable),
};
let align = self.align_of(variable_type);
let name = variable_name.as_str();
unsafe {
llvm::LLVMRustDIBuilderCreateVariable(
DIB(self),
dwarf_tag,
scope_metadata,
name.as_c_char_ptr(),
name.len(),
file_metadata,
loc.line,
type_metadata,
true,
DIFlags::FlagZero,
argument_index,
align.bits() as u32,
)
match variable_kind {
ArgumentVariable(arg_index) => unsafe {
llvm::LLVMDIBuilderCreateParameterVariable(
DIB(self),
scope_metadata,
name.as_ptr(),
name.len(),
arg_index as c_uint,
file_metadata,
loc.line,
type_metadata,
llvm::Bool::TRUE, // (preserve descriptor during optimizations)
DIFlags::FlagZero,
)
},
LocalVariable => unsafe {
llvm::LLVMDIBuilderCreateAutoVariable(
DIB(self),
scope_metadata,
name.as_ptr(),
name.len(),
file_metadata,
loc.line,
type_metadata,
llvm::Bool::TRUE, // (preserve descriptor during optimizations)
DIFlags::FlagZero,
align.bits() as u32,
)
},
}
}
}

2
compiler/rustc_codegen_llvm/src/debuginfo/utils.rs
View file

@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>(
builder: &DIBuilder<'ll>,
arr: &[Option<&'ll DIDescriptor>],
) -> &'ll DIArray {
unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) }
unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) }
}
#[inline]

4
compiler/rustc_codegen_llvm/src/declare.rs
View file

@ -26,7 +26,7 @@ use crate::abi::FnAbiLlvmExt;
use crate::common::AsCCharPtr;
use crate::context::{CodegenCx, GenericCx, SCx, SimpleCx};
use crate::llvm::AttributePlace::Function;
use crate::llvm::Visibility;
use crate::llvm::{FromGeneric, Visibility};
use crate::type_::Type;
use crate::value::Value;
use crate::{attributes, llvm};
@ -76,7 +76,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>(
attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx));
}
attrs.extend(attributes::non_lazy_bind_attr(cx));
attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess));
attributes::apply_to_llfn(llfn, Function, &attrs);

11
compiler/rustc_codegen_llvm/src/intrinsic.rs
View file

@ -18,7 +18,6 @@ use rustc_middle::{bug, span_bug};
use rustc_span::{Span, Symbol, sym};
use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate};
use rustc_target::callconv::PassMode;
use rustc_target::spec::PanicStrategy;
use tracing::debug;
use crate::abi::FnAbiLlvmExt;
@ -298,7 +297,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
let align = if name == sym::unaligned_volatile_load {
1
} else {
result.layout.align.abi.bytes() as u32
result.layout.align.bytes() as u32
};
unsafe {
llvm::LLVMSetAlignment(load, align);
@ -674,7 +673,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>(
catch_func: &'ll Value,
dest: PlaceRef<'tcx, &'ll Value>,
) {
if bx.sess().panic_strategy() == PanicStrategy::Abort {
if !bx.sess().panic_strategy().unwinds() {
let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
bx.call(try_func_ty, None, None, try_func, &[data], None, None);
// Return 0 unconditionally from the intrinsic call;
@ -1048,7 +1047,7 @@ fn codegen_emcc_try<'ll, 'tcx>(
// create an alloca and pass a pointer to that.
let ptr_size = bx.tcx().data_layout.pointer_size();
let ptr_align = bx.tcx().data_layout.pointer_align().abi;
let i8_align = bx.tcx().data_layout.i8_align.abi;
let i8_align = bx.tcx().data_layout.i8_align;
// Required in order for there to be no padding between the fields.
assert!(i8_align <= ptr_align);
let catch_data = bx.alloca(2 * ptr_size, ptr_align);
@ -1213,6 +1212,9 @@ fn codegen_autodiff<'ll, 'tcx>(
&mut diff_attrs.input_activity,
);
let fnc_tree =
rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized()));
// Build body
generate_enzyme_call(
bx,
@ -1223,6 +1225,7 @@ fn codegen_autodiff<'ll, 'tcx>(
&val_arr,
diff_attrs.clone(),
result,
fnc_tree,
);
}

14
compiler/rustc_codegen_llvm/src/lib.rs
View file

@ -68,6 +68,7 @@ mod llvm_util;
mod mono_item;
mod type_;
mod type_of;
mod typetree;
mod va_arg;
mod value;
@ -231,6 +232,10 @@ impl CodegenBackend for LlvmCodegenBackend {
crate::DEFAULT_LOCALE_RESOURCE
}
fn name(&self) -> &'static str {
"llvm"
}
fn init(&self, sess: &Session) {
llvm_util::init(sess); // Make sure llvm is inited
}
@ -349,7 +354,14 @@ impl CodegenBackend for LlvmCodegenBackend {
// Run the linker on any artifacts that resulted from the LLVM run.
// This should produce either a finished executable or library.
link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, metadata, outputs);
link_binary(
sess,
&LlvmArchiveBuilderBuilder,
codegen_results,
metadata,
outputs,
self.name(),
);
}
}

115
compiler/rustc_codegen_llvm/src/llvm/conversions.rs Normal file
View file

@ -0,0 +1,115 @@
//! Conversions from backend-independent data types to/from LLVM FFI types.
use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate};
use rustc_middle::ty::AtomicOrdering;
use rustc_session::config::DebugInfo;
use rustc_target::spec::SymbolVisibility;
use crate::llvm;
/// Helper trait for converting backend-independent types to LLVM-specific
/// types, for FFI purposes.
pub(crate) trait FromGeneric<T> {
fn from_generic(other: T) -> Self;
}
impl FromGeneric<SymbolVisibility> for llvm::Visibility {
fn from_generic(visibility: SymbolVisibility) -> Self {
match visibility {
SymbolVisibility::Hidden => Self::Hidden,
SymbolVisibility::Protected => Self::Protected,
SymbolVisibility::Interposable => Self::Default,
}
}
}
impl FromGeneric<IntPredicate> for llvm::IntPredicate {
fn from_generic(int_pred: IntPredicate) -> Self {
match int_pred {
IntPredicate::IntEQ => Self::IntEQ,
IntPredicate::IntNE => Self::IntNE,
IntPredicate::IntUGT => Self::IntUGT,
IntPredicate::IntUGE => Self::IntUGE,
IntPredicate::IntULT => Self::IntULT,
IntPredicate::IntULE => Self::IntULE,
IntPredicate::IntSGT => Self::IntSGT,
IntPredicate::IntSGE => Self::IntSGE,
IntPredicate::IntSLT => Self::IntSLT,
IntPredicate::IntSLE => Self::IntSLE,
}
}
}
impl FromGeneric<RealPredicate> for llvm::RealPredicate {
fn from_generic(real_pred: RealPredicate) -> Self {
match real_pred {
RealPredicate::RealPredicateFalse => Self::RealPredicateFalse,
RealPredicate::RealOEQ => Self::RealOEQ,
RealPredicate::RealOGT => Self::RealOGT,
RealPredicate::RealOGE => Self::RealOGE,
RealPredicate::RealOLT => Self::RealOLT,
RealPredicate::RealOLE => Self::RealOLE,
RealPredicate::RealONE => Self::RealONE,
RealPredicate::RealORD => Self::RealORD,
RealPredicate::RealUNO => Self::RealUNO,
RealPredicate::RealUEQ => Self::RealUEQ,
RealPredicate::RealUGT => Self::RealUGT,
RealPredicate::RealUGE => Self::RealUGE,
RealPredicate::RealULT => Self::RealULT,
RealPredicate::RealULE => Self::RealULE,
RealPredicate::RealUNE => Self::RealUNE,
RealPredicate::RealPredicateTrue => Self::RealPredicateTrue,
}
}
}
impl FromGeneric<AtomicRmwBinOp> for llvm::AtomicRmwBinOp {
fn from_generic(op: AtomicRmwBinOp) -> Self {
match op {
AtomicRmwBinOp::AtomicXchg => Self::AtomicXchg,
AtomicRmwBinOp::AtomicAdd => Self::AtomicAdd,
AtomicRmwBinOp::AtomicSub => Self::AtomicSub,
AtomicRmwBinOp::AtomicAnd => Self::AtomicAnd,
AtomicRmwBinOp::AtomicNand => Self::AtomicNand,
AtomicRmwBinOp::AtomicOr => Self::AtomicOr,
AtomicRmwBinOp::AtomicXor => Self::AtomicXor,
AtomicRmwBinOp::AtomicMax => Self::AtomicMax,
AtomicRmwBinOp::AtomicMin => Self::AtomicMin,
AtomicRmwBinOp::AtomicUMax => Self::AtomicUMax,
AtomicRmwBinOp::AtomicUMin => Self::AtomicUMin,
}
}
}
impl FromGeneric<AtomicOrdering> for llvm::AtomicOrdering {
fn from_generic(ordering: AtomicOrdering) -> Self {
match ordering {
AtomicOrdering::Relaxed => Self::Monotonic,
AtomicOrdering::Acquire => Self::Acquire,
AtomicOrdering::Release => Self::Release,
AtomicOrdering::AcqRel => Self::AcquireRelease,
AtomicOrdering::SeqCst => Self::SequentiallyConsistent,
}
}
}
impl FromGeneric<DebugInfo> for llvm::debuginfo::DebugEmissionKind {
fn from_generic(kind: DebugInfo) -> Self {
// We should be setting LLVM's emission kind to `LineTablesOnly` if
// we are compiling with "limited" debuginfo. However, some of the
// existing tools relied on slightly more debuginfo being generated than
// would be the case with `LineTablesOnly`, and we did not want to break
// these tools in a "drive-by fix", without a good idea or plan about
// what limited debuginfo should exactly look like. So for now we are
// instead adding a new debuginfo option "line-tables-only" so as to
// not break anything and to allow users to have 'limited' debug info.
//
// See https://github.com/rust-lang/rust/issues/60020 for details.
match kind {
DebugInfo::None => Self::NoDebug,
DebugInfo::LineDirectivesOnly => Self::DebugDirectivesOnly,
DebugInfo::LineTablesOnly => Self::LineTablesOnly,
DebugInfo::Limited | DebugInfo::Full => Self::FullDebug,
}
}
}

206
compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
View file

@ -3,9 +3,36 @@
use libc::{c_char, c_uint};
use super::MetadataKindId;
use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value};
use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value};
use crate::llvm::{Bool, Builder};
// TypeTree types
pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree;
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub(crate) struct EnzymeTypeTree {
_unused: [u8; 0],
}
#[repr(u32)]
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
#[allow(non_camel_case_types)]
pub(crate) enum CConcreteType {
DT_Anything = 0,
DT_Integer = 1,
DT_Pointer = 2,
DT_Half = 3,
DT_Float = 4,
DT_Double = 5,
DT_Unknown = 6,
DT_FP128 = 9,
}
pub(crate) struct TypeTree {
pub(crate) inner: CTypeTreeRef,
}
#[link(name = "llvm-wrapper", kind = "static")]
unsafe extern "C" {
// Enzyme
@ -68,10 +95,40 @@ pub(crate) mod Enzyme_AD {
use libc::c_void;
use super::{CConcreteType, CTypeTreeRef, Context};
unsafe extern "C" {
pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8);
pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char);
}
// TypeTree functions
unsafe extern "C" {
pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef;
pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef;
pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef;
pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef);
pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool;
pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64);
pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef);
pub(crate) fn EnzymeTypeTreeShiftIndiciesEq(
arg1: CTypeTreeRef,
data_layout: *const c_char,
offset: i64,
max_size: i64,
add_offset: u64,
);
pub(crate) fn EnzymeTypeTreeInsertEq(
CTT: CTypeTreeRef,
indices: *const i64,
len: usize,
ct: CConcreteType,
ctx: &Context,
);
pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char;
pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char);
}
unsafe extern "C" {
static mut EnzymePrintPerf: c_void;
static mut EnzymePrintActivity: c_void;
@ -141,6 +198,67 @@ pub(crate) use self::Fallback_AD::*;
pub(crate) mod Fallback_AD {
#![allow(unused_variables)]
use libc::c_char;
use super::{CConcreteType, CTypeTreeRef, Context};
// TypeTree function fallbacks
pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef {
unimplemented!()
}
pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef {
unimplemented!()
}
pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef {
unimplemented!()
}
pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq(
arg1: CTypeTreeRef,
data_layout: *const c_char,
offset: i64,
max_size: i64,
add_offset: u64,
) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeInsertEq(
CTT: CTypeTreeRef,
indices: *const i64,
len: usize,
ct: CConcreteType,
ctx: &Context,
) {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char {
unimplemented!()
}
pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) {
unimplemented!()
}
pub(crate) fn set_inline(val: bool) {
unimplemented!()
}
@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD {
unimplemented!()
}
}
impl TypeTree {
pub(crate) fn new() -> TypeTree {
let inner = unsafe { EnzymeNewTypeTree() };
TypeTree { inner }
}
pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree {
let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) };
TypeTree { inner }
}
pub(crate) fn merge(self, other: Self) -> Self {
unsafe {
EnzymeMergeTypeTree(self.inner, other.inner);
}
drop(other);
self
}
#[must_use]
pub(crate) fn shift(
self,
layout: &str,
offset: isize,
max_size: isize,
add_offset: usize,
) -> Self {
let layout = std::ffi::CString::new(layout).unwrap();
unsafe {
EnzymeTypeTreeShiftIndiciesEq(
self.inner,
layout.as_ptr(),
offset as i64,
max_size as i64,
add_offset as u64,
);
}
self
}
pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) {
unsafe {
EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx);
}
}
}
impl Clone for TypeTree {
fn clone(&self) -> Self {
let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) };
TypeTree { inner }
}
}
impl std::fmt::Display for TypeTree {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let ptr = unsafe { EnzymeTypeTreeToString(self.inner) };
let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) };
match cstr.to_str() {
Ok(x) => write!(f, "{}", x)?,
Err(err) => write!(f, "could not parse: {}", err)?,
}
// delete C string pointer
unsafe {
EnzymeTypeTreeToStringFree(ptr);
}
Ok(())
}
}
impl std::fmt::Debug for TypeTree {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
<Self as std::fmt::Display>::fmt(self, f)
}
}
impl Drop for TypeTree {
fn drop(&mut self) {
unsafe { EnzymeFreeTypeTree(self.inner) }
}
}

293
compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file

@ -20,15 +20,15 @@ use std::ptr;
use bitflags::bitflags;
use libc::{c_char, c_int, c_uchar, c_uint, c_ulonglong, c_void, size_t};
use rustc_macros::TryFromU32;
use rustc_target::spec::SymbolVisibility;
use super::RustString;
use super::debuginfo::{
DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags,
DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, DISubrange,
DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind,
DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram,
DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind,
};
use crate::llvm;
use crate::llvm::MetadataKindId;
/// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`,
/// which has a different ABI from Rust or C++ `bool`.
@ -219,16 +219,6 @@ pub(crate) enum Visibility {
Protected = 2,
}
impl Visibility {
pub(crate) fn from_generic(visibility: SymbolVisibility) -> Self {
match visibility {
SymbolVisibility::Hidden => Visibility::Hidden,
SymbolVisibility::Protected => Visibility::Protected,
SymbolVisibility::Interposable => Visibility::Default,
}
}
}
/// LLVMUnnamedAddr
#[repr(C)]
pub(crate) enum UnnamedAddr {
@ -318,24 +308,6 @@ pub(crate) enum IntPredicate {
IntSLE = 41,
}
impl IntPredicate {
pub(crate) fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self {
use rustc_codegen_ssa::common::IntPredicate as Common;
match intpre {
Common::IntEQ => Self::IntEQ,
Common::IntNE => Self::IntNE,
Common::IntUGT => Self::IntUGT,
Common::IntUGE => Self::IntUGE,
Common::IntULT => Self::IntULT,
Common::IntULE => Self::IntULE,
Common::IntSGT => Self::IntSGT,
Common::IntSGE => Self::IntSGE,
Common::IntSLT => Self::IntSLT,
Common::IntSLE => Self::IntSLE,
}
}
}
/// LLVMRealPredicate
#[derive(Copy, Clone)]
#[repr(C)]
@ -358,30 +330,6 @@ pub(crate) enum RealPredicate {
RealPredicateTrue = 15,
}
impl RealPredicate {
pub(crate) fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self {
use rustc_codegen_ssa::common::RealPredicate as Common;
match realp {
Common::RealPredicateFalse => Self::RealPredicateFalse,
Common::RealOEQ => Self::RealOEQ,
Common::RealOGT => Self::RealOGT,
Common::RealOGE => Self::RealOGE,
Common::RealOLT => Self::RealOLT,
Common::RealOLE => Self::RealOLE,
Common::RealONE => Self::RealONE,
Common::RealORD => Self::RealORD,
Common::RealUNO => Self::RealUNO,
Common::RealUEQ => Self::RealUEQ,
Common::RealUGT => Self::RealUGT,
Common::RealUGE => Self::RealUGE,
Common::RealULT => Self::RealULT,
Common::RealULE => Self::RealULE,
Common::RealUNE => Self::RealUNE,
Common::RealPredicateTrue => Self::RealPredicateTrue,
}
}
}
/// Must match the layout of `LLVMTypeKind`.
///
/// Use [`RawEnum<TypeKind>`] for values of `LLVMTypeKind` returned from LLVM,
@ -457,25 +405,6 @@ pub(crate) enum AtomicRmwBinOp {
AtomicUMin = 10,
}
impl AtomicRmwBinOp {
pub(crate) fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self {
use rustc_codegen_ssa::common::AtomicRmwBinOp as Common;
match op {
Common::AtomicXchg => Self::AtomicXchg,
Common::AtomicAdd => Self::AtomicAdd,
Common::AtomicSub => Self::AtomicSub,
Common::AtomicAnd => Self::AtomicAnd,
Common::AtomicNand => Self::AtomicNand,
Common::AtomicOr => Self::AtomicOr,
Common::AtomicXor => Self::AtomicXor,
Common::AtomicMax => Self::AtomicMax,
Common::AtomicMin => Self::AtomicMin,
Common::AtomicUMax => Self::AtomicUMax,
Common::AtomicUMin => Self::AtomicUMin,
}
}
}
/// LLVMAtomicOrdering
#[derive(Copy, Clone)]
#[repr(C)]
@ -492,19 +421,6 @@ pub(crate) enum AtomicOrdering {
SequentiallyConsistent = 7,
}
impl AtomicOrdering {
pub(crate) fn from_generic(ao: rustc_middle::ty::AtomicOrdering) -> Self {
use rustc_middle::ty::AtomicOrdering as Common;
match ao {
Common::Relaxed => Self::Monotonic,
Common::Acquire => Self::Acquire,
Common::Release => Self::Release,
Common::AcqRel => Self::AcquireRelease,
Common::SeqCst => Self::SequentiallyConsistent,
}
}
}
/// LLVMRustFileType
#[derive(Copy, Clone)]
#[repr(C)]
@ -513,31 +429,6 @@ pub(crate) enum FileType {
ObjectFile,
}
/// LLVMMetadataType
#[derive(Copy, Clone)]
#[repr(C)]
#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")]
pub(crate) enum MetadataType {
MD_dbg = 0,
MD_tbaa = 1,
MD_prof = 2,
MD_fpmath = 3,
MD_range = 4,
MD_tbaa_struct = 5,
MD_invariant_load = 6,
MD_alias_scope = 7,
MD_noalias = 8,
MD_nontemporal = 9,
MD_mem_parallel_loop_access = 10,
MD_nonnull = 11,
MD_unpredictable = 15,
MD_align = 17,
MD_type = 19,
MD_vcall_visibility = 28,
MD_noundef = 29,
MD_kcfi_type = 36,
}
/// Must match the layout of `LLVMInlineAsmDialect`.
#[derive(Copy, Clone, PartialEq)]
#[repr(C)]
@ -710,6 +601,7 @@ pub(crate) enum MemoryEffects {
None,
ReadOnly,
InaccessibleMemOnly,
ReadOnlyNotPure,
}
/// LLVMOpcode
@ -806,6 +698,8 @@ unsafe extern "C" {
pub(crate) type Metadata;
pub(crate) type BasicBlock;
pub(crate) type Comdat;
/// `&'ll DbgRecord` represents `LLVMDbgRecordRef`.
pub(crate) type DbgRecord;
}
#[repr(C)]
pub(crate) struct Builder<'a>(InvariantOpaque<'a>);
@ -890,7 +784,6 @@ pub(crate) mod debuginfo {
pub(crate) type DIVariable = DIDescriptor;
pub(crate) type DIGlobalVariableExpression = DIDescriptor;
pub(crate) type DIArray = DIDescriptor;
pub(crate) type DISubrange = DIDescriptor;
pub(crate) type DIEnumerator = DIDescriptor;
pub(crate) type DITemplateTypeParameter = DIDescriptor;
@ -962,28 +855,6 @@ pub(crate) mod debuginfo {
DebugDirectivesOnly,
}
impl DebugEmissionKind {
pub(crate) fn from_generic(kind: rustc_session::config::DebugInfo) -> Self {
// We should be setting LLVM's emission kind to `LineTablesOnly` if
// we are compiling with "limited" debuginfo. However, some of the
// existing tools relied on slightly more debuginfo being generated than
// would be the case with `LineTablesOnly`, and we did not want to break
// these tools in a "drive-by fix", without a good idea or plan about
// what limited debuginfo should exactly look like. So for now we are
// instead adding a new debuginfo option "line-tables-only" so as to
// not break anything and to allow users to have 'limited' debug info.
//
// See https://github.com/rust-lang/rust/issues/60020 for details.
use rustc_session::config::DebugInfo;
match kind {
DebugInfo::None => DebugEmissionKind::NoDebug,
DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly,
DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly,
DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug,
}
}
}
/// LLVMRustDebugNameTableKind
#[derive(Clone, Copy)]
#[repr(C)]
@ -1033,16 +904,6 @@ pub(crate) type GetSymbolsCallback =
unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void;
pub(crate) type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void;
#[derive(Copy, Clone)]
#[repr(transparent)]
pub(crate) struct MetadataKindId(c_uint);
impl From<MetadataType> for MetadataKindId {
fn from(value: MetadataType) -> Self {
Self(value as c_uint)
}
}
unsafe extern "C" {
// Create and destroy contexts.
pub(crate) fn LLVMContextDispose(C: &'static mut Context);
@ -1052,6 +913,8 @@ unsafe extern "C" {
SLen: c_uint,
) -> MetadataKindId;
pub(crate) fn LLVMDisposeTargetMachine(T: ptr::NonNull<TargetMachine>);
// Create modules.
pub(crate) fn LLVMModuleCreateWithNameInContext(
ModuleID: *const c_char,
@ -1091,7 +954,7 @@ unsafe extern "C" {
pub(crate) fn LLVMInt16TypeInContext(C: &Context) -> &Type;
pub(crate) fn LLVMInt32TypeInContext(C: &Context) -> &Type;
pub(crate) fn LLVMInt64TypeInContext(C: &Context) -> &Type;
pub(crate) fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type;
pub(crate) safe fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type;
pub(crate) fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint;
@ -1120,7 +983,7 @@ unsafe extern "C" {
) -> &'a Type;
// Operations on array, pointer, and vector types (sequence types)
pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
pub(crate) safe fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type;
@ -1135,7 +998,11 @@ unsafe extern "C" {
pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t);
pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value);
pub(crate) safe fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: MetadataKindId, Node: &'a Value);
pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
pub(crate) fn LLVMGlobalSetMetadata<'a>(
Val: &'a Value,
KindID: MetadataKindId,
Metadata: &'a Metadata,
);
pub(crate) safe fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;
// Operations on constants of any type
@ -1989,6 +1856,68 @@ unsafe extern "C" {
Scope: Option<&'ll Metadata>,
AlignInBits: u32, // (optional; default is 0)
) -> &'ll Metadata;
pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>(
Builder: &DIBuilder<'ll>,
LowerBound: i64,
Count: i64,
) -> &'ll Metadata;
pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>(
Builder: &DIBuilder<'ll>,
Data: *const Option<&'ll Metadata>,
NumElements: size_t,
) -> &'ll Metadata;
pub(crate) fn LLVMDIBuilderCreateExpression<'ll>(
Builder: &DIBuilder<'ll>,
Addr: *const u64,
Length: size_t,
) -> &'ll Metadata;
pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>(
Builder: &DIBuilder<'ll>,
Storage: &'ll Value,
VarInfo: &'ll Metadata,
Expr: &'ll Metadata,
DebugLoc: &'ll Metadata,
Block: &'ll BasicBlock,
) -> &'ll DbgRecord;
pub(crate) fn LLVMDIBuilderInsertDbgValueRecordAtEnd<'ll>(
Builder: &DIBuilder<'ll>,
Val: &'ll Value,
VarInfo: &'ll Metadata,
Expr: &'ll Metadata,
DebugLoc: &'ll Metadata,
Block: &'ll BasicBlock,
) -> &'ll DbgRecord;
pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>(
Builder: &DIBuilder<'ll>,
Scope: &'ll Metadata,
Name: *const c_uchar, // See "PTR_LEN_STR".
NameLen: size_t,
File: &'ll Metadata,
LineNo: c_uint,
Ty: &'ll Metadata,
AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations."
Flags: DIFlags,
AlignInBits: u32,
) -> &'ll Metadata;
pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>(
Builder: &DIBuilder<'ll>,
Scope: &'ll Metadata,
Name: *const c_uchar, // See "PTR_LEN_STR".
NameLen: size_t,
ArgNo: c_uint,
File: &'ll Metadata,
LineNo: c_uint,
Ty: &'ll Metadata,
AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations."
Flags: DIFlags,
) -> &'ll Metadata;
}
#[link(name = "llvm-wrapper", kind = "static")]
@ -2002,7 +1931,7 @@ unsafe extern "C" {
// Operations on all values
pub(crate) fn LLVMRustGlobalAddMetadata<'a>(
Val: &'a Value,
KindID: c_uint,
KindID: MetadataKindId,
Metadata: &'a Metadata,
);
pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool;
@ -2199,8 +2128,11 @@ unsafe extern "C" {
ConstraintsLen: size_t,
) -> bool;
/// A list of pointer-length strings is passed as two pointer-length slices,
/// one slice containing pointers and one slice containing their corresponding
/// lengths. The implementation will check that both slices have the same length.
pub(crate) fn LLVMRustCoverageWriteFilenamesToBuffer(
Filenames: *const *const c_char,
Filenames: *const *const c_uchar, // See "PTR_LEN_STR".
FilenamesLen: size_t,
Lengths: *const size_t,
LengthsLen: size_t,
@ -2223,18 +2155,25 @@ unsafe extern "C" {
pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar(
F: &Value,
FuncName: *const c_char,
FuncName: *const c_uchar, // See "PTR_LEN_STR".
FuncNameLen: size_t,
) -> &Value;
pub(crate) fn LLVMRustCoverageHashBytes(Bytes: *const c_char, NumBytes: size_t) -> u64;
pub(crate) fn LLVMRustCoverageHashBytes(
Bytes: *const c_uchar, // See "PTR_LEN_STR".
NumBytes: size_t,
) -> u64;
pub(crate) fn LLVMRustCoverageWriteCovmapSectionNameToString(M: &Module, OutStr: &RustString);
pub(crate) safe fn LLVMRustCoverageWriteCovmapSectionNameToString(
M: &Module,
OutStr: &RustString,
);
pub(crate) safe fn LLVMRustCoverageWriteCovfunSectionNameToString(
M: &Module,
OutStr: &RustString,
);
pub(crate) safe fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString);
pub(crate) fn LLVMRustCoverageWriteCovfunSectionNameToString(M: &Module, OutStr: &RustString);
pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString);
pub(crate) fn LLVMRustCoverageMappingVersion() -> u32;
pub(crate) safe fn LLVMRustCoverageMappingVersion() -> u32;
pub(crate) fn LLVMRustDebugMetadataVersion() -> u32;
pub(crate) fn LLVMRustVersionMajor() -> u32;
pub(crate) fn LLVMRustVersionMinor() -> u32;
@ -2355,43 +2294,6 @@ unsafe extern "C" {
AlignInBits: u32,
) -> &'a DIGlobalVariableExpression;
pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>(
Builder: &DIBuilder<'a>,
Tag: c_uint,
Scope: &'a DIDescriptor,
Name: *const c_char,
NameLen: size_t,
File: &'a DIFile,
LineNo: c_uint,
Ty: &'a DIType,
AlwaysPreserve: bool,
Flags: DIFlags,
ArgNo: c_uint,
AlignInBits: u32,
) -> &'a DIVariable;
pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>(
Builder: &DIBuilder<'a>,
Lo: i64,
Count: i64,
) -> &'a DISubrange;
pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>(
Builder: &DIBuilder<'a>,
Ptr: *const Option<&'a DIDescriptor>,
Count: c_uint,
) -> &'a DIArray;
pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>(
Builder: &DIBuilder<'a>,
Val: &'a Value,
VarInfo: &'a DIVariable,
AddrOps: *const u64,
AddrOpsCount: c_uint,
DL: &'a DILocation,
InsertAtEnd: &'a BasicBlock,
);
pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>(
Builder: &DIBuilder<'a>,
Name: *const c_char,
@ -2498,7 +2400,6 @@ unsafe extern "C" {
UseWasmEH: bool,
) -> *mut TargetMachine;
pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine);
pub(crate) fn LLVMRustAddLibraryInfo<'a>(
PM: &PassManager<'a>,
M: &'a Module,

71
compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs Normal file
View file

@ -0,0 +1,71 @@
use libc::c_uint;
pub(crate) use self::fixed_kinds::*;
#[derive(Copy, Clone)]
#[repr(transparent)]
pub(crate) struct MetadataKindId(c_uint);
macro_rules! declare_fixed_metadata_kinds {
(
$(
FIXED_MD_KIND($variant:ident, $value:literal)
)*
) => {
// Use a submodule to group all declarations into one `#[expect(..)]`.
#[expect(dead_code)]
mod fixed_kinds {
use super::MetadataKindId;
$(
#[expect(non_upper_case_globals)]
pub(crate) const $variant: MetadataKindId = MetadataKindId($value);
)*
}
};
}
// Must be kept in sync with the corresponding static assertions in `RustWrapper.cpp`.
declare_fixed_metadata_kinds! {
FIXED_MD_KIND(MD_dbg, 0)
FIXED_MD_KIND(MD_tbaa, 1)
FIXED_MD_KIND(MD_prof, 2)
FIXED_MD_KIND(MD_fpmath, 3)
FIXED_MD_KIND(MD_range, 4)
FIXED_MD_KIND(MD_tbaa_struct, 5)
FIXED_MD_KIND(MD_invariant_load, 6)
FIXED_MD_KIND(MD_alias_scope, 7)
FIXED_MD_KIND(MD_noalias, 8)
FIXED_MD_KIND(MD_nontemporal, 9)
FIXED_MD_KIND(MD_mem_parallel_loop_access, 10)
FIXED_MD_KIND(MD_nonnull, 11)
FIXED_MD_KIND(MD_dereferenceable, 12)
FIXED_MD_KIND(MD_dereferenceable_or_null, 13)
FIXED_MD_KIND(MD_make_implicit, 14)
FIXED_MD_KIND(MD_unpredictable, 15)
FIXED_MD_KIND(MD_invariant_group, 16)
FIXED_MD_KIND(MD_align, 17)
FIXED_MD_KIND(MD_loop, 18)
FIXED_MD_KIND(MD_type, 19)
FIXED_MD_KIND(MD_section_prefix, 20)
FIXED_MD_KIND(MD_absolute_symbol, 21)
FIXED_MD_KIND(MD_associated, 22)
FIXED_MD_KIND(MD_callees, 23)
FIXED_MD_KIND(MD_irr_loop, 24)
FIXED_MD_KIND(MD_access_group, 25)
FIXED_MD_KIND(MD_callback, 26)
FIXED_MD_KIND(MD_preserve_access_index, 27)
FIXED_MD_KIND(MD_vcall_visibility, 28)
FIXED_MD_KIND(MD_noundef, 29)
FIXED_MD_KIND(MD_annotation, 30)
FIXED_MD_KIND(MD_nosanitize, 31)
FIXED_MD_KIND(MD_func_sanitize, 32)
FIXED_MD_KIND(MD_exclude, 33)
FIXED_MD_KIND(MD_memprof, 34)
FIXED_MD_KIND(MD_callsite, 35)
FIXED_MD_KIND(MD_kcfi_type, 36)
FIXED_MD_KIND(MD_pcsections, 37)
FIXED_MD_KIND(MD_DIAssignID, 38)
FIXED_MD_KIND(MD_coro_outside_frame, 39)
FIXED_MD_KIND(MD_mmra, 40)
FIXED_MD_KIND(MD_noalias_addrspace, 41)
}

5
compiler/rustc_codegen_llvm/src/llvm/mod.rs
View file

@ -11,13 +11,16 @@ use rustc_llvm::RustString;
pub(crate) use self::CallConv::*;
pub(crate) use self::CodeGenOptSize::*;
pub(crate) use self::MetadataType::*;
pub(crate) use self::conversions::*;
pub(crate) use self::ffi::*;
pub(crate) use self::metadata_kind::*;
use crate::common::AsCCharPtr;
mod conversions;
pub(crate) mod diagnostic;
pub(crate) mod enzyme_ffi;
mod ffi;
mod metadata_kind;
pub(crate) use self::enzyme_ffi::*;

2
compiler/rustc_codegen_llvm/src/llvm_util.rs
View file

@ -106,7 +106,7 @@ unsafe fn configure_llvm(sess: &Session) {
if sess.target.os == "emscripten"
&& !sess.opts.unstable_opts.emscripten_wasm_eh
&& sess.panic_strategy() == PanicStrategy::Unwind
&& sess.panic_strategy().unwinds()
{
add("-enable-emscripten-cxx-exceptions", false);
}

72
compiler/rustc_codegen_llvm/src/type_.rs
View file

@ -63,7 +63,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
///x Creates an integer type with the given number of bits, e.g., i24
pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type {
unsafe { llvm::LLVMIntTypeInContext(self.llcx(), num_bits as c_uint) }
llvm::LLVMIntTypeInContext(self.llcx(), num_bits as c_uint)
}
pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type {
@ -178,7 +178,7 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
}
fn type_i128(&self) -> &'ll Type {
unsafe { llvm::LLVMIntTypeInContext(self.llcx(), 128) }
self.type_ix(128)
}
fn type_isize(&self) -> &'ll Type {
@ -210,11 +210,11 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
}
fn type_ptr(&self) -> &'ll Type {
self.type_ptr_ext(AddressSpace::ZERO)
llvm_type_ptr(self.llcx())
}
fn type_ptr_ext(&self, address_space: AddressSpace) -> &'ll Type {
unsafe { llvm::LLVMPointerTypeInContext(self.llcx(), address_space.0) }
llvm_type_ptr_in_address_space(self.llcx(), address_space)
}
fn element_type(&self, ty: &'ll Type) -> &'ll Type {
@ -253,15 +253,15 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
}
}
impl Type {
/// Creates an integer type with the given number of bits, e.g., i24
pub(crate) fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type {
unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) }
}
pub(crate) fn llvm_type_ptr(llcx: &llvm::Context) -> &Type {
llvm_type_ptr_in_address_space(llcx, AddressSpace::ZERO)
}
pub(crate) fn ptr_llcx(llcx: &llvm::Context) -> &Type {
unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::ZERO.0) }
}
pub(crate) fn llvm_type_ptr_in_address_space<'ll>(
llcx: &'ll llvm::Context,
addr_space: AddressSpace,
) -> &'ll Type {
llvm::LLVMPointerTypeInContext(llcx, addr_space.0)
}
impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
@ -302,26 +302,14 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
fn add_type_metadata(&self, function: &'ll Value, typeid: &[u8]) {
let typeid_metadata = self.create_metadata(typeid);
unsafe {
let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
llvm::LLVMRustGlobalAddMetadata(
function,
llvm::MD_type as c_uint,
llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()),
)
}
let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
self.global_add_metadata_node(function, llvm::MD_type, &v);
}
fn set_type_metadata(&self, function: &'ll Value, typeid: &[u8]) {
let typeid_metadata = self.create_metadata(typeid);
unsafe {
let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
llvm::LLVMGlobalSetMetadata(
function,
llvm::MD_type as c_uint,
llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()),
)
}
let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
self.global_set_metadata_node(function, llvm::MD_type, &v);
}
fn typeid_metadata(&self, typeid: &[u8]) -> Option<&'ll Metadata> {
@ -329,32 +317,12 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
}
fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) {
let kcfi_type_metadata = self.const_u32(kcfi_typeid);
unsafe {
llvm::LLVMRustGlobalAddMetadata(
function,
llvm::MD_kcfi_type as c_uint,
llvm::LLVMMDNodeInContext2(
self.llcx,
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
1,
),
)
}
let kcfi_type_metadata = [llvm::LLVMValueAsMetadata(self.const_u32(kcfi_typeid))];
self.global_add_metadata_node(function, llvm::MD_kcfi_type, &kcfi_type_metadata);
}
fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) {
let kcfi_type_metadata = self.const_u32(kcfi_typeid);
unsafe {
llvm::LLVMGlobalSetMetadata(
function,
llvm::MD_kcfi_type as c_uint,
llvm::LLVMMDNodeInContext2(
self.llcx,
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
1,
),
)
}
let kcfi_type_metadata = [llvm::LLVMValueAsMetadata(self.const_u32(kcfi_typeid))];
self.global_set_metadata_node(function, llvm::MD_kcfi_type, &kcfi_type_metadata);
}
}

122
compiler/rustc_codegen_llvm/src/typetree.rs Normal file
View file

@ -0,0 +1,122 @@
use rustc_ast::expand::typetree::FncTree;
#[cfg(feature = "llvm_enzyme")]
use {
crate::attributes,
rustc_ast::expand::typetree::TypeTree as RustTypeTree,
std::ffi::{CString, c_char, c_uint},
};
use crate::llvm::{self, Value};
#[cfg(feature = "llvm_enzyme")]
fn to_enzyme_typetree(
rust_typetree: RustTypeTree,
_data_layout: &str,
llcx: &llvm::Context,
) -> llvm::TypeTree {
let mut enzyme_tt = llvm::TypeTree::new();
process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx);
enzyme_tt
}
#[cfg(feature = "llvm_enzyme")]
fn process_typetree_recursive(
enzyme_tt: &mut llvm::TypeTree,
rust_typetree: &RustTypeTree,
parent_indices: &[i64],
llcx: &llvm::Context,
) {
for rust_type in &rust_typetree.0 {
let concrete_type = match rust_type.kind {
rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything,
rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer,
rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer,
rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half,
rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float,
rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double,
rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128,
rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown,
};
let mut indices = parent_indices.to_vec();
if !parent_indices.is_empty() {
indices.push(rust_type.offset as i64);
} else if rust_type.offset == -1 {
indices.push(-1);
} else {
indices.push(rust_type.offset as i64);
}
enzyme_tt.insert(&indices, concrete_type, llcx);
if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer
&& !rust_type.child.0.is_empty()
{
process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx);
}
}
}
#[cfg(feature = "llvm_enzyme")]
pub(crate) fn add_tt<'ll>(
llmod: &'ll llvm::Module,
llcx: &'ll llvm::Context,
fn_def: &'ll Value,
tt: FncTree,
) {
let inputs = tt.args;
let ret_tt: RustTypeTree = tt.ret;
let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) };
let llvm_data_layout =
std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes())
.expect("got a non-UTF8 data-layout from LLVM");
let attr_name = "enzyme_type";
let c_attr_name = CString::new(attr_name).unwrap();
for (i, input) in inputs.iter().enumerate() {
unsafe {
let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx);
let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner);
let c_str = std::ffi::CStr::from_ptr(c_str);
let attr = llvm::LLVMCreateStringAttribute(
llcx,
c_attr_name.as_ptr(),
c_attr_name.as_bytes().len() as c_uint,
c_str.as_ptr(),
c_str.to_bytes().len() as c_uint,
);
attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]);
llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr());
}
}
unsafe {
let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx);
let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner);
let c_str = std::ffi::CStr::from_ptr(c_str);
let ret_attr = llvm::LLVMCreateStringAttribute(
llcx,
c_attr_name.as_ptr(),
c_attr_name.as_bytes().len() as c_uint,
c_str.as_ptr(),
c_str.to_bytes().len() as c_uint,
);
attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]);
llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr());
}
}
#[cfg(not(feature = "llvm_enzyme"))]
pub(crate) fn add_tt<'ll>(
_llmod: &'ll llvm::Module,
_llcx: &'ll llvm::Context,
_fn_def: &'ll Value,
_tt: FncTree,
) {
unimplemented!()
}

13
compiler/rustc_codegen_llvm/src/va_arg.rs
View file

@ -193,7 +193,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
// the offset again.
bx.switch_to_block(maybe_reg);
if gr_type && layout.align.abi.bytes() > 8 {
if gr_type && layout.align.bytes() > 8 {
reg_off_v = bx.add(reg_off_v, bx.const_i32(15));
reg_off_v = bx.and(reg_off_v, bx.const_i32(-16));
}
@ -291,7 +291,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>(
bx.inbounds_ptradd(va_list_addr, bx.const_usize(1)) // fpr
};
let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align.abi);
let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align);
// "Align" the register count when the type is passed as `i64`.
if is_i64 || (is_f64 && is_soft_float_abi) {
@ -329,7 +329,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>(
// Increase the used-register count.
let reg_incr = if is_i64 || (is_f64 && is_soft_float_abi) { 2 } else { 1 };
let new_num_regs = bx.add(num_regs, bx.cx.const_u8(reg_incr));
bx.store(new_num_regs, num_regs_addr, dl.i8_align.abi);
bx.store(new_num_regs, num_regs_addr, dl.i8_align);
bx.br(end);
@ -339,7 +339,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>(
let mem_addr = {
bx.switch_to_block(in_mem);
bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align.abi);
bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align);
// Everything in the overflow area is rounded up to a size of at least 4.
let overflow_area_align = Align::from_bytes(4).unwrap();
@ -738,6 +738,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>(
src_align,
bx.const_u32(layout.layout.size().bytes() as u32),
MemFlags::empty(),
None,
);
tmp
} else {
@ -760,7 +761,7 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
// byte boundary if alignment needed by type exceeds 8 byte boundary.
// It isn't stated explicitly in the standard, but in practice we use
// alignment greater than 16 where necessary.
if layout.layout.align.abi.bytes() > 8 {
if layout.layout.align.bytes() > 8 {
unreachable!("all instances of VaArgSafe have an alignment <= 8");
}
@ -813,7 +814,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
let va_ndx_offset = va_reg_offset + 4;
let offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_ndx_offset));
let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi);
let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align);
let offset = round_up_to_alignment(bx, offset, layout.align.abi);
let slot_size = layout.size.align_to(Align::from_bytes(4).unwrap()).bytes() as i32;

4
compiler/rustc_codegen_ssa/Cargo.toml
View file

@ -8,9 +8,7 @@ edition = "2024"
ar_archive_writer = "0.5"
bitflags = "2.4.1"
bstr = "1.11.3"
# `cc` updates often break things, so we pin it here. Cargo enforces "max 1 semver-compat version
# per crate", so if you change this, you need to also change it in `rustc_llvm` and `rustc_windows_rc`.
cc = "=1.2.16"
find-msvc-tools = "0.1.2"
itertools = "0.12"
pathdiff = "0.2.0"
regex = "1.4"

4
compiler/rustc_codegen_ssa/messages.ftl
View file

@ -97,10 +97,10 @@ codegen_ssa_invalid_literal_value = invalid literal value
codegen_ssa_invalid_monomorphization_basic_float_type = invalid monomorphization of `{$name}` intrinsic: expected basic float type, found `{$ty}`
codegen_ssa_invalid_monomorphization_basic_integer_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer type, found `{$ty}`
codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer or pointer type, found `{$ty}`
codegen_ssa_invalid_monomorphization_basic_integer_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer type, found `{$ty}`
codegen_ssa_invalid_monomorphization_cannot_return = invalid monomorphization of `{$name}` intrinsic: cannot return `{$ret_ty}`, expected `u{$expected_int_bits}` or `[u8; {$expected_bytes}]`
codegen_ssa_invalid_monomorphization_cast_wide_pointer = invalid monomorphization of `{$name}` intrinsic: cannot cast wide pointer `{$ty}`

22
compiler/rustc_codegen_ssa/src/back/link.rs
View file

@ -9,7 +9,7 @@ use std::path::{Path, PathBuf};
use std::process::{Output, Stdio};
use std::{env, fmt, fs, io, mem, str};
use cc::windows_registry;
use find_msvc_tools;
use itertools::Itertools;
use regex::Regex;
use rustc_arena::TypedArena;
@ -47,8 +47,8 @@ use rustc_span::Symbol;
use rustc_target::spec::crt_objects::CrtObjects;
use rustc_target::spec::{
BinaryFormat, Cc, LinkOutputKind, LinkSelfContainedComponents, LinkSelfContainedDefault,
LinkerFeatures, LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy, RelocModel, RelroLevel,
SanitizerSet, SplitDebuginfo,
LinkerFeatures, LinkerFlavor, LinkerFlavorCli, Lld, RelocModel, RelroLevel, SanitizerSet,
SplitDebuginfo,
};
use tracing::{debug, info, warn};
@ -79,6 +79,7 @@ pub fn link_binary(
codegen_results: CodegenResults,
metadata: EncodedMetadata,
outputs: &OutputFilenames,
codegen_backend: &'static str,
) {
let _timer = sess.timer("link_binary");
let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
@ -154,6 +155,7 @@ pub fn link_binary(
&codegen_results,
&metadata,
path.as_ref(),
codegen_backend,
);
}
}
@ -680,6 +682,7 @@ fn link_natively(
codegen_results: &CodegenResults,
metadata: &EncodedMetadata,
tmpdir: &Path,
codegen_backend: &'static str,
) {
info!("preparing {:?} to {:?}", crate_type, out_filename);
let (linker_path, flavor) = linker_and_flavor(sess);
@ -705,6 +708,7 @@ fn link_natively(
codegen_results,
metadata,
self_contained_components,
codegen_backend,
);
linker::disable_localization(&mut cmd);
@ -877,9 +881,9 @@ fn link_natively(
// All Microsoft `link.exe` linking ror codes are
// four digit numbers in the range 1000 to 9999 inclusive
if is_msvc_link_exe && (code < 1000 || code > 9999) {
let is_vs_installed = windows_registry::find_vs_version().is_ok();
let is_vs_installed = find_msvc_tools::find_vs_version().is_ok();
let has_linker =
windows_registry::find_tool(&sess.target.arch, "link.exe").is_some();
find_msvc_tools::find_tool(&sess.target.arch, "link.exe").is_some();
sess.dcx().emit_note(errors::LinkExeUnexpectedError);
@ -2208,6 +2212,7 @@ fn linker_with_args(
codegen_results: &CodegenResults,
metadata: &EncodedMetadata,
self_contained_components: LinkSelfContainedComponents,
codegen_backend: &'static str,
) -> Command {
let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
let cmd = &mut *super::linker::get_linker(
@ -2216,6 +2221,7 @@ fn linker_with_args(
flavor,
self_contained_components.are_any_components_enabled(),
&codegen_results.crate_info.target_cpu,
codegen_backend,
);
let link_output_kind = link_output_kind(sess, crate_type);
@ -2512,10 +2518,10 @@ fn add_order_independent_options(
if sess.target.os == "emscripten" {
cmd.cc_arg(if sess.opts.unstable_opts.emscripten_wasm_eh {
"-fwasm-exceptions"
} else if sess.panic_strategy() == PanicStrategy::Abort {
"-sDISABLE_EXCEPTION_CATCHING=1"
} else {
} else if sess.panic_strategy().unwinds() {
"-sDISABLE_EXCEPTION_CATCHING=0"
} else {
"-sDISABLE_EXCEPTION_CATCHING=1"
});
}

Some files were not shown because too many files have changed in this diff Show more